fix(database): enforce Flyway migration governance
This commit is contained in:
xqzp2026
@@ -25,9 +25,10 @@ xiaoqu-agent-service/
|
||||
├── mvnw / .mvn/wrapper/ # Maven Wrapper（本机无全局 mvn）
|
||||
├── pom.xml
|
||||
├── sql/
|
||||
│ ├── V1__init.sql # 自有库 xiaoqu_agent_svc_d 建表（手工执行，未接 flyway）
|
||||
│ ├── V2__p4_admin.sql # 旧库升级管理员密码哈希字段
|
||||
│ └── seed-dev.sql # dev 种子数据（运营作用域授权，不含固定 ak）
|
||||
│ ├── README.md # migration 编写、回滚与不可变规则
|
||||
│ ├── bootstrap/create-database.sql # 首次创建数据库（结构不在这里维护）
|
||||
│ ├── rollback/ # 与每个版本配套的人工回滚方案
|
||||
│ └── seed-dev.sql # dev 种子数据，不进入 Flyway
|
||||
├── front/ # Vue 3 管理后台，Maven 打包时自动构建进 jar
|
||||
├── src/main/java/me/iiv/agent/
|
||||
│ ├── AgentServiceApplication.java
|
||||
@@ -42,6 +43,7 @@ xiaoqu-agent-service/
|
||||
│ ├── entity/m0、m2、svc mapper/m0、m1、m2、svc # codegen 生成物（提交进仓库）
|
||||
├── src/main/resources/
|
||||
│ ├── application.yml # MCP server: protocol=STREAMABLE, type=SYNC，端点 /mcp
|
||||
│ ├── db/migration/ # Flyway V1/V2... + 历史文件校验和清单
|
||||
│ └── application-dev.yml # 四数据源 + Redis，账号密码必须由环境变量提供
|
||||
└── src/test/java/me/iiv/agent/ # 权限回归测试 + run-once 代码生成器
|
||||
```
|
||||
@@ -86,8 +88,8 @@ xiaoqu-agent-service/
|
||||
## 启动
|
||||
|
||||
```bash
|
||||
# 首次：由数据库管理员建自有库；启动一次同步工具后，再创建 dev 作用域
|
||||
mysql -h192.168.1.181 -u<svc_db_admin> -p < sql/V1__init.sql
|
||||
# 首次仅由数据库管理员创建空库；表结构由应用启动时的 Flyway 自动迁移
|
||||
mysql -h192.168.1.181 -u<svc_db_admin> -p < sql/bootstrap/create-database.sql
|
||||
# 启动一次让 ToolSyncService 同步 tool 表后：
|
||||
mysql -h192.168.1.181 -u<svc_db_admin> -p xiaoqu_agent_svc_d < sql/seed-dev.sql
|
||||
|
||||
@@ -101,13 +103,15 @@ export M2_DB_USERNAME=<readonly_user>
|
||||
export M2_DB_PASSWORD=<password>
|
||||
export SVC_DB_USERNAME=<svc_user>
|
||||
export SVC_DB_PASSWORD=<password>
|
||||
# 已有 V1 库首次接入时设 true；确认 flyway_schema_history 后可设 false
|
||||
export SVC_FLYWAY_BASELINE_ON_MIGRATE=true
|
||||
export REDIS_PASSWORD=<password>
|
||||
# 仅首次、且 admin_user 为空时创建管理员；创建完成后可移除这三个变量
|
||||
export ADMIN_BOOTSTRAP_ACCOUNT=<admin_account>
|
||||
export ADMIN_BOOTSTRAP_PASSWORD=<strong_password>
|
||||
export ADMIN_BOOTSTRAP_NAME=<display_name>
|
||||
|
||||
./mvnw clean package -DskipTests
|
||||
./mvnw clean package
|
||||
java -jar target/xiaoqu-agent-service.jar
|
||||
# 或
|
||||
./mvnw spring-boot:run
|
||||
@@ -117,14 +121,15 @@ nohup java -jar target/xiaoqu-agent-service.jar > target/app.log 2>&1 &
|
||||
|
||||
- 不提供默认 profile、数据库账号或密码；缺失配置时不得启动。
|
||||
- dev 端口 **8088**，管理后台 `http://localhost:8088/`，MCP 端点 `http://localhost:8088/mcp`。
|
||||
- 已存在 V1 数据库时，部署 P4 前先执行 `mysql ... < sql/V2__p4_admin.sql`。
|
||||
- Flyway 在 Web 服务启动前执行 `validate + migrate`；checksum、版本或 SQL 执行失败时应用不会启动。
|
||||
- 新增 SQL 必须遵循 [sql/README.md](sql/README.md)，禁止修改已合并的历史 migration。
|
||||
|
||||
### 部署脚本（推荐，本机 dev 部署）
|
||||
|
||||
```bash
|
||||
cp deploy/.env.example deploy/.env.dev # 填写 dev 凭据（已 gitignore，不入库）
|
||||
./deploy/deploy.sh build # 构建 jar
|
||||
./deploy/deploy.sh restart # 停旧起新 + 健康检查（/mcp 返回 401/200 即就绪）
|
||||
./deploy/deploy.sh build # 全量测试 + 构建 jar（含 migration 治理测试）
|
||||
./deploy/deploy.sh restart # 停旧起新；启动先迁移，失败立即报告并阻断
|
||||
./deploy/deploy.sh status | log | stop # 状态 / 跟踪日志 / 停止
|
||||
```
|
||||
|
||||
|
||||
@@ -8,4 +8,6 @@ M2_DB_USERNAME=root
|
||||
M2_DB_PASSWORD=
|
||||
SVC_DB_USERNAME=root
|
||||
SVC_DB_PASSWORD=
|
||||
# 默认拒绝为非空库自动建基线；仅已有 V1 库首次接入时临时改为 true
|
||||
SVC_FLYWAY_BASELINE_ON_MIGRATE=false
|
||||
REDIS_PASSWORD=
|
||||
|
||||
@@ -28,6 +28,12 @@ is_running() {
|
||||
health_check() {
|
||||
# /mcp 未带 ak 会返回 401，带 ak 返回 200——两者都证明服务已起来
|
||||
for i in $(seq 1 90); do
|
||||
if ! is_running; then
|
||||
echo "[ERROR] 服务进程已退出，可能是数据库迁移或启动失败" >&2
|
||||
tail -40 "$LOG_FILE" >&2 || true
|
||||
rm -f "$PID_FILE"
|
||||
return 1
|
||||
fi
|
||||
code=$(curl -s -o /dev/null -w "%{http_code}" -X POST "http://127.0.0.1:$PORT/mcp" \
|
||||
-H 'Content-Type: application/json' \
|
||||
-H 'Accept: application/json, text/event-stream' \
|
||||
@@ -44,7 +50,7 @@ health_check() {
|
||||
|
||||
do_build() {
|
||||
cd "$APP_HOME"
|
||||
./mvnw clean package -DskipTests
|
||||
./mvnw clean package
|
||||
echo "[OK] 构建完成: $JAR"
|
||||
}
|
||||
|
||||
|
||||
@@ -64,6 +64,14 @@
|
||||
<groupId>com.mysql</groupId>
|
||||
<artifactId>mysql-connector-j</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.flywaydb</groupId>
|
||||
<artifactId>flyway-core</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.flywaydb</groupId>
|
||||
<artifactId>flyway-mysql</artifactId>
|
||||
</dependency>
|
||||
|
||||
<!-- Redis（db5，用户热缓存/字典，P1 仅做连通性自检） -->
|
||||
<dependency>
|
||||
@@ -82,6 +90,11 @@
|
||||
<artifactId>spring-boot-starter-test</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.h2database</groupId>
|
||||
<artifactId>h2</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<!-- codegen（run-once，仅 test scope，不进主启动流程） -->
|
||||
<dependency>
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
# 数据库迁移规范
|
||||
|
||||
自有库结构由 Flyway 管理，运行时只扫描 `src/main/resources/db/migration`。
|
||||
|
||||
1. 已合并的 `V<N>__description.sql` 永不修改、删除或改名；Flyway 与测试都会校验历史 checksum。
|
||||
2. 每次结构或数据变更新增下一个连续版本，并同步新增 `sql/rollback/V<N>__description.down.sql`。
|
||||
3. Migration 只操作当前 svc schema，禁止包含 `CREATE DATABASE`、`USE` 和环境专属数据。
|
||||
4. DDL 优先向后兼容；删除字段或表必须先经过废弃版本。大表变更必须给出锁表时间与分批方案。
|
||||
5. `seed-dev.sql` 仅用于开发数据，不进入 Flyway。数据库首次创建使用 `bootstrap/create-database.sql`。
|
||||
6. 合并前必须运行 `./mvnw clean package`；部署启动时 Flyway 会再次 validate 并按版本迁移，失败则应用不会启动。
|
||||
|
||||
`baseline-on-migrate` 默认关闭，避免连错非空库时被静默接管。已有 V1 数据库首次接入时，显式设置 `SVC_FLYWAY_BASELINE_ON_MIGRATE=true`：Flyway 记录 V1 基线并只执行 V2 以后版本；确认生成 `flyway_schema_history` 后立即恢复为 `false`。空数据库无需开启，会正常执行全部版本。
|
||||
@@ -0,0 +1,3 @@
|
||||
CREATE DATABASE IF NOT EXISTS xiaoqu_agent_svc_d
|
||||
DEFAULT CHARACTER SET utf8mb4
|
||||
COLLATE utf8mb4_general_ci;
|
||||
@@ -0,0 +1,8 @@
|
||||
-- 仅用于尚未承载业务数据的环境，按依赖逆序删除整个服务自有结构。
|
||||
DROP TABLE IF EXISTS admin_user;
|
||||
DROP TABLE IF EXISTS audit_log;
|
||||
DROP TABLE IF EXISTS token_scope;
|
||||
DROP TABLE IF EXISTS scope_tool;
|
||||
DROP TABLE IF EXISTS tool;
|
||||
DROP TABLE IF EXISTS agent_scope;
|
||||
DROP TABLE IF EXISTS agent_api_key;
|
||||
@@ -0,0 +1,15 @@
|
||||
-- PBKDF2 哈希长度超过 64 时拒绝回退，避免截断管理员密码。
|
||||
DELIMITER //
|
||||
CREATE PROCEDURE rollback_v2_p4_admin()
|
||||
BEGIN
|
||||
IF EXISTS (SELECT 1 FROM admin_user WHERE CHAR_LENGTH(password_hash) > 64) THEN
|
||||
SIGNAL SQLSTATE '45000'
|
||||
SET MESSAGE_TEXT = 'V2 rollback blocked: PBKDF2 password hashes already exist';
|
||||
END IF;
|
||||
ALTER TABLE admin_user
|
||||
MODIFY COLUMN password_hash CHAR(64) NOT NULL COMMENT '密码 SHA-256 hex';
|
||||
END //
|
||||
DELIMITER ;
|
||||
|
||||
CALL rollback_v2_p4_admin();
|
||||
DROP PROCEDURE rollback_v2_p4_admin;
|
||||
@@ -7,6 +7,7 @@ import org.apache.ibatis.session.SqlSessionFactory;
|
||||
import org.mybatis.spring.annotation.MapperScan;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.boot.autoconfigure.flyway.FlywayDataSource;
|
||||
import org.springframework.boot.jdbc.DataSourceBuilder;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
@@ -23,6 +24,7 @@ import javax.sql.DataSource;
|
||||
public class SvcDataSourceConfig {
|
||||
|
||||
@Bean(name = "svcDataSource")
|
||||
@FlywayDataSource
|
||||
@ConfigurationProperties(prefix = "app.datasource.svc")
|
||||
public DataSource svcDataSource() {
|
||||
return DataSourceBuilder.create().type(HikariDataSource.class).build();
|
||||
|
||||
@@ -18,6 +18,16 @@ app:
|
||||
spring:
|
||||
application:
|
||||
name: xiaoqu-agent-service
|
||||
flyway:
|
||||
enabled: true
|
||||
locations: classpath:db/migration
|
||||
baseline-on-migrate: ${SVC_FLYWAY_BASELINE_ON_MIGRATE:false}
|
||||
baseline-version: 1
|
||||
baseline-description: legacy V1 schema
|
||||
validate-on-migrate: true
|
||||
validate-migration-naming: true
|
||||
out-of-order: false
|
||||
clean-disabled: true
|
||||
ai:
|
||||
mcp:
|
||||
server:
|
||||
|
||||
+2
-6
@@ -1,13 +1,9 @@
|
||||
-- =============================================================
|
||||
-- xiaoqu-agent-service 自有库初始化（dev 环境）
|
||||
-- xiaoqu-agent-service 自有库基线结构
|
||||
-- 设计文档：doc/agent-mcp-service-design.md §7
|
||||
-- 执行：mysql -h192.168.1.181 -uroot -p < sql/V1__init.sql
|
||||
-- 由 Flyway 执行；数据库本身由 sql/bootstrap/create-database.sql 创建
|
||||
-- =============================================================
|
||||
|
||||
CREATE DATABASE IF NOT EXISTS xiaoqu_agent_svc_d DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
|
||||
|
||||
USE xiaoqu_agent_svc_d;
|
||||
|
||||
-- token（ak）：属于某个人（user_id 逻辑关联 m0.user），只存 SHA-256 哈希
|
||||
CREATE TABLE IF NOT EXISTS agent_api_key (
|
||||
id BIGINT NOT NULL AUTO_INCREMENT COMMENT '主键',
|
||||
-2
@@ -1,4 +1,2 @@
|
||||
USE xiaoqu_agent_svc_d;
|
||||
|
||||
ALTER TABLE admin_user
|
||||
MODIFY COLUMN password_hash VARCHAR(255) NOT NULL COMMENT 'PBKDF2 加盐密码哈希';
|
||||
@@ -0,0 +1,2 @@
|
||||
f8ec43a4fda81382d5675bc59fe7f4339f2e38a5b4241c260c3569e33b43a26d V1__init.sql
|
||||
0ad892601e891a18712b1b6a613c392c94e63492dc14f4f779375c092a3acab6 V2__p4_admin.sql
|
||||
@@ -0,0 +1,62 @@
|
||||
package me.iiv.agent.config;
|
||||
|
||||
import org.flywaydb.core.Flyway;
|
||||
import org.h2.jdbcx.JdbcDataSource;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.Statement;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
class FlywayMigrationTest {
|
||||
|
||||
@Test
|
||||
void emptyDatabaseMigratesThroughLatestVersion() throws Exception {
|
||||
JdbcDataSource dataSource = dataSource("empty");
|
||||
Flyway flyway = flyway(dataSource, false);
|
||||
|
||||
flyway.migrate();
|
||||
|
||||
assertEquals("2", flyway.info().current().getVersion().getVersion());
|
||||
assertEquals(255, passwordHashLength(dataSource));
|
||||
}
|
||||
|
||||
@Test
|
||||
void legacyV1DatabaseIsBaselinedThenUpgraded() throws Exception {
|
||||
JdbcDataSource dataSource = dataSource("legacy");
|
||||
try (Connection connection = dataSource.getConnection(); Statement statement = connection.createStatement()) {
|
||||
statement.execute("CREATE TABLE admin_user (id BIGINT AUTO_INCREMENT PRIMARY KEY, password_hash CHAR(64) NOT NULL)");
|
||||
}
|
||||
Flyway flyway = flyway(dataSource, true);
|
||||
|
||||
flyway.migrate();
|
||||
|
||||
assertEquals("2", flyway.info().current().getVersion().getVersion());
|
||||
assertEquals(255, passwordHashLength(dataSource));
|
||||
}
|
||||
|
||||
private static Flyway flyway(JdbcDataSource dataSource, boolean baseline) {
|
||||
return Flyway.configure().dataSource(dataSource).locations("classpath:db/migration")
|
||||
.baselineOnMigrate(baseline).baselineVersion("1").validateMigrationNaming(true)
|
||||
.cleanDisabled(true).load();
|
||||
}
|
||||
|
||||
private static JdbcDataSource dataSource(String name) {
|
||||
JdbcDataSource dataSource = new JdbcDataSource();
|
||||
dataSource.setURL("jdbc:h2:mem:flyway_" + name + ";MODE=MySQL;DATABASE_TO_LOWER=TRUE;DB_CLOSE_DELAY=-1");
|
||||
dataSource.setUser("sa");
|
||||
return dataSource;
|
||||
}
|
||||
|
||||
private static int passwordHashLength(JdbcDataSource dataSource) throws Exception {
|
||||
try (Connection connection = dataSource.getConnection();
|
||||
Statement statement = connection.createStatement();
|
||||
ResultSet result = statement.executeQuery("SELECT CHARACTER_MAXIMUM_LENGTH FROM INFORMATION_SCHEMA.COLUMNS "
|
||||
+ "WHERE TABLE_NAME='admin_user' AND COLUMN_NAME='password_hash'")) {
|
||||
result.next();
|
||||
return result.getInt(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
package me.iiv.agent.config;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.security.MessageDigest;
|
||||
import java.util.HexFormat;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.TreeMap;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.IntStream;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
class MigrationGovernanceTest {
|
||||
|
||||
private static final Path MIGRATIONS = Path.of("src/main/resources/db/migration");
|
||||
private static final Path ROLLBACKS = Path.of("sql/rollback");
|
||||
private static final Pattern NAME = Pattern.compile("V(\\d+)__[a-z0-9_]+\\.sql");
|
||||
|
||||
@Test
|
||||
void migrationsAreOrderedAndHaveRollbackPlans() throws Exception {
|
||||
assertTrue(Files.isDirectory(MIGRATIONS), "缺少 Flyway migration 目录");
|
||||
List<Path> files;
|
||||
try (var stream = Files.list(MIGRATIONS)) {
|
||||
files = stream.filter(path -> path.getFileName().toString().endsWith(".sql")).sorted().toList();
|
||||
}
|
||||
assertFalse(files.isEmpty(), "至少保留一个基线 migration");
|
||||
|
||||
Map<Integer, Path> versions = new TreeMap<>();
|
||||
for (Path file : files) {
|
||||
Matcher matcher = NAME.matcher(file.getFileName().toString());
|
||||
assertTrue(matcher.matches(), () -> "迁移文件名不合规: " + file.getFileName());
|
||||
int version = Integer.parseInt(matcher.group(1));
|
||||
assertFalse(versions.containsKey(version), () -> "迁移版本重复: V" + version);
|
||||
versions.put(version, file);
|
||||
String sql = Files.readString(file);
|
||||
assertFalse(Pattern.compile("(?im)^\\s*(USE|CREATE\\s+DATABASE)\\b").matcher(sql).find(),
|
||||
() -> "migration 只能操作当前 svc schema: " + file.getFileName());
|
||||
String rollback = file.getFileName().toString().replace(".sql", ".down.sql");
|
||||
assertTrue(Files.isRegularFile(ROLLBACKS.resolve(rollback)), () -> "缺少回滚方案: " + rollback);
|
||||
}
|
||||
List<Integer> expected = IntStream.rangeClosed(1, versions.size()).boxed().toList();
|
||||
assertEquals(expected, versions.keySet().stream().toList(), "迁移版本必须从 V1 连续递增");
|
||||
}
|
||||
|
||||
@Test
|
||||
void committedMigrationChecksumsMatchManifest() throws Exception {
|
||||
Path manifest = MIGRATIONS.resolve("checksums.sha256");
|
||||
assertTrue(Files.isRegularFile(manifest), "缺少 migration 校验和清单");
|
||||
Map<String, String> checksums = new TreeMap<>();
|
||||
for (String line : Files.readAllLines(manifest)) {
|
||||
if (line.isBlank() || line.startsWith("#")) continue;
|
||||
String[] parts = line.trim().split("\\s+", 2);
|
||||
assertEquals(2, parts.length, "校验和格式应为: sha256 文件名");
|
||||
checksums.put(parts[1], parts[0]);
|
||||
}
|
||||
Set<String> migrationNames;
|
||||
try (var stream = Files.list(MIGRATIONS)) {
|
||||
migrationNames = stream.map(path -> path.getFileName().toString())
|
||||
.filter(name -> name.endsWith(".sql")).collect(java.util.stream.Collectors.toSet());
|
||||
}
|
||||
assertEquals(migrationNames, checksums.keySet(), "校验和清单必须覆盖全部 migration");
|
||||
MessageDigest digest = MessageDigest.getInstance("SHA-256");
|
||||
for (Map.Entry<String, String> entry : checksums.entrySet()) {
|
||||
String actual = HexFormat.of().formatHex(digest.digest(Files.readAllBytes(MIGRATIONS.resolve(entry.getKey()))));
|
||||
assertEquals(entry.getValue(), actual, () -> "历史 migration 被修改: " + entry.getKey());
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user