From 87659f9de9f1444d65ab253366e809e8220eab7c Mon Sep 17 00:00:00 2001 From: xqzp2026 Date: Thu, 3 Sep 2026 18:28:10 +0930 Subject: [PATCH] fix(database): enforce Flyway migration governance --- xiaoqu-agent-service/README.md | 23 +++--- xiaoqu-agent-service/deploy/.env.example | 2 + xiaoqu-agent-service/deploy/deploy.sh | 8 +- xiaoqu-agent-service/pom.xml | 13 ++++ xiaoqu-agent-service/sql/README.md | 12 +++ .../sql/bootstrap/create-database.sql | 3 + .../sql/rollback/V1__init.down.sql | 8 ++ .../sql/rollback/V2__p4_admin.down.sql | 15 ++++ .../iiv/agent/config/SvcDataSourceConfig.java | 2 + .../src/main/resources/application.yml | 10 +++ .../main/resources/db/migration}/V1__init.sql | 8 +- .../resources/db/migration}/V2__p4_admin.sql | 2 - .../resources/db/migration/checksums.sha256 | 2 + .../iiv/agent/config/FlywayMigrationTest.java | 62 +++++++++++++++ .../agent/config/MigrationGovernanceTest.java | 76 +++++++++++++++++++ 15 files changed, 228 insertions(+), 18 deletions(-) create mode 100644 xiaoqu-agent-service/sql/README.md create mode 100644 xiaoqu-agent-service/sql/bootstrap/create-database.sql create mode 100644 xiaoqu-agent-service/sql/rollback/V1__init.down.sql create mode 100644 xiaoqu-agent-service/sql/rollback/V2__p4_admin.down.sql rename xiaoqu-agent-service/{sql => src/main/resources/db/migration}/V1__init.sql (96%) rename xiaoqu-agent-service/{sql => src/main/resources/db/migration}/V2__p4_admin.sql (82%) create mode 100644 xiaoqu-agent-service/src/main/resources/db/migration/checksums.sha256 create mode 100644 xiaoqu-agent-service/src/test/java/me/iiv/agent/config/FlywayMigrationTest.java create mode 100644 xiaoqu-agent-service/src/test/java/me/iiv/agent/config/MigrationGovernanceTest.java diff --git a/xiaoqu-agent-service/README.md b/xiaoqu-agent-service/README.md index 17b0caa..c92ff3c 100644 --- a/xiaoqu-agent-service/README.md +++ b/xiaoqu-agent-service/README.md @@ -25,9 +25,10 @@ xiaoqu-agent-service/ ├── mvnw / .mvn/wrapper/ # Maven Wrapper（本机无全局 mvn） ├── pom.xml ├── sql/ -│ ├── V1__init.sql # 自有库 xiaoqu_agent_svc_d 建表（手工执行，未接 flyway） -│ ├── V2__p4_admin.sql # 旧库升级管理员密码哈希字段 -│ └── seed-dev.sql # dev 种子数据（运营作用域授权，不含固定 ak） +│ ├── README.md # migration 编写、回滚与不可变规则 +│ ├── bootstrap/create-database.sql # 首次创建数据库（结构不在这里维护） +│ ├── rollback/ # 与每个版本配套的人工回滚方案 +│ └── seed-dev.sql # dev 种子数据，不进入 Flyway ├── front/ # Vue 3 管理后台，Maven 打包时自动构建进 jar ├── src/main/java/me/iiv/agent/ │ ├── AgentServiceApplication.java @@ -42,6 +43,7 @@ xiaoqu-agent-service/ │ ├── entity/m0、m2、svc mapper/m0、m1、m2、svc # codegen 生成物（提交进仓库） ├── src/main/resources/ │ ├── application.yml # MCP server: protocol=STREAMABLE, type=SYNC，端点 /mcp +│ ├── db/migration/ # Flyway V1/V2... + 历史文件校验和清单 │ └── application-dev.yml # 四数据源 + Redis，账号密码必须由环境变量提供 └── src/test/java/me/iiv/agent/ # 权限回归测试 + run-once 代码生成器 ``` @@ -86,8 +88,8 @@ xiaoqu-agent-service/ ## 启动 ```bash -# 首次：由数据库管理员建自有库；启动一次同步工具后，再创建 dev 作用域 -mysql -h192.168.1.181 -u -p < sql/V1__init.sql +# 首次仅由数据库管理员创建空库；表结构由应用启动时的 Flyway 自动迁移 +mysql -h192.168.1.181 -u -p < sql/bootstrap/create-database.sql # 启动一次让 ToolSyncService 同步 tool 表后： mysql -h192.168.1.181 -u -p xiaoqu_agent_svc_d < sql/seed-dev.sql @@ -101,13 +103,15 @@ export M2_DB_USERNAME= export M2_DB_PASSWORD= export SVC_DB_USERNAME= export SVC_DB_PASSWORD= +# 已有 V1 库首次接入时设 true；确认 flyway_schema_history 后可设 false +export SVC_FLYWAY_BASELINE_ON_MIGRATE=true export REDIS_PASSWORD= # 仅首次、且 admin_user 为空时创建管理员；创建完成后可移除这三个变量 export ADMIN_BOOTSTRAP_ACCOUNT= export ADMIN_BOOTSTRAP_PASSWORD= export ADMIN_BOOTSTRAP_NAME= -./mvnw clean package -DskipTests +./mvnw clean package java -jar target/xiaoqu-agent-service.jar # 或 ./mvnw spring-boot:run @@ -117,14 +121,15 @@ nohup java -jar target/xiaoqu-agent-service.jar > target/app.log 2>&1 & - 不提供默认 profile、数据库账号或密码；缺失配置时不得启动。 - dev 端口 **8088**，管理后台 `http://localhost:8088/`，MCP 端点 `http://localhost:8088/mcp`。 -- 已存在 V1 数据库时，部署 P4 前先执行 `mysql ... < sql/V2__p4_admin.sql`。 +- Flyway 在 Web 服务启动前执行 `validate + migrate`；checksum、版本或 SQL 执行失败时应用不会启动。 +- 新增 SQL 必须遵循 [sql/README.md](sql/README.md)，禁止修改已合并的历史 migration。 ### 部署脚本（推荐，本机 dev 部署） ```bash cp deploy/.env.example deploy/.env.dev # 填写 dev 凭据（已 gitignore，不入库） -./deploy/deploy.sh build # 构建 jar -./deploy/deploy.sh restart # 停旧起新 + 健康检查（/mcp 返回 401/200 即就绪） +./deploy/deploy.sh build # 全量测试 + 构建 jar（含 migration 治理测试） +./deploy/deploy.sh restart # 停旧起新；启动先迁移，失败立即报告并阻断 ./deploy/deploy.sh status | log | stop # 状态 / 跟踪日志 / 停止 ``` diff --git a/xiaoqu-agent-service/deploy/.env.example b/xiaoqu-agent-service/deploy/.env.example index bd6712a..82409d0 100644 --- a/xiaoqu-agent-service/deploy/.env.example +++ b/xiaoqu-agent-service/deploy/.env.example @@ -8,4 +8,6 @@ M2_DB_USERNAME=root M2_DB_PASSWORD= SVC_DB_USERNAME=root SVC_DB_PASSWORD= +# 默认拒绝为非空库自动建基线；仅已有 V1 库首次接入时临时改为 true +SVC_FLYWAY_BASELINE_ON_MIGRATE=false REDIS_PASSWORD= diff --git a/xiaoqu-agent-service/deploy/deploy.sh b/xiaoqu-agent-service/deploy/deploy.sh index 8e86c20..49e6981 100755 --- a/xiaoqu-agent-service/deploy/deploy.sh +++ b/xiaoqu-agent-service/deploy/deploy.sh @@ -28,6 +28,12 @@ is_running() { health_check() { # /mcp 未带 ak 会返回 401，带 ak 返回 200——两者都证明服务已起来 for i in $(seq 1 90); do + if ! is_running; then + echo "[ERROR] 服务进程已退出，可能是数据库迁移或启动失败" >&2 + tail -40 "$LOG_FILE" >&2 || true + rm -f "$PID_FILE" + return 1 + fi code=$(curl -s -o /dev/null -w "%{http_code}" -X POST "http://127.0.0.1:$PORT/mcp" \ -H 'Content-Type: application/json' \ -H 'Accept: application/json, text/event-stream' \ @@ -44,7 +50,7 @@ health_check() { do_build() { cd "$APP_HOME" - ./mvnw clean package -DskipTests + ./mvnw clean package echo "[OK] 构建完成: $JAR" } diff --git a/xiaoqu-agent-service/pom.xml b/xiaoqu-agent-service/pom.xml index 675c731..1625a90 100644 --- a/xiaoqu-agent-service/pom.xml +++ b/xiaoqu-agent-service/pom.xml @@ -64,6 +64,14 @@ com.mysql mysql-connector-j + + org.flywaydb + flyway-core + + + org.flywaydb + flyway-mysql + @@ -82,6 +90,11 @@ spring-boot-starter-test test + + com.h2database + h2 + test + diff --git a/xiaoqu-agent-service/sql/README.md b/xiaoqu-agent-service/sql/README.md new file mode 100644 index 0000000..1401d87 --- /dev/null +++ b/xiaoqu-agent-service/sql/README.md @@ -0,0 +1,12 @@ +# 数据库迁移规范 + +自有库结构由 Flyway 管理，运行时只扫描 `src/main/resources/db/migration`。 + +1. 已合并的 `V__description.sql` 永不修改、删除或改名；Flyway 与测试都会校验历史 checksum。 +2. 每次结构或数据变更新增下一个连续版本，并同步新增 `sql/rollback/V__description.down.sql`。 +3. Migration 只操作当前 svc schema，禁止包含 `CREATE DATABASE`、`USE` 和环境专属数据。 +4. DDL 优先向后兼容；删除字段或表必须先经过废弃版本。大表变更必须给出锁表时间与分批方案。 +5. `seed-dev.sql` 仅用于开发数据，不进入 Flyway。数据库首次创建使用 `bootstrap/create-database.sql`。 +6. 合并前必须运行 `./mvnw clean package`；部署启动时 Flyway 会再次 validate 并按版本迁移，失败则应用不会启动。 + +`baseline-on-migrate` 默认关闭，避免连错非空库时被静默接管。已有 V1 数据库首次接入时，显式设置 `SVC_FLYWAY_BASELINE_ON_MIGRATE=true`：Flyway 记录 V1 基线并只执行 V2 以后版本；确认生成 `flyway_schema_history` 后立即恢复为 `false`。空数据库无需开启，会正常执行全部版本。 diff --git a/xiaoqu-agent-service/sql/bootstrap/create-database.sql b/xiaoqu-agent-service/sql/bootstrap/create-database.sql new file mode 100644 index 0000000..38747f7 --- /dev/null +++ b/xiaoqu-agent-service/sql/bootstrap/create-database.sql @@ -0,0 +1,3 @@ +CREATE DATABASE IF NOT EXISTS xiaoqu_agent_svc_d + DEFAULT CHARACTER SET utf8mb4 + COLLATE utf8mb4_general_ci; diff --git a/xiaoqu-agent-service/sql/rollback/V1__init.down.sql b/xiaoqu-agent-service/sql/rollback/V1__init.down.sql new file mode 100644 index 0000000..aee03a7 --- /dev/null +++ b/xiaoqu-agent-service/sql/rollback/V1__init.down.sql @@ -0,0 +1,8 @@ +-- 仅用于尚未承载业务数据的环境，按依赖逆序删除整个服务自有结构。 +DROP TABLE IF EXISTS admin_user; +DROP TABLE IF EXISTS audit_log; +DROP TABLE IF EXISTS token_scope; +DROP TABLE IF EXISTS scope_tool; +DROP TABLE IF EXISTS tool; +DROP TABLE IF EXISTS agent_scope; +DROP TABLE IF EXISTS agent_api_key; diff --git a/xiaoqu-agent-service/sql/rollback/V2__p4_admin.down.sql b/xiaoqu-agent-service/sql/rollback/V2__p4_admin.down.sql new file mode 100644 index 0000000..82e343c --- /dev/null +++ b/xiaoqu-agent-service/sql/rollback/V2__p4_admin.down.sql @@ -0,0 +1,15 @@ +-- PBKDF2 哈希长度超过 64 时拒绝回退，避免截断管理员密码。 +DELIMITER // +CREATE PROCEDURE rollback_v2_p4_admin() +BEGIN + IF EXISTS (SELECT 1 FROM admin_user WHERE CHAR_LENGTH(password_hash) > 64) THEN + SIGNAL SQLSTATE '45000' + SET MESSAGE_TEXT = 'V2 rollback blocked: PBKDF2 password hashes already exist'; + END IF; + ALTER TABLE admin_user + MODIFY COLUMN password_hash CHAR(64) NOT NULL COMMENT '密码 SHA-256 hex'; +END // +DELIMITER ; + +CALL rollback_v2_p4_admin(); +DROP PROCEDURE rollback_v2_p4_admin; diff --git a/xiaoqu-agent-service/src/main/java/me/iiv/agent/config/SvcDataSourceConfig.java b/xiaoqu-agent-service/src/main/java/me/iiv/agent/config/SvcDataSourceConfig.java index 6d9adcb..ac2a9d5 100644 --- a/xiaoqu-agent-service/src/main/java/me/iiv/agent/config/SvcDataSourceConfig.java +++ b/xiaoqu-agent-service/src/main/java/me/iiv/agent/config/SvcDataSourceConfig.java @@ -7,6 +7,7 @@ import org.apache.ibatis.session.SqlSessionFactory; import org.mybatis.spring.annotation.MapperScan; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.boot.autoconfigure.flyway.FlywayDataSource; import org.springframework.boot.jdbc.DataSourceBuilder; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -23,6 +24,7 @@ import javax.sql.DataSource; public class SvcDataSourceConfig { @Bean(name = "svcDataSource") + @FlywayDataSource @ConfigurationProperties(prefix = "app.datasource.svc") public DataSource svcDataSource() { return DataSourceBuilder.create().type(HikariDataSource.class).build(); diff --git a/xiaoqu-agent-service/src/main/resources/application.yml b/xiaoqu-agent-service/src/main/resources/application.yml index 306047b..d85049a 100644 --- a/xiaoqu-agent-service/src/main/resources/application.yml +++ b/xiaoqu-agent-service/src/main/resources/application.yml @@ -18,6 +18,16 @@ app: spring: application: name: xiaoqu-agent-service + flyway: + enabled: true + locations: classpath:db/migration + baseline-on-migrate: ${SVC_FLYWAY_BASELINE_ON_MIGRATE:false} + baseline-version: 1 + baseline-description: legacy V1 schema + validate-on-migrate: true + validate-migration-naming: true + out-of-order: false + clean-disabled: true ai: mcp: server: diff --git a/xiaoqu-agent-service/sql/V1__init.sql b/xiaoqu-agent-service/src/main/resources/db/migration/V1__init.sql similarity index 96% rename from xiaoqu-agent-service/sql/V1__init.sql rename to xiaoqu-agent-service/src/main/resources/db/migration/V1__init.sql index f874a83..037d653 100644 --- a/xiaoqu-agent-service/sql/V1__init.sql +++ b/xiaoqu-agent-service/src/main/resources/db/migration/V1__init.sql @@ -1,13 +1,9 @@ -- ============================================================= --- xiaoqu-agent-service 自有库初始化（dev 环境） +-- xiaoqu-agent-service 自有库基线结构 -- 设计文档：doc/agent-mcp-service-design.md §7 --- 执行：mysql -h192.168.1.181 -uroot -p < sql/V1__init.sql +-- 由 Flyway 执行；数据库本身由 sql/bootstrap/create-database.sql 创建 -- ============================================================= -CREATE DATABASE IF NOT EXISTS xiaoqu_agent_svc_d DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; - -USE xiaoqu_agent_svc_d; - -- token（ak）：属于某个人（user_id 逻辑关联 m0.user），只存 SHA-256 哈希 CREATE TABLE IF NOT EXISTS agent_api_key ( id BIGINT NOT NULL AUTO_INCREMENT COMMENT '主键', diff --git a/xiaoqu-agent-service/sql/V2__p4_admin.sql b/xiaoqu-agent-service/src/main/resources/db/migration/V2__p4_admin.sql similarity index 82% rename from xiaoqu-agent-service/sql/V2__p4_admin.sql rename to xiaoqu-agent-service/src/main/resources/db/migration/V2__p4_admin.sql index be656ae..f97ed5f 100644 --- a/xiaoqu-agent-service/sql/V2__p4_admin.sql +++ b/xiaoqu-agent-service/src/main/resources/db/migration/V2__p4_admin.sql @@ -1,4 +1,2 @@ -USE xiaoqu_agent_svc_d; - ALTER TABLE admin_user MODIFY COLUMN password_hash VARCHAR(255) NOT NULL COMMENT 'PBKDF2 加盐密码哈希'; diff --git a/xiaoqu-agent-service/src/main/resources/db/migration/checksums.sha256 b/xiaoqu-agent-service/src/main/resources/db/migration/checksums.sha256 new file mode 100644 index 0000000..e693a0a --- /dev/null +++ b/xiaoqu-agent-service/src/main/resources/db/migration/checksums.sha256 @@ -0,0 +1,2 @@ +f8ec43a4fda81382d5675bc59fe7f4339f2e38a5b4241c260c3569e33b43a26d V1__init.sql +0ad892601e891a18712b1b6a613c392c94e63492dc14f4f779375c092a3acab6 V2__p4_admin.sql diff --git a/xiaoqu-agent-service/src/test/java/me/iiv/agent/config/FlywayMigrationTest.java b/xiaoqu-agent-service/src/test/java/me/iiv/agent/config/FlywayMigrationTest.java new file mode 100644 index 0000000..9790ad6 --- /dev/null +++ b/xiaoqu-agent-service/src/test/java/me/iiv/agent/config/FlywayMigrationTest.java @@ -0,0 +1,62 @@ +package me.iiv.agent.config; + +import org.flywaydb.core.Flyway; +import org.h2.jdbcx.JdbcDataSource; +import org.junit.jupiter.api.Test; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class FlywayMigrationTest { + + @Test + void emptyDatabaseMigratesThroughLatestVersion() throws Exception { + JdbcDataSource dataSource = dataSource("empty"); + Flyway flyway = flyway(dataSource, false); + + flyway.migrate(); + + assertEquals("2", flyway.info().current().getVersion().getVersion()); + assertEquals(255, passwordHashLength(dataSource)); + } + + @Test + void legacyV1DatabaseIsBaselinedThenUpgraded() throws Exception { + JdbcDataSource dataSource = dataSource("legacy"); + try (Connection connection = dataSource.getConnection(); Statement statement = connection.createStatement()) { + statement.execute("CREATE TABLE admin_user (id BIGINT AUTO_INCREMENT PRIMARY KEY, password_hash CHAR(64) NOT NULL)"); + } + Flyway flyway = flyway(dataSource, true); + + flyway.migrate(); + + assertEquals("2", flyway.info().current().getVersion().getVersion()); + assertEquals(255, passwordHashLength(dataSource)); + } + + private static Flyway flyway(JdbcDataSource dataSource, boolean baseline) { + return Flyway.configure().dataSource(dataSource).locations("classpath:db/migration") + .baselineOnMigrate(baseline).baselineVersion("1").validateMigrationNaming(true) + .cleanDisabled(true).load(); + } + + private static JdbcDataSource dataSource(String name) { + JdbcDataSource dataSource = new JdbcDataSource(); + dataSource.setURL("jdbc:h2:mem:flyway_" + name + ";MODE=MySQL;DATABASE_TO_LOWER=TRUE;DB_CLOSE_DELAY=-1"); + dataSource.setUser("sa"); + return dataSource; + } + + private static int passwordHashLength(JdbcDataSource dataSource) throws Exception { + try (Connection connection = dataSource.getConnection(); + Statement statement = connection.createStatement(); + ResultSet result = statement.executeQuery("SELECT CHARACTER_MAXIMUM_LENGTH FROM INFORMATION_SCHEMA.COLUMNS " + + "WHERE TABLE_NAME='admin_user' AND COLUMN_NAME='password_hash'")) { + result.next(); + return result.getInt(1); + } + } +} diff --git a/xiaoqu-agent-service/src/test/java/me/iiv/agent/config/MigrationGovernanceTest.java b/xiaoqu-agent-service/src/test/java/me/iiv/agent/config/MigrationGovernanceTest.java new file mode 100644 index 0000000..c4795d6 --- /dev/null +++ b/xiaoqu-agent-service/src/test/java/me/iiv/agent/config/MigrationGovernanceTest.java @@ -0,0 +1,76 @@ +package me.iiv.agent.config; + +import org.junit.jupiter.api.Test; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.util.HexFormat; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.IntStream; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class MigrationGovernanceTest { + + private static final Path MIGRATIONS = Path.of("src/main/resources/db/migration"); + private static final Path ROLLBACKS = Path.of("sql/rollback"); + private static final Pattern NAME = Pattern.compile("V(\\d+)__[a-z0-9_]+\\.sql"); + + @Test + void migrationsAreOrderedAndHaveRollbackPlans() throws Exception { + assertTrue(Files.isDirectory(MIGRATIONS), "缺少 Flyway migration 目录"); + List files; + try (var stream = Files.list(MIGRATIONS)) { + files = stream.filter(path -> path.getFileName().toString().endsWith(".sql")).sorted().toList(); + } + assertFalse(files.isEmpty(), "至少保留一个基线 migration"); + + Map versions = new TreeMap<>(); + for (Path file : files) { + Matcher matcher = NAME.matcher(file.getFileName().toString()); + assertTrue(matcher.matches(), () -> "迁移文件名不合规: " + file.getFileName()); + int version = Integer.parseInt(matcher.group(1)); + assertFalse(versions.containsKey(version), () -> "迁移版本重复: V" + version); + versions.put(version, file); + String sql = Files.readString(file); + assertFalse(Pattern.compile("(?im)^\\s*(USE|CREATE\\s+DATABASE)\\b").matcher(sql).find(), + () -> "migration 只能操作当前 svc schema: " + file.getFileName()); + String rollback = file.getFileName().toString().replace(".sql", ".down.sql"); + assertTrue(Files.isRegularFile(ROLLBACKS.resolve(rollback)), () -> "缺少回滚方案: " + rollback); + } + List expected = IntStream.rangeClosed(1, versions.size()).boxed().toList(); + assertEquals(expected, versions.keySet().stream().toList(), "迁移版本必须从 V1 连续递增"); + } + + @Test + void committedMigrationChecksumsMatchManifest() throws Exception { + Path manifest = MIGRATIONS.resolve("checksums.sha256"); + assertTrue(Files.isRegularFile(manifest), "缺少 migration 校验和清单"); + Map checksums = new TreeMap<>(); + for (String line : Files.readAllLines(manifest)) { + if (line.isBlank() || line.startsWith("#")) continue; + String[] parts = line.trim().split("\\s+", 2); + assertEquals(2, parts.length, "校验和格式应为: sha256 文件名"); + checksums.put(parts[1], parts[0]); + } + Set migrationNames; + try (var stream = Files.list(MIGRATIONS)) { + migrationNames = stream.map(path -> path.getFileName().toString()) + .filter(name -> name.endsWith(".sql")).collect(java.util.stream.Collectors.toSet()); + } + assertEquals(migrationNames, checksums.keySet(), "校验和清单必须覆盖全部 migration"); + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + for (Map.Entry entry : checksums.entrySet()) { + String actual = HexFormat.of().formatHex(digest.digest(Files.readAllBytes(MIGRATIONS.resolve(entry.getKey())))); + assertEquals(entry.getValue(), actual, () -> "历史 migration 被修改: " + entry.getKey()); + } + } +}