"""清理 dataset_registry 中已废弃的 MySQL/SQLite-era 旧 task 行。 ⚠️ 一次性脚本(2026-07-03):PG 迁移后 6 个旧 task 行 enabled=0 但未删除, 长期占位把 daily_check 报告撑到 18 行(噪音 1/3)。 清理策略(**保守** — 只删明确孤儿): 1. enabled=0 2. storage_layer IN ('sqlite', 'parquet', 'mysql', '') ← 新 PG-only 体系只有 'pg' 3. sync_script 路径不在 app.tasks 下(即非 app.tasks.task_xxx:run 形式) 执行: .venv/bin/python -m bin.archive.clean_legacy_registry_rows --dry-run .venv/bin/python -m bin.archive.clean_legacy_registry_rows # 真删 安全: - 脚本默认 --dry-run,必须显式 --apply 才会 DELETE - 删之前 dump 被删行的 dataset_id / name / storage_uri / sync_script 到 stdout - 事务:先 SELECT WHERE 命中 → 再 DELETE 同 WHERE,单次 commit;失败全回滚 不在清理范围(保留): - enabled=0 但 storage_layer='pg' 的行(可能临时禁用,不是孤儿) - enabled=1 的所有行(哪怕 sync_script 路径奇怪) """ from __future__ import annotations import argparse import sys from pathlib import Path from typing import Any _PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent if str(_PROJECT_ROOT) not in sys.path: sys.path.insert(0, str(_PROJECT_ROOT)) from sqlalchemy import delete, select # noqa: E402 from app.core.db import ops as db_ops # noqa: E402 from app.core.db.models import DatasetRegistry # noqa: E402 # 新 PG-only 体系下"合法" task 的 sync_script 路径前缀 _NEW_SCRIPT_PREFIX = "app.tasks." # 旧 storage_layer 值(PG 迁移前用过) _LEGACY_STORAGE_LAYERS = {"sqlite", "parquet", "mysql", ""} def find_legacy_rows() -> list[dict[str, Any]]: """找出所有 enabled=0 且 storage_layer 旧 或 sync_script 不在新体系下的行。""" db_ops._ensure_schema() # noqa: SLF001 with db_ops.get_session() as s: all_rows = s.execute( select(DatasetRegistry).order_by(DatasetRegistry.sort_order) ).scalars().all() legacy: list[dict[str, Any]] = [] for r in all_rows: if (r.enabled or 0) != 0: continue storage_layer = (r.storage_layer or "").lower() script = (r.sync_script or "").strip() # 命中条件 1: storage_layer 是旧值 hit_storage = storage_layer in _LEGACY_STORAGE_LAYERS # 命中条件 2: sync_script 不在新体系下 hit_script = not script.startswith(_NEW_SCRIPT_PREFIX) if hit_storage or hit_script: legacy.append({ "dataset_id": r.dataset_id, "name": r.name or "", "storage_layer": storage_layer, "storage_uri": r.storage_uri or "", "sync_script": script, "sort_order": r.sort_order, "status": r.status or "idle", }) return legacy def delete_legacy_rows(rows: list[dict[str, Any]]) -> int: """事务里 DELETE 同 WHERE;返回删除行数。""" if not rows: return 0 ids = [r["dataset_id"] for r in rows] db_ops._ensure_schema() # noqa: SLF001 with db_ops.get_session() as s: result = s.execute( delete(DatasetRegistry).where(DatasetRegistry.dataset_id.in_(ids)) ) s.commit() return result.rowcount or 0 def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description="清理 dataset_registry 中废弃的旧 task 行") parser.add_argument("--apply", action="store_true", help="真删(默认 dry-run)") args = parser.parse_args(argv) rows = find_legacy_rows() print(f"扫描到 {len(rows)} 行废弃行:") for r in rows: print( f" - dataset_id={r['dataset_id']:<20} " f"storage_layer={r['storage_layer']:<8} " f"sync_script={r['sync_script']!r}" ) if not args.apply: print(f"\n[DRY-RUN] 不执行 DELETE。确认无误后加 --apply 重跑。") return 0 if not rows: print("没有可清理的行。") return 0 n = delete_legacy_rows(rows) print(f"\n✅ 已删除 {n} 行。") return 0 if __name__ == "__main__": sys.exit(main())