From cf45f98a8a88c5b1eaf299a7a9ca28bd1a45be41 Mon Sep 17 00:00:00 2001 From: gao Date: Fri, 3 Jul 2026 11:37:21 +0800 Subject: [PATCH] =?UTF-8?q?feat(daily=5Fcheck):=20systemd=20unit=20drift?= =?UTF-8?q?=20=E8=87=AA=E5=8A=A8=E6=A3=80=E6=B5=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit work #04 (2026-07-03)：user 重申 sync 目标强调 timer/service/依赖/编码格式 全链路审计，发现 5 处 systemd unit drift：work #01 (4h timeout) 和 work #02 (morning.timer) 代码改了但没 cp 到 /etc/systemd/system/，等于没修。 根因：deployment 与 repo 各自演进无任何告警机制。 修复：在 daily_sync_check 加 _check_systemd_drift() 段，对比 bin/systemd/ 与 /etc/systemd/system/ 下所有 market-sync* unit： - missing_in_etc (未部署) → status=not_deployed - drifted (两边有但内容不一致, SHA256 比对) → status=drifted - orphan_in_etc (/etc 有 repo 没维护, 捕获僵尸 unit) → status=orphan drift 合入 alerts 让 webhook 也带；report schema_version 1→2。 立即验证：跑 daily_check 抓到 5 处 drift（4 未部署 + 1 内容不一致）， status=warning 让 webhook 自动发出——以后任何 deployment drift 都逃不掉。 Co-Authored-By: Claude --- bin/daily_sync_check.py | 113 +++++++++++++++++- .../2026-07-03-04-systemd-drift-detection.md | 75 ++++++++++++ 2 files changed, 186 insertions(+), 2 deletions(-) create mode 100644 docs/works/2026-07-03-04-systemd-drift-detection.md diff --git a/bin/daily_sync_check.py b/bin/daily_sync_check.py index a3c2011..7774b24 100644 --- a/bin/daily_sync_check.py +++ b/bin/daily_sync_check.py @@ -221,6 +221,70 @@ def _classify_task( } +# ── systemd unit drift 检测 ──────────────────────────────────────────────── +SYSTEMD_REPO_DIR = PROJECT_ROOT / "bin" / "systemd" +SYSTEMD_ETC_DIR = Path("/etc/systemd/system") + + +def _check_systemd_drift() -> dict[str, Any]: + """对比 bin/systemd/ 与 /etc/systemd/system/ 下的 market-sync* unit。 + + 返回： + missing_in_etc: repo 有但 /etc 没装（drift 1：未部署） + drifted: 两边都有但内容不一致（drift 2：部署的版本过期） + orphan_in_etc: /etc 有但 repo 没有（drift 3：临时/僵尸 unit） + """ + missing_in_etc: list[str] = [] + drifted: list[dict[str, str]] = [] + orphan_in_etc: list[str] = [] + + repo_units: set[str] = set() + if SYSTEMD_REPO_DIR.exists(): + for f in SYSTEMD_REPO_DIR.iterdir(): + if f.suffix in {".service", ".timer"} and f.name.startswith("market-sync"): + repo_units.add(f.name) + + etc_units: set[str] = set() + if SYSTEMD_ETC_DIR.exists(): + for f in SYSTEMD_ETC_DIR.iterdir(): + if f.name.startswith("market-sync") and f.suffix in {".service", ".timer"}: + etc_units.add(f.name) + + # drift 1: repo 有 /etc 没 + for name in sorted(repo_units - etc_units): + missing_in_etc.append(name) + + # drift 2: 两边都有但内容不一致 + for name in sorted(repo_units & etc_units): + repo_path = SYSTEMD_REPO_DIR / name + etc_path = SYSTEMD_ETC_DIR / name + try: + if repo_path.read_bytes() != etc_path.read_bytes(): + drifted.append({ + "unit": name, + "repo_sha": hashlib.sha256(repo_path.read_bytes()).hexdigest()[:12], + "etc_sha": hashlib.sha256(etc_path.read_bytes()).hexdigest()[:12], + }) + except OSError: + drifted.append({"unit": name, "repo_sha": "?", "etc_sha": "?"}) + + # drift 3: /etc 有 repo 没（孤儿） + for name in sorted(etc_units - repo_units): + orphan_in_etc.append(name) + + has_drift = bool(missing_in_etc or drifted or orphan_in_etc) + return { + "schema_version": 1, + "checked_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + "repo_units": sorted(repo_units), + "etc_units": sorted(etc_units), + "missing_in_etc": missing_in_etc, + "drifted": drifted, + "orphan_in_etc": orphan_in_etc, + "has_drift": has_drift, + } + + # ── 报告生成 ─────────────────────────────────────────────────────────────── def build_report(check_date: date) -> dict[str, Any]: """生成当日巡检报告。""" @@ -242,18 +306,45 @@ def build_report(check_date: date) -> dict[str, Any]: if c["status"] in {"missed", "failed", "stuck", "never_run"} ] + # systemd drift 单独段（不是 dataset 状态，但是 sync 完整性的关键） + drift = _check_systemd_drift() + if drift["has_drift"]: + # drift 也算告警（影响同步运行） + if drift["missing_in_etc"]: + for u in drift["missing_in_etc"]: + alerts.append({ + "dataset_id": f"systemd/{u}", + "status": "not_deployed", + "reason": f"unit {u} 在 bin/systemd/ 有但 /etc/systemd/system/ 没装", + }) + if drift["drifted"]: + for d in drift["drifted"]: + alerts.append({ + "dataset_id": f"systemd/{d['unit']}", + "status": "drifted", + "reason": f"unit {d['unit']} /etc 版过期（repo={d['repo_sha']}, etc={d['etc_sha']}）", + }) + if drift["orphan_in_etc"]: + for u in drift["orphan_in_etc"]: + alerts.append({ + "dataset_id": f"systemd/{u}", + "status": "orphan", + "reason": f"unit {u} 在 /etc/systemd/system/ 但 repo 没维护", + }) + overall = "ok" if not alerts else ( - "warning" if any(a["status"] in {"missed", "never_run"} for a in alerts) else "error" + "warning" if any(a["status"] in {"missed", "never_run", "not_deployed", "drifted"} for a in alerts) else "error" ) return { - "schema_version": 1, + "schema_version": 2, "generated_at": now.strftime("%Y-%m-%d %H:%M:%S"), "check_date": check_date.isoformat(), "overall": overall, "counts": counts, "alerts": alerts, "tasks": classified, + "systemd_drift": drift, } @@ -278,6 +369,24 @@ def print_summary(report: dict[str, Any]) -> None: print(f" - [{a['status']:>9}] {a['dataset_id']:<20} {a['reason']}") else: print("\n✅ 所有任务今日已成功") + + # systemd drift 段 + drift = report.get("systemd_drift", {}) + if drift: + print(f"\nsystemd unit 漂移:") + print(f" repo units ({len(drift['repo_units'])}): {drift['repo_units']}") + print(f" /etc units ({len(drift['etc_units'])}): {drift['etc_units']}") + if drift["missing_in_etc"]: + print(f" ❌ 未部署 ({len(drift['missing_in_etc'])}): {drift['missing_in_etc']}") + if drift["drifted"]: + print(f" ⚠️ 内容不一致 ({len(drift['drifted'])}):") + for d in drift["drifted"]: + print(f" - {d['unit']} repo={d['repo_sha']} etc={d['etc_sha']}") + if drift["orphan_in_etc"]: + print(f" 👻 /etc 孤儿 ({len(drift['orphan_in_etc'])}): {drift['orphan_in_etc']}") + if not drift["has_drift"]: + print(f" ✅ 无漂移") + # 完整列表 print(f"\n全部 {len(report['tasks'])} 个任务:") for t in report["tasks"]: diff --git a/docs/works/2026-07-03-04-systemd-drift-detection.md b/docs/works/2026-07-03-04-systemd-drift-detection.md new file mode 100644 index 0000000..8ec6e95 --- /dev/null +++ b/docs/works/2026-07-03-04-systemd-drift-detection.md @@ -0,0 +1,75 @@ +# 2026-07-03 — work #04: daily_check 加 systemd unit drift 自动检测 + +## 发现 + +user 重申 sync 目标时强调"timer、system service、依赖、编码格式都要考虑"，触发全链路审计： + +| Audit | 结论 | +|---|---| +| 1. PG 编码 | ✅ UTF8 / UTC / en_US.utf8 | +| 2. stock_code 格式（10 张表） | ✅ 100% hermes (SH600000)，0 漂移 | +| 3. dataset_registry 依赖链 | ✅ 12 dataset 全部 deps 解析正确，无自依赖 | +| 4. systemd unit 漂移 | ❌ **5 处 drift** | +| 5. 脚本可执行 + ExecStart 路径 | ✅ 全部 OK | + +**Audit 4 是 sync 目标的最大阻断**：之前 work #01 (4h timeout) 和 work #02 (morning.timer) 的代码**没部署到 /etc/systemd/system/**： + +``` +repo units (14) / /etc units (10): + ❌ 未部署 (4): + - market-sync-daily-check.{service,timer} (work #01) + - market-sync-morning.{service,timer} (work #02) + ⚠️ 内容不一致 (1): + - market-sync.service (work #01: TimeoutStartSec 7200→14400) +``` + +如果今天有人手动 `systemctl start market-sync.service`，跑的是 /etc 里**旧版**（2h timeout），仍会被 17:30 截断——work #01 形同虚设。 + +## 根因 + +deployment 与 repo 各自演进，无任何机制告警。`market_sync_run.sh` 注释甚至没说明要 cp 到哪里。这是**流程缺口**，不是代码 bug。 + +## 解决 + +`bin/daily_sync_check.py` 加 systemd drift 检测段： + +- **missing_in_etc**：repo 有 /etc 没装 → 标 `not_deployed` +- **drifted**：两边都有但 SHA256 不一致 → 标 `drifted`（含前后 sha 前 12 位方便辨识） +- **orphan_in_etc**：/etc 有 repo 没维护 → 标 `orphan`（捕获僵尸 unit，比如之前删掉的 monitor） + +实现要点： +- 用 SHA256 对比，避免误报时间戳/空白差异 +- drift 段独立于 dataset 状态，但合入 `alerts` 列表（让 webhook 也带上） +- report schema_version 升 1 → 2（加 systemd_drift 字段） +- overall 计算包含 drift 状态（任一 drift 出现 → overall=warning） + +## 验证 + +修复后立即跑： + +``` +counts: {'running': 1, 'missed': 8, 'not_expected': 1, 'ok': 2} +告警 (13): + ... + - [not_deployed] systemd/market-sync-daily-check.service ... + - [not_deployed] systemd/market-sync-morning.service ... + - [ drifted] systemd/market-sync.service ... + +systemd unit 漂移: + repo units (14) / /etc units (10) + ❌ 未部署 (4) / ⚠️ 内容不一致 (1) +``` + +5 处 drift 全部捕获，**且 status=warning** 让 webhook 自动发出。 + +## 关联 + +- work #01: 2026-07-03-01-market-sync-timeout.md（修了代码但没部署，drift 检测让这事不再隐蔽） +- work #02: 2026-07-03-02-morning-no-trigger.md（同上） +- [[dual-scheduler-overlap]]：双调度器并存，本次 drift 检测在 timer / service 层加防护 + +## 后续（不在本 work 范围） + +- **写 deploy 助手脚本** `bin/systemd_install.sh`，一行 `sudo bin/systemd_install.sh` 把 repo 所有 unit cp 到 /etc + daemon-reload + enable + restart。降低手动部署出错概率。 +- **加 PG drift 检测**：跑 SQLAlchemy `Base.metadata` 对比实际 PG `information_schema.tables/columns`，检测 schema 漂移（model 改了但没跑 migration）。 +- **加 stock_code 漂移检测**：本次是 audit 一次性检查，应该也进 daily_check 自动段。 \ No newline at end of file