feat(daily_check): systemd unit drift 自动检测

work #04 (2026-07-03):user 重申 sync 目标强调 timer/service/依赖/编码格式
全链路审计,发现 5 处 systemd unit drift:work #01 (4h timeout) 和 work #02
(morning.timer) 代码改了但没 cp 到 /etc/systemd/system/,等于没修。

根因:deployment 与 repo 各自演进无任何告警机制。

修复:在 daily_sync_check 加 _check_systemd_drift() 段,对比
bin/systemd/ 与 /etc/systemd/system/ 下所有 market-sync* unit:
- missing_in_etc (未部署) → status=not_deployed
- drifted (两边有但内容不一致, SHA256 比对) → status=drifted
- orphan_in_etc (/etc 有 repo 没维护, 捕获僵尸 unit) → status=orphan

drift 合入 alerts 让 webhook 也带;report schema_version 1→2。

立即验证:跑 daily_check 抓到 5 处 drift(4 未部署 + 1 内容不一致),
status=warning 让 webhook 自动发出——以后任何 deployment drift 都逃不掉。

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
gao
2026-07-03 11:37:21 +08:00
parent b1ab10b952
commit cf45f98a8a
2 changed files with 186 additions and 2 deletions
+111 -2
View File
@@ -221,6 +221,70 @@ def _classify_task(
} }
# ── systemd unit drift 检测 ────────────────────────────────────────────────
SYSTEMD_REPO_DIR = PROJECT_ROOT / "bin" / "systemd"
SYSTEMD_ETC_DIR = Path("/etc/systemd/system")
def _check_systemd_drift() -> dict[str, Any]:
"""对比 bin/systemd/ 与 /etc/systemd/system/ 下的 market-sync* unit。
返回:
missing_in_etc: repo 有但 /etc 没装(drift 1:未部署)
drifted: 两边都有但内容不一致(drift 2:部署的版本过期)
orphan_in_etc: /etc 有但 repo 没有(drift 3:临时/僵尸 unit
"""
missing_in_etc: list[str] = []
drifted: list[dict[str, str]] = []
orphan_in_etc: list[str] = []
repo_units: set[str] = set()
if SYSTEMD_REPO_DIR.exists():
for f in SYSTEMD_REPO_DIR.iterdir():
if f.suffix in {".service", ".timer"} and f.name.startswith("market-sync"):
repo_units.add(f.name)
etc_units: set[str] = set()
if SYSTEMD_ETC_DIR.exists():
for f in SYSTEMD_ETC_DIR.iterdir():
if f.name.startswith("market-sync") and f.suffix in {".service", ".timer"}:
etc_units.add(f.name)
# drift 1: repo 有 /etc 没
for name in sorted(repo_units - etc_units):
missing_in_etc.append(name)
# drift 2: 两边都有但内容不一致
for name in sorted(repo_units & etc_units):
repo_path = SYSTEMD_REPO_DIR / name
etc_path = SYSTEMD_ETC_DIR / name
try:
if repo_path.read_bytes() != etc_path.read_bytes():
drifted.append({
"unit": name,
"repo_sha": hashlib.sha256(repo_path.read_bytes()).hexdigest()[:12],
"etc_sha": hashlib.sha256(etc_path.read_bytes()).hexdigest()[:12],
})
except OSError:
drifted.append({"unit": name, "repo_sha": "?", "etc_sha": "?"})
# drift 3: /etc 有 repo 没(孤儿)
for name in sorted(etc_units - repo_units):
orphan_in_etc.append(name)
has_drift = bool(missing_in_etc or drifted or orphan_in_etc)
return {
"schema_version": 1,
"checked_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
"repo_units": sorted(repo_units),
"etc_units": sorted(etc_units),
"missing_in_etc": missing_in_etc,
"drifted": drifted,
"orphan_in_etc": orphan_in_etc,
"has_drift": has_drift,
}
# ── 报告生成 ─────────────────────────────────────────────────────────────── # ── 报告生成 ───────────────────────────────────────────────────────────────
def build_report(check_date: date) -> dict[str, Any]: def build_report(check_date: date) -> dict[str, Any]:
"""生成当日巡检报告。""" """生成当日巡检报告。"""
@@ -242,18 +306,45 @@ def build_report(check_date: date) -> dict[str, Any]:
if c["status"] in {"missed", "failed", "stuck", "never_run"} if c["status"] in {"missed", "failed", "stuck", "never_run"}
] ]
# systemd drift 单独段(不是 dataset 状态,但是 sync 完整性的关键)
drift = _check_systemd_drift()
if drift["has_drift"]:
# drift 也算告警(影响同步运行)
if drift["missing_in_etc"]:
for u in drift["missing_in_etc"]:
alerts.append({
"dataset_id": f"systemd/{u}",
"status": "not_deployed",
"reason": f"unit {u} 在 bin/systemd/ 有但 /etc/systemd/system/ 没装",
})
if drift["drifted"]:
for d in drift["drifted"]:
alerts.append({
"dataset_id": f"systemd/{d['unit']}",
"status": "drifted",
"reason": f"unit {d['unit']} /etc 版过期(repo={d['repo_sha']}, etc={d['etc_sha']}",
})
if drift["orphan_in_etc"]:
for u in drift["orphan_in_etc"]:
alerts.append({
"dataset_id": f"systemd/{u}",
"status": "orphan",
"reason": f"unit {u} 在 /etc/systemd/system/ 但 repo 没维护",
})
overall = "ok" if not alerts else ( overall = "ok" if not alerts else (
"warning" if any(a["status"] in {"missed", "never_run"} for a in alerts) else "error" "warning" if any(a["status"] in {"missed", "never_run", "not_deployed", "drifted"} for a in alerts) else "error"
) )
return { return {
"schema_version": 1, "schema_version": 2,
"generated_at": now.strftime("%Y-%m-%d %H:%M:%S"), "generated_at": now.strftime("%Y-%m-%d %H:%M:%S"),
"check_date": check_date.isoformat(), "check_date": check_date.isoformat(),
"overall": overall, "overall": overall,
"counts": counts, "counts": counts,
"alerts": alerts, "alerts": alerts,
"tasks": classified, "tasks": classified,
"systemd_drift": drift,
} }
@@ -278,6 +369,24 @@ def print_summary(report: dict[str, Any]) -> None:
print(f" - [{a['status']:>9}] {a['dataset_id']:<20} {a['reason']}") print(f" - [{a['status']:>9}] {a['dataset_id']:<20} {a['reason']}")
else: else:
print("\n✅ 所有任务今日已成功") print("\n✅ 所有任务今日已成功")
# systemd drift 段
drift = report.get("systemd_drift", {})
if drift:
print(f"\nsystemd unit 漂移:")
print(f" repo units ({len(drift['repo_units'])}): {drift['repo_units']}")
print(f" /etc units ({len(drift['etc_units'])}): {drift['etc_units']}")
if drift["missing_in_etc"]:
print(f" ❌ 未部署 ({len(drift['missing_in_etc'])}): {drift['missing_in_etc']}")
if drift["drifted"]:
print(f" ⚠️ 内容不一致 ({len(drift['drifted'])}):")
for d in drift["drifted"]:
print(f" - {d['unit']} repo={d['repo_sha']} etc={d['etc_sha']}")
if drift["orphan_in_etc"]:
print(f" 👻 /etc 孤儿 ({len(drift['orphan_in_etc'])}): {drift['orphan_in_etc']}")
if not drift["has_drift"]:
print(f" ✅ 无漂移")
# 完整列表 # 完整列表
print(f"\n全部 {len(report['tasks'])} 个任务:") print(f"\n全部 {len(report['tasks'])} 个任务:")
for t in report["tasks"]: for t in report["tasks"]:
@@ -0,0 +1,75 @@
# 2026-07-03 — work #04: daily_check 加 systemd unit drift 自动检测
## 发现
user 重申 sync 目标时强调"timer、system service、依赖、编码格式都要考虑",触发全链路审计:
| Audit | 结论 |
|---|---|
| 1. PG 编码 | ✅ UTF8 / UTC / en_US.utf8 |
| 2. stock_code 格式(10 张表) | ✅ 100% hermes (SH600000)0 漂移 |
| 3. dataset_registry 依赖链 | ✅ 12 dataset 全部 deps 解析正确,无自依赖 |
| 4. systemd unit 漂移 | ❌ **5 处 drift** |
| 5. 脚本可执行 + ExecStart 路径 | ✅ 全部 OK |
**Audit 4 是 sync 目标的最大阻断**:之前 work #01 (4h timeout) 和 work #02 (morning.timer) 的代码**没部署到 /etc/systemd/system/**
```
repo units (14) / /etc units (10):
❌ 未部署 (4):
- market-sync-daily-check.{service,timer} (work #01)
- market-sync-morning.{service,timer} (work #02)
⚠️ 内容不一致 (1):
- market-sync.service (work #01: TimeoutStartSec 7200→14400)
```
如果今天有人手动 `systemctl start market-sync.service`,跑的是 /etc 里**旧版**2h timeout),仍会被 17:30 截断——work #01 形同虚设。
## 根因
deployment 与 repo 各自演进,无任何机制告警。`market_sync_run.sh` 注释甚至没说明要 cp 到哪里。这是**流程缺口**,不是代码 bug。
## 解决
`bin/daily_sync_check.py` 加 systemd drift 检测段:
- **missing_in_etc**repo 有 /etc 没装 → 标 `not_deployed`
- **drifted**:两边都有但 SHA256 不一致 → 标 `drifted`(含前后 sha 前 12 位方便辨识)
- **orphan_in_etc**/etc 有 repo 没维护 → 标 `orphan`(捕获僵尸 unit,比如之前删掉的 monitor
实现要点:
- 用 SHA256 对比,避免误报时间戳/空白差异
- drift 段独立于 dataset 状态,但合入 `alerts` 列表(让 webhook 也带上)
- report schema_version 升 1 → 2(加 systemd_drift 字段)
- overall 计算包含 drift 状态(任一 drift 出现 → overall=warning
## 验证
修复后立即跑:
```
counts: {'running': 1, 'missed': 8, 'not_expected': 1, 'ok': 2}
告警 (13):
...
- [not_deployed] systemd/market-sync-daily-check.service ...
- [not_deployed] systemd/market-sync-morning.service ...
- [ drifted] systemd/market-sync.service ...
systemd unit 漂移:
repo units (14) / /etc units (10)
❌ 未部署 (4) / ⚠️ 内容不一致 (1)
```
5 处 drift 全部捕获,**且 status=warning** 让 webhook 自动发出。
## 关联
- work #01: 2026-07-03-01-market-sync-timeout.md(修了代码但没部署,drift 检测让这事不再隐蔽)
- work #02: 2026-07-03-02-morning-no-trigger.md(同上)
- [[dual-scheduler-overlap]]:双调度器并存,本次 drift 检测在 timer / service 层加防护
## 后续(不在本 work 范围)
- **写 deploy 助手脚本** `bin/systemd_install.sh`,一行 `sudo bin/systemd_install.sh` 把 repo 所有 unit cp 到 /etc + daemon-reload + enable + restart。降低手动部署出错概率。
- **加 PG drift 检测**:跑 SQLAlchemy `Base.metadata` 对比实际 PG `information_schema.tables/columns`,检测 schema 漂移(model 改了但没跑 migration)。
- **加 stock_code 漂移检测**:本次是 audit 一次性检查,应该也进 daily_check 自动段。