"""每日同步任务巡检。 读 PG `dataset_registry`，对比今日预期调度窗口，给出每个 task 的健康状态， 写 `logs/daily_check_YYYYMMDD_HHMMSS.json` 报告，仅在有失败/漏跑时 POST webhook。 用法： .venv/bin/python bin/daily_sync_check.py # 默认检查"今天" .venv/bin/python bin/daily_sync_check.py --dry-run # 只生成报告，不 POST .venv/bin/python bin/daily_sync_check.py --report-only # 完全不读 secrets / 不 POST .venv/bin/python bin/daily_sync_check.py --date 2026-07-02 # 检查指定日期 为什么是 stand-alone（不放在 app.tasks 下）： - 巡检逻辑独立于 sync 业务，混在 app.tasks 会让注册表臃肿 - 独立脚本便于 cron / systemd timer / /loop 三种触发方式共用 """ from __future__ import annotations import argparse import hashlib import hmac import json import logging import os import re import sys import time from datetime import date, datetime, timedelta from pathlib import Path from typing import Any, Optional import requests import sqlalchemy as sa # ── 路径与日志 ───────────────────────────────────────────────────────────── PROJECT_ROOT = Path(__file__).resolve().parent.parent if str(PROJECT_ROOT) not in sys.path: sys.path.insert(0, str(PROJECT_ROOT)) LOG_DIR = PROJECT_ROOT / "logs" SECRETS_FILE = PROJECT_ROOT / "config" / "daily_check_secrets.env" LOG = logging.getLogger("daily-sync-check") # ── 运行模式 ─────────────────────────────────────────────────────────────── def _runtime_mode() -> str: """读取当前运行模式：docker | systemd。 优先读环境变量 RUNTIME_MODE，未设置时尝试从 .env 读取，默认 docker。 """ mode = os.environ.get("RUNTIME_MODE", "") if not mode: env_file = PROJECT_ROOT / ".env" if env_file.exists(): try: for line in env_file.read_text(encoding="utf-8").splitlines(): line = line.strip() if line.startswith("RUNTIME_MODE="): mode = line.split("=", 1)[1].strip().strip('"').strip("'") break except Exception: pass return (mode or "docker").lower() # ── 预期调度表 ───────────────────────────────────────────────────────────── # 与 [[market-data-overview]] 保持一致；同步触发点（HH:MM）按 Asia/Shanghai 解释。 # weekend_only=True 表示只在周六/周日期望运行（实际仅 stock_node 周六一次）。 SCHEDULE: dict[str, dict[str, Any]] = { "stock_basic": {"window_end": "09:30", "weekend_only": False}, "kline_index": {"window_end": "15:35", "weekend_only": False}, "kline_daily": {"window_end": "15:45", "weekend_only": False}, "kline_5min": {"window_end": "16:05", "weekend_only": False}, "market_regime": {"window_end": "16:20", "weekend_only": False}, "share_snapshot": {"window_end": "12:00", "weekend_only": True}, # 周六 "moneyflow": {"window_end": "21:40", "weekend_only": False}, "longhubang": {"window_end": "22:05", "weekend_only": False}, "stock_node": {"window_end": "12:00", "weekend_only": True}, # 周六 "mairui_ma_daily": {"window_end": "17:00", "weekend_only": False}, # 本地派生 "mairui_indicators": {"window_end": "17:10", "weekend_only": False}, # mairui MACD/KDJ/BOLL } # ── 状态判定 ─────────────────────────────────────────────────────────────── def _parse_ts(s: Optional[str]) -> Optional[datetime]: """dataset_registry 时间字段是 '%Y-%m-%d %H:%M:%S' 或 None。""" if not s: return None for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%dT%H:%M:%S%z"): try: return datetime.strptime(s, fmt) except ValueError: continue return None def _classify_task( row: dict[str, Any], check_date: date, now: datetime, ) -> dict[str, Any]: """判定单个 task 在 check_date 的健康状态。 返回字段： status: ok | missed | failed | running | never_run | not_expected reason: 简短中文原因 last_success_at / last_failure_at / last_error / needs_resync: 原值透传 """ dataset_id = row["dataset_id"] enabled = (row.get("enabled") or 0) == 1 spec = SCHEDULE.get(dataset_id) is_weekend = check_date.weekday() >= 5 if not enabled: return { "dataset_id": dataset_id, "name": row.get("name", ""), "status": "disabled", "reason": "enabled=0", "last_success_at": row.get("last_success_at"), "last_failure_at": row.get("last_failure_at"), "last_error": row.get("last_error"), "needs_resync": row.get("needs_resync", 0), "status_field": row.get("status", "idle"), } # weekend 期望判定 if spec and spec["weekend_only"] and not is_weekend: return { "dataset_id": dataset_id, "name": row.get("name", ""), "status": "not_expected", "reason": "周末专属任务，今天不期望", "last_success_at": row.get("last_success_at"), "last_failure_at": row.get("last_failure_at"), "last_error": row.get("last_error"), "needs_resync": row.get("needs_resync", 0), "status_field": row.get("status", "idle"), } if spec and not spec["weekend_only"] and is_weekend: return { "dataset_id": dataset_id, "name": row.get("name", ""), "status": "not_expected", "reason": "周末不调度", "last_success_at": row.get("last_success_at"), "last_failure_at": row.get("last_failure_at"), "last_error": row.get("last_error"), "needs_resync": row.get("needs_resync", 0), "status_field": row.get("status", "idle"), } last_success = _parse_ts(row.get("last_success_at")) last_failure = _parse_ts(row.get("last_failure_at")) status_field = row.get("status", "idle") # 期望窗口是否已过？未到窗口不应判 missed if spec and spec.get("window_end") and not status_field == "running": try: hh, mm = spec["window_end"].split(":") window_end_dt = datetime.combine(check_date, datetime.min.time()).replace( hour=int(hh), minute=int(mm) ) if now < window_end_dt: return { "dataset_id": dataset_id, "name": row.get("name", ""), "status": "not_yet", "reason": f"期望窗口 {spec['window_end']} 未到（now={now.strftime('%H:%M')}）", "last_success_at": row.get("last_success_at"), "last_failure_at": row.get("last_failure_at"), "last_error": row.get("last_error"), "needs_resync": row.get("needs_resync", 0), "status_field": status_field, } except (ValueError, AttributeError): pass # 解析失败回落到旧逻辑 # 还在跑（卡死检测） if status_field == "running": started = _parse_ts(row.get("started_at")) if started and (now - started) > timedelta(hours=2): return { "dataset_id": dataset_id, "name": row.get("name", ""), "status": "stuck", "reason": f"running > 2h, started_at={row.get('started_at')}", "last_success_at": row.get("last_success_at"), "last_failure_at": row.get("last_failure_at"), "last_error": row.get("last_error"), "needs_resync": row.get("needs_resync", 0), "status_field": status_field, } return { "dataset_id": dataset_id, "name": row.get("name", ""), "status": "running", "reason": f"status=running, started_at={row.get('started_at')}", "last_success_at": row.get("last_success_at"), "last_failure_at": row.get("last_failure_at"), "last_error": row.get("last_error"), "needs_resync": row.get("needs_resync", 0), "status_field": status_field, } # 从未跑过 if last_success is None and last_failure is None: return { "dataset_id": dataset_id, "name": row.get("name", ""), "status": "never_run", "reason": "从未同步过", "last_success_at": row.get("last_success_at"), "last_failure_at": row.get("last_failure_at"), "last_error": row.get("last_error"), "needs_resync": row.get("needs_resync", 0), "status_field": status_field, } # 今日是否有成功？ today_start = datetime.combine(check_date, datetime.min.time()) if last_success and last_success >= today_start: return { "dataset_id": dataset_id, "name": row.get("name", ""), "status": "ok", "reason": f"今日已成功 @ {last_success.strftime('%H:%M:%S')}", "last_success_at": row.get("last_success_at"), "last_failure_at": row.get("last_failure_at"), "last_error": row.get("last_error"), "needs_resync": row.get("needs_resync", 0), "status_field": status_field, } # 今日失败过？ if last_failure and last_failure >= today_start and ( not last_success or last_failure > last_success ): return { "dataset_id": dataset_id, "name": row.get("name", ""), "status": "failed", "reason": f"今日失败 @ {last_failure.strftime('%H:%M:%S')}: " f"{(row.get('last_error') or '')[:120]}", "last_success_at": row.get("last_success_at"), "last_failure_at": row.get("last_failure_at"), "last_error": row.get("last_error"), "needs_resync": row.get("needs_resync", 0), "status_field": status_field, } # 期望窗口已过但今日没成功 return { "dataset_id": dataset_id, "name": row.get("name", ""), "status": "missed", "reason": ( f"今日未成功（last_success={last_success.strftime('%Y-%m-%d %H:%M') if last_success else 'never'}）" ), "last_success_at": row.get("last_success_at"), "last_failure_at": row.get("last_failure_at"), "last_error": row.get("last_error"), "needs_resync": row.get("needs_resync", 0), "status_field": status_field, } # ── systemd unit drift 检测 ──────────────────────────────────────────────── SYSTEMD_REPO_DIR = PROJECT_ROOT / "deploy" / "systemd" / "units" SYSTEMD_LEGACY_REPO_DIR = PROJECT_ROOT / "bin" / "systemd" SYSTEMD_ETC_DIR = Path("/etc/systemd/system") def _check_systemd_drift() -> dict[str, Any]: """对比 deploy/systemd/units/ 与 /etc/systemd/system/ 下的 market-sync* unit。 兼容旧路径 bin/systemd/：如果 deploy/systemd/units/ 不存在则回退。 返回： missing_in_etc: repo 有但 /etc 没装（drift 1：未部署） drifted: 两边都有但内容不一致（drift 2：部署的版本过期） orphan_in_etc: /etc 有但 repo 没有（drift 3：临时/僵尸 unit） """ repo_dir = SYSTEMD_REPO_DIR if SYSTEMD_REPO_DIR.exists() else SYSTEMD_LEGACY_REPO_DIR missing_in_etc: list[str] = [] drifted: list[dict[str, str]] = [] orphan_in_etc: list[str] = [] repo_units: set[str] = set() if repo_dir.exists(): for f in repo_dir.iterdir(): if f.suffix in {".service", ".timer"} and f.name.startswith("market-sync"): repo_units.add(f.name) etc_units: set[str] = set() if SYSTEMD_ETC_DIR.exists(): for f in SYSTEMD_ETC_DIR.iterdir(): if f.name.startswith("market-sync") and f.suffix in {".service", ".timer"}: etc_units.add(f.name) # drift 1: repo 有 /etc 没 for name in sorted(repo_units - etc_units): missing_in_etc.append(name) # drift 2: 两边都有但内容不一致 for name in sorted(repo_units & etc_units): repo_path = repo_dir / name etc_path = SYSTEMD_ETC_DIR / name # 跳过被 mask 的 unit：/etc 下是 /dev/null 的符号链接，表示 systemd 故意禁用 # 这是 deploy.sh 对 market-sync-worker.service 的标准操作，不应报 drift if etc_path.is_symlink() and etc_path.resolve() == Path("/dev/null"): continue try: if repo_path.read_bytes() != etc_path.read_bytes(): drifted.append({ "unit": name, "repo_sha": hashlib.sha256(repo_path.read_bytes()).hexdigest()[:12], "etc_sha": hashlib.sha256(etc_path.read_bytes()).hexdigest()[:12], }) except OSError: drifted.append({"unit": name, "repo_sha": "?", "etc_sha": "?"}) # drift 3: /etc 有 repo 没（孤儿） for name in sorted(etc_units - repo_units): orphan_in_etc.append(name) has_drift = bool(missing_in_etc or drifted or orphan_in_etc) return { "schema_version": 1, "checked_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), "repo_units": sorted(repo_units), "etc_units": sorted(etc_units), "missing_in_etc": missing_in_etc, "drifted": drifted, "orphan_in_etc": orphan_in_etc, "has_drift": has_drift, } # ── 数据一致性检查 ───────────────────────────────────────────────────────── def _check_data_consistency() -> dict[str, Any]: """检查上游 kline_stock 与下游衍生表是否同步。 当前覆盖： - kline_stock_ma_daily 是否比 kline_stock 缺行/缺日期 - market_regime_daily 是否比 kline_stock 缺最近交易日 返回： alerts: 可读告警列表 details: 原始指标 """ from app.core.config import settings alerts: list[dict[str, str]] = [] details: dict[str, Any] = {"checked": False} try: engine = sa.create_engine(settings.pg_sqlalchemy_url()) with engine.connect() as conn: # 1) kline_stock vs kline_stock_ma_daily kline_max = conn.execute( sa.text("SELECT MAX(trade_date) FROM market_data.kline_stock") ).scalar() ma_max = conn.execute( sa.text("SELECT MAX(trade_date) FROM market_data.kline_stock_ma_daily") ).scalar() missing_rows = conn.execute( sa.text(""" SELECT COUNT(*) FROM ( SELECT stock_code, trade_date FROM market_data.kline_stock EXCEPT SELECT stock_code, trade_date FROM market_data.kline_stock_ma_daily ) t """) ).scalar() or 0 details["kline_stock_max_date"] = str(kline_max) if kline_max else None details["kline_stock_ma_daily_max_date"] = str(ma_max) if ma_max else None details["kline_ma_missing_rows"] = missing_rows if kline_max and ma_max and ma_max < kline_max: alerts.append({ "dataset_id": "data_consistency/kline_ma_daily", "status": "stale", "reason": f"kline_stock_ma_daily 最新日期 {ma_max} 落后于 kline_stock {kline_max}", }) elif missing_rows and missing_rows > 0: alerts.append({ "dataset_id": "data_consistency/kline_ma_daily", "status": "gap", "reason": f"kline_stock 有 {missing_rows} 行未覆盖到 kline_stock_ma_daily", }) # 2) kline_stock vs market_regime_daily（最近 3 个交易日） regime_max = conn.execute( sa.text("SELECT MAX(trade_date) FROM market_data.market_regime_daily") ).scalar() details["market_regime_daily_max_date"] = str(regime_max) if regime_max else None if kline_max and regime_max and regime_max < kline_max: alerts.append({ "dataset_id": "data_consistency/market_regime", "status": "stale", "reason": f"market_regime_daily 最新日期 {regime_max} 落后于 kline_stock {kline_max}", }) details["checked"] = True except Exception as e: details["error"] = str(e) alerts.append({ "dataset_id": "data_consistency/check_failed", "status": "error", "reason": f"数据一致性检查异常: {e}", }) return {"alerts": alerts, "details": details} # ── 报告生成 ─────────────────────────────────────────────────────────────── def build_report(check_date: date) -> dict[str, Any]: """生成当日巡检报告。""" from app.core.db import ops as db_ops # 延迟导入:避免单独跑这个脚本时加载 .env now = datetime.now() rows = db_ops.fetch_dataset_registry_rows() classified = [_classify_task(r, check_date, now) for r in rows] # 汇总计数 counts: dict[str, int] = {} for c in classified: counts[c["status"]] = counts.get(c["status"], 0) + 1 # 告警列表：只挑非 ok / non-expected 的 alerts = [ {"dataset_id": c["dataset_id"], "status": c["status"], "reason": c["reason"]} for c in classified if c["status"] in {"missed", "failed", "stuck", "never_run"} ] # systemd drift 单独段（不是 dataset 状态，但是 sync 完整性的关键） drift = _check_systemd_drift() if drift["has_drift"]: # drift 也算告警（影响同步运行） if drift["missing_in_etc"]: for u in drift["missing_in_etc"]: alerts.append({ "dataset_id": f"systemd/{u}", "status": "not_deployed", "reason": f"unit {u} 在 deploy/systemd/units/ 有但 /etc/systemd/system/ 没装", }) if drift["drifted"]: for d in drift["drifted"]: alerts.append({ "dataset_id": f"systemd/{d['unit']}", "status": "drifted", "reason": f"unit {d['unit']} /etc 版过期（repo={d['repo_sha']}, etc={d['etc_sha']}）", }) if drift["orphan_in_etc"]: for u in drift["orphan_in_etc"]: alerts.append({ "dataset_id": f"systemd/{u}", "status": "orphan", "reason": f"unit {u} 在 /etc/systemd/system/ 但 repo 没维护", }) # 数据一致性检查（上游回填后下游未跟进） consistency = _check_data_consistency() alerts.extend(consistency["alerts"]) overall = "ok" if not alerts else ( "warning" if any(a["status"] in {"missed", "never_run", "not_deployed", "drifted"} for a in alerts) else "error" ) return { "schema_version": 2, "generated_at": now.strftime("%Y-%m-%d %H:%M:%S"), "check_date": check_date.isoformat(), "runtime_mode": _runtime_mode(), "overall": overall, "counts": counts, "alerts": alerts, "tasks": classified, "systemd_drift": drift, "data_consistency": consistency["details"], } def write_report(report: dict[str, Any]) -> Path: """把报告写到 logs/daily_check_YYYYMMDD_HHMMSS.json。""" LOG_DIR.mkdir(parents=True, exist_ok=True) ts = datetime.now().strftime("%Y%m%d_%H%M%S") path = LOG_DIR / f"daily_check_{ts}.json" path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8") return path def print_summary(report: dict[str, Any]) -> None: """stdout 一眼看懂的汇总。""" print("=" * 78) print(f"每日同步巡检 date={report['check_date']} runtime_mode={report.get('runtime_mode', 'unknown')} overall={report['overall']}") print("=" * 78) print(f"counts: {report['counts']}") if report["alerts"]: print(f"\n告警 ({len(report['alerts'])}):") for a in report["alerts"]: print(f" - [{a['status']:>9}] {a['dataset_id']:<20} {a['reason']}") else: print("\n✅ 所有任务今日已成功") # systemd drift 段 drift = report.get("systemd_drift", {}) if drift: print(f"\nsystemd unit 漂移:") print(f" repo units ({len(drift['repo_units'])}): {drift['repo_units']}") print(f" /etc units ({len(drift['etc_units'])}): {drift['etc_units']}") if drift["missing_in_etc"]: print(f" ❌ 未部署 ({len(drift['missing_in_etc'])}): {drift['missing_in_etc']}") if drift["drifted"]: print(f" ⚠️ 内容不一致 ({len(drift['drifted'])}):") for d in drift["drifted"]: print(f" - {d['unit']} repo={d['repo_sha']} etc={d['etc_sha']}") if drift["orphan_in_etc"]: print(f" 👻 /etc 孤儿 ({len(drift['orphan_in_etc'])}): {drift['orphan_in_etc']}") if not drift["has_drift"]: print(f" ✅ 无漂移") # 完整列表 print(f"\n全部 {len(report['tasks'])} 个任务:") for t in report["tasks"]: icon = { "ok": "✅", "missed": "❌", "failed": "❌", "stuck": "⚠️", "running": "🔄", "never_run": "❓", "not_expected": "⏸ ", "not_yet": "⏳", "disabled": "🚫", }.get(t["status"], "·") print(f" {icon} [{t['status']:>13}] {t['dataset_id']:<20} {t['reason']}") # ── Webhook ──────────────────────────────────────────────────────────────── SIG_HEADER_CANDIDATES: list[tuple[str, str]] = [ ("X-Hub-Signature-256", "sha256={sig}"), ("X-Webhook-Signature", "{sig}"), ("X-Gitlab-Token", "{secret}"), ] HTTP_TIMEOUT = 10 def _load_secrets() -> dict[str, str]: out: dict[str, str] = {} if not SECRETS_FILE.exists(): return out try: for raw in SECRETS_FILE.read_text(encoding="utf-8").splitlines(): line = raw.strip() if not line or line.startswith("#"): continue if "=" not in line: continue k, v = line.split("=", 1) k = k.strip() v = v.strip().strip('"').strip("'") if k and v: out[k] = v except Exception as e: LOG.error("failed to parse %s: %s", SECRETS_FILE, e) return out def post_webhook(url: str, secret: str, payload: dict[str, Any], *, dry_run: bool = False) -> int: body = json.dumps(payload, ensure_ascii=False).encode("utf-8") sig = hmac.new(secret.encode("utf-8"), body, hashlib.sha256).hexdigest() if dry_run: print(f"[webhook DRY-RUN] {url}") print(json.dumps(payload, ensure_ascii=False, indent=2)) return 200 for hdr_name, value_fmt in SIG_HEADER_CANDIDATES: sig_value = value_fmt.format(sig=sig, secret=secret) headers = {"Content-Type": "application/json", hdr_name: sig_value} try: r = requests.post(url, data=body, headers=headers, timeout=HTTP_TIMEOUT) if r.status_code in (401, 403): LOG.info("auth header %s rejected (%s) — trying next", hdr_name, r.status_code) continue return r.status_code except requests.RequestException as e: LOG.error("POST %s failed with %s: %s", url, hdr_name, e) return 0 return 0 def maybe_alert(report: dict[str, Any], secrets: dict[str, str], *, dry_run: bool) -> None: """只在 overall != ok 时 POST。""" if report["overall"] == "ok": LOG.info("全部 ok，跳过 webhook") return url = secrets.get("FAILURE_WEBHOOK_URL") secret = secrets.get("FAILURE_WEBHOOK_SECRET") if not url or not secret: LOG.warning("FAILURE_WEBHOOK_URL/SECRET 未配置，跳过 POST") return payload = { "task": "daily_check", "service": "bin/daily_sync_check.py", "status": report["overall"], "check_date": report["check_date"], "counts": report["counts"], "alerts": report["alerts"], "generated_at": report["generated_at"], } status = post_webhook(url, secret, payload, dry_run=dry_run) LOG.info("webhook POST → %s", status) # ── CLI ──────────────────────────────────────────────────────────────────── def main(argv: Optional[list] = None) -> int: parser = argparse.ArgumentParser(description="每日同步巡检") parser.add_argument("--date", help="检查日期 YYYY-MM-DD（默认今天）") parser.add_argument("--dry-run", action="store_true", help="webhook 走 dry-run（仍写报告）") parser.add_argument("--report-only", action="store_true", help="不读 secrets、不 POST") args = parser.parse_args(argv) logging.basicConfig( level=logging.INFO, format="%(asctime)s %(levelname)-7s %(name)s: %(message)s", datefmt="%Y-%m-%dT%H:%M:%S", ) if args.date: check_date = datetime.strptime(args.date, "%Y-%m-%d").date() else: check_date = date.today() report = build_report(check_date) path = write_report(report) print_summary(report) LOG.info("报告已写: %s", path) if not args.report_only: secrets = _load_secrets() maybe_alert(report, secrets, dry_run=args.dry_run) return 0 if report["overall"] == "ok" else 1 if __name__ == "__main__": sys.exit(main())