Files
market_sync/bin/daily_sync_check.py
T
gao f7a2cbe5a8 fix(daily_check): 期望窗口未到不应判 missed
work #07 (2026-07-03):daily_check 在日内运行时(如 09:00/12:00/15:00)
误把'还没到时间'的任务标 missed。SCHEDULE 里有 window_end 但 _classify_task
没用到——直接判 last_success vs today_start。

修复:判定 missed 之前先比较 now vs window_end;未到则返回 not_yet
(新状态,不进 alerts),已过但今日未成功走原 missed 路径。解析失败
回落旧逻辑不引入风险。

验证 4 个用例:12:00 kline_daily→not_yet, 18:00 kline_daily→missed,
18:00 longhubang→not_yet, 22:30 longhubang→missed。全部正确。

Co-Authored-By: Claude <noreply@anthropic.com>
2026-07-03 18:25:50 +08:00

541 lines
21 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""每日同步任务巡检。
读 PG `dataset_registry`,对比今日预期调度窗口,给出每个 task 的健康状态,
写 `logs/daily_check_YYYYMMDD_HHMMSS.json` 报告,仅在有失败/漏跑时 POST webhook。
用法:
.venv/bin/python bin/daily_sync_check.py # 默认检查"今天"
.venv/bin/python bin/daily_sync_check.py --dry-run # 只生成报告,不 POST
.venv/bin/python bin/daily_sync_check.py --report-only # 完全不读 secrets / 不 POST
.venv/bin/python bin/daily_sync_check.py --date 2026-07-02 # 检查指定日期
为什么是 stand-alone(不放在 app.tasks 下):
- 巡检逻辑独立于 sync 业务,混在 app.tasks 会让注册表臃肿
- 独立脚本便于 cron / systemd timer / /loop 三种触发方式共用
"""
from __future__ import annotations
import argparse
import hashlib
import hmac
import json
import logging
import os
import re
import sys
import time
from datetime import date, datetime, timedelta
from pathlib import Path
from typing import Any, Optional
import requests
# ── 路径与日志 ─────────────────────────────────────────────────────────────
PROJECT_ROOT = Path(__file__).resolve().parent.parent
if str(PROJECT_ROOT) not in sys.path:
sys.path.insert(0, str(PROJECT_ROOT))
LOG_DIR = PROJECT_ROOT / "logs"
SECRETS_FILE = PROJECT_ROOT / "config" / "daily_check_secrets.env"
LOG = logging.getLogger("daily-sync-check")
# ── 预期调度表 ─────────────────────────────────────────────────────────────
# 与 [[market-data-overview]] 保持一致;同步触发点(HH:MM)按 Asia/Shanghai 解释。
# weekend_only=True 表示只在周六/周日期望运行(实际仅 stock_node 周六一次)。
SCHEDULE: dict[str, dict[str, Any]] = {
"stock_basic": {"window_end": "09:30", "weekend_only": False},
"industry_sector": {"window_end": "09:45", "weekend_only": False},
"kline_index": {"window_end": "15:35", "weekend_only": False},
"kline_daily": {"window_end": "15:45", "weekend_only": False},
"kline_5min": {"window_end": "16:05", "weekend_only": False},
"market_regime": {"window_end": "16:20", "weekend_only": False},
"share_snapshot": {"window_end": "16:25", "weekend_only": False},
"moneyflow": {"window_end": "21:40", "weekend_only": False},
"longhubang": {"window_end": "22:05", "weekend_only": False},
"stock_node": {"window_end": "12:00", "weekend_only": True}, # 周六
"mairui_ma_daily": {"window_end": "17:00", "weekend_only": False}, # 本地派生
}
# ── 状态判定 ───────────────────────────────────────────────────────────────
def _parse_ts(s: Optional[str]) -> Optional[datetime]:
"""dataset_registry 时间字段是 '%Y-%m-%d %H:%M:%S' 或 None。"""
if not s:
return None
for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%dT%H:%M:%S%z"):
try:
return datetime.strptime(s, fmt)
except ValueError:
continue
return None
def _classify_task(
row: dict[str, Any],
check_date: date,
now: datetime,
) -> dict[str, Any]:
"""判定单个 task 在 check_date 的健康状态。
返回字段:
status: ok | missed | failed | running | never_run | not_expected
reason: 简短中文原因
last_success_at / last_failure_at / last_error / needs_resync: 原值透传
"""
dataset_id = row["dataset_id"]
enabled = (row.get("enabled") or 0) == 1
spec = SCHEDULE.get(dataset_id)
is_weekend = check_date.weekday() >= 5
if not enabled:
return {
"dataset_id": dataset_id,
"name": row.get("name", ""),
"status": "disabled",
"reason": "enabled=0",
"last_success_at": row.get("last_success_at"),
"last_failure_at": row.get("last_failure_at"),
"last_error": row.get("last_error"),
"needs_resync": row.get("needs_resync", 0),
"status_field": row.get("status", "idle"),
}
# weekend 期望判定
if spec and spec["weekend_only"] and not is_weekend:
return {
"dataset_id": dataset_id,
"name": row.get("name", ""),
"status": "not_expected",
"reason": "周末专属任务,今天不期望",
"last_success_at": row.get("last_success_at"),
"last_failure_at": row.get("last_failure_at"),
"last_error": row.get("last_error"),
"needs_resync": row.get("needs_resync", 0),
"status_field": row.get("status", "idle"),
}
if spec and not spec["weekend_only"] and is_weekend:
return {
"dataset_id": dataset_id,
"name": row.get("name", ""),
"status": "not_expected",
"reason": "周末不调度",
"last_success_at": row.get("last_success_at"),
"last_failure_at": row.get("last_failure_at"),
"last_error": row.get("last_error"),
"needs_resync": row.get("needs_resync", 0),
"status_field": row.get("status", "idle"),
}
last_success = _parse_ts(row.get("last_success_at"))
last_failure = _parse_ts(row.get("last_failure_at"))
status_field = row.get("status", "idle")
# 期望窗口是否已过?未到窗口不应判 missed
if spec and spec.get("window_end") and not status_field == "running":
try:
hh, mm = spec["window_end"].split(":")
window_end_dt = datetime.combine(check_date, datetime.min.time()).replace(
hour=int(hh), minute=int(mm)
)
if now < window_end_dt:
return {
"dataset_id": dataset_id,
"name": row.get("name", ""),
"status": "not_yet",
"reason": f"期望窗口 {spec['window_end']} 未到(now={now.strftime('%H:%M')}",
"last_success_at": row.get("last_success_at"),
"last_failure_at": row.get("last_failure_at"),
"last_error": row.get("last_error"),
"needs_resync": row.get("needs_resync", 0),
"status_field": status_field,
}
except (ValueError, AttributeError):
pass # 解析失败回落到旧逻辑
# 还在跑(卡死检测)
if status_field == "running":
started = _parse_ts(row.get("started_at"))
if started and (now - started) > timedelta(hours=2):
return {
"dataset_id": dataset_id,
"name": row.get("name", ""),
"status": "stuck",
"reason": f"running > 2h, started_at={row.get('started_at')}",
"last_success_at": row.get("last_success_at"),
"last_failure_at": row.get("last_failure_at"),
"last_error": row.get("last_error"),
"needs_resync": row.get("needs_resync", 0),
"status_field": status_field,
}
return {
"dataset_id": dataset_id,
"name": row.get("name", ""),
"status": "running",
"reason": f"status=running, started_at={row.get('started_at')}",
"last_success_at": row.get("last_success_at"),
"last_failure_at": row.get("last_failure_at"),
"last_error": row.get("last_error"),
"needs_resync": row.get("needs_resync", 0),
"status_field": status_field,
}
# 从未跑过
if last_success is None and last_failure is None:
return {
"dataset_id": dataset_id,
"name": row.get("name", ""),
"status": "never_run",
"reason": "从未同步过",
"last_success_at": row.get("last_success_at"),
"last_failure_at": row.get("last_failure_at"),
"last_error": row.get("last_error"),
"needs_resync": row.get("needs_resync", 0),
"status_field": status_field,
}
# 今日是否有成功?
today_start = datetime.combine(check_date, datetime.min.time())
if last_success and last_success >= today_start:
return {
"dataset_id": dataset_id,
"name": row.get("name", ""),
"status": "ok",
"reason": f"今日已成功 @ {last_success.strftime('%H:%M:%S')}",
"last_success_at": row.get("last_success_at"),
"last_failure_at": row.get("last_failure_at"),
"last_error": row.get("last_error"),
"needs_resync": row.get("needs_resync", 0),
"status_field": status_field,
}
# 今日失败过?
if last_failure and last_failure >= today_start and (
not last_success or last_failure > last_success
):
return {
"dataset_id": dataset_id,
"name": row.get("name", ""),
"status": "failed",
"reason": f"今日失败 @ {last_failure.strftime('%H:%M:%S')}: "
f"{(row.get('last_error') or '')[:120]}",
"last_success_at": row.get("last_success_at"),
"last_failure_at": row.get("last_failure_at"),
"last_error": row.get("last_error"),
"needs_resync": row.get("needs_resync", 0),
"status_field": status_field,
}
# 期望窗口已过但今日没成功
return {
"dataset_id": dataset_id,
"name": row.get("name", ""),
"status": "missed",
"reason": (
f"今日未成功(last_success={last_success.strftime('%Y-%m-%d %H:%M') if last_success else 'never'}"
),
"last_success_at": row.get("last_success_at"),
"last_failure_at": row.get("last_failure_at"),
"last_error": row.get("last_error"),
"needs_resync": row.get("needs_resync", 0),
"status_field": status_field,
}
# ── systemd unit drift 检测 ────────────────────────────────────────────────
SYSTEMD_REPO_DIR = PROJECT_ROOT / "bin" / "systemd"
SYSTEMD_ETC_DIR = Path("/etc/systemd/system")
def _check_systemd_drift() -> dict[str, Any]:
"""对比 bin/systemd/ 与 /etc/systemd/system/ 下的 market-sync* unit。
返回:
missing_in_etc: repo 有但 /etc 没装(drift 1:未部署)
drifted: 两边都有但内容不一致(drift 2:部署的版本过期)
orphan_in_etc: /etc 有但 repo 没有(drift 3:临时/僵尸 unit
"""
missing_in_etc: list[str] = []
drifted: list[dict[str, str]] = []
orphan_in_etc: list[str] = []
repo_units: set[str] = set()
if SYSTEMD_REPO_DIR.exists():
for f in SYSTEMD_REPO_DIR.iterdir():
if f.suffix in {".service", ".timer"} and f.name.startswith("market-sync"):
repo_units.add(f.name)
etc_units: set[str] = set()
if SYSTEMD_ETC_DIR.exists():
for f in SYSTEMD_ETC_DIR.iterdir():
if f.name.startswith("market-sync") and f.suffix in {".service", ".timer"}:
etc_units.add(f.name)
# drift 1: repo 有 /etc 没
for name in sorted(repo_units - etc_units):
missing_in_etc.append(name)
# drift 2: 两边都有但内容不一致
for name in sorted(repo_units & etc_units):
repo_path = SYSTEMD_REPO_DIR / name
etc_path = SYSTEMD_ETC_DIR / name
try:
if repo_path.read_bytes() != etc_path.read_bytes():
drifted.append({
"unit": name,
"repo_sha": hashlib.sha256(repo_path.read_bytes()).hexdigest()[:12],
"etc_sha": hashlib.sha256(etc_path.read_bytes()).hexdigest()[:12],
})
except OSError:
drifted.append({"unit": name, "repo_sha": "?", "etc_sha": "?"})
# drift 3: /etc 有 repo 没(孤儿)
for name in sorted(etc_units - repo_units):
orphan_in_etc.append(name)
has_drift = bool(missing_in_etc or drifted or orphan_in_etc)
return {
"schema_version": 1,
"checked_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
"repo_units": sorted(repo_units),
"etc_units": sorted(etc_units),
"missing_in_etc": missing_in_etc,
"drifted": drifted,
"orphan_in_etc": orphan_in_etc,
"has_drift": has_drift,
}
# ── 报告生成 ───────────────────────────────────────────────────────────────
def build_report(check_date: date) -> dict[str, Any]:
"""生成当日巡检报告。"""
from app.core.db import ops as db_ops # 延迟导入:避免单独跑这个脚本时加载 .env
now = datetime.now()
rows = db_ops.fetch_dataset_registry_rows()
classified = [_classify_task(r, check_date, now) for r in rows]
# 汇总计数
counts: dict[str, int] = {}
for c in classified:
counts[c["status"]] = counts.get(c["status"], 0) + 1
# 告警列表:只挑非 ok / non-expected 的
alerts = [
{"dataset_id": c["dataset_id"], "status": c["status"], "reason": c["reason"]}
for c in classified
if c["status"] in {"missed", "failed", "stuck", "never_run"}
]
# systemd drift 单独段(不是 dataset 状态,但是 sync 完整性的关键)
drift = _check_systemd_drift()
if drift["has_drift"]:
# drift 也算告警(影响同步运行)
if drift["missing_in_etc"]:
for u in drift["missing_in_etc"]:
alerts.append({
"dataset_id": f"systemd/{u}",
"status": "not_deployed",
"reason": f"unit {u} 在 bin/systemd/ 有但 /etc/systemd/system/ 没装",
})
if drift["drifted"]:
for d in drift["drifted"]:
alerts.append({
"dataset_id": f"systemd/{d['unit']}",
"status": "drifted",
"reason": f"unit {d['unit']} /etc 版过期(repo={d['repo_sha']}, etc={d['etc_sha']}",
})
if drift["orphan_in_etc"]:
for u in drift["orphan_in_etc"]:
alerts.append({
"dataset_id": f"systemd/{u}",
"status": "orphan",
"reason": f"unit {u} 在 /etc/systemd/system/ 但 repo 没维护",
})
overall = "ok" if not alerts else (
"warning" if any(a["status"] in {"missed", "never_run", "not_deployed", "drifted"} for a in alerts) else "error"
)
return {
"schema_version": 2,
"generated_at": now.strftime("%Y-%m-%d %H:%M:%S"),
"check_date": check_date.isoformat(),
"overall": overall,
"counts": counts,
"alerts": alerts,
"tasks": classified,
"systemd_drift": drift,
}
def write_report(report: dict[str, Any]) -> Path:
"""把报告写到 logs/daily_check_YYYYMMDD_HHMMSS.json。"""
LOG_DIR.mkdir(parents=True, exist_ok=True)
ts = datetime.now().strftime("%Y%m%d_%H%M%S")
path = LOG_DIR / f"daily_check_{ts}.json"
path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
return path
def print_summary(report: dict[str, Any]) -> None:
"""stdout 一眼看懂的汇总。"""
print("=" * 78)
print(f"每日同步巡检 date={report['check_date']} overall={report['overall']}")
print("=" * 78)
print(f"counts: {report['counts']}")
if report["alerts"]:
print(f"\n告警 ({len(report['alerts'])}):")
for a in report["alerts"]:
print(f" - [{a['status']:>9}] {a['dataset_id']:<20} {a['reason']}")
else:
print("\n✅ 所有任务今日已成功")
# systemd drift 段
drift = report.get("systemd_drift", {})
if drift:
print(f"\nsystemd unit 漂移:")
print(f" repo units ({len(drift['repo_units'])}): {drift['repo_units']}")
print(f" /etc units ({len(drift['etc_units'])}): {drift['etc_units']}")
if drift["missing_in_etc"]:
print(f" ❌ 未部署 ({len(drift['missing_in_etc'])}): {drift['missing_in_etc']}")
if drift["drifted"]:
print(f" ⚠️ 内容不一致 ({len(drift['drifted'])}):")
for d in drift["drifted"]:
print(f" - {d['unit']} repo={d['repo_sha']} etc={d['etc_sha']}")
if drift["orphan_in_etc"]:
print(f" 👻 /etc 孤儿 ({len(drift['orphan_in_etc'])}): {drift['orphan_in_etc']}")
if not drift["has_drift"]:
print(f" ✅ 无漂移")
# 完整列表
print(f"\n全部 {len(report['tasks'])} 个任务:")
for t in report["tasks"]:
icon = {
"ok": "",
"missed": "",
"failed": "",
"stuck": "⚠️",
"running": "🔄",
"never_run": "",
"not_expected": "",
"not_yet": "",
"disabled": "🚫",
}.get(t["status"], "·")
print(f" {icon} [{t['status']:>13}] {t['dataset_id']:<20} {t['reason']}")
# ── Webhook ────────────────────────────────────────────────────────────────
SIG_HEADER_CANDIDATES: list[tuple[str, str]] = [
("X-Hub-Signature-256", "sha256={sig}"),
("X-Webhook-Signature", "{sig}"),
("X-Gitlab-Token", "{secret}"),
]
HTTP_TIMEOUT = 10
def _load_secrets() -> dict[str, str]:
out: dict[str, str] = {}
if not SECRETS_FILE.exists():
return out
try:
for raw in SECRETS_FILE.read_text(encoding="utf-8").splitlines():
line = raw.strip()
if not line or line.startswith("#"):
continue
if "=" not in line:
continue
k, v = line.split("=", 1)
k = k.strip()
v = v.strip().strip('"').strip("'")
if k and v:
out[k] = v
except Exception as e:
LOG.error("failed to parse %s: %s", SECRETS_FILE, e)
return out
def post_webhook(url: str, secret: str, payload: dict[str, Any], *, dry_run: bool = False) -> int:
body = json.dumps(payload, ensure_ascii=False).encode("utf-8")
sig = hmac.new(secret.encode("utf-8"), body, hashlib.sha256).hexdigest()
if dry_run:
print(f"[webhook DRY-RUN] {url}")
print(json.dumps(payload, ensure_ascii=False, indent=2))
return 200
for hdr_name, value_fmt in SIG_HEADER_CANDIDATES:
sig_value = value_fmt.format(sig=sig, secret=secret)
headers = {"Content-Type": "application/json", hdr_name: sig_value}
try:
r = requests.post(url, data=body, headers=headers, timeout=HTTP_TIMEOUT)
if r.status_code in (401, 403):
LOG.info("auth header %s rejected (%s) — trying next", hdr_name, r.status_code)
continue
return r.status_code
except requests.RequestException as e:
LOG.error("POST %s failed with %s: %s", url, hdr_name, e)
return 0
return 0
def maybe_alert(report: dict[str, Any], secrets: dict[str, str], *, dry_run: bool) -> None:
"""只在 overall != ok 时 POST。"""
if report["overall"] == "ok":
LOG.info("全部 ok,跳过 webhook")
return
url = secrets.get("FAILURE_WEBHOOK_URL")
secret = secrets.get("FAILURE_WEBHOOK_SECRET")
if not url or not secret:
LOG.warning("FAILURE_WEBHOOK_URL/SECRET 未配置,跳过 POST")
return
payload = {
"task": "daily_check",
"service": "bin/daily_sync_check.py",
"status": report["overall"],
"check_date": report["check_date"],
"counts": report["counts"],
"alerts": report["alerts"],
"generated_at": report["generated_at"],
}
status = post_webhook(url, secret, payload, dry_run=dry_run)
LOG.info("webhook POST → %s", status)
# ── CLI ────────────────────────────────────────────────────────────────────
def main(argv: Optional[list] = None) -> int:
parser = argparse.ArgumentParser(description="每日同步巡检")
parser.add_argument("--date", help="检查日期 YYYY-MM-DD(默认今天)")
parser.add_argument("--dry-run", action="store_true", help="webhook 走 dry-run(仍写报告)")
parser.add_argument("--report-only", action="store_true", help="不读 secrets、不 POST")
args = parser.parse_args(argv)
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s %(levelname)-7s %(name)s: %(message)s",
datefmt="%Y-%m-%dT%H:%M:%S",
)
if args.date:
check_date = datetime.strptime(args.date, "%Y-%m-%d").date()
else:
check_date = date.today()
report = build_report(check_date)
path = write_report(report)
print_summary(report)
LOG.info("报告已写: %s", path)
if not args.report_only:
secrets = _load_secrets()
maybe_alert(report, secrets, dry_run=args.dry_run)
return 0 if report["overall"] == "ok" else 1
if __name__ == "__main__":
sys.exit(main())