fix: market-sync 2h timeout 截断 + 加 daily-check 巡检
work #01 (2026-07-03)：昨天 15:30 runall 中 kline_5min 全量 69min 把 2h 窗口吃满，share_snapshot / market_regime 被 systemd SIGTERM 截断； 今天又叠加 worker.py 未长驻，导致 10 个 task missed。 变更： - bin/systemd/market-sync.service: TimeoutStartSec 7200 → 14400 (4h) - bin/daily_sync_check.py: 新增每日巡检，读 PG dataset_registry， 对比 SCHEDULE 判定 ok/missed/failed/stuck/never_run，写 JSON 报告， 失败时 POST HMAC webhook - bin/systemd/market-sync-daily-check.{service,timer}: 23:00 daily 触发 daily_check（替代 session-only 的 /loop 方案） - config/daily_check_secrets.env.template: webhook 配置模板 - docs/works/2026-07-03-01-market-sync-timeout.md: 完整 work 记录 （根因 + 中期/长期改进建议） Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
gao
@@ -0,0 +1,408 @@
|
||||
"""每日同步任务巡检。
|
||||
|
||||
读 PG `dataset_registry`，对比今日预期调度窗口，给出每个 task 的健康状态，
|
||||
写 `logs/daily_check_YYYYMMDD_HHMMSS.json` 报告，仅在有失败/漏跑时 POST webhook。
|
||||
|
||||
用法：
|
||||
.venv/bin/python bin/daily_sync_check.py # 默认检查"今天"
|
||||
.venv/bin/python bin/daily_sync_check.py --dry-run # 只生成报告，不 POST
|
||||
.venv/bin/python bin/daily_sync_check.py --report-only # 完全不读 secrets / 不 POST
|
||||
.venv/bin/python bin/daily_sync_check.py --date 2026-07-02 # 检查指定日期
|
||||
|
||||
为什么是 stand-alone（不放在 app.tasks 下）：
|
||||
- 巡检逻辑独立于 sync 业务，混在 app.tasks 会让注册表臃肿
|
||||
- 独立脚本便于 cron / systemd timer / /loop 三种触发方式共用
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from datetime import date, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional
|
||||
|
||||
import requests
|
||||
|
||||
|
||||
# ── 路径与日志 ─────────────────────────────────────────────────────────────
|
||||
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||||
if str(PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
LOG_DIR = PROJECT_ROOT / "logs"
|
||||
SECRETS_FILE = PROJECT_ROOT / "config" / "daily_check_secrets.env"
|
||||
|
||||
LOG = logging.getLogger("daily-sync-check")
|
||||
|
||||
|
||||
# ── 预期调度表 ─────────────────────────────────────────────────────────────
|
||||
# 与 [[market-data-overview]] 保持一致；同步触发点（HH:MM）按 Asia/Shanghai 解释。
|
||||
# weekend_only=True 表示只在周六/周日期望运行（实际仅 stock_node 周六一次）。
|
||||
SCHEDULE: dict[str, dict[str, Any]] = {
|
||||
"stock_basic": {"window_end": "09:30", "weekend_only": False},
|
||||
"industry_sector": {"window_end": "09:45", "weekend_only": False},
|
||||
"kline_index": {"window_end": "15:35", "weekend_only": False},
|
||||
"kline_daily": {"window_end": "15:45", "weekend_only": False},
|
||||
"kline_5min": {"window_end": "16:05", "weekend_only": False},
|
||||
"market_regime": {"window_end": "16:20", "weekend_only": False},
|
||||
"share_snapshot": {"window_end": "16:25", "weekend_only": False},
|
||||
"moneyflow": {"window_end": "21:40", "weekend_only": False},
|
||||
"longhubang": {"window_end": "22:05", "weekend_only": False},
|
||||
"stock_node": {"window_end": "12:00", "weekend_only": True}, # 周六
|
||||
}
|
||||
|
||||
|
||||
# ── 状态判定 ───────────────────────────────────────────────────────────────
|
||||
def _parse_ts(s: Optional[str]) -> Optional[datetime]:
|
||||
"""dataset_registry 时间字段是 '%Y-%m-%d %H:%M:%S' 或 None。"""
|
||||
if not s:
|
||||
return None
|
||||
for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%dT%H:%M:%S%z"):
|
||||
try:
|
||||
return datetime.strptime(s, fmt)
|
||||
except ValueError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def _classify_task(
|
||||
row: dict[str, Any],
|
||||
check_date: date,
|
||||
now: datetime,
|
||||
) -> dict[str, Any]:
|
||||
"""判定单个 task 在 check_date 的健康状态。
|
||||
|
||||
返回字段：
|
||||
status: ok | missed | failed | running | never_run | not_expected
|
||||
reason: 简短中文原因
|
||||
last_success_at / last_failure_at / last_error / needs_resync: 原值透传
|
||||
"""
|
||||
dataset_id = row["dataset_id"]
|
||||
enabled = (row.get("enabled") or 0) == 1
|
||||
spec = SCHEDULE.get(dataset_id)
|
||||
is_weekend = check_date.weekday() >= 5
|
||||
|
||||
if not enabled:
|
||||
return {
|
||||
"dataset_id": dataset_id,
|
||||
"name": row.get("name", ""),
|
||||
"status": "disabled",
|
||||
"reason": "enabled=0",
|
||||
"last_success_at": row.get("last_success_at"),
|
||||
"last_failure_at": row.get("last_failure_at"),
|
||||
"last_error": row.get("last_error"),
|
||||
"needs_resync": row.get("needs_resync", 0),
|
||||
"status_field": row.get("status", "idle"),
|
||||
}
|
||||
|
||||
# weekend 期望判定
|
||||
if spec and spec["weekend_only"] and not is_weekend:
|
||||
return {
|
||||
"dataset_id": dataset_id,
|
||||
"name": row.get("name", ""),
|
||||
"status": "not_expected",
|
||||
"reason": "周末专属任务，今天不期望",
|
||||
"last_success_at": row.get("last_success_at"),
|
||||
"last_failure_at": row.get("last_failure_at"),
|
||||
"last_error": row.get("last_error"),
|
||||
"needs_resync": row.get("needs_resync", 0),
|
||||
"status_field": row.get("status", "idle"),
|
||||
}
|
||||
if spec and not spec["weekend_only"] and is_weekend:
|
||||
return {
|
||||
"dataset_id": dataset_id,
|
||||
"name": row.get("name", ""),
|
||||
"status": "not_expected",
|
||||
"reason": "周末不调度",
|
||||
"last_success_at": row.get("last_success_at"),
|
||||
"last_failure_at": row.get("last_failure_at"),
|
||||
"last_error": row.get("last_error"),
|
||||
"needs_resync": row.get("needs_resync", 0),
|
||||
"status_field": row.get("status", "idle"),
|
||||
}
|
||||
|
||||
last_success = _parse_ts(row.get("last_success_at"))
|
||||
last_failure = _parse_ts(row.get("last_failure_at"))
|
||||
status_field = row.get("status", "idle")
|
||||
|
||||
# 还在跑（卡死检测）
|
||||
if status_field == "running":
|
||||
started = _parse_ts(row.get("started_at"))
|
||||
if started and (now - started) > timedelta(hours=2):
|
||||
return {
|
||||
"dataset_id": dataset_id,
|
||||
"name": row.get("name", ""),
|
||||
"status": "stuck",
|
||||
"reason": f"running > 2h, started_at={row.get('started_at')}",
|
||||
"last_success_at": row.get("last_success_at"),
|
||||
"last_failure_at": row.get("last_failure_at"),
|
||||
"last_error": row.get("last_error"),
|
||||
"needs_resync": row.get("needs_resync", 0),
|
||||
"status_field": status_field,
|
||||
}
|
||||
return {
|
||||
"dataset_id": dataset_id,
|
||||
"name": row.get("name", ""),
|
||||
"status": "running",
|
||||
"reason": f"status=running, started_at={row.get('started_at')}",
|
||||
"last_success_at": row.get("last_success_at"),
|
||||
"last_failure_at": row.get("last_failure_at"),
|
||||
"last_error": row.get("last_error"),
|
||||
"needs_resync": row.get("needs_resync", 0),
|
||||
"status_field": status_field,
|
||||
}
|
||||
|
||||
# 从未跑过
|
||||
if last_success is None and last_failure is None:
|
||||
return {
|
||||
"dataset_id": dataset_id,
|
||||
"name": row.get("name", ""),
|
||||
"status": "never_run",
|
||||
"reason": "从未同步过",
|
||||
"last_success_at": row.get("last_success_at"),
|
||||
"last_failure_at": row.get("last_failure_at"),
|
||||
"last_error": row.get("last_error"),
|
||||
"needs_resync": row.get("needs_resync", 0),
|
||||
"status_field": status_field,
|
||||
}
|
||||
|
||||
# 今日是否有成功？
|
||||
today_start = datetime.combine(check_date, datetime.min.time())
|
||||
if last_success and last_success >= today_start:
|
||||
return {
|
||||
"dataset_id": dataset_id,
|
||||
"name": row.get("name", ""),
|
||||
"status": "ok",
|
||||
"reason": f"今日已成功 @ {last_success.strftime('%H:%M:%S')}",
|
||||
"last_success_at": row.get("last_success_at"),
|
||||
"last_failure_at": row.get("last_failure_at"),
|
||||
"last_error": row.get("last_error"),
|
||||
"needs_resync": row.get("needs_resync", 0),
|
||||
"status_field": status_field,
|
||||
}
|
||||
|
||||
# 今日失败过？
|
||||
if last_failure and last_failure >= today_start and (
|
||||
not last_success or last_failure > last_success
|
||||
):
|
||||
return {
|
||||
"dataset_id": dataset_id,
|
||||
"name": row.get("name", ""),
|
||||
"status": "failed",
|
||||
"reason": f"今日失败 @ {last_failure.strftime('%H:%M:%S')}: "
|
||||
f"{(row.get('last_error') or '')[:120]}",
|
||||
"last_success_at": row.get("last_success_at"),
|
||||
"last_failure_at": row.get("last_failure_at"),
|
||||
"last_error": row.get("last_error"),
|
||||
"needs_resync": row.get("needs_resync", 0),
|
||||
"status_field": status_field,
|
||||
}
|
||||
|
||||
# 期望窗口已过但今日没成功
|
||||
return {
|
||||
"dataset_id": dataset_id,
|
||||
"name": row.get("name", ""),
|
||||
"status": "missed",
|
||||
"reason": (
|
||||
f"今日未成功（last_success={last_success.strftime('%Y-%m-%d %H:%M') if last_success else 'never'}）"
|
||||
),
|
||||
"last_success_at": row.get("last_success_at"),
|
||||
"last_failure_at": row.get("last_failure_at"),
|
||||
"last_error": row.get("last_error"),
|
||||
"needs_resync": row.get("needs_resync", 0),
|
||||
"status_field": status_field,
|
||||
}
|
||||
|
||||
|
||||
# ── 报告生成 ───────────────────────────────────────────────────────────────
|
||||
def build_report(check_date: date) -> dict[str, Any]:
|
||||
"""生成当日巡检报告。"""
|
||||
from app.core.db import ops as db_ops # 延迟导入:避免单独跑这个脚本时加载 .env
|
||||
|
||||
now = datetime.now()
|
||||
rows = db_ops.fetch_dataset_registry_rows()
|
||||
classified = [_classify_task(r, check_date, now) for r in rows]
|
||||
|
||||
# 汇总计数
|
||||
counts: dict[str, int] = {}
|
||||
for c in classified:
|
||||
counts[c["status"]] = counts.get(c["status"], 0) + 1
|
||||
|
||||
# 告警列表：只挑非 ok / non-expected 的
|
||||
alerts = [
|
||||
{"dataset_id": c["dataset_id"], "status": c["status"], "reason": c["reason"]}
|
||||
for c in classified
|
||||
if c["status"] in {"missed", "failed", "stuck", "never_run"}
|
||||
]
|
||||
|
||||
overall = "ok" if not alerts else (
|
||||
"warning" if any(a["status"] in {"missed", "never_run"} for a in alerts) else "error"
|
||||
)
|
||||
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"generated_at": now.strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"check_date": check_date.isoformat(),
|
||||
"overall": overall,
|
||||
"counts": counts,
|
||||
"alerts": alerts,
|
||||
"tasks": classified,
|
||||
}
|
||||
|
||||
|
||||
def write_report(report: dict[str, Any]) -> Path:
|
||||
"""把报告写到 logs/daily_check_YYYYMMDD_HHMMSS.json。"""
|
||||
LOG_DIR.mkdir(parents=True, exist_ok=True)
|
||||
ts = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
path = LOG_DIR / f"daily_check_{ts}.json"
|
||||
path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
def print_summary(report: dict[str, Any]) -> None:
|
||||
"""stdout 一眼看懂的汇总。"""
|
||||
print("=" * 78)
|
||||
print(f"每日同步巡检 date={report['check_date']} overall={report['overall']}")
|
||||
print("=" * 78)
|
||||
print(f"counts: {report['counts']}")
|
||||
if report["alerts"]:
|
||||
print(f"\n告警 ({len(report['alerts'])}):")
|
||||
for a in report["alerts"]:
|
||||
print(f" - [{a['status']:>9}] {a['dataset_id']:<20} {a['reason']}")
|
||||
else:
|
||||
print("\n✅ 所有任务今日已成功")
|
||||
# 完整列表
|
||||
print(f"\n全部 {len(report['tasks'])} 个任务:")
|
||||
for t in report["tasks"]:
|
||||
icon = {
|
||||
"ok": "✅",
|
||||
"missed": "❌",
|
||||
"failed": "❌",
|
||||
"stuck": "⚠️",
|
||||
"running": "🔄",
|
||||
"never_run": "❓",
|
||||
"not_expected": "⏸ ",
|
||||
"disabled": "🚫",
|
||||
}.get(t["status"], "·")
|
||||
print(f" {icon} [{t['status']:>13}] {t['dataset_id']:<20} {t['reason']}")
|
||||
|
||||
|
||||
# ── Webhook ────────────────────────────────────────────────────────────────
|
||||
SIG_HEADER_CANDIDATES: list[tuple[str, str]] = [
|
||||
("X-Hub-Signature-256", "sha256={sig}"),
|
||||
("X-Webhook-Signature", "{sig}"),
|
||||
("X-Gitlab-Token", "{secret}"),
|
||||
]
|
||||
HTTP_TIMEOUT = 10
|
||||
|
||||
|
||||
def _load_secrets() -> dict[str, str]:
|
||||
out: dict[str, str] = {}
|
||||
if not SECRETS_FILE.exists():
|
||||
return out
|
||||
try:
|
||||
for raw in SECRETS_FILE.read_text(encoding="utf-8").splitlines():
|
||||
line = raw.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
if "=" not in line:
|
||||
continue
|
||||
k, v = line.split("=", 1)
|
||||
k = k.strip()
|
||||
v = v.strip().strip('"').strip("'")
|
||||
if k and v:
|
||||
out[k] = v
|
||||
except Exception as e:
|
||||
LOG.error("failed to parse %s: %s", SECRETS_FILE, e)
|
||||
return out
|
||||
|
||||
|
||||
def post_webhook(url: str, secret: str, payload: dict[str, Any], *, dry_run: bool = False) -> int:
|
||||
body = json.dumps(payload, ensure_ascii=False).encode("utf-8")
|
||||
sig = hmac.new(secret.encode("utf-8"), body, hashlib.sha256).hexdigest()
|
||||
|
||||
if dry_run:
|
||||
print(f"[webhook DRY-RUN] {url}")
|
||||
print(json.dumps(payload, ensure_ascii=False, indent=2))
|
||||
return 200
|
||||
|
||||
for hdr_name, value_fmt in SIG_HEADER_CANDIDATES:
|
||||
sig_value = value_fmt.format(sig=sig, secret=secret)
|
||||
headers = {"Content-Type": "application/json", hdr_name: sig_value}
|
||||
try:
|
||||
r = requests.post(url, data=body, headers=headers, timeout=HTTP_TIMEOUT)
|
||||
if r.status_code in (401, 403):
|
||||
LOG.info("auth header %s rejected (%s) — trying next", hdr_name, r.status_code)
|
||||
continue
|
||||
return r.status_code
|
||||
except requests.RequestException as e:
|
||||
LOG.error("POST %s failed with %s: %s", url, hdr_name, e)
|
||||
return 0
|
||||
return 0
|
||||
|
||||
|
||||
def maybe_alert(report: dict[str, Any], secrets: dict[str, str], *, dry_run: bool) -> None:
|
||||
"""只在 overall != ok 时 POST。"""
|
||||
if report["overall"] == "ok":
|
||||
LOG.info("全部 ok，跳过 webhook")
|
||||
return
|
||||
|
||||
url = secrets.get("FAILURE_WEBHOOK_URL")
|
||||
secret = secrets.get("FAILURE_WEBHOOK_SECRET")
|
||||
if not url or not secret:
|
||||
LOG.warning("FAILURE_WEBHOOK_URL/SECRET 未配置，跳过 POST")
|
||||
return
|
||||
|
||||
payload = {
|
||||
"task": "daily_check",
|
||||
"service": "bin/daily_sync_check.py",
|
||||
"status": report["overall"],
|
||||
"check_date": report["check_date"],
|
||||
"counts": report["counts"],
|
||||
"alerts": report["alerts"],
|
||||
"generated_at": report["generated_at"],
|
||||
}
|
||||
status = post_webhook(url, secret, payload, dry_run=dry_run)
|
||||
LOG.info("webhook POST → %s", status)
|
||||
|
||||
|
||||
# ── CLI ────────────────────────────────────────────────────────────────────
|
||||
def main(argv: Optional[list] = None) -> int:
|
||||
parser = argparse.ArgumentParser(description="每日同步巡检")
|
||||
parser.add_argument("--date", help="检查日期 YYYY-MM-DD（默认今天）")
|
||||
parser.add_argument("--dry-run", action="store_true", help="webhook 走 dry-run（仍写报告）")
|
||||
parser.add_argument("--report-only", action="store_true", help="不读 secrets、不 POST")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s %(levelname)-7s %(name)s: %(message)s",
|
||||
datefmt="%Y-%m-%dT%H:%M:%S",
|
||||
)
|
||||
|
||||
if args.date:
|
||||
check_date = datetime.strptime(args.date, "%Y-%m-%d").date()
|
||||
else:
|
||||
check_date = date.today()
|
||||
|
||||
report = build_report(check_date)
|
||||
path = write_report(report)
|
||||
print_summary(report)
|
||||
LOG.info("报告已写: %s", path)
|
||||
|
||||
if not args.report_only:
|
||||
secrets = _load_secrets()
|
||||
maybe_alert(report, secrets, dry_run=args.dry_run)
|
||||
|
||||
return 0 if report["overall"] == "ok" else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user