Files
market_sync/bin/daily_sync_check.py
T
gao b351bd7595 feat: 新增 mairui 历史分时 MA 日 K 级别 sync task
work #06 (2026-07-03):user 请求加 mairui /hsdata 历史分时 MA 同步(日 K 级别)。

mairui 端点探测:/d/ma, /d/ma5/10/20, /15/ma, /30/ma, /60/ma 端点结构存在
但当前免费 licence 返 数据不存在;基础 K 线 (/d/n, /15/n, /30/n, /60/n) 正常。

策略:本地从 kline_stock 计算(pandas per-stock rolling),写新表
kline_stock_ma_daily,source=local_kline_proxy 标识本地派生。
mairui URL 留作未来升级 licence 后切 API 用。

变更:
- app/core/db/models.py: KlineStockMADaily ORM model
- app/core/db/ops.py: upsert_kline_stock_ma_daily_rows (批量 5000/批)
- app/tasks/task_mairui_ma_daily.py: SyncMairuiMADaily (全量重算)
- app/tasks/__init__.py: 注册到 TASKS dict
- app/core/sync/registry.py: SYNC_DEFINITION (sort_order=90, dep=kline_daily)
- app/core/scheduler/scheduler.py: schedule_mairui_ma_daily @ 16:30 + register_sync_jobs
- bin/daily_sync_check.py: SCHEDULE entry (window_end=17:00)

烟测:11,684,592 行, 5510 只, 1370s (23min),MA5/10/20/60 全部计算。
SH600519 样本:ma5=1194.18 ma10=1191.03 ma20=1211.94 ma60=1296.20 (2026-07-03)

未来优化(不在本 work):
- 增量模式(每日只算最近 1-2 天)→ 23min → 30s
- 升级 mairui licence 切到 /d/maN API
- 加 EMA / BOLL / KDJ 等其他指标

Co-Authored-By: Claude <noreply@anthropic.com>
2026-07-03 17:36:49 +08:00

518 lines
20 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""每日同步任务巡检。
读 PG `dataset_registry`,对比今日预期调度窗口,给出每个 task 的健康状态,
写 `logs/daily_check_YYYYMMDD_HHMMSS.json` 报告,仅在有失败/漏跑时 POST webhook。
用法:
.venv/bin/python bin/daily_sync_check.py # 默认检查"今天"
.venv/bin/python bin/daily_sync_check.py --dry-run # 只生成报告,不 POST
.venv/bin/python bin/daily_sync_check.py --report-only # 完全不读 secrets / 不 POST
.venv/bin/python bin/daily_sync_check.py --date 2026-07-02 # 检查指定日期
为什么是 stand-alone(不放在 app.tasks 下):
- 巡检逻辑独立于 sync 业务,混在 app.tasks 会让注册表臃肿
- 独立脚本便于 cron / systemd timer / /loop 三种触发方式共用
"""
from __future__ import annotations
import argparse
import hashlib
import hmac
import json
import logging
import os
import re
import sys
import time
from datetime import date, datetime, timedelta
from pathlib import Path
from typing import Any, Optional
import requests
# ── 路径与日志 ─────────────────────────────────────────────────────────────
PROJECT_ROOT = Path(__file__).resolve().parent.parent
if str(PROJECT_ROOT) not in sys.path:
sys.path.insert(0, str(PROJECT_ROOT))
LOG_DIR = PROJECT_ROOT / "logs"
SECRETS_FILE = PROJECT_ROOT / "config" / "daily_check_secrets.env"
LOG = logging.getLogger("daily-sync-check")
# ── 预期调度表 ─────────────────────────────────────────────────────────────
# 与 [[market-data-overview]] 保持一致;同步触发点(HH:MM)按 Asia/Shanghai 解释。
# weekend_only=True 表示只在周六/周日期望运行(实际仅 stock_node 周六一次)。
SCHEDULE: dict[str, dict[str, Any]] = {
"stock_basic": {"window_end": "09:30", "weekend_only": False},
"industry_sector": {"window_end": "09:45", "weekend_only": False},
"kline_index": {"window_end": "15:35", "weekend_only": False},
"kline_daily": {"window_end": "15:45", "weekend_only": False},
"kline_5min": {"window_end": "16:05", "weekend_only": False},
"market_regime": {"window_end": "16:20", "weekend_only": False},
"share_snapshot": {"window_end": "16:25", "weekend_only": False},
"moneyflow": {"window_end": "21:40", "weekend_only": False},
"longhubang": {"window_end": "22:05", "weekend_only": False},
"stock_node": {"window_end": "12:00", "weekend_only": True}, # 周六
"mairui_ma_daily": {"window_end": "17:00", "weekend_only": False}, # 本地派生
}
# ── 状态判定 ───────────────────────────────────────────────────────────────
def _parse_ts(s: Optional[str]) -> Optional[datetime]:
"""dataset_registry 时间字段是 '%Y-%m-%d %H:%M:%S' 或 None。"""
if not s:
return None
for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%dT%H:%M:%S%z"):
try:
return datetime.strptime(s, fmt)
except ValueError:
continue
return None
def _classify_task(
row: dict[str, Any],
check_date: date,
now: datetime,
) -> dict[str, Any]:
"""判定单个 task 在 check_date 的健康状态。
返回字段:
status: ok | missed | failed | running | never_run | not_expected
reason: 简短中文原因
last_success_at / last_failure_at / last_error / needs_resync: 原值透传
"""
dataset_id = row["dataset_id"]
enabled = (row.get("enabled") or 0) == 1
spec = SCHEDULE.get(dataset_id)
is_weekend = check_date.weekday() >= 5
if not enabled:
return {
"dataset_id": dataset_id,
"name": row.get("name", ""),
"status": "disabled",
"reason": "enabled=0",
"last_success_at": row.get("last_success_at"),
"last_failure_at": row.get("last_failure_at"),
"last_error": row.get("last_error"),
"needs_resync": row.get("needs_resync", 0),
"status_field": row.get("status", "idle"),
}
# weekend 期望判定
if spec and spec["weekend_only"] and not is_weekend:
return {
"dataset_id": dataset_id,
"name": row.get("name", ""),
"status": "not_expected",
"reason": "周末专属任务,今天不期望",
"last_success_at": row.get("last_success_at"),
"last_failure_at": row.get("last_failure_at"),
"last_error": row.get("last_error"),
"needs_resync": row.get("needs_resync", 0),
"status_field": row.get("status", "idle"),
}
if spec and not spec["weekend_only"] and is_weekend:
return {
"dataset_id": dataset_id,
"name": row.get("name", ""),
"status": "not_expected",
"reason": "周末不调度",
"last_success_at": row.get("last_success_at"),
"last_failure_at": row.get("last_failure_at"),
"last_error": row.get("last_error"),
"needs_resync": row.get("needs_resync", 0),
"status_field": row.get("status", "idle"),
}
last_success = _parse_ts(row.get("last_success_at"))
last_failure = _parse_ts(row.get("last_failure_at"))
status_field = row.get("status", "idle")
# 还在跑(卡死检测)
if status_field == "running":
started = _parse_ts(row.get("started_at"))
if started and (now - started) > timedelta(hours=2):
return {
"dataset_id": dataset_id,
"name": row.get("name", ""),
"status": "stuck",
"reason": f"running > 2h, started_at={row.get('started_at')}",
"last_success_at": row.get("last_success_at"),
"last_failure_at": row.get("last_failure_at"),
"last_error": row.get("last_error"),
"needs_resync": row.get("needs_resync", 0),
"status_field": status_field,
}
return {
"dataset_id": dataset_id,
"name": row.get("name", ""),
"status": "running",
"reason": f"status=running, started_at={row.get('started_at')}",
"last_success_at": row.get("last_success_at"),
"last_failure_at": row.get("last_failure_at"),
"last_error": row.get("last_error"),
"needs_resync": row.get("needs_resync", 0),
"status_field": status_field,
}
# 从未跑过
if last_success is None and last_failure is None:
return {
"dataset_id": dataset_id,
"name": row.get("name", ""),
"status": "never_run",
"reason": "从未同步过",
"last_success_at": row.get("last_success_at"),
"last_failure_at": row.get("last_failure_at"),
"last_error": row.get("last_error"),
"needs_resync": row.get("needs_resync", 0),
"status_field": status_field,
}
# 今日是否有成功?
today_start = datetime.combine(check_date, datetime.min.time())
if last_success and last_success >= today_start:
return {
"dataset_id": dataset_id,
"name": row.get("name", ""),
"status": "ok",
"reason": f"今日已成功 @ {last_success.strftime('%H:%M:%S')}",
"last_success_at": row.get("last_success_at"),
"last_failure_at": row.get("last_failure_at"),
"last_error": row.get("last_error"),
"needs_resync": row.get("needs_resync", 0),
"status_field": status_field,
}
# 今日失败过?
if last_failure and last_failure >= today_start and (
not last_success or last_failure > last_success
):
return {
"dataset_id": dataset_id,
"name": row.get("name", ""),
"status": "failed",
"reason": f"今日失败 @ {last_failure.strftime('%H:%M:%S')}: "
f"{(row.get('last_error') or '')[:120]}",
"last_success_at": row.get("last_success_at"),
"last_failure_at": row.get("last_failure_at"),
"last_error": row.get("last_error"),
"needs_resync": row.get("needs_resync", 0),
"status_field": status_field,
}
# 期望窗口已过但今日没成功
return {
"dataset_id": dataset_id,
"name": row.get("name", ""),
"status": "missed",
"reason": (
f"今日未成功(last_success={last_success.strftime('%Y-%m-%d %H:%M') if last_success else 'never'}"
),
"last_success_at": row.get("last_success_at"),
"last_failure_at": row.get("last_failure_at"),
"last_error": row.get("last_error"),
"needs_resync": row.get("needs_resync", 0),
"status_field": status_field,
}
# ── systemd unit drift 检测 ────────────────────────────────────────────────
SYSTEMD_REPO_DIR = PROJECT_ROOT / "bin" / "systemd"
SYSTEMD_ETC_DIR = Path("/etc/systemd/system")
def _check_systemd_drift() -> dict[str, Any]:
"""对比 bin/systemd/ 与 /etc/systemd/system/ 下的 market-sync* unit。
返回:
missing_in_etc: repo 有但 /etc 没装(drift 1:未部署)
drifted: 两边都有但内容不一致(drift 2:部署的版本过期)
orphan_in_etc: /etc 有但 repo 没有(drift 3:临时/僵尸 unit
"""
missing_in_etc: list[str] = []
drifted: list[dict[str, str]] = []
orphan_in_etc: list[str] = []
repo_units: set[str] = set()
if SYSTEMD_REPO_DIR.exists():
for f in SYSTEMD_REPO_DIR.iterdir():
if f.suffix in {".service", ".timer"} and f.name.startswith("market-sync"):
repo_units.add(f.name)
etc_units: set[str] = set()
if SYSTEMD_ETC_DIR.exists():
for f in SYSTEMD_ETC_DIR.iterdir():
if f.name.startswith("market-sync") and f.suffix in {".service", ".timer"}:
etc_units.add(f.name)
# drift 1: repo 有 /etc 没
for name in sorted(repo_units - etc_units):
missing_in_etc.append(name)
# drift 2: 两边都有但内容不一致
for name in sorted(repo_units & etc_units):
repo_path = SYSTEMD_REPO_DIR / name
etc_path = SYSTEMD_ETC_DIR / name
try:
if repo_path.read_bytes() != etc_path.read_bytes():
drifted.append({
"unit": name,
"repo_sha": hashlib.sha256(repo_path.read_bytes()).hexdigest()[:12],
"etc_sha": hashlib.sha256(etc_path.read_bytes()).hexdigest()[:12],
})
except OSError:
drifted.append({"unit": name, "repo_sha": "?", "etc_sha": "?"})
# drift 3: /etc 有 repo 没(孤儿)
for name in sorted(etc_units - repo_units):
orphan_in_etc.append(name)
has_drift = bool(missing_in_etc or drifted or orphan_in_etc)
return {
"schema_version": 1,
"checked_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
"repo_units": sorted(repo_units),
"etc_units": sorted(etc_units),
"missing_in_etc": missing_in_etc,
"drifted": drifted,
"orphan_in_etc": orphan_in_etc,
"has_drift": has_drift,
}
# ── 报告生成 ───────────────────────────────────────────────────────────────
def build_report(check_date: date) -> dict[str, Any]:
"""生成当日巡检报告。"""
from app.core.db import ops as db_ops # 延迟导入:避免单独跑这个脚本时加载 .env
now = datetime.now()
rows = db_ops.fetch_dataset_registry_rows()
classified = [_classify_task(r, check_date, now) for r in rows]
# 汇总计数
counts: dict[str, int] = {}
for c in classified:
counts[c["status"]] = counts.get(c["status"], 0) + 1
# 告警列表:只挑非 ok / non-expected 的
alerts = [
{"dataset_id": c["dataset_id"], "status": c["status"], "reason": c["reason"]}
for c in classified
if c["status"] in {"missed", "failed", "stuck", "never_run"}
]
# systemd drift 单独段(不是 dataset 状态,但是 sync 完整性的关键)
drift = _check_systemd_drift()
if drift["has_drift"]:
# drift 也算告警(影响同步运行)
if drift["missing_in_etc"]:
for u in drift["missing_in_etc"]:
alerts.append({
"dataset_id": f"systemd/{u}",
"status": "not_deployed",
"reason": f"unit {u} 在 bin/systemd/ 有但 /etc/systemd/system/ 没装",
})
if drift["drifted"]:
for d in drift["drifted"]:
alerts.append({
"dataset_id": f"systemd/{d['unit']}",
"status": "drifted",
"reason": f"unit {d['unit']} /etc 版过期(repo={d['repo_sha']}, etc={d['etc_sha']}",
})
if drift["orphan_in_etc"]:
for u in drift["orphan_in_etc"]:
alerts.append({
"dataset_id": f"systemd/{u}",
"status": "orphan",
"reason": f"unit {u} 在 /etc/systemd/system/ 但 repo 没维护",
})
overall = "ok" if not alerts else (
"warning" if any(a["status"] in {"missed", "never_run", "not_deployed", "drifted"} for a in alerts) else "error"
)
return {
"schema_version": 2,
"generated_at": now.strftime("%Y-%m-%d %H:%M:%S"),
"check_date": check_date.isoformat(),
"overall": overall,
"counts": counts,
"alerts": alerts,
"tasks": classified,
"systemd_drift": drift,
}
def write_report(report: dict[str, Any]) -> Path:
"""把报告写到 logs/daily_check_YYYYMMDD_HHMMSS.json。"""
LOG_DIR.mkdir(parents=True, exist_ok=True)
ts = datetime.now().strftime("%Y%m%d_%H%M%S")
path = LOG_DIR / f"daily_check_{ts}.json"
path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
return path
def print_summary(report: dict[str, Any]) -> None:
"""stdout 一眼看懂的汇总。"""
print("=" * 78)
print(f"每日同步巡检 date={report['check_date']} overall={report['overall']}")
print("=" * 78)
print(f"counts: {report['counts']}")
if report["alerts"]:
print(f"\n告警 ({len(report['alerts'])}):")
for a in report["alerts"]:
print(f" - [{a['status']:>9}] {a['dataset_id']:<20} {a['reason']}")
else:
print("\n✅ 所有任务今日已成功")
# systemd drift 段
drift = report.get("systemd_drift", {})
if drift:
print(f"\nsystemd unit 漂移:")
print(f" repo units ({len(drift['repo_units'])}): {drift['repo_units']}")
print(f" /etc units ({len(drift['etc_units'])}): {drift['etc_units']}")
if drift["missing_in_etc"]:
print(f" ❌ 未部署 ({len(drift['missing_in_etc'])}): {drift['missing_in_etc']}")
if drift["drifted"]:
print(f" ⚠️ 内容不一致 ({len(drift['drifted'])}):")
for d in drift["drifted"]:
print(f" - {d['unit']} repo={d['repo_sha']} etc={d['etc_sha']}")
if drift["orphan_in_etc"]:
print(f" 👻 /etc 孤儿 ({len(drift['orphan_in_etc'])}): {drift['orphan_in_etc']}")
if not drift["has_drift"]:
print(f" ✅ 无漂移")
# 完整列表
print(f"\n全部 {len(report['tasks'])} 个任务:")
for t in report["tasks"]:
icon = {
"ok": "",
"missed": "",
"failed": "",
"stuck": "⚠️",
"running": "🔄",
"never_run": "",
"not_expected": "",
"disabled": "🚫",
}.get(t["status"], "·")
print(f" {icon} [{t['status']:>13}] {t['dataset_id']:<20} {t['reason']}")
# ── Webhook ────────────────────────────────────────────────────────────────
SIG_HEADER_CANDIDATES: list[tuple[str, str]] = [
("X-Hub-Signature-256", "sha256={sig}"),
("X-Webhook-Signature", "{sig}"),
("X-Gitlab-Token", "{secret}"),
]
HTTP_TIMEOUT = 10
def _load_secrets() -> dict[str, str]:
out: dict[str, str] = {}
if not SECRETS_FILE.exists():
return out
try:
for raw in SECRETS_FILE.read_text(encoding="utf-8").splitlines():
line = raw.strip()
if not line or line.startswith("#"):
continue
if "=" not in line:
continue
k, v = line.split("=", 1)
k = k.strip()
v = v.strip().strip('"').strip("'")
if k and v:
out[k] = v
except Exception as e:
LOG.error("failed to parse %s: %s", SECRETS_FILE, e)
return out
def post_webhook(url: str, secret: str, payload: dict[str, Any], *, dry_run: bool = False) -> int:
body = json.dumps(payload, ensure_ascii=False).encode("utf-8")
sig = hmac.new(secret.encode("utf-8"), body, hashlib.sha256).hexdigest()
if dry_run:
print(f"[webhook DRY-RUN] {url}")
print(json.dumps(payload, ensure_ascii=False, indent=2))
return 200
for hdr_name, value_fmt in SIG_HEADER_CANDIDATES:
sig_value = value_fmt.format(sig=sig, secret=secret)
headers = {"Content-Type": "application/json", hdr_name: sig_value}
try:
r = requests.post(url, data=body, headers=headers, timeout=HTTP_TIMEOUT)
if r.status_code in (401, 403):
LOG.info("auth header %s rejected (%s) — trying next", hdr_name, r.status_code)
continue
return r.status_code
except requests.RequestException as e:
LOG.error("POST %s failed with %s: %s", url, hdr_name, e)
return 0
return 0
def maybe_alert(report: dict[str, Any], secrets: dict[str, str], *, dry_run: bool) -> None:
"""只在 overall != ok 时 POST。"""
if report["overall"] == "ok":
LOG.info("全部 ok,跳过 webhook")
return
url = secrets.get("FAILURE_WEBHOOK_URL")
secret = secrets.get("FAILURE_WEBHOOK_SECRET")
if not url or not secret:
LOG.warning("FAILURE_WEBHOOK_URL/SECRET 未配置,跳过 POST")
return
payload = {
"task": "daily_check",
"service": "bin/daily_sync_check.py",
"status": report["overall"],
"check_date": report["check_date"],
"counts": report["counts"],
"alerts": report["alerts"],
"generated_at": report["generated_at"],
}
status = post_webhook(url, secret, payload, dry_run=dry_run)
LOG.info("webhook POST → %s", status)
# ── CLI ────────────────────────────────────────────────────────────────────
def main(argv: Optional[list] = None) -> int:
parser = argparse.ArgumentParser(description="每日同步巡检")
parser.add_argument("--date", help="检查日期 YYYY-MM-DD(默认今天)")
parser.add_argument("--dry-run", action="store_true", help="webhook 走 dry-run(仍写报告)")
parser.add_argument("--report-only", action="store_true", help="不读 secrets、不 POST")
args = parser.parse_args(argv)
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s %(levelname)-7s %(name)s: %(message)s",
datefmt="%Y-%m-%dT%H:%M:%S",
)
if args.date:
check_date = datetime.strptime(args.date, "%Y-%m-%d").date()
else:
check_date = date.today()
report = build_report(check_date)
path = write_report(report)
print_summary(report)
LOG.info("报告已写: %s", path)
if not args.report_only:
secrets = _load_secrets()
maybe_alert(report, secrets, dry_run=args.dry_run)
return 0 if report["overall"] == "ok" else 1
if __name__ == "__main__":
sys.exit(main())