kyugao/market_sync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: baseline — ORM migration + systemd deploy + MCP server + daily check

Browse Source 
- ORM migration: models/ops 重构
- deploy: 标准化 systemd timer/service 部署体系
- MCP server: 5 个只读工具(任务/状态/数据集)
- daily check: 数据一致性巡检
- watch: task_watch 后台监控
- kline_daily: 麦蕊优先,时间门禁15:00
- 删除: task_industry_sector, task_sector_features
This commit is contained in:
gao
2026-07-21 16:37:00 +08:00
parent 91e83a3b0b
commit 8f016f25df
94 changed files with 4117 additions and 1176 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bin/daily_sync_check.py
+127 -9
View File
@@ -29,6 +29,7 @@ from pathlib import Path
from typing import Any, Optional
import requests
import sqlalchemy as sa
# ── 路径与日志 ─────────────────────────────────────────────────────────────
@@ -42,21 +43,43 @@ SECRETS_FILE = PROJECT_ROOT / "config" / "daily_check_secrets.env"
LOG = logging.getLogger("daily-sync-check")
# ── 运行模式 ───────────────────────────────────────────────────────────────
def _runtime_mode() -> str:
"""读取当前运行模式：docker | systemd。
优先读环境变量 RUNTIME_MODE，未设置时尝试从 .env 读取，默认 docker。
"""
mode = os.environ.get("RUNTIME_MODE", "")
if not mode:
env_file = PROJECT_ROOT / ".env"
if env_file.exists():
try:
for line in env_file.read_text(encoding="utf-8").splitlines():
line = line.strip()
if line.startswith("RUNTIME_MODE="):
mode = line.split("=", 1)[1].strip().strip('"').strip("'")
break
except Exception:
pass
return (mode or "docker").lower()
# ── 预期调度表 ─────────────────────────────────────────────────────────────
# 与 [[market-data-overview]] 保持一致；同步触发点（HH:MM）按 Asia/Shanghai 解释。
# weekend_only=True 表示只在周六/周日期望运行（实际仅 stock_node 周六一次）。
SCHEDULE: dict[str, dict[str, Any]] = {
"stock_basic": {"window_end": "09:30", "weekend_only": False},
"industry_sector": {"window_end": "09:45", "weekend_only": False},
"kline_index": {"window_end": "15:35", "weekend_only": False},
"kline_daily": {"window_end": "15:45", "weekend_only": False},
"kline_5min": {"window_end": "16:05", "weekend_only": False},
"market_regime": {"window_end": "16:20", "weekend_only": False},
"share_snapshot": {"window_end": "16:25", "weekend_only": False},
"share_snapshot": {"window_end": "12:00", "weekend_only": True}, # 周六
"moneyflow": {"window_end": "21:40", "weekend_only": False},
"longhubang": {"window_end": "22:05", "weekend_only": False},
"stock_node": {"window_end": "12:00", "weekend_only": True}, # 周六
"mairui_ma_daily": {"window_end": "17:00", "weekend_only": False}, # 本地派生
"mairui_indicators": {"window_end": "17:10", "weekend_only": False}, # mairui MACD/KDJ/BOLL
}
@@ -245,25 +268,29 @@ def _classify_task(
# ── systemd unit drift 检测 ────────────────────────────────────────────────
SYSTEMD_REPO_DIR = PROJECT_ROOT / "bin" / "systemd"
SYSTEMD_REPO_DIR = PROJECT_ROOT / "deploy" / "systemd" / "units"
SYSTEMD_LEGACY_REPO_DIR = PROJECT_ROOT / "bin" / "systemd"
SYSTEMD_ETC_DIR = Path("/etc/systemd/system")
def _check_systemd_drift() -> dict[str, Any]:
"""对比 bin/systemd/ 与 /etc/systemd/system/ 下的 market-sync* unit。
"""对比 deploy/systemd/units/ 与 /etc/systemd/system/ 下的 market-sync* unit。
兼容旧路径 bin/systemd/：如果 deploy/systemd/units/ 不存在则回退。
返回：
missing_in_etc: repo 有但 /etc 没装（drift 1：未部署）
drifted: 两边都有但内容不一致（drift 2：部署的版本过期）
orphan_in_etc: /etc 有但 repo 没有（drift 3：临时/僵尸 unit）
"""
repo_dir = SYSTEMD_REPO_DIR if SYSTEMD_REPO_DIR.exists() else SYSTEMD_LEGACY_REPO_DIR
missing_in_etc: list[str] = []
drifted: list[dict[str, str]] = []
orphan_in_etc: list[str] = []
repo_units: set[str] = set()
if SYSTEMD_REPO_DIR.exists():
for f in SYSTEMD_REPO_DIR.iterdir():
if repo_dir.exists():
for f in repo_dir.iterdir():
if f.suffix in {".service", ".timer"} and f.name.startswith("market-sync"):
repo_units.add(f.name)
@@ -279,8 +306,12 @@ def _check_systemd_drift() -> dict[str, Any]:
# drift 2: 两边都有但内容不一致
for name in sorted(repo_units & etc_units):
repo_path = SYSTEMD_REPO_DIR / name
repo_path = repo_dir / name
etc_path = SYSTEMD_ETC_DIR / name
# 跳过被 mask 的 unit：/etc 下是 /dev/null 的符号链接，表示 systemd 故意禁用
# 这是 deploy.sh 对 market-sync-worker.service 的标准操作，不应报 drift
if etc_path.is_symlink() and etc_path.resolve() == Path("/dev/null"):
continue
try:
if repo_path.read_bytes() != etc_path.read_bytes():
drifted.append({
@@ -308,6 +339,87 @@ def _check_systemd_drift() -> dict[str, Any]:
}
# ── 数据一致性检查 ─────────────────────────────────────────────────────────
def _check_data_consistency() -> dict[str, Any]:
"""检查上游 kline_stock 与下游衍生表是否同步。
当前覆盖：
- kline_stock_ma_daily 是否比 kline_stock 缺行/缺日期
- market_regime_daily 是否比 kline_stock 缺最近交易日
返回：
alerts: 可读告警列表
details: 原始指标
"""
from app.core.config import settings
alerts: list[dict[str, str]] = []
details: dict[str, Any] = {"checked": False}
try:
engine = sa.create_engine(settings.pg_sqlalchemy_url())
with engine.connect() as conn:
# 1) kline_stock vs kline_stock_ma_daily
kline_max = conn.execute(
sa.text("SELECT MAX(trade_date) FROM market_data.kline_stock")
).scalar()
ma_max = conn.execute(
sa.text("SELECT MAX(trade_date) FROM market_data.kline_stock_ma_daily")
).scalar()
missing_rows = conn.execute(
sa.text("""
SELECT COUNT(*) FROM (
SELECT stock_code, trade_date FROM market_data.kline_stock
EXCEPT
SELECT stock_code, trade_date FROM market_data.kline_stock_ma_daily
) t
""")
).scalar() or 0
details["kline_stock_max_date"] = str(kline_max) if kline_max else None
details["kline_stock_ma_daily_max_date"] = str(ma_max) if ma_max else None
details["kline_ma_missing_rows"] = missing_rows
if kline_max and ma_max and ma_max < kline_max:
alerts.append({
"dataset_id": "data_consistency/kline_ma_daily",
"status": "stale",
"reason": f"kline_stock_ma_daily 最新日期 {ma_max} 落后于 kline_stock {kline_max}",
})
elif missing_rows and missing_rows > 0:
alerts.append({
"dataset_id": "data_consistency/kline_ma_daily",
"status": "gap",
"reason": f"kline_stock 有 {missing_rows} 行未覆盖到 kline_stock_ma_daily",
})
# 2) kline_stock vs market_regime_daily（最近 3 个交易日）
regime_max = conn.execute(
sa.text("SELECT MAX(trade_date) FROM market_data.market_regime_daily")
).scalar()
details["market_regime_daily_max_date"] = str(regime_max) if regime_max else None
if kline_max and regime_max and regime_max < kline_max:
alerts.append({
"dataset_id": "data_consistency/market_regime",
"status": "stale",
"reason": f"market_regime_daily 最新日期 {regime_max} 落后于 kline_stock {kline_max}",
})
details["checked"] = True
except Exception as e:
details["error"] = str(e)
alerts.append({
"dataset_id": "data_consistency/check_failed",
"status": "error",
"reason": f"数据一致性检查异常: {e}",
})
return {"alerts": alerts, "details": details}
# ── 报告生成 ───────────────────────────────────────────────────────────────
def build_report(check_date: date) -> dict[str, Any]:
"""生成当日巡检报告。"""
@@ -338,7 +450,7 @@ def build_report(check_date: date) -> dict[str, Any]:
alerts.append({
"dataset_id": f"systemd/{u}",
"status": "not_deployed",
"reason": f"unit {u} 在 bin/systemd/ 有但 /etc/systemd/system/ 没装",
"reason": f"unit {u} 在 deploy/systemd/units/ 有但 /etc/systemd/system/ 没装",
})
if drift["drifted"]:
for d in drift["drifted"]:
@@ -355,6 +467,10 @@ def build_report(check_date: date) -> dict[str, Any]:
"reason": f"unit {u} 在 /etc/systemd/system/ 但 repo 没维护",
})
# 数据一致性检查（上游回填后下游未跟进）
consistency = _check_data_consistency()
alerts.extend(consistency["alerts"])
overall = "ok" if not alerts else (
"warning" if any(a["status"] in {"missed", "never_run", "not_deployed", "drifted"} for a in alerts) else "error"
)
@@ -363,11 +479,13 @@ def build_report(check_date: date) -> dict[str, Any]:
"schema_version": 2,
"generated_at": now.strftime("%Y-%m-%d %H:%M:%S"),
"check_date": check_date.isoformat(),
"runtime_mode": _runtime_mode(),
"overall": overall,
"counts": counts,
"alerts": alerts,
"tasks": classified,
"systemd_drift": drift,
"data_consistency": consistency["details"],
}
@@ -383,7 +501,7 @@ def write_report(report: dict[str, Any]) -> Path:
def print_summary(report: dict[str, Any]) -> None:
"""stdout 一眼看懂的汇总。"""
print("=" * 78)
print(f"每日同步巡检 date={report['check_date']} overall={report['overall']}")
print(f"每日同步巡检 date={report['check_date']} runtime_mode={report.get('runtime_mode', 'unknown')} overall={report['overall']}")
print("=" * 78)
print(f"counts: {report['counts']}")
if report["alerts"]: