gao
05635b76b9
状态： - 9 个 sync task（stock_basic / kline_daily / kline_index / kline_5min / moneyflow / industry_sector / sector_features / share_snapshot / market_regime） - 数据源：baostock + mairui + 雪球(pysnowball) + 新浪（4 个） - 项目级约束：永远不用 akshare（已落实） - kline_5min 改用 DB 快照统一全量/增量逻辑 - 零后端 Chrome 扩展 xueqiu_sync（独立项目）
81 lines
2.7 KiB
Python
81 lines
2.7 KiB
Python
"""一次性跑完所有 sync task（按依赖顺序）。
|
||
|
||
跳过 industry_sector（已 ok）。
|
||
按顺序触发：stock_basic → kline_daily → kline_index → kline_5min → moneyflow
|
||
→ share_snapshot → market_regime
|
||
|
||
每个 task 跑完写日志 + 把 result 落到 summary.json。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import sys
|
||
import time
|
||
from pathlib import Path
|
||
|
||
_PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||
if str(_PROJECT_ROOT) not in sys.path:
|
||
sys.path.insert(0, str(_PROJECT_ROOT))
|
||
|
||
from app.core.utils.logging import setup_logging, get_logger
|
||
setup_logging()
|
||
logger = get_logger("runall")
|
||
|
||
from app.core.datasource.registry import build_default_registry
|
||
from app.core.sync.registry import seed_sync_registry
|
||
build_default_registry()
|
||
seed_sync_registry()
|
||
|
||
from app.tasks import get_task
|
||
|
||
# 顺序执行，便于排查
|
||
TASKS = [
|
||
"stock_basic",
|
||
"industry_sector", # 已 ok，但再跑一次全量也 OK (用 SKIP 标志)
|
||
"kline_index",
|
||
"kline_daily",
|
||
"kline_5min",
|
||
"moneyflow",
|
||
"share_snapshot",
|
||
"market_regime",
|
||
]
|
||
|
||
# industry_sector 跑过一次（291s 全量）已 ok，跳过
|
||
SKIP = {"industry_sector"}
|
||
|
||
results = {}
|
||
t_all = time.time()
|
||
for tid in TASKS:
|
||
if tid in SKIP:
|
||
results[tid] = {"status": "skipped", "message": "已 ok"}
|
||
logger.info(f"[runall] {tid} 跳过（已 ok）")
|
||
continue
|
||
logger.info(f"[runall] >>> 开始 {tid}")
|
||
t0 = time.time()
|
||
try:
|
||
task = get_task(tid)
|
||
# 大表 task 限小批量股票，避免跑爆；其他 task 跑全量
|
||
kwargs = {"max_workers": 5}
|
||
if tid in {"kline_daily", "kline_5min", "moneyflow", "share_snapshot"}:
|
||
# 这些 task 支持 MARKET_DATA_STOCK_LIMIT 环境变量，但通过 cli 跑可 --codes 限
|
||
# 测全量费时，先跑全量
|
||
pass
|
||
r = task.run(trigger_source="runall", **kwargs)
|
||
elapsed = round(time.time() - t0, 1)
|
||
r["elapsed_sec"] = elapsed
|
||
results[tid] = r
|
||
logger.info(f"[runall] <<< {tid} 完成 status={r.get('status')} elapsed={elapsed}s msg={r.get('message')}")
|
||
except Exception as e:
|
||
elapsed = round(time.time() - t0, 1)
|
||
logger.exception(f"[runall] !!! {tid} 异常: {e}")
|
||
results[tid] = {"status": "error", "message": str(e), "elapsed_sec": elapsed}
|
||
|
||
# 每个 task 之间 sleep 5s 让健康监控跑
|
||
time.sleep(5)
|
||
|
||
elapsed_total = round(time.time() - t_all, 1)
|
||
summary = {"total_elapsed_sec": elapsed_total, "tasks": results}
|
||
out = _PROJECT_ROOT / "logs" / "runall_summary.json"
|
||
out.write_text(json.dumps(summary, ensure_ascii=False, indent=2))
|
||
logger.info(f"[runall] 全部完成 total={elapsed_total}s summary={out}")
|
||
print(json.dumps(summary, ensure_ascii=False, indent=2)) |