chore: 重构前基线 — 9 个 sync task 全部 ok + akshare 移除 + mairui 资金流接入

状态:
- 9 个 sync task(stock_basic / kline_daily / kline_index / kline_5min /
  moneyflow / industry_sector / sector_features / share_snapshot / market_regime)
- 数据源:baostock + mairui + 雪球(pysnowball) + 新浪(4 个)
- 项目级约束:永远不用 akshare(已落实)
- kline_5min 改用 DB 快照统一全量/增量逻辑
- 零后端 Chrome 扩展 xueqiu_sync(独立项目)
This commit is contained in:
gao
2026-06-15 15:36:09 +08:00
commit 05635b76b9
55 changed files with 6307 additions and 0 deletions
+5
View File
@@ -0,0 +1,5 @@
"""进程入口点。
- `cli` — 手动触发/状态查询命令行(`python -m app.entrypoints.cli ...`
- `worker` — 纯调度模式,无 Web 界面(`python -m app.entrypoints.worker`
"""
+132
View File
@@ -0,0 +1,132 @@
"""CLI 入口:手动触发 / 状态查询。"""
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
_PROJECT_ROOT = Path(__file__).resolve().parent.parent
if str(_PROJECT_ROOT) not in sys.path:
sys.path.insert(0, str(_PROJECT_ROOT))
def main():
parser = argparse.ArgumentParser(
description="market_data_sync CLI",
)
sub = parser.add_subparsers(dest="cmd", required=True)
# sync <task_id>
p_sync = sub.add_parser("sync", help="手动执行一次同步任务")
p_sync.add_argument("task_id", help="dataset_id, 例如 kline_daily")
p_sync.add_argument("--codes", help="指定股票代码,逗号分隔", default=None)
p_sync.add_argument("--start", help="起始日期 YYYY-MM-DD", default=None)
p_sync.add_argument("--end", help="结束日期 YYYY-MM-DD", default=None)
p_sync.add_argument("--workers", type=int, default=10, help="并发数")
sub.add_parser("list", help="列出所有同步任务")
sub.add_parser("status", help="查看整体状态(调度器 / 数据源 / 同步任务)")
sub.add_parser("datasources", help="查看数据源健康度")
sub.add_parser("schedule", help="查看计划任务")
sub.add_parser("reseed", help="重新 seed 默认计划任务到 config 表")
args = parser.parse_args()
if args.cmd == "sync":
from app.core.utils.logging import setup_logging
setup_logging()
from app.tasks import get_task
from app.core.datasource.registry import build_default_registry
from app.core.sync.registry import seed_sync_registry
build_default_registry()
seed_sync_registry() # 确保 dataset_registry 有这条任务定义
try:
task = get_task(args.task_id)
except KeyError as e:
print(f"错误: {e}")
sys.exit(1)
kwargs = {}
if args.codes:
kwargs["codes"] = [c.strip() for c in args.codes.split(",") if c.strip()]
if args.start:
kwargs["start"] = args.start
if args.end:
kwargs["end"] = args.end
if args.workers:
kwargs["max_workers"] = args.workers
result = task.run(trigger_source="cli", **kwargs)
print(json.dumps(result, ensure_ascii=False, indent=2))
sys.exit(0 if result.get("status") in ("ok", "warning") else 2)
if args.cmd == "list":
from app.tasks import TASKS
print("可用同步任务:")
for tid, cls in TASKS.items():
doc = (cls.__doc__ or cls.__name__).split("\n")[0]
print(f" {tid:25s} {doc}")
return
if args.cmd == "status":
from app.core.utils.logging import setup_logging
setup_logging()
from app.core.scheduler import get_scheduler_status
from app.core.datasource.registry import build_default_registry, get_health_status
from app.core.sync import get_registry_status
build_default_registry()
print("\n=== 调度器 ===")
s = get_scheduler_status()
print(json.dumps(s, ensure_ascii=False, indent=2, default=str))
print("\n=== 数据源健康度 ===")
for r in get_health_status():
mark = "" if r.get("success") else ""
print(f" {mark} {r.get('name', r.get('key')):30s} {r.get('message', '')}")
print("\n=== 同步任务状态 ===")
for r in get_registry_status():
print(f" {r['dataset_id']:25s} status={r.get('status'):10s} "
f"lastRun={r.get('last_success_at') or r.get('last_failure_at') or '-':20s} "
f"msg={r.get('message', '')[:60]}")
return
if args.cmd == "datasources":
from app.core.utils.logging import setup_logging
setup_logging()
from app.core.datasource.registry import build_default_registry, get_health_status, run_health_check
build_default_registry()
print("正在跑连通性测试...")
run_health_check()
for r in get_health_status():
mark = "" if r.get("success") else ""
print(f" {mark} {r.get('name', r.get('key')):30s} provides={r.get('provides', [])} {r.get('message', '')}")
return
if args.cmd == "schedule":
from app.core.utils.logging import setup_logging
setup_logging()
from app.core.scheduler import get_scheduler_status, seed_schedule_configs
seed_schedule_configs()
s = get_scheduler_status()
print(f"调度器运行中: {s['running']}")
for j in s["jobs"]:
enabled = "" if j.get("enabled") else ""
trigger = j.get("time") or f"every {j.get('intervalSeconds')}s"
print(f" {enabled} {j.get('name', j['key']):30s} {trigger:15s} "
f"condition={j.get('condition', 'always'):12s} "
f"last={j.get('lastStatus', '-')}")
return
if args.cmd == "reseed":
from app.core.utils.logging import setup_logging
setup_logging()
from app.core.scheduler.scheduler import seed_schedule_configs
from app.core.sync.registry import seed_sync_registry
from app.core.datasource.registry import seed_datasource_configs
seed_datasource_configs()
seed_sync_registry()
seed_schedule_configs()
print("已重新 seeddatasources + sync_registry + schedules")
return
if __name__ == "__main__":
main()
+73
View File
@@ -0,0 +1,73 @@
"""纯 worker 模式(无 web)— 启动调度器 + 健康监控。
用法:python -m app.worker
"""
from __future__ import annotations
import json
import sys
import time
from pathlib import Path
_PROJECT_ROOT = Path(__file__).resolve().parent.parent
if str(_PROJECT_ROOT) not in sys.path:
sys.path.insert(0, str(_PROJECT_ROOT))
from app.core.config import settings
from app.core.db import ops as db_ops
from app.core.utils.logging import get_logger, setup_logging
setup_logging()
logger = get_logger("worker")
def main():
logger.info("[worker] market_data_sync worker 启动")
# 1. 注册数据源 + seed config
from app.core.datasource.registry import build_default_registry, seed_datasource_configs
build_default_registry()
seed_datasource_configs()
# 2. seed sync registry + 恢复卡死任务
from app.core.sync.registry import recover_interrupted_syncs, seed_sync_registry
seed_sync_registry()
n = recover_interrupted_syncs()
if n > 0:
logger.warning(f"[worker] 恢复了 {n} 个中断的同步任务")
# 3. seed 节假日
if settings.holidays_list:
db_ops.upsert_config(
"trading_calendar_holidays",
json.dumps(settings.holidays_list, ensure_ascii=False),
category="general",
description="A 股休市日",
)
# 4. seed 默认计划任务
if settings.scheduler_auto_seed:
from app.core.scheduler.scheduler import seed_schedule_configs
seed_schedule_configs()
# 5. 注册 sync job + 启动调度器 + 健康监控
from app.core.scheduler.scheduler import register_sync_jobs, start_scheduler
from app.core.datasource.registry import start_health_monitor
register_sync_jobs()
start_scheduler()
start_health_monitor()
logger.info("[worker] 已启动调度器 + 健康监控,按 Ctrl+C 退出")
try:
while True:
time.sleep(60)
except KeyboardInterrupt:
logger.info("[worker] 收到 SIGINT,退出")
from app.core.scheduler.scheduler import stop_scheduler
stop_scheduler()
sys.exit(0)
if __name__ == "__main__":
main()