chore: 重构前基线 — 9 个 sync task 全部 ok + akshare 移除 + mairui 资金流接入
状态: - 9 个 sync task(stock_basic / kline_daily / kline_index / kline_5min / moneyflow / industry_sector / sector_features / share_snapshot / market_regime) - 数据源:baostock + mairui + 雪球(pysnowball) + 新浪(4 个) - 项目级约束:永远不用 akshare(已落实) - kline_5min 改用 DB 快照统一全量/增量逻辑 - 零后端 Chrome 扩展 xueqiu_sync(独立项目)
This commit is contained in:
@@ -0,0 +1,5 @@
|
||||
"""进程入口点。
|
||||
|
||||
- `cli` — 手动触发/状态查询命令行(`python -m app.entrypoints.cli ...`)
|
||||
- `worker` — 纯调度模式,无 Web 界面(`python -m app.entrypoints.worker`)
|
||||
"""
|
||||
@@ -0,0 +1,132 @@
|
||||
"""CLI 入口:手动触发 / 状态查询。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
_PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||||
if str(_PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(_PROJECT_ROOT))
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="market_data_sync CLI",
|
||||
)
|
||||
sub = parser.add_subparsers(dest="cmd", required=True)
|
||||
|
||||
# sync <task_id>
|
||||
p_sync = sub.add_parser("sync", help="手动执行一次同步任务")
|
||||
p_sync.add_argument("task_id", help="dataset_id, 例如 kline_daily")
|
||||
p_sync.add_argument("--codes", help="指定股票代码,逗号分隔", default=None)
|
||||
p_sync.add_argument("--start", help="起始日期 YYYY-MM-DD", default=None)
|
||||
p_sync.add_argument("--end", help="结束日期 YYYY-MM-DD", default=None)
|
||||
p_sync.add_argument("--workers", type=int, default=10, help="并发数")
|
||||
|
||||
sub.add_parser("list", help="列出所有同步任务")
|
||||
sub.add_parser("status", help="查看整体状态(调度器 / 数据源 / 同步任务)")
|
||||
sub.add_parser("datasources", help="查看数据源健康度")
|
||||
sub.add_parser("schedule", help="查看计划任务")
|
||||
sub.add_parser("reseed", help="重新 seed 默认计划任务到 config 表")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.cmd == "sync":
|
||||
from app.core.utils.logging import setup_logging
|
||||
setup_logging()
|
||||
from app.tasks import get_task
|
||||
from app.core.datasource.registry import build_default_registry
|
||||
from app.core.sync.registry import seed_sync_registry
|
||||
build_default_registry()
|
||||
seed_sync_registry() # 确保 dataset_registry 有这条任务定义
|
||||
try:
|
||||
task = get_task(args.task_id)
|
||||
except KeyError as e:
|
||||
print(f"错误: {e}")
|
||||
sys.exit(1)
|
||||
kwargs = {}
|
||||
if args.codes:
|
||||
kwargs["codes"] = [c.strip() for c in args.codes.split(",") if c.strip()]
|
||||
if args.start:
|
||||
kwargs["start"] = args.start
|
||||
if args.end:
|
||||
kwargs["end"] = args.end
|
||||
if args.workers:
|
||||
kwargs["max_workers"] = args.workers
|
||||
result = task.run(trigger_source="cli", **kwargs)
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
sys.exit(0 if result.get("status") in ("ok", "warning") else 2)
|
||||
|
||||
if args.cmd == "list":
|
||||
from app.tasks import TASKS
|
||||
print("可用同步任务:")
|
||||
for tid, cls in TASKS.items():
|
||||
doc = (cls.__doc__ or cls.__name__).split("\n")[0]
|
||||
print(f" {tid:25s} {doc}")
|
||||
return
|
||||
|
||||
if args.cmd == "status":
|
||||
from app.core.utils.logging import setup_logging
|
||||
setup_logging()
|
||||
from app.core.scheduler import get_scheduler_status
|
||||
from app.core.datasource.registry import build_default_registry, get_health_status
|
||||
from app.core.sync import get_registry_status
|
||||
build_default_registry()
|
||||
print("\n=== 调度器 ===")
|
||||
s = get_scheduler_status()
|
||||
print(json.dumps(s, ensure_ascii=False, indent=2, default=str))
|
||||
print("\n=== 数据源健康度 ===")
|
||||
for r in get_health_status():
|
||||
mark = "✓" if r.get("success") else "✗"
|
||||
print(f" {mark} {r.get('name', r.get('key')):30s} {r.get('message', '')}")
|
||||
print("\n=== 同步任务状态 ===")
|
||||
for r in get_registry_status():
|
||||
print(f" {r['dataset_id']:25s} status={r.get('status'):10s} "
|
||||
f"lastRun={r.get('last_success_at') or r.get('last_failure_at') or '-':20s} "
|
||||
f"msg={r.get('message', '')[:60]}")
|
||||
return
|
||||
|
||||
if args.cmd == "datasources":
|
||||
from app.core.utils.logging import setup_logging
|
||||
setup_logging()
|
||||
from app.core.datasource.registry import build_default_registry, get_health_status, run_health_check
|
||||
build_default_registry()
|
||||
print("正在跑连通性测试...")
|
||||
run_health_check()
|
||||
for r in get_health_status():
|
||||
mark = "✓" if r.get("success") else "✗"
|
||||
print(f" {mark} {r.get('name', r.get('key')):30s} provides={r.get('provides', [])} {r.get('message', '')}")
|
||||
return
|
||||
|
||||
if args.cmd == "schedule":
|
||||
from app.core.utils.logging import setup_logging
|
||||
setup_logging()
|
||||
from app.core.scheduler import get_scheduler_status, seed_schedule_configs
|
||||
seed_schedule_configs()
|
||||
s = get_scheduler_status()
|
||||
print(f"调度器运行中: {s['running']}")
|
||||
for j in s["jobs"]:
|
||||
enabled = "✓" if j.get("enabled") else "✗"
|
||||
trigger = j.get("time") or f"every {j.get('intervalSeconds')}s"
|
||||
print(f" {enabled} {j.get('name', j['key']):30s} {trigger:15s} "
|
||||
f"condition={j.get('condition', 'always'):12s} "
|
||||
f"last={j.get('lastStatus', '-')}")
|
||||
return
|
||||
|
||||
if args.cmd == "reseed":
|
||||
from app.core.utils.logging import setup_logging
|
||||
setup_logging()
|
||||
from app.core.scheduler.scheduler import seed_schedule_configs
|
||||
from app.core.sync.registry import seed_sync_registry
|
||||
from app.core.datasource.registry import seed_datasource_configs
|
||||
seed_datasource_configs()
|
||||
seed_sync_registry()
|
||||
seed_schedule_configs()
|
||||
print("已重新 seed:datasources + sync_registry + schedules")
|
||||
return
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,73 @@
|
||||
"""纯 worker 模式(无 web)— 启动调度器 + 健康监控。
|
||||
|
||||
用法:python -m app.worker
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
_PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||||
if str(_PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(_PROJECT_ROOT))
|
||||
|
||||
from app.core.config import settings
|
||||
from app.core.db import ops as db_ops
|
||||
from app.core.utils.logging import get_logger, setup_logging
|
||||
|
||||
setup_logging()
|
||||
logger = get_logger("worker")
|
||||
|
||||
|
||||
def main():
|
||||
logger.info("[worker] market_data_sync worker 启动")
|
||||
|
||||
# 1. 注册数据源 + seed config
|
||||
from app.core.datasource.registry import build_default_registry, seed_datasource_configs
|
||||
build_default_registry()
|
||||
seed_datasource_configs()
|
||||
|
||||
# 2. seed sync registry + 恢复卡死任务
|
||||
from app.core.sync.registry import recover_interrupted_syncs, seed_sync_registry
|
||||
seed_sync_registry()
|
||||
n = recover_interrupted_syncs()
|
||||
if n > 0:
|
||||
logger.warning(f"[worker] 恢复了 {n} 个中断的同步任务")
|
||||
|
||||
# 3. seed 节假日
|
||||
if settings.holidays_list:
|
||||
db_ops.upsert_config(
|
||||
"trading_calendar_holidays",
|
||||
json.dumps(settings.holidays_list, ensure_ascii=False),
|
||||
category="general",
|
||||
description="A 股休市日",
|
||||
)
|
||||
|
||||
# 4. seed 默认计划任务
|
||||
if settings.scheduler_auto_seed:
|
||||
from app.core.scheduler.scheduler import seed_schedule_configs
|
||||
seed_schedule_configs()
|
||||
|
||||
# 5. 注册 sync job + 启动调度器 + 健康监控
|
||||
from app.core.scheduler.scheduler import register_sync_jobs, start_scheduler
|
||||
from app.core.datasource.registry import start_health_monitor
|
||||
register_sync_jobs()
|
||||
start_scheduler()
|
||||
start_health_monitor()
|
||||
|
||||
logger.info("[worker] 已启动调度器 + 健康监控,按 Ctrl+C 退出")
|
||||
|
||||
try:
|
||||
while True:
|
||||
time.sleep(60)
|
||||
except KeyboardInterrupt:
|
||||
logger.info("[worker] 收到 SIGINT,退出")
|
||||
from app.core.scheduler.scheduler import stop_scheduler
|
||||
stop_scheduler()
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user