chore: 重构前基线 — 9 个 sync task 全部 ok + akshare 移除 + mairui 资金流接入

状态:
- 9 个 sync task(stock_basic / kline_daily / kline_index / kline_5min /
  moneyflow / industry_sector / sector_features / share_snapshot / market_regime)
- 数据源:baostock + mairui + 雪球(pysnowball) + 新浪(4 个)
- 项目级约束:永远不用 akshare(已落实)
- kline_5min 改用 DB 快照统一全量/增量逻辑
- 零后端 Chrome 扩展 xueqiu_sync(独立项目)
This commit is contained in:
gao
2026-06-15 15:36:09 +08:00
commit 05635b76b9
55 changed files with 6307 additions and 0 deletions
+1
View File
@@ -0,0 +1 @@
"""routes 子包入口。"""
+47
View File
@@ -0,0 +1,47 @@
"""数据源管理:列出 / 测试 / 启用状态查询。"""
from __future__ import annotations
from fastapi import APIRouter, HTTPException
from app.core.datasource.base import registry as ds_registry
from app.core.datasource.registry import (
get_health_status,
is_source_ready,
run_health_check,
)
router = APIRouter(prefix="/api/datasources", tags=["datasources"])
@router.get("")
def list_datasources():
"""列出所有注册数据源 + 状态。"""
items = []
for src in ds_registry.all():
ok, reason = is_source_ready(src.key)
health = next((h for h in get_health_status() if h["key"] == src.key), None)
items.append({
"key": src.key,
"name": src.name,
"provides": src.provides,
"requiresCredential": src.requires_credential,
"credentialKey": src.credential_key,
"ready": ok,
"readyReason": reason,
"health": health,
})
return {"datasources": items}
@router.post("/test/{source_key}")
def test_one(source_key: str):
if ds_registry.get(source_key) is None:
raise HTTPException(404, f"数据源 {source_key} 未注册")
result = run_health_check(source_key)
return {"result": result[0] if result else None}
@router.post("/test-all")
def test_all():
results = run_health_check()
return {"results": results}
+12
View File
@@ -0,0 +1,12 @@
"""健康检查路由。"""
from __future__ import annotations
from fastapi import APIRouter
router = APIRouter(prefix="/api/health", tags=["health"])
@router.get("")
def health_check():
"""服务存活检查。"""
return {"status": "ok", "service": "market_data_sync", "version": "0.1.0"}
+68
View File
@@ -0,0 +1,68 @@
"""计划任务管理:列出 / 启用 / 禁用 / 改时间。"""
from __future__ import annotations
import json
from fastapi import APIRouter, HTTPException
from pydantic import BaseModel
from app.core.db import ops as db_ops
from app.core.scheduler import (
get_scheduler_status,
is_trading_day,
next_trading_day,
seed_schedule_configs,
)
router = APIRouter(prefix="/api/schedule", tags=["schedule"])
class ScheduleUpdate(BaseModel):
enabled: bool | None = None
time: str | None = None
intervalSeconds: int | None = None
condition: str | None = None
@router.get("")
def list_schedules():
return {"scheduler": get_scheduler_status(), "isTradingDay": is_trading_day(),
"nextTradingDay": next_trading_day().isoformat()}
@router.post("/reseed")
def reseed():
seed_schedule_configs()
return {"status": "ok"}
@router.patch("/{key}")
def update_schedule(key: str, body: ScheduleUpdate):
target = None
for r in db_ops.fetch_all_config():
if r.get("key") == key and r.get("category") == "schedule":
target = r
break
if target is None:
raise HTTPException(404, f"未知 schedule key: {key}")
try:
obj = json.loads(target.get("value", "") or "{}")
except (json.JSONDecodeError, TypeError):
obj = {}
if body.enabled is not None:
obj["enabled"] = body.enabled
if body.time is not None:
obj["time"] = body.time
if body.intervalSeconds is not None:
obj["intervalSeconds"] = body.intervalSeconds
if body.condition is not None:
if body.condition not in ("always", "trading_day"):
raise HTTPException(400, f"condition 仅支持 always / trading_day")
obj["condition"] = body.condition
db_ops.upsert_config(
key, json.dumps(obj, ensure_ascii=False),
category="schedule", description=target.get("description", ""),
)
return {"status": "ok", "key": key, "value": obj}
+82
View File
@@ -0,0 +1,82 @@
"""同步任务管理:手动触发 / 状态查询 / 失败重试。"""
from __future__ import annotations
import threading
from typing import Any
from fastapi import APIRouter, HTTPException, Query
from app.core.sync import get_registry_status
from app.tasks import get_task, TASKS
router = APIRouter(prefix="/api/sync", tags=["sync"])
# 防重入:同一 dataset_id 只能有一个 run 在跑
_running_locks: dict[str, threading.Lock] = {}
_locks_guard = threading.Lock()
def _get_lock(dataset_id: str) -> threading.Lock:
with _locks_guard:
if dataset_id not in _running_locks:
_running_locks[dataset_id] = threading.Lock()
return _running_locks[dataset_id]
@router.get("/tasks")
def list_tasks():
return {"tasks": list(TASKS.keys())}
@router.get("/registry")
def registry_status():
return {"registry": get_registry_status()}
@router.get("/registry/{dataset_id}")
def registry_one(dataset_id: str):
rows = get_registry_status()
for r in rows:
if r["dataset_id"] == dataset_id:
return r
raise HTTPException(404, f"未知 dataset_id: {dataset_id}")
@router.post("/run/{dataset_id}")
def run_task(dataset_id: str, force: bool = Query(False)):
"""同步触发一次。如果已在跑且 force=False 则返回 409。"""
if dataset_id not in TASKS:
raise HTTPException(404, f"未知 dataset_id: {dataset_id},可选: {list(TASKS)}")
lk = _get_lock(dataset_id)
if not lk.acquire(blocking=False):
if not force:
raise HTTPException(409, f"{dataset_id} 正在运行中")
# force=True 时阻塞等
lk.acquire()
def _do():
try:
task = get_task(dataset_id)
return task.run(trigger_source="manual")
finally:
lk.release()
thread = threading.Thread(target=_do, daemon=True, name=f"manual-{dataset_id}")
thread.start()
return {"status": "accepted", "dataset_id": dataset_id, "trigger": "manual"}
@router.post("/reset/{dataset_id}")
def reset_status(dataset_id: str):
"""清除 needs_resync / 错误状态,允许下次调度重试。"""
if dataset_id not in TASKS:
raise HTTPException(404, f"未知 dataset_id: {dataset_id}")
from app.core.db import ops as db_ops
db_ops.update_dataset_registry_state(
dataset_id,
status="idle",
needs_resync=0,
last_error=None,
message="手动重置",
)
return {"status": "ok", "dataset_id": dataset_id}