chore: 重构前基线 — 9 个 sync task 全部 ok + akshare 移除 + mairui 资金流接入
状态: - 9 个 sync task(stock_basic / kline_daily / kline_index / kline_5min / moneyflow / industry_sector / sector_features / share_snapshot / market_regime) - 数据源:baostock + mairui + 雪球(pysnowball) + 新浪(4 个) - 项目级约束:永远不用 akshare(已落实) - kline_5min 改用 DB 快照统一全量/增量逻辑 - 零后端 Chrome 扩展 xueqiu_sync(独立项目)
This commit is contained in:
@@ -0,0 +1 @@
|
||||
"""routes 子包入口。"""
|
||||
@@ -0,0 +1,47 @@
|
||||
"""数据源管理:列出 / 测试 / 启用状态查询。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, HTTPException
|
||||
|
||||
from app.core.datasource.base import registry as ds_registry
|
||||
from app.core.datasource.registry import (
|
||||
get_health_status,
|
||||
is_source_ready,
|
||||
run_health_check,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix="/api/datasources", tags=["datasources"])
|
||||
|
||||
|
||||
@router.get("")
|
||||
def list_datasources():
|
||||
"""列出所有注册数据源 + 状态。"""
|
||||
items = []
|
||||
for src in ds_registry.all():
|
||||
ok, reason = is_source_ready(src.key)
|
||||
health = next((h for h in get_health_status() if h["key"] == src.key), None)
|
||||
items.append({
|
||||
"key": src.key,
|
||||
"name": src.name,
|
||||
"provides": src.provides,
|
||||
"requiresCredential": src.requires_credential,
|
||||
"credentialKey": src.credential_key,
|
||||
"ready": ok,
|
||||
"readyReason": reason,
|
||||
"health": health,
|
||||
})
|
||||
return {"datasources": items}
|
||||
|
||||
|
||||
@router.post("/test/{source_key}")
|
||||
def test_one(source_key: str):
|
||||
if ds_registry.get(source_key) is None:
|
||||
raise HTTPException(404, f"数据源 {source_key} 未注册")
|
||||
result = run_health_check(source_key)
|
||||
return {"result": result[0] if result else None}
|
||||
|
||||
|
||||
@router.post("/test-all")
|
||||
def test_all():
|
||||
results = run_health_check()
|
||||
return {"results": results}
|
||||
@@ -0,0 +1,12 @@
|
||||
"""健康检查路由。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter
|
||||
|
||||
router = APIRouter(prefix="/api/health", tags=["health"])
|
||||
|
||||
|
||||
@router.get("")
|
||||
def health_check():
|
||||
"""服务存活检查。"""
|
||||
return {"status": "ok", "service": "market_data_sync", "version": "0.1.0"}
|
||||
@@ -0,0 +1,68 @@
|
||||
"""计划任务管理:列出 / 启用 / 禁用 / 改时间。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from pydantic import BaseModel
|
||||
|
||||
from app.core.db import ops as db_ops
|
||||
from app.core.scheduler import (
|
||||
get_scheduler_status,
|
||||
is_trading_day,
|
||||
next_trading_day,
|
||||
seed_schedule_configs,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix="/api/schedule", tags=["schedule"])
|
||||
|
||||
|
||||
class ScheduleUpdate(BaseModel):
|
||||
enabled: bool | None = None
|
||||
time: str | None = None
|
||||
intervalSeconds: int | None = None
|
||||
condition: str | None = None
|
||||
|
||||
|
||||
@router.get("")
|
||||
def list_schedules():
|
||||
return {"scheduler": get_scheduler_status(), "isTradingDay": is_trading_day(),
|
||||
"nextTradingDay": next_trading_day().isoformat()}
|
||||
|
||||
|
||||
@router.post("/reseed")
|
||||
def reseed():
|
||||
seed_schedule_configs()
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@router.patch("/{key}")
|
||||
def update_schedule(key: str, body: ScheduleUpdate):
|
||||
target = None
|
||||
for r in db_ops.fetch_all_config():
|
||||
if r.get("key") == key and r.get("category") == "schedule":
|
||||
target = r
|
||||
break
|
||||
if target is None:
|
||||
raise HTTPException(404, f"未知 schedule key: {key}")
|
||||
try:
|
||||
obj = json.loads(target.get("value", "") or "{}")
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
obj = {}
|
||||
|
||||
if body.enabled is not None:
|
||||
obj["enabled"] = body.enabled
|
||||
if body.time is not None:
|
||||
obj["time"] = body.time
|
||||
if body.intervalSeconds is not None:
|
||||
obj["intervalSeconds"] = body.intervalSeconds
|
||||
if body.condition is not None:
|
||||
if body.condition not in ("always", "trading_day"):
|
||||
raise HTTPException(400, f"condition 仅支持 always / trading_day")
|
||||
obj["condition"] = body.condition
|
||||
|
||||
db_ops.upsert_config(
|
||||
key, json.dumps(obj, ensure_ascii=False),
|
||||
category="schedule", description=target.get("description", ""),
|
||||
)
|
||||
return {"status": "ok", "key": key, "value": obj}
|
||||
@@ -0,0 +1,82 @@
|
||||
"""同步任务管理:手动触发 / 状态查询 / 失败重试。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Query
|
||||
|
||||
from app.core.sync import get_registry_status
|
||||
from app.tasks import get_task, TASKS
|
||||
|
||||
router = APIRouter(prefix="/api/sync", tags=["sync"])
|
||||
|
||||
# 防重入:同一 dataset_id 只能有一个 run 在跑
|
||||
_running_locks: dict[str, threading.Lock] = {}
|
||||
_locks_guard = threading.Lock()
|
||||
|
||||
|
||||
def _get_lock(dataset_id: str) -> threading.Lock:
|
||||
with _locks_guard:
|
||||
if dataset_id not in _running_locks:
|
||||
_running_locks[dataset_id] = threading.Lock()
|
||||
return _running_locks[dataset_id]
|
||||
|
||||
|
||||
@router.get("/tasks")
|
||||
def list_tasks():
|
||||
return {"tasks": list(TASKS.keys())}
|
||||
|
||||
|
||||
@router.get("/registry")
|
||||
def registry_status():
|
||||
return {"registry": get_registry_status()}
|
||||
|
||||
|
||||
@router.get("/registry/{dataset_id}")
|
||||
def registry_one(dataset_id: str):
|
||||
rows = get_registry_status()
|
||||
for r in rows:
|
||||
if r["dataset_id"] == dataset_id:
|
||||
return r
|
||||
raise HTTPException(404, f"未知 dataset_id: {dataset_id}")
|
||||
|
||||
|
||||
@router.post("/run/{dataset_id}")
|
||||
def run_task(dataset_id: str, force: bool = Query(False)):
|
||||
"""同步触发一次。如果已在跑且 force=False 则返回 409。"""
|
||||
if dataset_id not in TASKS:
|
||||
raise HTTPException(404, f"未知 dataset_id: {dataset_id},可选: {list(TASKS)}")
|
||||
lk = _get_lock(dataset_id)
|
||||
if not lk.acquire(blocking=False):
|
||||
if not force:
|
||||
raise HTTPException(409, f"{dataset_id} 正在运行中")
|
||||
# force=True 时阻塞等
|
||||
lk.acquire()
|
||||
|
||||
def _do():
|
||||
try:
|
||||
task = get_task(dataset_id)
|
||||
return task.run(trigger_source="manual")
|
||||
finally:
|
||||
lk.release()
|
||||
|
||||
thread = threading.Thread(target=_do, daemon=True, name=f"manual-{dataset_id}")
|
||||
thread.start()
|
||||
return {"status": "accepted", "dataset_id": dataset_id, "trigger": "manual"}
|
||||
|
||||
|
||||
@router.post("/reset/{dataset_id}")
|
||||
def reset_status(dataset_id: str):
|
||||
"""清除 needs_resync / 错误状态,允许下次调度重试。"""
|
||||
if dataset_id not in TASKS:
|
||||
raise HTTPException(404, f"未知 dataset_id: {dataset_id}")
|
||||
from app.core.db import ops as db_ops
|
||||
db_ops.update_dataset_registry_state(
|
||||
dataset_id,
|
||||
status="idle",
|
||||
needs_resync=0,
|
||||
last_error=None,
|
||||
message="手动重置",
|
||||
)
|
||||
return {"status": "ok", "dataset_id": dataset_id}
|
||||
Reference in New Issue
Block a user