chore: 重构前基线 — 9 个 sync task 全部 ok + akshare 移除 + mairui 资金流接入
状态： - 9 个 sync task（stock_basic / kline_daily / kline_index / kline_5min / moneyflow / industry_sector / sector_features / share_snapshot / market_regime） - 数据源：baostock + mairui + 雪球(pysnowball) + 新浪（4 个） - 项目级约束：永远不用 akshare（已落实） - kline_5min 改用 DB 快照统一全量/增量逻辑 - 零后端 Chrome 扩展 xueqiu_sync（独立项目）
This commit is contained in:
gao
@@ -0,0 +1 @@
|
||||
"""API 子包入口。"""
|
||||
@@ -0,0 +1,94 @@
|
||||
"""FastAPI 入口 + 启动钩子。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# 确保项目根在 sys.path
|
||||
_PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
|
||||
if str(_PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(_PROJECT_ROOT))
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
|
||||
from app.core.config import settings
|
||||
from app.core.utils.logging import get_logger, setup_logging
|
||||
from app.api.routes import health, sync, datasources, schedule
|
||||
|
||||
setup_logging()
|
||||
logger = get_logger("api")
|
||||
|
||||
app = FastAPI(
|
||||
title="Market Data Sync API",
|
||||
version="0.1.0",
|
||||
description="股票市场数据定时同步与管理",
|
||||
)
|
||||
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=["*"],
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
app.include_router(health.router)
|
||||
app.include_router(sync.router)
|
||||
app.include_router(datasources.router)
|
||||
app.include_router(schedule.router)
|
||||
|
||||
|
||||
# ── 启动钩子 ──────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@app.on_event("startup")
|
||||
def _startup():
|
||||
logger.info("[startup] market_data_sync 启动")
|
||||
|
||||
# 1. 注册数据源
|
||||
from app.core.datasource.registry import build_default_registry
|
||||
from app.core.datasource.base import registry
|
||||
build_default_registry()
|
||||
logger.info(f"[startup] 已注册 {len(registry.all())} 个数据源")
|
||||
|
||||
# 2. seed 数据源定义到 config 表
|
||||
from app.core.datasource.registry import seed_datasource_configs
|
||||
seed_datasource_configs()
|
||||
|
||||
# 3. seed 默认数据集注册
|
||||
from app.core.sync.registry import seed_sync_registry
|
||||
seed_sync_registry()
|
||||
|
||||
# 4. 启动时恢复卡死的 sync 任务
|
||||
from app.core.sync.registry import recover_interrupted_syncs
|
||||
n = recover_interrupted_syncs()
|
||||
if n > 0:
|
||||
logger.warning(f"[startup] 恢复了 {n} 个中断的同步任务")
|
||||
|
||||
# 5. seed 节假日
|
||||
from app.core.db import ops as db_ops
|
||||
if settings.holidays_list:
|
||||
db_ops.upsert_config(
|
||||
"trading_calendar_holidays",
|
||||
json.dumps(settings.holidays_list, ensure_ascii=False),
|
||||
category="general",
|
||||
description="A 股休市日",
|
||||
)
|
||||
|
||||
# 6. seed 默认计划任务
|
||||
if settings.scheduler_auto_seed:
|
||||
from app.core.scheduler.scheduler import seed_schedule_configs
|
||||
seed_schedule_configs()
|
||||
|
||||
# 7. 注册 sync job + 启动调度器
|
||||
from app.core.scheduler.scheduler import register_sync_jobs, start_scheduler
|
||||
register_sync_jobs()
|
||||
start_scheduler()
|
||||
|
||||
# 8. 启动数据源健康监控
|
||||
from app.core.datasource.registry import start_health_monitor
|
||||
start_health_monitor()
|
||||
|
||||
logger.info("[startup] 完成")
|
||||
@@ -0,0 +1 @@
|
||||
"""routes 子包入口。"""
|
||||
@@ -0,0 +1,47 @@
|
||||
"""数据源管理：列出 / 测试 / 启用状态查询。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, HTTPException
|
||||
|
||||
from app.core.datasource.base import registry as ds_registry
|
||||
from app.core.datasource.registry import (
|
||||
get_health_status,
|
||||
is_source_ready,
|
||||
run_health_check,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix="/api/datasources", tags=["datasources"])
|
||||
|
||||
|
||||
@router.get("")
|
||||
def list_datasources():
|
||||
"""列出所有注册数据源 + 状态。"""
|
||||
items = []
|
||||
for src in ds_registry.all():
|
||||
ok, reason = is_source_ready(src.key)
|
||||
health = next((h for h in get_health_status() if h["key"] == src.key), None)
|
||||
items.append({
|
||||
"key": src.key,
|
||||
"name": src.name,
|
||||
"provides": src.provides,
|
||||
"requiresCredential": src.requires_credential,
|
||||
"credentialKey": src.credential_key,
|
||||
"ready": ok,
|
||||
"readyReason": reason,
|
||||
"health": health,
|
||||
})
|
||||
return {"datasources": items}
|
||||
|
||||
|
||||
@router.post("/test/{source_key}")
|
||||
def test_one(source_key: str):
|
||||
if ds_registry.get(source_key) is None:
|
||||
raise HTTPException(404, f"数据源 {source_key} 未注册")
|
||||
result = run_health_check(source_key)
|
||||
return {"result": result[0] if result else None}
|
||||
|
||||
|
||||
@router.post("/test-all")
|
||||
def test_all():
|
||||
results = run_health_check()
|
||||
return {"results": results}
|
||||
@@ -0,0 +1,12 @@
|
||||
"""健康检查路由。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter
|
||||
|
||||
router = APIRouter(prefix="/api/health", tags=["health"])
|
||||
|
||||
|
||||
@router.get("")
|
||||
def health_check():
|
||||
"""服务存活检查。"""
|
||||
return {"status": "ok", "service": "market_data_sync", "version": "0.1.0"}
|
||||
@@ -0,0 +1,68 @@
|
||||
"""计划任务管理：列出 / 启用 / 禁用 / 改时间。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from pydantic import BaseModel
|
||||
|
||||
from app.core.db import ops as db_ops
|
||||
from app.core.scheduler import (
|
||||
get_scheduler_status,
|
||||
is_trading_day,
|
||||
next_trading_day,
|
||||
seed_schedule_configs,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix="/api/schedule", tags=["schedule"])
|
||||
|
||||
|
||||
class ScheduleUpdate(BaseModel):
|
||||
enabled: bool | None = None
|
||||
time: str | None = None
|
||||
intervalSeconds: int | None = None
|
||||
condition: str | None = None
|
||||
|
||||
|
||||
@router.get("")
|
||||
def list_schedules():
|
||||
return {"scheduler": get_scheduler_status(), "isTradingDay": is_trading_day(),
|
||||
"nextTradingDay": next_trading_day().isoformat()}
|
||||
|
||||
|
||||
@router.post("/reseed")
|
||||
def reseed():
|
||||
seed_schedule_configs()
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@router.patch("/{key}")
|
||||
def update_schedule(key: str, body: ScheduleUpdate):
|
||||
target = None
|
||||
for r in db_ops.fetch_all_config():
|
||||
if r.get("key") == key and r.get("category") == "schedule":
|
||||
target = r
|
||||
break
|
||||
if target is None:
|
||||
raise HTTPException(404, f"未知 schedule key: {key}")
|
||||
try:
|
||||
obj = json.loads(target.get("value", "") or "{}")
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
obj = {}
|
||||
|
||||
if body.enabled is not None:
|
||||
obj["enabled"] = body.enabled
|
||||
if body.time is not None:
|
||||
obj["time"] = body.time
|
||||
if body.intervalSeconds is not None:
|
||||
obj["intervalSeconds"] = body.intervalSeconds
|
||||
if body.condition is not None:
|
||||
if body.condition not in ("always", "trading_day"):
|
||||
raise HTTPException(400, f"condition 仅支持 always / trading_day")
|
||||
obj["condition"] = body.condition
|
||||
|
||||
db_ops.upsert_config(
|
||||
key, json.dumps(obj, ensure_ascii=False),
|
||||
category="schedule", description=target.get("description", ""),
|
||||
)
|
||||
return {"status": "ok", "key": key, "value": obj}
|
||||
@@ -0,0 +1,82 @@
|
||||
"""同步任务管理：手动触发 / 状态查询 / 失败重试。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Query
|
||||
|
||||
from app.core.sync import get_registry_status
|
||||
from app.tasks import get_task, TASKS
|
||||
|
||||
router = APIRouter(prefix="/api/sync", tags=["sync"])
|
||||
|
||||
# 防重入：同一 dataset_id 只能有一个 run 在跑
|
||||
_running_locks: dict[str, threading.Lock] = {}
|
||||
_locks_guard = threading.Lock()
|
||||
|
||||
|
||||
def _get_lock(dataset_id: str) -> threading.Lock:
|
||||
with _locks_guard:
|
||||
if dataset_id not in _running_locks:
|
||||
_running_locks[dataset_id] = threading.Lock()
|
||||
return _running_locks[dataset_id]
|
||||
|
||||
|
||||
@router.get("/tasks")
|
||||
def list_tasks():
|
||||
return {"tasks": list(TASKS.keys())}
|
||||
|
||||
|
||||
@router.get("/registry")
|
||||
def registry_status():
|
||||
return {"registry": get_registry_status()}
|
||||
|
||||
|
||||
@router.get("/registry/{dataset_id}")
|
||||
def registry_one(dataset_id: str):
|
||||
rows = get_registry_status()
|
||||
for r in rows:
|
||||
if r["dataset_id"] == dataset_id:
|
||||
return r
|
||||
raise HTTPException(404, f"未知 dataset_id: {dataset_id}")
|
||||
|
||||
|
||||
@router.post("/run/{dataset_id}")
|
||||
def run_task(dataset_id: str, force: bool = Query(False)):
|
||||
"""同步触发一次。如果已在跑且 force=False 则返回 409。"""
|
||||
if dataset_id not in TASKS:
|
||||
raise HTTPException(404, f"未知 dataset_id: {dataset_id}，可选: {list(TASKS)}")
|
||||
lk = _get_lock(dataset_id)
|
||||
if not lk.acquire(blocking=False):
|
||||
if not force:
|
||||
raise HTTPException(409, f"{dataset_id} 正在运行中")
|
||||
# force=True 时阻塞等
|
||||
lk.acquire()
|
||||
|
||||
def _do():
|
||||
try:
|
||||
task = get_task(dataset_id)
|
||||
return task.run(trigger_source="manual")
|
||||
finally:
|
||||
lk.release()
|
||||
|
||||
thread = threading.Thread(target=_do, daemon=True, name=f"manual-{dataset_id}")
|
||||
thread.start()
|
||||
return {"status": "accepted", "dataset_id": dataset_id, "trigger": "manual"}
|
||||
|
||||
|
||||
@router.post("/reset/{dataset_id}")
|
||||
def reset_status(dataset_id: str):
|
||||
"""清除 needs_resync / 错误状态，允许下次调度重试。"""
|
||||
if dataset_id not in TASKS:
|
||||
raise HTTPException(404, f"未知 dataset_id: {dataset_id}")
|
||||
from app.core.db import ops as db_ops
|
||||
db_ops.update_dataset_registry_state(
|
||||
dataset_id,
|
||||
status="idle",
|
||||
needs_resync=0,
|
||||
last_error=None,
|
||||
message="手动重置",
|
||||
)
|
||||
return {"status": "ok", "dataset_id": dataset_id}
|
||||
Reference in New Issue
Block a user