chore: 重构前基线 — 9 个 sync task 全部 ok + akshare 移除 + mairui 资金流接入

状态： - 9 个 sync task（stock_basic / kline_daily / kline_index / kline_5min / moneyflow / industry_sector / sector_features / share_snapshot / market_regime） - 数据源：baostock + mairui + 雪球(pysnowball) + 新浪（4 个） - 项目级约束：永远不用 akshare（已落实） - kline_5min 改用 DB 快照统一全量/增量逻辑 - 零后端 Chrome 扩展 xueqiu_sync（独立项目）
2026-06-15 15:36:09 +08:00
commit 05635b76b9
55 changed files with 6307 additions and 0 deletions
@@ -0,0 +1,22 @@
+"""数据源层：DataSource 抽象 + 适配器 + 注册表 + 健康检查。"""
+from app.core.datasource.base import DataSource, FetchResult, registry
+from app.core.datasource.registry import (
+    build_default_registry,
+    run_health_check,
+    get_health_status,
+    is_source_ready,
+    seed_datasource_configs,
+    pick_source,
+)
+
+__all__ = [
+    "DataSource",
+    "FetchResult",
+    "registry",
+    "build_default_registry",
+    "run_health_check",
+    "get_health_status",
+    "is_source_ready",
+    "seed_datasource_configs",
+    "pick_source",
+]
@@ -0,0 +1,108 @@
+"""DataSource 抽象基类 + 全局注册器。
+
+每个数据源实现一个 DataSource 子类，注册到全局 registry。
+fetch_* 方法返回 FetchResult，便于上层做交叉校验。"""
+from __future__ import annotations
+
+import abc
+from dataclasses import dataclass, field
+from typing import Any, Callable, Optional
+
+import pandas as pd
+
+
+@dataclass
+class FetchResult:
+    """数据源拉取结果。"""
+    df: pd.DataFrame
+    source_key: str = ""
+    validated: bool = False
+    detail: str = ""
+
+    @property
+    def empty(self) -> bool:
+        return self.df is None or self.df.empty
+
+
+class DataSource(abc.ABC):
+    """数据源抽象。
+
+    子类必须实现：
+      - key / name / provides / requires_credential
+      - is_available(): bool  是否就绪（凭证已配 + 通过连通性测试）
+      - health_check(): dict  {success, message}
+      - fetch_* 方法：每个提供的数据类型一个
+    """
+
+    key: str = ""
+    name: str = ""
+    provides: list[str] = []  # e.g. ["kline_daily", "stock_basic", "index_daily", "share", "industry", "kline_5min", "moneyflow"]
+    requires_credential: bool = False
+    credential_key: str = ""   # e.g. "XUEQIU_TOKEN"
+
+    @abc.abstractmethod
+    def is_available(self) -> tuple[bool, str]:
+        """返回 (是否就绪, 原因)。"""
+        ...
+
+    @abc.abstractmethod
+    def health_check(self) -> dict[str, Any]:
+        """连通性测试，返回 {success: bool, message: str}。"""
+        ...
+
+    # ── 通用可选方法（按 provides 类型实现）──
+
+    def fetch_stock_basic(self) -> list[dict[str, Any]]:
+        """全市场股票基础信息。每条: {code, name, exchange, list_date, listing_status}"""
+        raise NotImplementedError
+
+    def fetch_kline_daily(self, code6: str, start: str, end: str) -> pd.DataFrame:
+        """日 K 线。返回标准列：trade_date, open, high, low, close, volume"""
+        raise NotImplementedError
+
+    def fetch_index_daily(self, index_code: str, start: str, end: str) -> pd.DataFrame:
+        """指数日 K。"""
+        raise NotImplementedError
+
+    def fetch_share_snapshot(self, code6: str) -> Optional[dict[str, Any]]:
+        """单只股票的最新股本快照。返回 {total_share, float_share, trade_date}（单位：亿股）"""
+        raise NotImplementedError
+
+    def fetch_industry_map(self) -> list[dict[str, Any]]:
+        """股票-行业映射。每条: {code, industry_name, industry_classification, update_date}"""
+        raise NotImplementedError
+
+    def fetch_kline_5min(self, code6: str, start: str, end: str) -> pd.DataFrame:
+        """5 分钟 K 线。"""
+        raise NotImplementedError
+
+    def fetch_moneyflow(self, code6: str, start: str, end: str) -> pd.DataFrame:
+        """资金流。每条: {trade_date, main_net_inflow, large_net_inflow, medium_net_inflow, small_net_inflow}"""
+        raise NotImplementedError
+
+
+# ── 全局注册器（程序启动时往里塞）──────────────────────────────────
+
+class _GlobalRegistry:
+    def __init__(self):
+        self._sources: dict[str, DataSource] = {}
+
+    def register(self, source: DataSource) -> None:
+        if not source.key:
+            raise ValueError(f"DataSource {source!r} missing .key")
+        self._sources[source.key] = source
+
+    def get(self, key: str) -> Optional[DataSource]:
+        return self._sources.get(key)
+
+    def all(self) -> list[DataSource]:
+        return list(self._sources.values())
+
+    def by_provides(self, capability: str) -> list[DataSource]:
+        return [s for s in self._sources.values() if capability in s.provides]
+
+    def clear(self) -> None:
+        self._sources.clear()
+
+
+registry = _GlobalRegistry()
@@ -0,0 +1,250 @@
+"""数据源注册表 + 健康度监控。
+
+模式照搬 dashboard/api/services/datasource_health.py，但适配新项目结构。
+"""
+from __future__ import annotations
+
+import json
+import os
+import threading
+import time
+from datetime import datetime, timezone
+from typing import Any, Optional
+
+from app.core.config import settings
+from app.core.db import ops as db_ops
+from app.core.datasource.base import DataSource, registry
+from app.core.utils.logging import get_logger
+
+logger = get_logger("datasource_health")
+
+
+# ── 数据源定义持久化到 config 表（datasource category）──────────────────
+
+
+_DATASOURCE_SEED: list[tuple[str, dict, str]] = [
+    (
+        "datasource_xinlang",
+        {
+            "name": "新浪财经",
+            "provides": ["kline_daily", "index_daily"],
+            "requiresCredential": False,
+            "note": "免费 K 线主数据源",
+        },
+        "日 K 线 + 指数 K 线主源",
+    ),
+    (
+        "datasource_baostock",
+        {
+            "name": "Baostock",
+            "provides": ["kline_daily", "stock_basic", "industry"],
+            "requiresCredential": False,
+            "note": "免费，提供股票基础信息 + 行业映射 + K 线复核",
+        },
+        "股票基础信息 + 行业映射 + 复核源",
+    ),
+    (
+        "datasource_xueqiu",
+        {
+            "name": "雪球",
+            "provides": ["kline_daily", "share"],
+            "requiresCredential": True,
+            "credentialKey": "XUEQIU_TOKEN",
+            "note": "通过 pysnowball 补齐股本快照与 K 线复核",
+        },
+        "股本快照（需 Token）+ 复核源",
+    ),
+    (
+        "datasource_mairui",
+        {
+            "name": "麦蕊智数（mairui.club）",
+            "provides": ["kline_daily", "kline_5min", "index_daily"],
+            "requiresCredential": True,
+            "credentialKey": "MAIRUI_LICENCE",
+            "note": "K 线主源（日线 + 5min + 指数），免费 licence 1 分钟 300 次",
+        },
+        "K 线主源（日线 + 5min + 指数）",
+    ),
+]
+
+
+def seed_datasource_configs() -> None:
+    """把数据源定义写入 config 表 category='datasource'。"""
+    for key, obj, desc in _DATASOURCE_SEED:
+        # 取已存在的 credentialValue（防止覆盖用户填的 token）
+        existing = next(
+            (r for r in db_ops.fetch_all_config()
+             if r.get("key") == key and r.get("category") == "datasource"),
+            None,
+        )
+        if existing and not obj.get("credentialValue"):
+            try:
+                old = json.loads(existing.get("value", "") or "{}")
+                if old.get("credentialValue"):
+                    obj["credentialValue"] = old["credentialValue"]
+            except Exception:
+                pass
+        db_ops.upsert_config(
+            key, json.dumps(obj, ensure_ascii=False), category="datasource", description=desc,
+        )
+
+
+def _read_datasource_configs() -> dict[str, dict]:
+    """从 config 表读出所有数据源定义。"""
+    result = {}
+    for r in db_ops.fetch_all_config():
+        if r.get("category") != "datasource":
+            continue
+        try:
+            obj = json.loads(r.get("value", "") or "{}")
+            obj["_key"] = r["key"]
+            result[r["key"]] = obj
+        except (json.JSONDecodeError, TypeError):
+            continue
+    return result
+
+
+# ── 健康监控状态（线程安全 dict）──────────────────────────────────────
+
+
+_health_lock = threading.Lock()
+_health_results: dict[str, dict[str, Any]] = {}
+_health_started = False
+_health_thread: Optional[threading.Thread] = None
+_HEALTH_INTERVAL = 30 * 60  # 30 分钟
+
+
+def _now_iso() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def run_health_check(source_key: Optional[str] = None) -> list[dict[str, Any]]:
+    """跑一次连通性测试，更新内部缓存。
+
+    Args:
+        source_key: 跑单个；为 None 跑所有已注册的数据源。
+    """
+    results: list[dict[str, Any]] = []
+    targets = [source_key] if source_key else [s.key for s in registry.all()]
+
+    for key in targets:
+        src = registry.get(key)
+        if src is None:
+            results.append({
+                "key": key,
+                "success": False,
+                "message": f"数据源 {key} 未注册",
+                "testedAt": _now_iso(),
+            })
+            continue
+        try:
+            r = src.health_check()
+            results.append({
+                "key": key,
+                "name": src.name,
+                "success": bool(r.get("success", False)),
+                "message": str(r.get("message", "")),
+                "testedAt": _now_iso(),
+                "provides": src.provides,
+                "requiresCredential": src.requires_credential,
+            })
+        except Exception as e:
+            results.append({
+                "key": key,
+                "name": getattr(src, "name", key),
+                "success": False,
+                "message": f"健康检查异常: {e}",
+                "testedAt": _now_iso(),
+            })
+
+    with _health_lock:
+        for r in results:
+            _health_results[r["key"]] = r
+    return results
+
+
+def get_health_status(source_key: Optional[str] = None) -> list[dict[str, Any]]:
+    """读取最近一次健康检查结果（不重新跑）。"""
+    with _health_lock:
+        if source_key:
+            r = _health_results.get(source_key)
+            return [dict(r)] if r else []
+        return [dict(_health_results[k]) for k in sorted(_health_results)]
+
+
+def is_source_ready(source_key: str) -> tuple[bool, str]:
+    """综合判断：是否在 registry + 凭证已配 + 健康检查通过。"""
+    src = registry.get(source_key)
+    if src is None:
+        return False, f"数据源 {source_key} 未注册"
+    if src.requires_credential:
+        token = os.environ.get(src.credential_key, "").strip()
+        if not token:
+            return False, f"凭证 {src.credential_key} 未配置"
+    ok, reason = src.is_available()
+    if not ok:
+        return False, reason
+    with _health_lock:
+        last = _health_results.get(source_key)
+    if last is None:
+        return False, "健康检查尚未完成"
+    if not last.get("success"):
+        return False, f"连通性测试失败: {last.get('message', '未知错误')}"
+    return True, "就绪"
+
+
+def pick_source(capability: str) -> Optional[DataSource]:
+    """从所有就绪数据源里挑一个能提供某能力的数据源，按注册顺序。"""
+    for s in registry.all():
+        if capability in s.provides:
+            ok, _ = is_source_ready(s.key)
+            if ok:
+                return s
+    return None
+
+
+# ── 后台线程定期跑健康检查 ──────────────────────────────────────────
+
+
+def _health_loop() -> None:
+    while True:
+        try:
+            run_health_check()
+            logger.info("[datasource_health] 全数据源健康检查完成")
+        except Exception as e:
+            logger.error(f"[datasource_health] 健康检查异常: {e}")
+        time.sleep(_HEALTH_INTERVAL)
+
+
+def start_health_monitor() -> None:
+    global _health_started, _health_thread
+    with _health_lock:
+        if _health_started:
+            return
+        _health_started = True
+        _health_thread = threading.Thread(target=_health_loop, daemon=True, name="datasource-health")
+        _health_thread.start()
+
+
+# ── 启动时构建默认 registry ──────────────────────────────────────────
+
+
+def build_default_registry() -> None:
+    """启动时调用：把默认 4 个数据源注册到全局 registry。"""
+    # 避免重复注册
+    if registry.all():
+        return
+    from app.sources.sina import SinaSource
+    from app.sources.baostock import BaostockSource
+    from app.sources.xueqiu import XueqiuSource
+    from app.sources.mairui import MairuiSource
+
+    registry.register(SinaSource())
+    registry.register(BaostockSource())
+    registry.register(MairuiSource())
+    if settings.xueqiu_token:
+        registry.register(XueqiuSource())
+    else:
+        # 即使没 token 也注册，is_available() 会拦截
+        registry.register(XueqiuSource())
+    # akshare 已移除（项目级决策：永远不用）
@@ -0,0 +1,115 @@
+"""数据源公共工具：代码转换、K 线标准化、日期过滤。
+
+参考 dashboard/api/services/datasource/fetch_kline.py，但适配新项目。"""
+from __future__ import annotations
+
+import re
+
+import pandas as pd
+
+KLINE_COLS = ["trade_date", "open", "high", "low", "close", "volume"]
+KLINE_5MIN_COLS = ["bar_time", "open", "high", "low", "close", "volume", "amount", "turnover_rate"]
+
+
+def to_code6(code: str) -> str:
+    """'SH600000' / 'sh.600000' / '600000' / 'SH600000.SH' → '600000'"""
+    c = str(code).strip().upper()
+    c = re.sub(r"^(SH|SZ|BJ)", "", c)
+    c = c.split(".")[0]
+    return c.zfill(6)
+
+
+def code6_to_exchange(code6: str) -> str:
+    """6 位代码 → 交易所前缀。"""
+    if code6.startswith(("5", "6", "9")):
+        return "SH"
+    if code6.startswith(("4", "8")):
+        return "BJ"
+    return "SZ"
+
+
+def code6_to_sina(code6: str) -> str:
+    """新浪财经 symbol：'sh600036' / 'sz000001'。"""
+    return f"{code6_to_exchange(code6).lower()}{code6}"
+
+
+def code6_to_baostock(code6: str) -> str:
+    """Baostock 风格：'sh.600000'。"""
+    return f"{code6_to_exchange(code6).lower()}.{code6}"
+
+
+def code6_to_xueqiu(code6: str) -> str:
+    """雪球 symbol：'SH600036'。"""
+    return f"{code6_to_exchange(code6)}{code6}"
+
+
+def code6_to_mairui(code6: str) -> str:
+    """麦蕊智数 symbol：'600036.SH'。"""
+    return f"{code6}.{code6_to_exchange(code6)}"
+
+
+def normalize_kline(df: pd.DataFrame) -> pd.DataFrame:
+    """标准化日 K 线 DataFrame。
+
+    输入列名可能是 date / day / vol 等，输出固定为 KLINE_COLS。
+    含 OHLC 合法性校验（high >= max(o,l,c), low <= min(o,h,c)）。
+    """
+    if df is None or df.empty:
+        return pd.DataFrame(columns=KLINE_COLS)
+    df = df.rename(columns={"date": "trade_date", "day": "trade_date", "vol": "volume"}).copy()
+    missing = [c for c in KLINE_COLS if c not in df.columns]
+    if missing:
+        return pd.DataFrame(columns=KLINE_COLS)
+    df = df[KLINE_COLS]
+    df["trade_date"] = pd.to_datetime(df["trade_date"], errors="coerce").astype("datetime64[ns]")
+    for col in ["open", "high", "low", "close", "volume"]:
+        df[col] = pd.to_numeric(df[col], errors="coerce")
+    df = df.dropna(subset=["trade_date", "open", "high", "low", "close"])
+    df["volume"] = df["volume"].fillna(0)
+    valid = (
+        (df["open"] > 0) & (df["high"] > 0) & (df["low"] > 0) & (df["close"] > 0)
+        & (df["volume"] >= 0)
+        & (df["high"] >= df[["open", "low", "close"]].max(axis=1))
+        & (df["low"] <= df[["open", "high", "close"]].min(axis=1))
+    )
+    df = df.loc[valid].sort_values("trade_date")
+    return df.drop_duplicates(subset=["trade_date"], keep="last").reset_index(drop=True)
+
+
+def filter_date_range(df: pd.DataFrame, start: str, end: str) -> pd.DataFrame:
+    if df.empty:
+        return df
+    df = df[
+        (df["trade_date"] >= pd.Timestamp(start))
+        & (df["trade_date"] <= pd.Timestamp(end))
+    ]
+    return df.reset_index(drop=True)
+
+
+def normalize_5min(df: pd.DataFrame) -> pd.DataFrame:
+    """标准化 5 分钟 K 线。"""
+    if df is None or df.empty:
+        return pd.DataFrame(columns=KLINE_5MIN_COLS)
+    df = df.rename(columns={
+        "时间": "bar_time", "time": "bar_time", "date": "bar_time",
+        "成交量": "volume", "成交额": "amount", "换手率": "turnover_rate",
+    }).copy()
+    if "bar_time" not in df.columns:
+        return pd.DataFrame(columns=KLINE_5MIN_COLS)
+    df["bar_time"] = pd.to_datetime(df["bar_time"], errors="coerce")
+    for col in ["open", "high", "low", "close", "volume", "amount", "turnover_rate"]:
+        if col in df.columns:
+            df[col] = pd.to_numeric(df[col], errors="coerce")
+    df = df.dropna(subset=["bar_time"])
+    keep = ["bar_time"] + [c for c in KLINE_5MIN_COLS[1:] if c in df.columns]
+    df = df[keep].sort_values("bar_time").drop_duplicates(subset=["bar_time"], keep="last")
+    return df.reset_index(drop=True)
+
+
+def is_a_share_code(code6: str) -> bool:
+    """判断 6 位代码是否是主板/创业板/科创板（排除北证 8 字头、可转债等）。"""
+    if not code6 or len(code6) != 6 or not code6.isdigit():
+        return False
+    if code6.startswith(("4", "8")):
+        return False
+    return True