chore: 重构前基线 — 9 个 sync task 全部 ok + akshare 移除 + mairui 资金流接入

状态： - 9 个 sync task（stock_basic / kline_daily / kline_index / kline_5min / moneyflow / industry_sector / sector_features / share_snapshot / market_regime） - 数据源：baostock + mairui + 雪球(pysnowball) + 新浪（4 个） - 项目级约束：永远不用 akshare（已落实） - kline_5min 改用 DB 快照统一全量/增量逻辑 - 零后端 Chrome 扩展 xueqiu_sync（独立项目）
2026-06-15 15:36:09 +08:00
commit 05635b76b9
55 changed files with 6307 additions and 0 deletions
@@ -0,0 +1,251 @@
+"""同步任务：全市场日 K 线。
+
+数据流：
+  1. 选主源（按 priority 选第一个 is_available 的源）
+  2. 增量判断：读 kline_stock.MAX(trade_date) 决定每只股票 start
+  3. **Producer-Consumer 模式**：
+     - N 个 fetcher 线程并发拉 K 线，把 (code, rows) 塞到 Queue
+     - 1 个 writer 线程攒满 BATCH_SIZE 就 executemany + commit 一次
+  4. 更新 stocks.kline_synced_at
+
+设计要点：
+  - 避免每只股票 commit（5000 只 = 5000 次 commit 太慢）
+  - 减少并发数（避免新浪限流 / MySQL 连接数爆）
+  - fetcher 间用 Queue 解耦，写库串行避免锁竞争
+"""
+from __future__ import annotations
+
+import os
+import queue
+import threading
+import time
+from datetime import datetime, timedelta
+from typing import Any
+
+import pandas as pd
+
+from app.core.db import ops as db_ops
+from app.core.datasource.base import registry as ds_registry
+from app.core.datasource.utils import is_a_share_code, to_code6
+from app.core.sync.base import SyncTask, _effective_sync_end
+from app.core.utils.logging import get_logger
+
+logger = get_logger("sync.kline_daily")
+
+# 主源优先级：mairui（5 RPS，免费 + 稳定 + 单接口含5min）→ 雪球 → Baostock → 新浪
+# 经验：sina 在单日窗口 (start=end=某天) 会返回空，雪球更可靠
+PRIMARY_PRIORITY = ["datasource_xueqiu", "datasource_mairui", "datasource_xinlang"]
+DEFAULT_START = "2018-01-01"
+# 写库批量：攒够 BATCH_SIZE 行就 executemany 一次 + commit
+BATCH_SIZE = 2000
+# fetcher 并发：实测单线程最稳（5000 只 × 0.2s = 17 分钟，足够）
+# 雪球 token 限速 10 RPS，单线程 0.1s/只 已经打满。多 worker 会触发风控 hang
+DEFAULT_FETCH_WORKERS = 1
+
+
+class SyncKlineDaily(SyncTask):
+    dataset_id = "kline_daily"
+
+    def _run(
+        self,
+        *,
+        trigger_source: str = "manual",
+        start: str | None = None,
+        end: str | None = None,
+        codes: list[str] | None = None,
+        incremental: bool = True,
+        max_workers: int = DEFAULT_FETCH_WORKERS,
+        **kwargs,
+    ) -> dict[str, Any]:
+        # 1. 选主源
+        primary = self._pick_primary()
+        if primary is None:
+            return {"status": "error", "message": "所有 K 线数据源均不可用"}
+
+        self._progress(
+            message=f"使用 {primary.name} 同步日K线 (fetcher={max_workers}, batch={BATCH_SIZE})...",
+            current_step=primary.key,
+        )
+
+        # 2. 拉股票列表
+        if codes:
+            stock_codes = [to_code6(c) for c in codes]
+        else:
+            stock_codes = [
+                c for c in db_ops.iter_stock_codes(active_only=True)
+                if is_a_share_code(c)
+            ]
+        limit_raw = os.environ.get("MARKET_DATA_STOCK_LIMIT", "").strip()
+        if limit_raw.isdigit() and int(limit_raw) > 0:
+            stock_codes = stock_codes[: int(limit_raw)]
+
+        if not stock_codes:
+            return {"status": "error", "message": "无股票代码（stocks 表为空？）"}
+
+        # 3. 增量判断
+        _end = end or _effective_sync_end()
+        end_date_obj = datetime.strptime(_end, "%Y-%m-%d").date()
+        jobs: list[tuple[str, str]] = []
+        if incremental and not start:
+            for c6 in stock_codes:
+                last = db_ops.get_stock_kline_max_date(c6)
+                if last is None:
+                    fetch_start = DEFAULT_START
+                elif datetime.strptime(last, "%Y-%m-%d").date() >= end_date_obj:
+                    continue
+                else:
+                    fetch_start = (datetime.strptime(last, "%Y-%m-%d").date() - timedelta(days=5)).strftime("%Y-%m-%d")
+                jobs.append((c6, fetch_start))
+        else:
+            fetch_start = start or DEFAULT_START
+            jobs = [(c, fetch_start) for c in stock_codes]
+
+        total = len(jobs)
+        skipped = len(stock_codes) - total
+        self._progress(
+            message=f"开始同步 {total} 只股票 (跳过 {skipped} 只已最新)",
+            current=0, total=total,
+        )
+
+        # 4. 并发 fetch + 攒 batch 写库（线程安全）
+        # mairui 等外部 API 无并发问题；DB 写串行加锁
+        from concurrent.futures import ThreadPoolExecutor, as_completed
+        t0 = time.time()
+        batch_lock = threading.Lock()
+        batch_rows: list[dict] = []
+        pending_codes: dict[str, str] = {}
+        ok_lock = threading.Lock()
+        ok_cnt = [0]
+        fail_cnt = [0]
+        rows_cnt = [0]
+
+        def _flush():
+            with batch_lock:
+                if not batch_rows:
+                    return
+                rows_to_write = list(batch_rows)
+                codes_to_update = dict(pending_codes)
+                batch_rows.clear()
+                pending_codes.clear()
+            try:
+                db_ops.upsert_kline_stock(rows_to_write)
+            except Exception as e:
+                logger.error(f"batch upsert 失败 ({len(rows_to_write)} 行): {e}")
+                return
+            for c6, latest in codes_to_update.items():
+                try:
+                    db_ops.update_stock_kline_synced_at(c6, latest)
+                except Exception as e:
+                    logger.warning(f"update synced_at 失败 {c6}: {e}")
+
+        def _fetch_one(code6: str, fetch_start: str) -> tuple[str, list[dict] | None, str | None]:
+            try:
+                df = primary.fetch_kline_daily(code6, fetch_start, _end)
+            except Exception as e:
+                return (code6, None, str(e)[:120])
+            if df is None or df.empty:
+                return (code6, None, "no data")
+            rows = []
+            for _, r in df.iterrows():
+                td = r["trade_date"]
+                rows.append({
+                    "stock_code": code6,
+                    "trade_date": td.strftime("%Y-%m-%d") if hasattr(td, "strftime") else str(td)[:10],
+                    "open": float(r["open"]),
+                    "high": float(r["high"]),
+                    "low": float(r["low"]),
+                    "close": float(r["close"]),
+                    "volume": float(r["volume"]),
+                })
+            latest = df["trade_date"].max()
+            latest = latest.strftime("%Y-%m-%d") if hasattr(latest, "strftime") else str(latest)[:10]
+            return (code6, rows, latest)
+
+        total = len(jobs)
+        with ThreadPoolExecutor(max_workers=max_workers) as pool:
+            futures = {pool.submit(_fetch_one, c6, fs): c6 for c6, fs in jobs}
+            done_cnt = 0
+            for future in as_completed(futures):
+                done_cnt += 1
+                code6, rows, latest = future.result()
+                if rows is None:
+                    with ok_lock:
+                        fail_cnt[0] += 1
+                    logger.warning(f"[kline {code6}] {latest}")
+                    continue
+                with batch_lock:
+                    batch_rows.extend(rows)
+                    pending_codes[code6] = latest
+                    cur_size = len(batch_rows)
+                with ok_lock:
+                    ok_cnt[0] += 1
+                    rows_cnt[0] += len(rows)
+
+                if cur_size >= BATCH_SIZE:
+                    _flush()
+
+                if done_cnt % 50 == 0 or done_cnt == total:
+                    elapsed = time.time() - t0
+                    rate = done_cnt / elapsed if elapsed > 0 else 0
+                    self._progress(
+                        message=f"进度 {done_cnt}/{total} OK:{ok_cnt[0]} FAIL:{fail_cnt[0]} {rate:.1f}只/秒",
+                        current=done_cnt, total=total, current_step=code6,
+                    )
+
+        # 收尾 flush
+        _flush()
+
+        elapsed = round(time.time() - t0, 1)
+        msg = f"日K线 {ok_cnt[0]}成 {fail_cnt[0]}败 {skipped}跳 共{rows_cnt[0]}行, {elapsed}s"
+        return {
+            "status": "ok" if fail_cnt[0] == 0 else "warning",
+            "message": msg,
+            "ok": ok_cnt[0],
+            "fail": fail_cnt[0],
+            "skip": skipped,
+            "rows": rows_cnt[0],
+            "elapsed_sec": elapsed,
+            "primary_source": primary.key,
+        }
+
+    def _pick_primary(self):
+        from app.core.datasource.registry import is_source_ready, run_health_check, get_health_status
+        if not get_health_status():
+            run_health_check()
+        for key in PRIMARY_PRIORITY:
+            ok, _ = is_source_ready(key)
+            if ok:
+                return ds_registry.get(key)
+        return None
+
+    def _sync_one(self, primary, code6: str, start: str, end: str) -> dict:
+        """保留这个方法以兼容外部调用（已不用，但单只测试可能用到）"""
+        try:
+            df = primary.fetch_kline_daily(code6, start, end)
+        except Exception as e:
+            return {"status": "fail", "error": str(e)}
+        if df is None or df.empty:
+            return {"status": "fail", "error": "no data"}
+        rows = []
+        for _, r in df.iterrows():
+            td = r["trade_date"]
+            rows.append({
+                "stock_code": code6,
+                "trade_date": td.strftime("%Y-%m-%d") if hasattr(td, "strftime") else str(td)[:10],
+                "open": float(r["open"]),
+                "high": float(r["high"]),
+                "low": float(r["low"]),
+                "close": float(r["close"]),
+                "volume": float(r["volume"]),
+            })
+        try:
+            db_ops.upsert_kline_stock(rows)
+        except Exception as e:
+            return {"status": "fail", "error": f"db write: {e}"}
+        if rows:
+            latest = max(r["trade_date"] for r in rows)
+            try:
+                db_ops.update_stock_kline_synced_at(code6, latest)
+            except Exception:
+                pass
+        return {"status": "ok", "rows": len(rows)}