chore: 重构前基线 — 9 个 sync task 全部 ok + akshare 移除 + mairui 资金流接入

状态：
- 9 个 sync task（stock_basic / kline_daily / kline_index / kline_5min /
  moneyflow / industry_sector / sector_features / share_snapshot / market_regime）
- 数据源：baostock + mairui + 雪球(pysnowball) + 新浪（4 个）
- 项目级约束：永远不用 akshare（已落实）
- kline_5min 改用 DB 快照统一全量/增量逻辑
- 零后端 Chrome 扩展 xueqiu_sync（独立项目）

.env.example

@@ -0,0 +1,38 @@
+# ============== MySQL ==============
+# 主数据库（market_data_sync 写入的目标库）
+MYSQL_HOST=127.0.0.1
+MYSQL_PORT=3306
+MYSQL_USER=root
+MYSQL_PASSWORD=
+MYSQL_DATABASE=market_data_sync_db
+
+# 简版库（对照测试，可选）
+MYSQL_DATABASE_LITE=grid_seeker
+
+# 业务主库（保留 — 同步任务不再写入）
+MYSQL_DATABASE_BUSINESS=grid_seeker_model_base
+
+# ============== 调度 ==============
+SCHEDULER_TICK_SECONDS=5
+SCHEDULER_TIMEZONE=Asia/Shanghai
+SCHEDULER_AUTO_SEED=true
+
+# ============== API 服务 ==============
+API_HOST=0.0.0.0
+API_PORT=8100
+
+# ============== 日志 ==============
+LOG_DIR=logs
+LOG_LEVEL=INFO
+
+# ============== 数据源开关 ==============
+DS_BAOSTOCK_ENABLED=true
+DS_SINA_ENABLED=true
+
+# ============== 雪球 ==============
+# K 线主源 + 股本快照
+XUEQIU_TOKEN=
+
+# ============== 交易日历 ==============
+# A 股休市日（逗号分隔 YYYY-MM-DD，pydantic 字段名是 trading_holidays）
+TRADING_HOLIDAYS=

.gitignore

@@ -0,0 +1,27 @@
+# Python
+.venv/
+__pycache__/
+*.pyc
+*.pyo
+*.pyd
+.pytest_cache/
+*.egg-info/
+build/
+dist/
+
+# Logs & runtime data
+logs/
+data/
+*.log
+
+# Environment / secrets
+.env
+.env.local
+
+# IDE
+.vscode/
+.idea/
+.DS_Store
+
+# Market data caches
+data_uat/

README.md

@@ -0,0 +1,77 @@
+# market_data_sync
+
+A 股市场数据**定时同步与管理**框架。
+
+把 `akshare` / `baostock` / 新浪 / 雪球 / 麦蕊智数 等多个数据源接入到 MySQL 库，提供：
+- 后台线程调度器（time-based + interval-based，交易日感知）
+- 多数据源 + 自动降级 + 健康度监控
+- `dataset_registry` 同步状态机（运行中 / 成功 / 失败 / 中断恢复）
+- FastAPI 管理接口（手动触发、状态查询、失败重试）
+- CLI 入口（手动跑任务、查看状态）
+
+> 📖 详细架构说明见 [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md)
+
+## 快速开始
+
+```bash
+# 1. 准备环境（首次）
+cp .env.example .env                 # 编辑 .env 填写真实 MySQL 凭据 + XUEQIU_TOKEN
+chmod +x start.sh
+./start.sh                           # 自动创建 .venv + 装依赖 + 启动 API（http://localhost:8100）
+
+# 2. 常用命令
+./start.sh worker                    # 仅启动调度器（无 web）
+./start.sh list                      # 列出所有同步任务
+./start.sh sync kline_daily          # 手动触发一次（dataset_id 即可）
+./start.sh status                    # 查看调度器/数据源/同步任务状态
+./start.sh datasources               # 查看数据源健康度
+./start.sh reseed                    # 重新 seed 默认配置到 config 表
+
+# 3. 一键全量同步（首次接入）
+python bin/runall_once.py
+
+# 4. 跑测试
+.venv/bin/pytest tests/ -v
+```
+
+## 项目结构
+
+```
+app/
+├── core/        # 核心抽象（config / db / sync / scheduler / datasource base）
+├── sources/     # 数据源实现（5 个）
+├── tasks/       # 同步任务实现（8 个）
+├── api/         # FastAPI 管理接口
+└── entrypoints/ # 进程入口（cli + worker）
+bin/             # 一次性脚本
+docs/            # 架构文档
+tests/           # 单元测试
+```
+
+## 内置同步任务
+
+| dataset_id | 触发时间 | 说明 |
+|---|---|---|
+| `stock_basic` | 交易日 09:00 | 全市场股票基础信息 + 股本 |
+| `kline_daily` | 交易日 15:40 | 增量日 K 线（OHLCV，多源交叉） |
+| `kline_index` | 交易日 15:50 | 六大指数日线 |
+| `kline_5min` | 交易日 16:00 | 增量 5 分钟 K 线 |
+| `moneyflow` | 交易日 16:30 | 资金流（主力/大/中/小单净额） |
+| `industry_sector` | 周一 09:30 | 股票-行业映射（Baostock） |
+| `share_snapshot` | 交易日 09:30 | 股本快照（雪球） |
+| `market_regime` | 交易日 16:00 | 市场情绪（衍生源） |
+
+## 添加新数据源
+
+1. 在 `app/sources/` 新建 `my_source.py`，继承 `DataSource`
+2. 在 `app/core/datasource/registry.py` 的 `build_default_registry()` 注册
+
+## 添加新同步任务
+
+1. 在 `app/tasks/` 新建 `task_xxx.py`，继承 `SyncTask`，设置 `dataset_id`
+2. 在 `app/tasks/__init__.py` 的 `TASKS` 字典注册
+3. （可选）在 `app/core/sync/registry.py` 的 `SYNC_DEFINITIONS` 加条目
+
+## License
+
+私有

app/__init__.py

@@ -0,0 +1,2 @@
+"""market_data_sync package."""
+__version__ = "0.1.0"

app/api/__init__.py

@@ -0,0 +1 @@
+"""API 子包入口。"""

app/api/main.py

@@ -0,0 +1,94 @@
+"""FastAPI 入口 + 启动钩子。"""
+from __future__ import annotations
+
+import json
+import os
+import sys
+from pathlib import Path
+
+# 确保项目根在 sys.path
+_PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
+if str(_PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(_PROJECT_ROOT))
+
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+
+from app.core.config import settings
+from app.core.utils.logging import get_logger, setup_logging
+from app.api.routes import health, sync, datasources, schedule
+
+setup_logging()
+logger = get_logger("api")
+
+app = FastAPI(
+    title="Market Data Sync API",
+    version="0.1.0",
+    description="股票市场数据定时同步与管理",
+)
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+app.include_router(health.router)
+app.include_router(sync.router)
+app.include_router(datasources.router)
+app.include_router(schedule.router)
+
+
+# ── 启动钩子 ──────────────────────────────────────────────────────────
+
+
+@app.on_event("startup")
+def _startup():
+    logger.info("[startup] market_data_sync 启动")
+
+    # 1. 注册数据源
+    from app.core.datasource.registry import build_default_registry
+    from app.core.datasource.base import registry
+    build_default_registry()
+    logger.info(f"[startup] 已注册 {len(registry.all())} 个数据源")
+
+    # 2. seed 数据源定义到 config 表
+    from app.core.datasource.registry import seed_datasource_configs
+    seed_datasource_configs()
+
+    # 3. seed 默认数据集注册
+    from app.core.sync.registry import seed_sync_registry
+    seed_sync_registry()
+
+    # 4. 启动时恢复卡死的 sync 任务
+    from app.core.sync.registry import recover_interrupted_syncs
+    n = recover_interrupted_syncs()
+    if n > 0:
+        logger.warning(f"[startup] 恢复了 {n} 个中断的同步任务")
+
+    # 5. seed 节假日
+    from app.core.db import ops as db_ops
+    if settings.holidays_list:
+        db_ops.upsert_config(
+            "trading_calendar_holidays",
+            json.dumps(settings.holidays_list, ensure_ascii=False),
+            category="general",
+            description="A 股休市日",
+        )
+
+    # 6. seed 默认计划任务
+    if settings.scheduler_auto_seed:
+        from app.core.scheduler.scheduler import seed_schedule_configs
+        seed_schedule_configs()
+
+    # 7. 注册 sync job + 启动调度器
+    from app.core.scheduler.scheduler import register_sync_jobs, start_scheduler
+    register_sync_jobs()
+    start_scheduler()
+
+    # 8. 启动数据源健康监控
+    from app.core.datasource.registry import start_health_monitor
+    start_health_monitor()
+
+    logger.info("[startup] 完成")

app/api/routes/__init__.py

@@ -0,0 +1 @@
+"""routes 子包入口。"""

app/api/routes/datasources.py

@@ -0,0 +1,47 @@
+"""数据源管理：列出 / 测试 / 启用状态查询。"""
+from __future__ import annotations
+
+from fastapi import APIRouter, HTTPException
+
+from app.core.datasource.base import registry as ds_registry
+from app.core.datasource.registry import (
+    get_health_status,
+    is_source_ready,
+    run_health_check,
+)
+
+router = APIRouter(prefix="/api/datasources", tags=["datasources"])
+
+
+@router.get("")
+def list_datasources():
+    """列出所有注册数据源 + 状态。"""
+    items = []
+    for src in ds_registry.all():
+        ok, reason = is_source_ready(src.key)
+        health = next((h for h in get_health_status() if h["key"] == src.key), None)
+        items.append({
+            "key": src.key,
+            "name": src.name,
+            "provides": src.provides,
+            "requiresCredential": src.requires_credential,
+            "credentialKey": src.credential_key,
+            "ready": ok,
+            "readyReason": reason,
+            "health": health,
+        })
+    return {"datasources": items}
+
+
+@router.post("/test/{source_key}")
+def test_one(source_key: str):
+    if ds_registry.get(source_key) is None:
+        raise HTTPException(404, f"数据源 {source_key} 未注册")
+    result = run_health_check(source_key)
+    return {"result": result[0] if result else None}
+
+
+@router.post("/test-all")
+def test_all():
+    results = run_health_check()
+    return {"results": results}

app/api/routes/health.py

@@ -0,0 +1,12 @@
+"""健康检查路由。"""
+from __future__ import annotations
+
+from fastapi import APIRouter
+
+router = APIRouter(prefix="/api/health", tags=["health"])
+
+
+@router.get("")
+def health_check():
+    """服务存活检查。"""
+    return {"status": "ok", "service": "market_data_sync", "version": "0.1.0"}

app/api/routes/schedule.py

@@ -0,0 +1,68 @@
+"""计划任务管理：列出 / 启用 / 禁用 / 改时间。"""
+from __future__ import annotations
+
+import json
+
+from fastapi import APIRouter, HTTPException
+from pydantic import BaseModel
+
+from app.core.db import ops as db_ops
+from app.core.scheduler import (
+    get_scheduler_status,
+    is_trading_day,
+    next_trading_day,
+    seed_schedule_configs,
+)
+
+router = APIRouter(prefix="/api/schedule", tags=["schedule"])
+
+
+class ScheduleUpdate(BaseModel):
+    enabled: bool | None = None
+    time: str | None = None
+    intervalSeconds: int | None = None
+    condition: str | None = None
+
+
+@router.get("")
+def list_schedules():
+    return {"scheduler": get_scheduler_status(), "isTradingDay": is_trading_day(),
+            "nextTradingDay": next_trading_day().isoformat()}
+
+
+@router.post("/reseed")
+def reseed():
+    seed_schedule_configs()
+    return {"status": "ok"}
+
+
+@router.patch("/{key}")
+def update_schedule(key: str, body: ScheduleUpdate):
+    target = None
+    for r in db_ops.fetch_all_config():
+        if r.get("key") == key and r.get("category") == "schedule":
+            target = r
+            break
+    if target is None:
+        raise HTTPException(404, f"未知 schedule key: {key}")
+    try:
+        obj = json.loads(target.get("value", "") or "{}")
+    except (json.JSONDecodeError, TypeError):
+        obj = {}
+
+    if body.enabled is not None:
+        obj["enabled"] = body.enabled
+    if body.time is not None:
+        obj["time"] = body.time
+    if body.intervalSeconds is not None:
+        obj["intervalSeconds"] = body.intervalSeconds
+    if body.condition is not None:
+        if body.condition not in ("always", "trading_day"):
+            raise HTTPException(400, f"condition 仅支持 always / trading_day")
+        obj["condition"] = body.condition
+
+    db_ops.upsert_config(
+        key, json.dumps(obj, ensure_ascii=False),
+        category="schedule", description=target.get("description", ""),
+    )
+    return {"status": "ok", "key": key, "value": obj}

app/api/routes/sync.py

@@ -0,0 +1,82 @@
+"""同步任务管理：手动触发 / 状态查询 / 失败重试。"""
+from __future__ import annotations
+
+import threading
+from typing import Any
+
+from fastapi import APIRouter, HTTPException, Query
+
+from app.core.sync import get_registry_status
+from app.tasks import get_task, TASKS
+
+router = APIRouter(prefix="/api/sync", tags=["sync"])
+
+# 防重入：同一 dataset_id 只能有一个 run 在跑
+_running_locks: dict[str, threading.Lock] = {}
+_locks_guard = threading.Lock()
+
+
+def _get_lock(dataset_id: str) -> threading.Lock:
+    with _locks_guard:
+        if dataset_id not in _running_locks:
+            _running_locks[dataset_id] = threading.Lock()
+        return _running_locks[dataset_id]
+
+
+@router.get("/tasks")
+def list_tasks():
+    return {"tasks": list(TASKS.keys())}
+
+
+@router.get("/registry")
+def registry_status():
+    return {"registry": get_registry_status()}
+
+
+@router.get("/registry/{dataset_id}")
+def registry_one(dataset_id: str):
+    rows = get_registry_status()
+    for r in rows:
+        if r["dataset_id"] == dataset_id:
+            return r
+    raise HTTPException(404, f"未知 dataset_id: {dataset_id}")
+
+
+@router.post("/run/{dataset_id}")
+def run_task(dataset_id: str, force: bool = Query(False)):
+    """同步触发一次。如果已在跑且 force=False 则返回 409。"""
+    if dataset_id not in TASKS:
+        raise HTTPException(404, f"未知 dataset_id: {dataset_id}，可选: {list(TASKS)}")
+    lk = _get_lock(dataset_id)
+    if not lk.acquire(blocking=False):
+        if not force:
+            raise HTTPException(409, f"{dataset_id} 正在运行中")
+        # force=True 时阻塞等
+        lk.acquire()
+
+    def _do():
+        try:
+            task = get_task(dataset_id)
+            return task.run(trigger_source="manual")
+        finally:
+            lk.release()
+
+    thread = threading.Thread(target=_do, daemon=True, name=f"manual-{dataset_id}")
+    thread.start()
+    return {"status": "accepted", "dataset_id": dataset_id, "trigger": "manual"}
+
+
+@router.post("/reset/{dataset_id}")
+def reset_status(dataset_id: str):
+    """清除 needs_resync / 错误状态，允许下次调度重试。"""
+    if dataset_id not in TASKS:
+        raise HTTPException(404, f"未知 dataset_id: {dataset_id}")
+    from app.core.db import ops as db_ops
+    db_ops.update_dataset_registry_state(
+        dataset_id,
+        status="idle",
+        needs_resync=0,
+        last_error=None,
+        message="手动重置",
+    )
+    return {"status": "ok", "dataset_id": dataset_id}

app/core/config.py

@@ -0,0 +1,126 @@
+"""统一配置：从 .env / 环境变量读 MySQL、调度器、数据源开关等参数。"""
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import Optional
+
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+try:
+    from dotenv import load_dotenv
+    _HAS_DOTENV = True
+except ImportError:
+    _HAS_DOTENV = False
+
+
+_PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
+
+# 启动时自动把 .env 加载到 os.environ，使下游 `os.environ.get(...)` 代码
+# （如 app/sources/mairui.py / xueqiu.py）能拿到 credentials。
+if _HAS_DOTENV:
+    _env_file = _PROJECT_ROOT / ".env"
+    if _env_file.exists():
+        load_dotenv(_env_file, override=False)
+
+
+class Settings(BaseSettings):
+    """应用配置。所有字段都可以通过环境变量或 .env 文件覆盖。"""
+
+    # MySQL
+    mysql_host: str = "127.0.0.1"
+    mysql_port: int = 3306
+    mysql_user: str = "root"
+    mysql_password: str = ""
+    # 同步任务写入的目标库（原始市场数据）
+    mysql_database: str = "market_data_sync_db"
+    # 简版库（对照测试，可选）
+    mysql_database_lite: str = "grid_seeker"
+    # 业务主库（保留— 同步任务不再写入）
+    mysql_database_business: str = "grid_seeker_model_base"
+
+    # Scheduler
+    scheduler_tick_seconds: int = 5
+    scheduler_timezone: str = "Asia/Shanghai"
+    scheduler_auto_seed: bool = True
+
+    # API
+    api_host: str = "0.0.0.0"
+    api_port: int = 8100
+
+    # Logging
+    log_level: str = "INFO"
+    log_dir: str = "./logs"
+
+    # Datasources
+    ds_baostock_enabled: bool = True
+    ds_sina_enabled: bool = True
+    xueqiu_token: str = ""
+
+    # Trading calendar
+    trading_holidays: str = ""
+
+    model_config = SettingsConfigDict(
+        env_file=str(_PROJECT_ROOT / ".env"),
+        env_file_encoding="utf-8",
+        case_sensitive=False,
+        extra="ignore",
+    )
+
+    @property
+    def project_root(self) -> Path:
+        return _PROJECT_ROOT
+
+    @property
+    def log_path(self) -> Path:
+        p = Path(self.log_dir)
+        if not p.is_absolute():
+            p = _PROJECT_ROOT / p
+        p.mkdir(parents=True, exist_ok=True)
+        return p
+
+    @property
+    def holidays_list(self) -> list[str]:
+        if not self.trading_holidays:
+            return []
+        return [d.strip() for d in self.trading_holidays.split(",") if d.strip()]
+
+    def mysql_url(self, database: Optional[str] = None) -> str:
+        """构造 SQLAlchemy URL。"""
+        db = database or self.mysql_database
+        return (
+            f"mysql+pymysql://{self.mysql_user}:{self.mysql_password}"
+            f"@{self.mysql_host}:{self.mysql_port}/{db}?charset=utf8mb4"
+        )
+
+    def pymysql_connect_kwargs(self, database: Optional[str] = None) -> dict:
+        """构造 pymysql.connect 的关键字参数。
+
+        内网自签名证书场景自动禁用 SSL。
+        """
+        db = database or self.mysql_database
+        return {
+            "host": self.mysql_host,
+            "port": self.mysql_port,
+            "user": self.mysql_user,
+            "password": self.mysql_password,
+            "database": db,
+            "charset": "utf8mb4",
+            "autocommit": False,
+            # 兼容自签名证书的内网 MySQL — 禁用 SSL
+            "ssl": None,
+        }
+
+
+_settings: Optional[Settings] = None
+
+
+def get_settings() -> Settings:
+    """全局单例。"""
+    global _settings
+    if _settings is None:
+        _settings = Settings()
+    return _settings
+
+
+settings = get_settings()

app/core/datasource/__init__.py

@@ -0,0 +1,22 @@
+"""数据源层：DataSource 抽象 + 适配器 + 注册表 + 健康检查。"""
+from app.core.datasource.base import DataSource, FetchResult, registry
+from app.core.datasource.registry import (
+    build_default_registry,
+    run_health_check,
+    get_health_status,
+    is_source_ready,
+    seed_datasource_configs,
+    pick_source,
+)
+
+__all__ = [
+    "DataSource",
+    "FetchResult",
+    "registry",
+    "build_default_registry",
+    "run_health_check",
+    "get_health_status",
+    "is_source_ready",
+    "seed_datasource_configs",
+    "pick_source",
+]

app/core/datasource/base.py

@@ -0,0 +1,108 @@
+"""DataSource 抽象基类 + 全局注册器。
+
+每个数据源实现一个 DataSource 子类，注册到全局 registry。
+fetch_* 方法返回 FetchResult，便于上层做交叉校验。"""
+from __future__ import annotations
+
+import abc
+from dataclasses import dataclass, field
+from typing import Any, Callable, Optional
+
+import pandas as pd
+
+
+@dataclass
+class FetchResult:
+    """数据源拉取结果。"""
+    df: pd.DataFrame
+    source_key: str = ""
+    validated: bool = False
+    detail: str = ""
+
+    @property
+    def empty(self) -> bool:
+        return self.df is None or self.df.empty
+
+
+class DataSource(abc.ABC):
+    """数据源抽象。
+
+    子类必须实现：
+      - key / name / provides / requires_credential
+      - is_available(): bool  是否就绪（凭证已配 + 通过连通性测试）
+      - health_check(): dict  {success, message}
+      - fetch_* 方法：每个提供的数据类型一个
+    """
+
+    key: str = ""
+    name: str = ""
+    provides: list[str] = []  # e.g. ["kline_daily", "stock_basic", "index_daily", "share", "industry", "kline_5min", "moneyflow"]
+    requires_credential: bool = False
+    credential_key: str = ""   # e.g. "XUEQIU_TOKEN"
+
+    @abc.abstractmethod
+    def is_available(self) -> tuple[bool, str]:
+        """返回 (是否就绪, 原因)。"""
+        ...
+
+    @abc.abstractmethod
+    def health_check(self) -> dict[str, Any]:
+        """连通性测试，返回 {success: bool, message: str}。"""
+        ...
+
+    # ── 通用可选方法（按 provides 类型实现）──
+
+    def fetch_stock_basic(self) -> list[dict[str, Any]]:
+        """全市场股票基础信息。每条: {code, name, exchange, list_date, listing_status}"""
+        raise NotImplementedError
+
+    def fetch_kline_daily(self, code6: str, start: str, end: str) -> pd.DataFrame:
+        """日 K 线。返回标准列：trade_date, open, high, low, close, volume"""
+        raise NotImplementedError
+
+    def fetch_index_daily(self, index_code: str, start: str, end: str) -> pd.DataFrame:
+        """指数日 K。"""
+        raise NotImplementedError
+
+    def fetch_share_snapshot(self, code6: str) -> Optional[dict[str, Any]]:
+        """单只股票的最新股本快照。返回 {total_share, float_share, trade_date}（单位：亿股）"""
+        raise NotImplementedError
+
+    def fetch_industry_map(self) -> list[dict[str, Any]]:
+        """股票-行业映射。每条: {code, industry_name, industry_classification, update_date}"""
+        raise NotImplementedError
+
+    def fetch_kline_5min(self, code6: str, start: str, end: str) -> pd.DataFrame:
+        """5 分钟 K 线。"""
+        raise NotImplementedError
+
+    def fetch_moneyflow(self, code6: str, start: str, end: str) -> pd.DataFrame:
+        """资金流。每条: {trade_date, main_net_inflow, large_net_inflow, medium_net_inflow, small_net_inflow}"""
+        raise NotImplementedError
+
+
+# ── 全局注册器（程序启动时往里塞）──────────────────────────────────
+
+class _GlobalRegistry:
+    def __init__(self):
+        self._sources: dict[str, DataSource] = {}
+
+    def register(self, source: DataSource) -> None:
+        if not source.key:
+            raise ValueError(f"DataSource {source!r} missing .key")
+        self._sources[source.key] = source
+
+    def get(self, key: str) -> Optional[DataSource]:
+        return self._sources.get(key)
+
+    def all(self) -> list[DataSource]:
+        return list(self._sources.values())
+
+    def by_provides(self, capability: str) -> list[DataSource]:
+        return [s for s in self._sources.values() if capability in s.provides]
+
+    def clear(self) -> None:
+        self._sources.clear()
+
+
+registry = _GlobalRegistry()

app/core/datasource/registry.py

@@ -0,0 +1,250 @@
+"""数据源注册表 + 健康度监控。
+
+模式照搬 dashboard/api/services/datasource_health.py，但适配新项目结构。
+"""
+from __future__ import annotations
+
+import json
+import os
+import threading
+import time
+from datetime import datetime, timezone
+from typing import Any, Optional
+
+from app.core.config import settings
+from app.core.db import ops as db_ops
+from app.core.datasource.base import DataSource, registry
+from app.core.utils.logging import get_logger
+
+logger = get_logger("datasource_health")
+
+
+# ── 数据源定义持久化到 config 表（datasource category）──────────────────
+
+
+_DATASOURCE_SEED: list[tuple[str, dict, str]] = [
+    (
+        "datasource_xinlang",
+        {
+            "name": "新浪财经",
+            "provides": ["kline_daily", "index_daily"],
+            "requiresCredential": False,
+            "note": "免费 K 线主数据源",
+        },
+        "日 K 线 + 指数 K 线主源",
+    ),
+    (
+        "datasource_baostock",
+        {
+            "name": "Baostock",
+            "provides": ["kline_daily", "stock_basic", "industry"],
+            "requiresCredential": False,
+            "note": "免费，提供股票基础信息 + 行业映射 + K 线复核",
+        },
+        "股票基础信息 + 行业映射 + 复核源",
+    ),
+    (
+        "datasource_xueqiu",
+        {
+            "name": "雪球",
+            "provides": ["kline_daily", "share"],
+            "requiresCredential": True,
+            "credentialKey": "XUEQIU_TOKEN",
+            "note": "通过 pysnowball 补齐股本快照与 K 线复核",
+        },
+        "股本快照（需 Token）+ 复核源",
+    ),
+    (
+        "datasource_mairui",
+        {
+            "name": "麦蕊智数（mairui.club）",
+            "provides": ["kline_daily", "kline_5min", "index_daily"],
+            "requiresCredential": True,
+            "credentialKey": "MAIRUI_LICENCE",
+            "note": "K 线主源（日线 + 5min + 指数），免费 licence 1 分钟 300 次",
+        },
+        "K 线主源（日线 + 5min + 指数）",
+    ),
+]
+
+
+def seed_datasource_configs() -> None:
+    """把数据源定义写入 config 表 category='datasource'。"""
+    for key, obj, desc in _DATASOURCE_SEED:
+        # 取已存在的 credentialValue（防止覆盖用户填的 token）
+        existing = next(
+            (r for r in db_ops.fetch_all_config()
+             if r.get("key") == key and r.get("category") == "datasource"),
+            None,
+        )
+        if existing and not obj.get("credentialValue"):
+            try:
+                old = json.loads(existing.get("value", "") or "{}")
+                if old.get("credentialValue"):
+                    obj["credentialValue"] = old["credentialValue"]
+            except Exception:
+                pass
+        db_ops.upsert_config(
+            key, json.dumps(obj, ensure_ascii=False), category="datasource", description=desc,
+        )
+
+
+def _read_datasource_configs() -> dict[str, dict]:
+    """从 config 表读出所有数据源定义。"""
+    result = {}
+    for r in db_ops.fetch_all_config():
+        if r.get("category") != "datasource":
+            continue
+        try:
+            obj = json.loads(r.get("value", "") or "{}")
+            obj["_key"] = r["key"]
+            result[r["key"]] = obj
+        except (json.JSONDecodeError, TypeError):
+            continue
+    return result
+
+
+# ── 健康监控状态（线程安全 dict）──────────────────────────────────────
+
+
+_health_lock = threading.Lock()
+_health_results: dict[str, dict[str, Any]] = {}
+_health_started = False
+_health_thread: Optional[threading.Thread] = None
+_HEALTH_INTERVAL = 30 * 60  # 30 分钟
+
+
+def _now_iso() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def run_health_check(source_key: Optional[str] = None) -> list[dict[str, Any]]:
+    """跑一次连通性测试，更新内部缓存。
+
+    Args:
+        source_key: 跑单个；为 None 跑所有已注册的数据源。
+    """
+    results: list[dict[str, Any]] = []
+    targets = [source_key] if source_key else [s.key for s in registry.all()]
+
+    for key in targets:
+        src = registry.get(key)
+        if src is None:
+            results.append({
+                "key": key,
+                "success": False,
+                "message": f"数据源 {key} 未注册",
+                "testedAt": _now_iso(),
+            })
+            continue
+        try:
+            r = src.health_check()
+            results.append({
+                "key": key,
+                "name": src.name,
+                "success": bool(r.get("success", False)),
+                "message": str(r.get("message", "")),
+                "testedAt": _now_iso(),
+                "provides": src.provides,
+                "requiresCredential": src.requires_credential,
+            })
+        except Exception as e:
+            results.append({
+                "key": key,
+                "name": getattr(src, "name", key),
+                "success": False,
+                "message": f"健康检查异常: {e}",
+                "testedAt": _now_iso(),
+            })
+
+    with _health_lock:
+        for r in results:
+            _health_results[r["key"]] = r
+    return results
+
+
+def get_health_status(source_key: Optional[str] = None) -> list[dict[str, Any]]:
+    """读取最近一次健康检查结果（不重新跑）。"""
+    with _health_lock:
+        if source_key:
+            r = _health_results.get(source_key)
+            return [dict(r)] if r else []
+        return [dict(_health_results[k]) for k in sorted(_health_results)]
+
+
+def is_source_ready(source_key: str) -> tuple[bool, str]:
+    """综合判断：是否在 registry + 凭证已配 + 健康检查通过。"""
+    src = registry.get(source_key)
+    if src is None:
+        return False, f"数据源 {source_key} 未注册"
+    if src.requires_credential:
+        token = os.environ.get(src.credential_key, "").strip()
+        if not token:
+            return False, f"凭证 {src.credential_key} 未配置"
+    ok, reason = src.is_available()
+    if not ok:
+        return False, reason
+    with _health_lock:
+        last = _health_results.get(source_key)
+    if last is None:
+        return False, "健康检查尚未完成"
+    if not last.get("success"):
+        return False, f"连通性测试失败: {last.get('message', '未知错误')}"
+    return True, "就绪"
+
+
+def pick_source(capability: str) -> Optional[DataSource]:
+    """从所有就绪数据源里挑一个能提供某能力的数据源，按注册顺序。"""
+    for s in registry.all():
+        if capability in s.provides:
+            ok, _ = is_source_ready(s.key)
+            if ok:
+                return s
+    return None
+
+
+# ── 后台线程定期跑健康检查 ──────────────────────────────────────────
+
+
+def _health_loop() -> None:
+    while True:
+        try:
+            run_health_check()
+            logger.info("[datasource_health] 全数据源健康检查完成")
+        except Exception as e:
+            logger.error(f"[datasource_health] 健康检查异常: {e}")
+        time.sleep(_HEALTH_INTERVAL)
+
+
+def start_health_monitor() -> None:
+    global _health_started, _health_thread
+    with _health_lock:
+        if _health_started:
+            return
+        _health_started = True
+        _health_thread = threading.Thread(target=_health_loop, daemon=True, name="datasource-health")
+        _health_thread.start()
+
+
+# ── 启动时构建默认 registry ──────────────────────────────────────────
+
+
+def build_default_registry() -> None:
+    """启动时调用：把默认 4 个数据源注册到全局 registry。"""
+    # 避免重复注册
+    if registry.all():
+        return
+    from app.sources.sina import SinaSource
+    from app.sources.baostock import BaostockSource
+    from app.sources.xueqiu import XueqiuSource
+    from app.sources.mairui import MairuiSource
+
+    registry.register(SinaSource())
+    registry.register(BaostockSource())
+    registry.register(MairuiSource())
+    if settings.xueqiu_token:
+        registry.register(XueqiuSource())
+    else:
+        # 即使没 token 也注册，is_available() 会拦截
+        registry.register(XueqiuSource())
+    # akshare 已移除（项目级决策：永远不用）

app/core/datasource/utils.py

@@ -0,0 +1,115 @@
+"""数据源公共工具：代码转换、K 线标准化、日期过滤。
+
+参考 dashboard/api/services/datasource/fetch_kline.py，但适配新项目。"""
+from __future__ import annotations
+
+import re
+
+import pandas as pd
+
+KLINE_COLS = ["trade_date", "open", "high", "low", "close", "volume"]
+KLINE_5MIN_COLS = ["bar_time", "open", "high", "low", "close", "volume", "amount", "turnover_rate"]
+
+
+def to_code6(code: str) -> str:
+    """'SH600000' / 'sh.600000' / '600000' / 'SH600000.SH' → '600000'"""
+    c = str(code).strip().upper()
+    c = re.sub(r"^(SH|SZ|BJ)", "", c)
+    c = c.split(".")[0]
+    return c.zfill(6)
+
+
+def code6_to_exchange(code6: str) -> str:
+    """6 位代码 → 交易所前缀。"""
+    if code6.startswith(("5", "6", "9")):
+        return "SH"
+    if code6.startswith(("4", "8")):
+        return "BJ"
+    return "SZ"
+
+
+def code6_to_sina(code6: str) -> str:
+    """新浪财经 symbol：'sh600036' / 'sz000001'。"""
+    return f"{code6_to_exchange(code6).lower()}{code6}"
+
+
+def code6_to_baostock(code6: str) -> str:
+    """Baostock 风格：'sh.600000'。"""
+    return f"{code6_to_exchange(code6).lower()}.{code6}"
+
+
+def code6_to_xueqiu(code6: str) -> str:
+    """雪球 symbol：'SH600036'。"""
+    return f"{code6_to_exchange(code6)}{code6}"
+
+
+def code6_to_mairui(code6: str) -> str:
+    """麦蕊智数 symbol：'600036.SH'。"""
+    return f"{code6}.{code6_to_exchange(code6)}"
+
+
+def normalize_kline(df: pd.DataFrame) -> pd.DataFrame:
+    """标准化日 K 线 DataFrame。
+
+    输入列名可能是 date / day / vol 等，输出固定为 KLINE_COLS。
+    含 OHLC 合法性校验（high >= max(o,l,c), low <= min(o,h,c)）。
+    """
+    if df is None or df.empty:
+        return pd.DataFrame(columns=KLINE_COLS)
+    df = df.rename(columns={"date": "trade_date", "day": "trade_date", "vol": "volume"}).copy()
+    missing = [c for c in KLINE_COLS if c not in df.columns]
+    if missing:
+        return pd.DataFrame(columns=KLINE_COLS)
+    df = df[KLINE_COLS]
+    df["trade_date"] = pd.to_datetime(df["trade_date"], errors="coerce").astype("datetime64[ns]")
+    for col in ["open", "high", "low", "close", "volume"]:
+        df[col] = pd.to_numeric(df[col], errors="coerce")
+    df = df.dropna(subset=["trade_date", "open", "high", "low", "close"])
+    df["volume"] = df["volume"].fillna(0)
+    valid = (
+        (df["open"] > 0) & (df["high"] > 0) & (df["low"] > 0) & (df["close"] > 0)
+        & (df["volume"] >= 0)
+        & (df["high"] >= df[["open", "low", "close"]].max(axis=1))
+        & (df["low"] <= df[["open", "high", "close"]].min(axis=1))
+    )
+    df = df.loc[valid].sort_values("trade_date")
+    return df.drop_duplicates(subset=["trade_date"], keep="last").reset_index(drop=True)
+
+
+def filter_date_range(df: pd.DataFrame, start: str, end: str) -> pd.DataFrame:
+    if df.empty:
+        return df
+    df = df[
+        (df["trade_date"] >= pd.Timestamp(start))
+        & (df["trade_date"] <= pd.Timestamp(end))
+    ]
+    return df.reset_index(drop=True)
+
+
+def normalize_5min(df: pd.DataFrame) -> pd.DataFrame:
+    """标准化 5 分钟 K 线。"""
+    if df is None or df.empty:
+        return pd.DataFrame(columns=KLINE_5MIN_COLS)
+    df = df.rename(columns={
+        "时间": "bar_time", "time": "bar_time", "date": "bar_time",
+        "成交量": "volume", "成交额": "amount", "换手率": "turnover_rate",
+    }).copy()
+    if "bar_time" not in df.columns:
+        return pd.DataFrame(columns=KLINE_5MIN_COLS)
+    df["bar_time"] = pd.to_datetime(df["bar_time"], errors="coerce")
+    for col in ["open", "high", "low", "close", "volume", "amount", "turnover_rate"]:
+        if col in df.columns:
+            df[col] = pd.to_numeric(df[col], errors="coerce")
+    df = df.dropna(subset=["bar_time"])
+    keep = ["bar_time"] + [c for c in KLINE_5MIN_COLS[1:] if c in df.columns]
+    df = df[keep].sort_values("bar_time").drop_duplicates(subset=["bar_time"], keep="last")
+    return df.reset_index(drop=True)
+
+
+def is_a_share_code(code6: str) -> bool:
+    """判断 6 位代码是否是主板/创业板/科创板（排除北证 8 字头、可转债等）。"""
+    if not code6 or len(code6) != 6 or not code6.isdigit():
+        return False
+    if code6.startswith(("4", "8")):
+        return False
+    return True

app/core/db/__init__.py

@@ -0,0 +1,4 @@
+"""DB 子包：连接管理 + schema + 业务 CRUD。"""
+from app.core.db.connection import get_mysql, init_mysql_schema, close_mysql
+
+__all__ = ["get_mysql", "init_mysql_schema", "close_mysql"]

app/core/db/connection.py

@@ -0,0 +1,104 @@
+"""MySQL 连接管理（thread-local，模式照搬 dashboard，但适配 market_data_sync 项目）。
+
+要点：
+- pymysql 连接不是线程安全的，每个线程各持一份
+- 内网自签名证书场景：传 ssl_disabled=True 跳过 SSL 校验
+- DATE / DATETIME / TIMESTAMP 统一转字符串，避免下游类型处理
+- DictCursor：返回字典风格行，方便业务拼装
+"""
+from __future__ import annotations
+
+import threading
+from typing import Optional
+
+import pymysql
+import pymysql.constants.FIELD_TYPE
+from pymysql.converters import conversions as _pymysql_conv, convert_date, convert_datetime
+from pymysql.cursors import DictCursor
+
+from app.core.config import settings
+
+_local = threading.local()
+
+
+def _strdate(obj):
+    d = convert_date(obj)
+    return d.strftime("%Y-%m-%d") if d else None
+
+
+def _strdatetime(obj):
+    d = convert_datetime(obj)
+    return d.strftime("%Y-%m-%d %H:%M:%S") if d else None
+
+
+# 包装默认 converter，让 DATE / DATETIME / TIMESTAMP 一律出字符串
+_conv = _pymysql_conv.copy()
+_conv[pymysql.constants.FIELD_TYPE.DATE] = _strdate
+_conv[pymysql.constants.FIELD_TYPE.DATETIME] = _strdatetime
+_conv[pymysql.constants.FIELD_TYPE.TIMESTAMP] = _strdatetime
+
+
+def get_mysql(database: Optional[str] = None) -> pymysql.connections.Connection:
+    """取当前线程的 MySQL 连接。无则新建。
+
+    关键修复：高并发场景下，连接可能被服务端超时断开 (wait_timeout)。
+    用 ping(reconnect=True) 让 pymysql 自动重连，避免 stale conn 导致
+    2013 Lost connection 错误。
+    """
+    attr = f"conn_{database or 'default'}"
+    conn = getattr(_local, attr, None)
+    if conn is not None:
+        try:
+            conn.ping(reconnect=True)
+        except Exception:
+            try:
+                conn.close()
+            except Exception:
+                pass
+            conn = None
+    if conn is None:
+        kwargs = dict(
+            host=settings.mysql_host,
+            port=settings.mysql_port,
+            user=settings.mysql_user,
+            password=settings.mysql_password,
+            database=database or settings.mysql_database,
+            charset="utf8mb4",
+            cursorclass=DictCursor,
+            autocommit=False,
+            conv=_conv,
+        )
+        # 内网自签名证书：禁用 SSL（用户场景）
+        kwargs["ssl"] = None
+        # 高并发时设较短 read_timeout / write_timeout
+        kwargs["read_timeout"] = 30
+        kwargs["write_timeout"] = 30
+        conn = pymysql.connect(**kwargs)
+        setattr(_local, attr, conn)
+    return conn
+
+
+def close_mysql() -> None:
+    """关闭当前线程的所有连接。"""
+    for attr in list(vars(_local).keys()):
+        conn = getattr(_local, attr, None)
+        if conn is not None:
+            try:
+                conn.close()
+            except Exception:
+                pass
+            try:
+                delattr(_local, attr)
+            except AttributeError:
+                pass
+
+
+# ── Schema DDL 入口 ────────────────────────────────────────────────────────
+
+
+def init_mysql_schema() -> None:
+    """集中创建/补齐所有表。被 db/ops.py 在第一次访问时调用。"""
+    # 延后导入避免循环依赖
+    from app.core.db.schema import ensure_all_tables
+
+    ensure_all_tables(get_mysql())

app/core/db/ops.py

@@ -0,0 +1,728 @@
+"""MySQL 业务 CRUD 函数集合（参考 dashboard/api/services/mysql_ops.py）。
+
+本文件只包含 market_data_sync 同步任务需要的最小函数集，不涉及 dashboard 的
+accounts / positions / trade_record / scoring 等业务表。
+
+设计要点：
+- 每个 ensure_xxx_table() 调用会触发 init_mysql_schema()（幂等）
+- 批量 upsert 走 executemany，效率高
+- 统一使用 DictCursor
+"""
+from __future__ import annotations
+
+import json
+from contextlib import contextmanager
+from datetime import datetime
+from typing import Any, Iterable, Optional
+
+from app.core.db.connection import get_mysql, init_mysql_schema
+
+
+# ── 内部 helpers ────────────────────────────────────────────────────────
+
+
+def _now_local_ts() -> str:
+    return datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+
+
+def _today_str() -> str:
+    return datetime.now().strftime("%Y-%m-%d")
+
+
+def _date_or_none(val) -> str | None:
+    if val is None:
+        return None
+    s = str(val).strip()
+    return s if s else None
+
+
+@contextmanager
+def _cursor():
+    """带 init_mysql_schema 的 cursor 上下文。"""
+    init_mysql_schema()
+    conn = get_mysql()
+    cur = conn.cursor()
+    try:
+        yield cur
+        conn.commit()
+    except Exception:
+        conn.rollback()
+        raise
+    finally:
+        cur.close()
+
+
+# ── config 表 ───────────────────────────────────────────────────────────
+
+
+def fetch_all_config() -> list[dict[str, Any]]:
+    init_mysql_schema()
+    conn = get_mysql()
+    with conn.cursor() as cur:
+        cur.execute("SELECT `key`, `value`, category, description, updated_at FROM config")
+        return cur.fetchall()
+
+
+def fetch_config_by_key(key: str) -> dict[str, Any] | None:
+    """按 key 查单条 config（比 fetch_all_config 高效）。"""
+    init_mysql_schema()
+    conn = get_mysql()
+    with conn.cursor() as cur:
+        cur.execute(
+            "SELECT `key`, `value`, category, description, updated_at FROM config WHERE `key` = %s",
+            (key,),
+        )
+        return cur.fetchone()
+
+
+def fetch_configs_by_category(category: str) -> list[dict[str, Any]]:
+    """按 category 查 config（比 fetch_all_config + 内存过滤高效）。"""
+    init_mysql_schema()
+    conn = get_mysql()
+    with conn.cursor() as cur:
+        cur.execute(
+            "SELECT `key`, `value`, category, description, updated_at FROM config WHERE category = %s",
+            (category,),
+        )
+        return cur.fetchall()
+
+
+def upsert_config(key: str, value: str, category: str = "general", description: str = "") -> None:
+    with _cursor() as cur:
+        cur.execute(
+            """
+            INSERT INTO config (`key`, `value`, category, description)
+            VALUES (%s, %s, %s, %s)
+            ON DUPLICATE KEY UPDATE
+                `value` = VALUES(`value`),
+                category = VALUES(category),
+                description = VALUES(description)
+            """,
+            (key, value, category, description),
+        )
+
+
+def delete_config(key: str) -> None:
+    with _cursor() as cur:
+        cur.execute("DELETE FROM config WHERE `key` = %s", (key,))
+
+
+# ── dataset_registry ────────────────────────────────────────────────────
+
+
+def upsert_dataset_registry_rows(rows: list[dict[str, Any]]) -> None:
+    """批量 upsert 同步任务定义到 dataset_registry。"""
+    if not rows:
+        return
+    with _cursor() as cur:
+        for r in rows:
+            dep_json = r.get("dependency_ids")
+            if isinstance(dep_json, (list, dict)):
+                dep_json = json.dumps(dep_json, ensure_ascii=False)
+            cur.execute(
+                """
+                INSERT INTO dataset_registry
+                    (dataset_id, name, description, storage_uri, storage_layer,
+                     management_role, source, sync_script, dependency_ids,
+                     enabled, sort_order)
+                VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
+                ON DUPLICATE KEY UPDATE
+                    name = VALUES(name),
+                    description = VALUES(description),
+                    storage_uri = VALUES(storage_uri),
+                    storage_layer = VALUES(storage_layer),
+                    management_role = VALUES(management_role),
+                    source = VALUES(source),
+                    sync_script = VALUES(sync_script),
+                    dependency_ids = VALUES(dependency_ids),
+                    enabled = VALUES(enabled),
+                    sort_order = VALUES(sort_order)
+                """,
+                (
+                    r.get("dataset_id", ""),
+                    r.get("name", ""),
+                    r.get("description", ""),
+                    r.get("storage_uri", ""),
+                    r.get("storage_layer", ""),
+                    r.get("management_role", ""),
+                    r.get("source", ""),
+                    r.get("sync_script", ""),
+                    dep_json or "[]",
+                    r.get("enabled", 1),
+                    r.get("sort_order", 0),
+                ),
+            )
+
+
+def fetch_dataset_registry_rows() -> list[dict[str, Any]]:
+    init_mysql_schema()
+    conn = get_mysql()
+    with conn.cursor() as cur:
+        cur.execute("SELECT * FROM dataset_registry ORDER BY sort_order, dataset_id")
+        return cur.fetchall()
+
+
+def fetch_dataset_registry_map() -> dict[str, dict[str, Any]]:
+    rows = fetch_dataset_registry_rows()
+    return {r["dataset_id"]: r for r in rows}
+
+
+def update_dataset_registry_state(dataset_id: str, **kwargs: Any) -> None:
+    """按需更新 dataset_registry 的状态字段。"""
+    allowed = {
+        "name", "description", "storage_uri", "storage_layer", "management_role",
+        "source", "sync_script", "dependency_ids", "enabled", "sort_order",
+        "status", "trigger_source", "started_at", "finished_at",
+        "last_success_at", "last_failure_at", "message", "last_error",
+        "needs_resync", "progress_current", "progress_total", "current_step",
+        "updated_at",
+    }
+    payload = {k: v for k, v in kwargs.items() if k in allowed}
+    if not payload:
+        return
+    if "updated_at" not in payload:
+        payload["updated_at"] = _now_local_ts()
+    set_clause = ", ".join(f"`{k}` = %s" for k in payload)
+    values = list(payload.values()) + [dataset_id]
+    with _cursor() as cur:
+        cur.execute(
+            f"UPDATE dataset_registry SET {set_clause} WHERE dataset_id = %s",
+            values,
+        )
+
+
+def recover_interrupted_dataset_registry() -> int:
+    """把状态卡在 running 的同步任务标记为 failed（启动时调用）。"""
+    with _cursor() as cur:
+        cur.execute(
+            """
+            UPDATE dataset_registry
+            SET status = 'failed',
+                finished_at = %s,
+                last_failure_at = %s,
+                last_error = CONCAT('进程在状态为 running 时被中断，已自动恢复。', IFNULL(last_error, '')),
+                message = CONCAT('[recovered] ', IFNULL(message, '同步被中断')),
+                needs_resync = 1
+            WHERE status = 'running'
+            """,
+            (_now_local_ts(), _now_local_ts()),
+        )
+        return cur.rowcount
+
+
+# ── stocks ──────────────────────────────────────────────────────────────
+
+
+def upsert_stock(
+    code: str,
+    name: str,
+    exchange: str,
+    list_date: str = "",
+    listing_status: str = "normal",
+    industry: str = "",
+) -> None:
+    """upsert 一只股票基础信息（单条）。
+
+    注意：MySQL 9.7.0 在 ON DUPLICATE KEY UPDATE 阶段对 DATE 字段的 VALUES()/new.col
+    求值存在 bug（会强制把空串塞进去，触发 1292 严格模式错误）。所以这里：
+    - list_date 字段在首次 INSERT 时写入；UPDATE 分支不更新（保持原值或 NULL）
+    - industry 字段也避开这个 bug
+
+    批量写入请用 upsert_stocks_bulk()，性能高 10-20 倍。
+    """
+    del list_date  # 显式不接受 list_date 更新（旧值保留）
+    del industry   # 同上
+    with _cursor() as cur:
+        cur.execute(
+            """
+            INSERT INTO stocks (code, name, exchange, listing_status)
+            VALUES (%s, %s, %s, %s)
+            ON DUPLICATE KEY UPDATE
+                name = VALUES(name),
+                exchange = VALUES(exchange),
+                listing_status = VALUES(listing_status)
+            """,
+            (code, name, exchange, listing_status),
+        )
+
+
+def upsert_stocks_bulk(rows: list[dict[str, Any]], chunk_size: int = 500) -> int:
+    """批量 upsert 股票基础信息。
+
+    每条 row 需有: code, name, exchange, listing_status（可选 list_date / industry）
+    性能：~5000 只股票从 ~50s 降到 ~3s。
+    """
+    if not rows:
+        return 0
+    sql = (
+        "INSERT INTO stocks (code, name, exchange, listing_status) "
+        "VALUES (%s, %s, %s, %s) "
+        "ON DUPLICATE KEY UPDATE "
+        "  name = VALUES(name), "
+        "  exchange = VALUES(exchange), "
+        "  listing_status = VALUES(listing_status)"
+    )
+    with _cursor() as cur:
+        total = 0
+        for i in range(0, len(rows), chunk_size):
+            chunk = rows[i: i + chunk_size]
+            values = [
+                (r.get("code", ""), r.get("name", ""), r.get("exchange", ""), r.get("listing_status", "normal"))
+                for r in chunk
+            ]
+            cur.executemany(sql, values)
+            total += len(chunk)
+        return total
+
+
+def update_stock_share_snapshot(code: str, total_share: float, float_share: float, trade_date: str = "") -> None:
+    with _cursor() as cur:
+        cur.execute(
+            """
+            UPDATE stocks
+            SET total_share = %s, float_share = %s,
+                share_updated_at = COALESCE(NULLIF(%s, ''), CURDATE())
+            WHERE code = %s
+            """,
+            (float(total_share), float(float_share), trade_date or "", code),
+        )
+
+
+def update_stock_kline_synced_at(code: str, synced_at: str) -> None:
+    with _cursor() as cur:
+        cur.execute(
+            "UPDATE stocks SET kline_synced_at = %s WHERE code = %s",
+            (_date_or_none(synced_at), code),
+        )
+
+
+def fetch_all_stocks() -> list[dict[str, Any]]:
+    init_mysql_schema()
+    conn = get_mysql()
+    with conn.cursor() as cur:
+        cur.execute("SELECT * FROM stocks ORDER BY code")
+        return cur.fetchall()
+
+
+def fetch_stock_by_code(code: str) -> Optional[dict[str, Any]]:
+    init_mysql_schema()
+    conn = get_mysql()
+    with conn.cursor() as cur:
+        cur.execute("SELECT * FROM stocks WHERE code = %s", (code,))
+        row = cur.fetchone()
+        return dict(row) if row else None
+
+
+def stock_count() -> int:
+    init_mysql_schema()
+    conn = get_mysql()
+    with conn.cursor() as cur:
+        cur.execute("SELECT COUNT(*) AS cnt FROM stocks")
+        row = cur.fetchone()
+        return int(row["cnt"]) if row else 0
+
+
+def iter_stock_codes(active_only: bool = True) -> Iterable[str]:
+    """yield 6 位股票代码（去除 SH/SZ/BJ 前缀）。"""
+    for s in fetch_all_stocks():
+        if active_only and s.get("listing_status") == "delisted":
+            continue
+        code = str(s.get("code", "")).strip()
+        if not code:
+            continue
+        for prefix in ("SH", "SZ", "BJ"):
+            if code.startswith(prefix):
+                code = code[len(prefix):]
+                break
+        if code.isdigit() and len(code) == 6:
+            yield code
+
+
+# ── indices ─────────────────────────────────────────────────────────────
+
+
+def upsert_index(
+    index_code: str,
+    index_name: str,
+    market: str = "",
+    category: str = "",
+    source: str = "",
+    enabled: bool = True,
+) -> None:
+    with _cursor() as cur:
+        cur.execute(
+            """
+            INSERT INTO indices (index_code, index_name, market, category, source, enabled)
+            VALUES (%s, %s, %s, %s, %s, %s)
+            ON DUPLICATE KEY UPDATE
+                index_name = VALUES(index_name),
+                market = VALUES(market),
+                category = VALUES(category),
+                source = VALUES(source),
+                enabled = VALUES(enabled)
+            """,
+            (index_code, index_name, market, category, source, int(enabled)),
+        )
+
+
+# ── kline 通用批量 upsert ──────────────────────────────────────────────
+
+
+def _bulk_upsert(
+    cur,
+    table: str,
+    columns: list[str],
+    rows: list[dict[str, Any]],
+    conflict_keys: list[str],
+    chunk_size: int = 1000,
+) -> int:
+    """通用批量 INSERT ... ON DUPLICATE KEY UPDATE 工具。"""
+    if not rows:
+        return 0
+    placeholders = ", ".join(["%s"] * len(columns))
+    col_list = ", ".join(f"`{c}`" for c in columns)
+    update_clause = ", ".join(
+        f"`{c}` = VALUES(`{c}`)"
+        for c in columns
+        if c not in conflict_keys
+    )
+    sql = (
+        f"INSERT INTO {table} ({col_list}) VALUES ({placeholders}) "
+        f"ON DUPLICATE KEY UPDATE {update_clause}"
+    )
+    total = 0
+    for i in range(0, len(rows), chunk_size):
+        chunk = rows[i: i + chunk_size]
+        values = [
+            tuple(_date_or_none(r.get(c)) if c.endswith("date") else r.get(c) for c in columns)
+            for r in chunk
+        ]
+        cur.executemany(sql, values)
+        total += len(chunk)
+    return total
+
+
+def upsert_kline_stock(rows: list[dict[str, Any]]) -> int:
+    """rows: stock_code, trade_date, open, high, low, close, volume"""
+    with _cursor() as cur:
+        return _bulk_upsert(
+            cur, "kline_stock",
+            ["stock_code", "trade_date", "open", "high", "low", "close", "volume"],
+            rows, conflict_keys=["stock_code", "trade_date"],
+        )
+
+
+def upsert_kline_index(rows: list[dict[str, Any]]) -> int:
+    """rows: index_code, trade_date, open, high, low, close, volume"""
+    with _cursor() as cur:
+        return _bulk_upsert(
+            cur, "kline_index",
+            ["index_code", "trade_date", "open", "high", "low", "close", "volume"],
+            rows, conflict_keys=["index_code", "trade_date"],
+        )
+
+
+def upsert_kline_5min(rows: list[dict[str, Any]]) -> int:
+    """rows: stock_code, bar_time, open, high, low, close, volume, amount, turnover_rate"""
+    with _cursor() as cur:
+        return _bulk_upsert(
+            cur, "kline_5min",
+            ["stock_code", "bar_time", "open", "high", "low", "close", "volume", "amount", "turnover_rate"],
+            rows, conflict_keys=["stock_code", "bar_time"],
+        )
+
+
+def get_kline_5min_snapshots() -> dict[str, Optional[datetime]]:
+    """一次 SQL 拉 kline_5min 全表每只股票的最新 bar_time。
+
+    返回：{stock_code (6位): max(bar_time) 或 None}
+    用于 kline_5min 任务的"全量/增量"统一规划：
+      - DB 里有 → 增量（从 max - 2 天 到今天）
+      - DB 里没 → 全量（从 mairui 历史深度起点 到今天）
+
+    DB 里 stock_code 形如 "000001.SZ"（mairui 写入格式），函数剥掉
+    交易所后缀统一为 6 位。bar_time 是 DATETIME 字段，pymysql DictCursor
+    在某些连接参数下会返回 str，这里手动解析为 datetime（解析失败置 None）。
+    """
+    init_mysql_schema()
+    conn = get_mysql()
+    out: dict[str, Optional[datetime]] = {}
+    with conn.cursor() as cur:
+        cur.execute("SELECT stock_code, MAX(bar_time) AS latest FROM kline_5min GROUP BY stock_code")
+        for r in cur.fetchall():
+            raw_code = str(r["stock_code"] or "").strip()
+            code6 = raw_code.split(".")[0]  # "000001.SZ" → "000001"
+            if len(code6) != 6 or not code6.isdigit():
+                continue
+            latest = r["latest"]
+            if isinstance(latest, datetime):
+                pass
+            elif isinstance(latest, str) and latest:
+                try:
+                    latest = datetime.strptime(latest[:19], "%Y-%m-%d %H:%M:%S")
+                except ValueError:
+                    latest = None
+            else:
+                latest = None
+            out[code6] = latest
+    return out
+
+
+def upsert_moneyflow(rows: list[dict[str, Any]]) -> int:
+    """rows: stock_code, trade_date, main_net_inflow, large_net_inflow, medium_net_inflow, small_net_inflow"""
+    with _cursor() as cur:
+        return _bulk_upsert(
+            cur, "moneyflow",
+            ["stock_code", "trade_date", "main_net_inflow", "large_net_inflow", "medium_net_inflow", "small_net_inflow"],
+            rows, conflict_keys=["stock_code", "trade_date"],
+        )
+
+
+def upsert_share(rows: list[dict[str, Any]]) -> int:
+    """rows: stock_code, trade_date, total_share, float_share"""
+    with _cursor() as cur:
+        return _bulk_upsert(
+            cur, "share",
+            ["stock_code", "trade_date", "total_share", "float_share"],
+            rows, conflict_keys=["stock_code", "trade_date"],
+        )
+
+
+# ── 行业 / 概念板块 ─────────────────────────────────────────────────────
+
+
+def replace_all_industries(rows: list[dict[str, Any]]) -> None:
+    """全量替换 industry 表。rows: code, industry_name, industry_classification, update_date"""
+    if not rows:
+        return
+    with _cursor() as cur:
+        cur.execute("DELETE FROM industry")
+        for r in rows:
+            cur.execute(
+                """
+                INSERT INTO industry (code, industry_name, industry_classification, update_date)
+                VALUES (%s, %s, %s, %s)
+                ON DUPLICATE KEY UPDATE
+                    industry_name = VALUES(industry_name),
+                    industry_classification = VALUES(industry_classification),
+                    update_date = VALUES(update_date)
+                """,
+                (
+                    str(r.get("code", "")).zfill(6),
+                    r.get("industry_name") or None,
+                    r.get("industry_classification") or None,
+                    _date_or_none(r.get("update_date")),
+                ),
+            )
+
+
+def fetch_all_industries() -> list[dict[str, Any]]:
+    init_mysql_schema()
+    conn = get_mysql()
+    with conn.cursor() as cur:
+        cur.execute("SELECT code, industry_name, industry_classification, update_date FROM industry")
+        return cur.fetchall()
+
+
+def replace_all_sectors(rows: list[dict[str, Any]]) -> None:
+    """rows: sector_key, sector_name, taxonomy, level, source, enabled"""
+    if not rows:
+        return
+    with _cursor() as cur:
+        cur.execute("DELETE FROM sectors")
+        for r in rows:
+            cur.execute(
+                """
+                INSERT INTO sectors (sector_key, sector_name, taxonomy, level, source, enabled)
+                VALUES (%s, %s, %s, %s, %s, %s)
+                ON DUPLICATE KEY UPDATE
+                    sector_name = VALUES(sector_name),
+                    taxonomy = VALUES(taxonomy),
+                    level = VALUES(level),
+                    source = VALUES(source),
+                    enabled = VALUES(enabled)
+                """,
+                (
+                    r.get("sector_key", ""),
+                    r.get("sector_name", ""),
+                    r.get("taxonomy", ""),
+                    r.get("level", ""),
+                    r.get("source", ""),
+                    int(r.get("enabled", 1)),
+                ),
+            )
+
+
+def replace_all_stock_sector_map(rows: list[dict[str, Any]]) -> None:
+    """rows: stock_code, sector_key"""
+    if not rows:
+        return
+    with _cursor() as cur:
+        cur.execute("DELETE FROM stock_sector_map")
+        for r in rows:
+            cur.execute(
+                """
+                INSERT INTO stock_sector_map (stock_code, sector_key)
+                VALUES (%s, %s)
+                ON DUPLICATE KEY UPDATE sector_key = VALUES(sector_key)
+                """,
+                (str(r.get("stock_code", "")).zfill(6), r.get("sector_key", "")),
+            )
+
+
+# ── 行业聚合（衍生）─────────────────────────────────────────────────────
+
+
+def replace_all_sector_indices(rows: list[dict[str, Any]]) -> None:
+    """rows: trade_date, sector_name, close, sector_amplitude"""
+    if not rows:
+        return
+    with _cursor() as cur:
+        for r in rows:
+            cur.execute(
+                """
+                INSERT INTO sector_indices (trade_date, sector_name, `close`, sector_amplitude)
+                VALUES (%s, %s, %s, %s)
+                ON DUPLICATE KEY UPDATE
+                    `close` = VALUES(`close`),
+                    sector_amplitude = VALUES(sector_amplitude)
+                """,
+                (
+                    _date_or_none(r.get("trade_date")),
+                    r.get("sector_name", ""),
+                    float(r.get("close") or 0),
+                    float(r.get("sector_amplitude") or 0),
+                ),
+            )
+
+
+def replace_all_sector_features(rows: list[dict[str, Any]]) -> None:
+    """rows: trade_date, sector_name, sector_ret, sector_amplitude, close, ema10, ema20, ema200, score"""
+    if not rows:
+        return
+    with _cursor() as cur:
+        for r in rows:
+            cur.execute(
+                """
+                INSERT INTO sector_features_daily
+                    (trade_date, sector_name, sector_ret, sector_amplitude, `close`, ema10, ema20, ema200, score)
+                VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
+                ON DUPLICATE KEY UPDATE
+                    sector_ret = VALUES(sector_ret),
+                    sector_amplitude = VALUES(sector_amplitude),
+                    `close` = VALUES(`close`),
+                    ema10 = VALUES(ema10),
+                    ema20 = VALUES(ema20),
+                    ema200 = VALUES(ema200),
+                    score = VALUES(score)
+                """,
+                (
+                    _date_or_none(r.get("trade_date")),
+                    r.get("sector_name", ""),
+                    float(r.get("sector_ret") or 0),
+                    float(r.get("sector_amplitude") or 0),
+                    float(r.get("close") or 0),
+                    float(r.get("ema10") or 0),
+                    float(r.get("ema20") or 0),
+                    float(r.get("ema200") or 0),
+                    int(r.get("score") or 0),
+                ),
+            )
+
+
+# ── 市场情绪（衍生）─────────────────────────────────────────────────────
+
+
+def upsert_market_regime_rows(rows: list[dict[str, Any]]) -> None:
+    """rows: trade_date, advancers, decliners, advance_ratio, turnover, turnover_avg_5d, turnover_ratio_5d, source, is_extreme_panic"""
+    if not rows:
+        return
+    with _cursor() as cur:
+        for r in rows:
+            cur.execute(
+                """
+                INSERT INTO market_regime_daily
+                    (trade_date, advancers, decliners, advance_ratio, turnover,
+                     turnover_avg_5d, turnover_ratio_5d, source, is_extreme_panic)
+                VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
+                ON DUPLICATE KEY UPDATE
+                    advancers = VALUES(advancers),
+                    decliners = VALUES(decliners),
+                    advance_ratio = VALUES(advance_ratio),
+                    turnover = VALUES(turnover),
+                    turnover_avg_5d = VALUES(turnover_avg_5d),
+                    turnover_ratio_5d = VALUES(turnover_ratio_5d),
+                    source = VALUES(source),
+                    is_extreme_panic = VALUES(is_extreme_panic)
+                """,
+                (
+                    _date_or_none(r.get("trade_date")),
+                    float(r.get("advancers") or 0),
+                    float(r.get("decliners") or 0),
+                    float(r.get("advance_ratio") or 0),
+                    float(r.get("turnover") or 0),
+                    float(r.get("turnover_avg_5d") or 0),
+                    float(r.get("turnover_ratio_5d") or 0),
+                    r.get("source", ""),
+                    int(r.get("is_extreme_panic") or 0),
+                ),
+            )
+
+
+# ── kline 查询（同步时用于判断增量起点）─────────────────────────────────
+
+
+def get_stock_kline_max_date(code: str) -> Optional[str]:
+    init_mysql_schema()
+    conn = get_mysql()
+    with conn.cursor() as cur:
+        cur.execute(
+            "SELECT MAX(trade_date) AS d FROM kline_stock WHERE stock_code = %s",
+            (code,),
+        )
+        row = cur.fetchone()
+        if row and row.get("d"):
+            return str(row["d"])
+    return None
+
+
+def get_index_kline_max_date(index_code: str) -> Optional[str]:
+    init_mysql_schema()
+    conn = get_mysql()
+    with conn.cursor() as cur:
+        cur.execute(
+            "SELECT MAX(trade_date) AS d FROM kline_index WHERE index_code = %s",
+            (index_code,),
+        )
+        row = cur.fetchone()
+        if row and row.get("d"):
+            return str(row["d"])
+    return None
+
+
+def get_5min_max_bar_time(code: str) -> Optional[str]:
+    init_mysql_schema()
+    conn = get_mysql()
+    with conn.cursor() as cur:
+        cur.execute(
+            "SELECT MAX(bar_time) AS d FROM kline_5min WHERE stock_code = %s",
+            (code,),
+        )
+        row = cur.fetchone()
+        if row and row.get("d"):
+            return str(row["d"])
+    return None
+
+
+def get_moneyflow_max_date(code: str) -> Optional[str]:
+    init_mysql_schema()
+    conn = get_mysql()
+    with conn.cursor() as cur:
+        cur.execute(
+            "SELECT MAX(trade_date) AS d FROM moneyflow WHERE stock_code = %s",
+            (code,),
+        )
+        row = cur.fetchone()
+        if row and row.get("d"):
+            return str(row["d"])
+    return None

app/core/db/schema.py

@@ -0,0 +1,264 @@
+"""Schema DDL — 集中所有 CREATE TABLE IF NOT EXISTS。
+
+参考 dashboard/api/services/mysql_conn.py::init_mysql_schema()，但补齐 dashboard 缺的几张表
+（kline_stock / kline_5min / kline_index / moneyflow / share / sector_features_daily / sector_indices
+/ stock_scores / registry），这些是 grid_seeker_model_base 生产库已存在但 mysql_conn.py 没建的表。
+
+所有 DDL 写在同一个文件，方便审阅、diff 和补字段。"""
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import pymysql.connections
+
+
+# DDL 列表（顺序无关，全部 IF NOT EXISTS）
+_DDL: list[str] = [
+    # ─── config ─────────────────────────────────────────────────────────
+    """
+    CREATE TABLE IF NOT EXISTS config (
+        `key`        VARCHAR(128) PRIMARY KEY,
+        `value`      TEXT         NOT NULL,
+        category     VARCHAR(32)  NOT NULL DEFAULT 'general',
+        description  VARCHAR(255) DEFAULT '',
+        updated_at   DATETIME     DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
+    """,
+    # ─── dataset_registry (同步状态机) ────────────────────────────────────
+    """
+    CREATE TABLE IF NOT EXISTS dataset_registry (
+        dataset_id       VARCHAR(64)  PRIMARY KEY,
+        name             VARCHAR(128) NOT NULL DEFAULT '',
+        description      TEXT,
+        storage_uri      VARCHAR(255) DEFAULT '',
+        storage_layer    VARCHAR(32)  DEFAULT '',
+        management_role  VARCHAR(32)  DEFAULT '',
+        source           VARCHAR(255) DEFAULT '',
+        sync_script      VARCHAR(255) DEFAULT '',
+        dependency_ids   VARCHAR(4096) DEFAULT '[]',
+        enabled          INT          DEFAULT 1,
+        sort_order       INT          DEFAULT 0,
+        status           VARCHAR(16)  DEFAULT 'idle',
+        trigger_source   VARCHAR(32)  DEFAULT '',
+        started_at       DATETIME     DEFAULT NULL,
+        finished_at      DATETIME     DEFAULT NULL,
+        last_success_at  DATETIME     DEFAULT NULL,
+        last_failure_at  DATETIME     DEFAULT NULL,
+        message          TEXT,
+        last_error       TEXT,
+        needs_resync     INT          DEFAULT 0,
+        progress_current INT          DEFAULT 0,
+        progress_total   INT          DEFAULT 0,
+        current_step     VARCHAR(255) DEFAULT '',
+        updated_at       DATETIME     DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+        INDEX idx_dataset_registry_sort_order (sort_order)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
+    """,
+    # ─── stocks (基础信息 + 股本字段) ─────────────────────────────────────
+    """
+    CREATE TABLE IF NOT EXISTS stocks (
+        code             VARCHAR(10)  PRIMARY KEY,
+        name             VARCHAR(32)  NOT NULL DEFAULT '',
+        exchange         VARCHAR(8)   NOT NULL DEFAULT '',
+        list_date        DATE         DEFAULT NULL,
+        listing_status   VARCHAR(16)  NOT NULL DEFAULT 'normal',
+        industry         VARCHAR(64)  DEFAULT '',
+        total_share      DOUBLE       DEFAULT 0,
+        float_share      DOUBLE       DEFAULT 0,
+        share_updated_at DATE         DEFAULT NULL,
+        kline_synced_at  DATE         DEFAULT NULL,
+        updated_at       DATETIME     DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
+    """,
+    # ─── indices (指数字典) ───────────────────────────────────────────────
+    """
+    CREATE TABLE IF NOT EXISTS indices (
+        index_code  VARCHAR(10)  PRIMARY KEY,
+        index_name  VARCHAR(64)  NOT NULL DEFAULT '',
+        market      VARCHAR(16)  DEFAULT '',
+        category    VARCHAR(16)  DEFAULT '',
+        source      VARCHAR(32)  DEFAULT '',
+        enabled     INT          DEFAULT 1,
+        updated_at  DATETIME     DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
+    """,
+    # ─── kline_stock (日 K 线，按股票) ────────────────────────────────────
+    """
+    CREATE TABLE IF NOT EXISTS kline_stock (
+        stock_code VARCHAR(10)  NOT NULL,
+        trade_date DATE         NOT NULL,
+        `open`     DOUBLE       NOT NULL,
+        `high`     DOUBLE       NOT NULL,
+        `low`      DOUBLE       NOT NULL,
+        `close`    DOUBLE       NOT NULL,
+        volume     DOUBLE       NOT NULL,
+        PRIMARY KEY (stock_code, trade_date),
+        INDEX idx_kline_stock_date (trade_date)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
+    """,
+    # ─── kline_index (指数日 K 线) ────────────────────────────────────────
+    """
+    CREATE TABLE IF NOT EXISTS kline_index (
+        index_code VARCHAR(10)  NOT NULL,
+        trade_date DATE         NOT NULL,
+        `open`     DOUBLE       NOT NULL,
+        `high`     DOUBLE       NOT NULL,
+        `low`      DOUBLE       NOT NULL,
+        `close`    DOUBLE       NOT NULL,
+        volume     DOUBLE       NOT NULL,
+        PRIMARY KEY (index_code, trade_date),
+        INDEX idx_kline_index_date (trade_date)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
+    """,
+    # ─── kline_5min (5 分钟 K 线) ────────────────────────────────────────
+    """
+    CREATE TABLE IF NOT EXISTS kline_5min (
+        stock_code    VARCHAR(10)  NOT NULL,
+        bar_time      DATETIME     NOT NULL,
+        `open`        DOUBLE       DEFAULT NULL,
+        `high`        DOUBLE       DEFAULT NULL,
+        `low`         DOUBLE       DEFAULT NULL,
+        `close`       DOUBLE       DEFAULT NULL,
+        volume        DOUBLE       DEFAULT NULL,
+        amount        DOUBLE       DEFAULT NULL,
+        turnover_rate DOUBLE       DEFAULT NULL,
+        PRIMARY KEY (stock_code, bar_time),
+        INDEX idx_kline_5min_bar_time (bar_time)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
+    """,
+    # ─── moneyflow (资金流，主力/大/中/小单净额) ──────────────────────────
+    """
+    CREATE TABLE IF NOT EXISTS moneyflow (
+        stock_code        VARCHAR(10) NOT NULL,
+        trade_date        DATE        NOT NULL,
+        main_net_inflow   DOUBLE      DEFAULT 0,
+        large_net_inflow  DOUBLE      DEFAULT 0,
+        medium_net_inflow DOUBLE      DEFAULT 0,
+        small_net_inflow  DOUBLE      DEFAULT 0,
+        PRIMARY KEY (stock_code, trade_date),
+        INDEX idx_moneyflow_date (trade_date)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
+    """,
+    # ─── share (股本快照) ────────────────────────────────────────────────
+    """
+    CREATE TABLE IF NOT EXISTS share (
+        stock_code   VARCHAR(10) NOT NULL,
+        trade_date   DATE        NOT NULL,
+        total_share  DOUBLE      NOT NULL,
+        float_share  DOUBLE      NOT NULL,
+        PRIMARY KEY (stock_code, trade_date)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
+    """,
+    # ─── sectors (行业字典) ──────────────────────────────────────────────
+    """
+    CREATE TABLE IF NOT EXISTS sectors (
+        sector_key  VARCHAR(64)  PRIMARY KEY,
+        sector_name VARCHAR(64)  NOT NULL DEFAULT '',
+        taxonomy    VARCHAR(64)  DEFAULT '',
+        level       VARCHAR(16)  DEFAULT '',
+        source      VARCHAR(32)  DEFAULT '',
+        enabled     INT          DEFAULT 1,
+        updated_at  DATETIME     DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+        INDEX idx_sectors_sector_name (sector_name)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
+    """,
+    # ─── stock_sector_map (股票→行业映射) ─────────────────────────────────
+    """
+    CREATE TABLE IF NOT EXISTS stock_sector_map (
+        stock_code  VARCHAR(6)   PRIMARY KEY,
+        sector_key  VARCHAR(64)  NOT NULL DEFAULT '',
+        updated_at  DATETIME     DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+        INDEX idx_stock_sector_map_sector_key (sector_key)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
+    """,
+    # ─── industry (股票-行业映射，datasource 实际写入用这张) ──────────────
+    """
+    CREATE TABLE IF NOT EXISTS industry (
+        code                    VARCHAR(6)   PRIMARY KEY,
+        industry_name           VARCHAR(64)  DEFAULT '',
+        industry_classification VARCHAR(32)  DEFAULT '',
+        update_date             DATE         DEFAULT NULL,
+        updated_at              DATETIME     DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+        INDEX idx_industry_industry_name (industry_name)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
+    """,
+    # ─── sector_indices (行业聚合指数时序) ─────────────────────────────────
+    """
+    CREATE TABLE IF NOT EXISTS sector_indices (
+        trade_date        DATE         NOT NULL,
+        sector_name       VARCHAR(64)  NOT NULL,
+        `close`           DOUBLE       DEFAULT 0,
+        sector_amplitude  DOUBLE       DEFAULT 0,
+        PRIMARY KEY (trade_date, sector_name),
+        INDEX idx_sector_indices_sector_name (sector_name)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
+    """,
+    # ─── sector_features_daily (行业特征衍生) ─────────────────────────────
+    """
+    CREATE TABLE IF NOT EXISTS sector_features_daily (
+        trade_date        DATE         NOT NULL,
+        sector_name       VARCHAR(64)  NOT NULL,
+        sector_ret        DOUBLE       DEFAULT 0,
+        sector_amplitude  DOUBLE       DEFAULT 0,
+        `close`           DOUBLE       DEFAULT 0,
+        ema10             DOUBLE       DEFAULT 0,
+        ema20             DOUBLE       DEFAULT 0,
+        ema200            DOUBLE       DEFAULT 0,
+        score             INT          DEFAULT 0,
+        PRIMARY KEY (trade_date, sector_name),
+        INDEX idx_sector_features_sector_name (sector_name)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
+    """,
+    # ─── market_regime_daily (市场情绪) ───────────────────────────────────
+    """
+    CREATE TABLE IF NOT EXISTS market_regime_daily (
+        trade_date         DATE         PRIMARY KEY,
+        advancers          DOUBLE       DEFAULT 0,
+        decliners          DOUBLE       DEFAULT 0,
+        advance_ratio      DOUBLE       DEFAULT 0,
+        turnover           DOUBLE       DEFAULT 0,
+        turnover_avg_5d    DOUBLE       DEFAULT 0,
+        turnover_ratio_5d  DOUBLE       DEFAULT 0,
+        source             VARCHAR(32)  DEFAULT '',
+        is_extreme_panic   INT          DEFAULT 0,
+        updated_at         DATETIME     DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+        INDEX idx_market_regime_daily_date (trade_date)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
+    """,
+    # ─── registry (模型注册，dashboard 业务无关但库里有) ─────────────────
+    """
+    CREATE TABLE IF NOT EXISTS registry (
+        model_name     VARCHAR(64)  NOT NULL,
+        role           VARCHAR(32)  NOT NULL,
+        version        VARCHAR(16)  DEFAULT NULL,
+        artifact_path  VARCHAR(256) NOT NULL,
+        architecture   VARCHAR(32)  DEFAULT NULL,
+        enabled        TINYINT(1)   DEFAULT 1,
+        updated_at     DATETIME     DEFAULT CURRENT_TIMESTAMP,
+        PRIMARY KEY (model_name, role)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
+    """,
+    # ─── stock_scores (评分快照，dashboard 业务无关但库里有) ─────────────
+    """
+    CREATE TABLE IF NOT EXISTS stock_scores (
+        `date`               DATE         NOT NULL,
+        stock_code           VARCHAR(10)  NOT NULL,
+        predicted_rounds     DOUBLE       DEFAULT 0,
+        stacking_probability DOUBLE       DEFAULT 0,
+        meta_ranker_score    DOUBLE       DEFAULT NULL,
+        latest_close         DOUBLE       DEFAULT 0,
+        `rank`               INT          DEFAULT 0,
+        PRIMARY KEY (`date`, stock_code),
+        INDEX idx_stock_scores_date (`date`)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
+    """,
+]
+
+
+def ensure_all_tables(conn) -> None:
+    """遍历 _DDL 列表，逐条执行 CREATE TABLE IF NOT EXISTS。"""
+    with conn.cursor() as cur:
+        for ddl in _DDL:
+            cur.execute(ddl)
+    conn.commit()

app/core/scheduler/__init__.py

@@ -0,0 +1,20 @@
+"""scheduler 子包入口。"""
+from app.core.scheduler.scheduler import (
+    start_scheduler,
+    stop_scheduler,
+    get_scheduler_status,
+    seed_schedule_configs,
+    register_job,
+    is_trading_day,
+    next_trading_day,
+)
+
+__all__ = [
+    "start_scheduler",
+    "stop_scheduler",
+    "get_scheduler_status",
+    "seed_schedule_configs",
+    "register_job",
+    "is_trading_day",
+    "next_trading_day",
+]

app/core/scheduler/scheduler.py

@@ -0,0 +1,392 @@
+"""轻量级定时任务调度器（照搬 dashboard/api/services/scheduler.py 模式）。
+
+后台线程每 N 秒扫描 config 表中 category='schedule' 的任务，到达触发时间时
+执行对应的 job 函数。执行记录写回 config 表。
+
+任务定义字段（存在 config 表，value 是 JSON）：
+  {
+    "name": "...",
+    "time": "15:40",                  # 触发时间（HH:MM），intervalSeconds=0 时必填
+    "intervalSeconds": 0,             # > 0 时按秒触发
+    "condition": "trading_day",       # always | trading_day
+    "job": "sync_kline_daily",        # 已注册的 job 名
+    "enabled": true,
+    "lastRun": "", "lastStatus": "", "lastMessage": ""
+  }
+"""
+from __future__ import annotations
+
+import json
+import threading
+import time
+from datetime import date, datetime, timedelta
+from typing import Any, Callable, Optional
+
+from app.core.config import settings
+from app.core.db import ops as db_ops
+from app.core.utils.logging import get_logger
+
+logger = get_logger("scheduler")
+
+
+# ── 交易日判断 ─────────────────────────────────────────────────────────
+
+_KNOWN_HOLIDAYS: set[str] = set()
+
+
+def _init_holidays() -> None:
+    """从 config 表读出休市日（key=trading_calendar_holidays, category=general）。"""
+    global _KNOWN_HOLIDAYS
+    r = db_ops.fetch_config_by_key("trading_calendar_holidays")
+    if r and r.get("category") == "general":
+        try:
+            holidays = json.loads(r.get("value", "") or "[]")
+            if isinstance(holidays, list):
+                _KNOWN_HOLIDAYS = {h for h in holidays if isinstance(h, str)}
+        except (json.JSONDecodeError, TypeError):
+            pass
+
+
+def is_trading_day(d: Optional[date] = None) -> bool:
+    """判断 A 股交易日（简化：周一至周五 + 非 _KNOWN_HOLIDAYS）。"""
+    if d is None:
+        d = date.today()
+    if d.weekday() >= 5:
+        return False
+    if d.isoformat() in _KNOWN_HOLIDAYS:
+        return False
+    return True
+
+
+def next_trading_day(after: Optional[date] = None) -> date:
+    d = after or date.today()
+    while not is_trading_day(d):
+        d += timedelta(days=1)
+    return d
+
+
+# ── Job registry ──────────────────────────────────────────────────────
+
+_job_registry: dict[str, Callable[[], dict]] = {}
+
+
+def register_job(name: str):
+    """装饰器：注册一个 job 函数。函数应返回 {status, message, ...}。"""
+    def decorator(fn):
+        _job_registry[name] = fn
+        return fn
+    return decorator
+
+
+# ── Scheduler 状态 ────────────────────────────────────────────────────
+
+_lock = threading.Lock()
+_running = False
+_thread: Optional[threading.Thread] = None
+
+# 配置缓存：避免每次 tick 全量查 config 表
+_config_cache_ts: float = 0.0
+_config_cache: list[dict] = []
+_CONFIG_CACHE_TTL = 10.0  # 缓存 10 秒，减少 DB 查询
+
+
+def get_scheduler_status() -> dict[str, Any]:
+    with _lock:
+        return {
+            "running": _running,
+            "jobs": _load_job_statuses(),
+        }
+
+
+def _load_schedule_configs() -> list[dict]:
+    """从 config 表读出所有 category='schedule' 的任务定义（带缓存）。"""
+    global _config_cache_ts, _config_cache
+    now = time.time()
+    if now - _config_cache_ts < _CONFIG_CACHE_TTL and _config_cache:
+        return _config_cache
+    result = []
+    for r in db_ops.fetch_configs_by_category("schedule"):
+        try:
+            obj = json.loads(r.get("value", "") or "{}")
+            obj["_key"] = r["key"]
+            result.append(obj)
+        except (json.JSONDecodeError, TypeError):
+            continue
+    _config_cache = result
+    _config_cache_ts = now
+    return result
+
+
+def _load_job_statuses() -> list[dict]:
+    cfgs = _load_schedule_configs()
+    return [
+        {
+            "key": c["_key"],
+            "name": c.get("name", ""),
+            "time": c.get("time", ""),
+            "intervalSeconds": c.get("intervalSeconds", 0),
+            "condition": c.get("condition", "always"),
+            "job": c.get("job", ""),
+            "enabled": c.get("enabled", True),
+            "lastRun": c.get("lastRun", ""),
+            "lastStatus": c.get("lastStatus", ""),
+            "lastMessage": c.get("lastMessage", ""),
+        }
+        for c in cfgs
+    ]
+
+
+def update_job_status(key: str, status: str, message: str) -> None:
+    """更新 job 执行状态 — 使用 fetch_config_by_key 高效查找。"""
+    r = db_ops.fetch_config_by_key(key)
+    if r is None or r.get("category") != "schedule":
+        return
+    try:
+        obj = json.loads(r.get("value", "") or "{}")
+    except (json.JSONDecodeError, TypeError):
+        obj = {}
+    obj["lastRun"] = datetime.now().isoformat(timespec="seconds")
+    obj["lastStatus"] = status
+    obj["lastMessage"] = message
+    db_ops.upsert_config(
+        key, json.dumps(obj, ensure_ascii=False),
+        category="schedule", description=r.get("description", ""),
+    )
+    # 更新后立即刷新缓存
+    global _config_cache_ts
+    _config_cache_ts = 0.0
+
+
+# ── 调度循环 ──────────────────────────────────────────────────────────
+
+def _loop() -> None:
+    global _running
+    _init_holidays()
+
+    last_fired: dict[str, str] = {}      # key → date string
+    last_fired_ts: dict[str, float] = {} # key → timestamp
+
+    while _running:
+        try:
+            now = datetime.now()
+            today_str = now.strftime("%Y-%m-%d")
+            current_time = now.strftime("%H:%M")
+            now_ts = time.time()
+
+            for cfg in _load_schedule_configs():
+                if not cfg.get("enabled", True):
+                    continue
+
+                key = cfg["_key"]
+                job_time = cfg.get("time", "")
+                condition = cfg.get("condition", "always")
+                job_name = cfg.get("job", "")
+                interval = int(cfg.get("intervalSeconds", 0) or 0)
+                persisted_last_run = str(cfg.get("lastRun", "") or "").strip()
+
+                # 启动时记住今天已跑过的时间触发任务
+                if not interval and persisted_last_run and key not in last_fired:
+                    try:
+                        persisted_date = datetime.fromisoformat(persisted_last_run).strftime("%Y-%m-%d")
+                    except ValueError:
+                        persisted_date = persisted_last_run[:10]
+                    if persisted_date == today_str:
+                        last_fired[key] = today_str
+
+                if interval > 0:
+                    if now_ts - last_fired_ts.get(key, 0) < interval:
+                        continue
+                else:
+                    if current_time < job_time:
+                        continue
+                    if last_fired.get(key) == today_str:
+                        continue
+
+                if condition == "trading_day" and not is_trading_day():
+                    continue
+
+                # 触发
+                fn = _job_registry.get(job_name)
+                if fn is None:
+                    update_job_status(key, "error", f"未注册的 job: {job_name}")
+                    continue
+
+                logger.info(f"[scheduler] 触发 {cfg.get('name', key)} @ {current_time}")
+                try:
+                    result = fn() or {}
+                    status = result.get("status", "ok")
+                    message = result.get("message", "")
+                    update_job_status(key, status, message)
+                    logger.info(f"[scheduler] 完成 {cfg.get('name', key)} → {status}: {message}")
+                except Exception as e:
+                    err = f"{type(e).__name__}: {e}"
+                    update_job_status(key, "error", err[:200])
+                    logger.error(f"[scheduler] 失败 {cfg.get('name', key)}: {err}")
+
+                last_fired[key] = today_str
+                last_fired_ts[key] = now_ts
+        except Exception as e:
+            logger.error(f"[scheduler] loop 异常: {e}")
+
+        time.sleep(max(5, settings.scheduler_tick_seconds))
+
+
+def start_scheduler() -> None:
+    global _running, _thread
+    with _lock:
+        if _running:
+            return
+        _running = True
+        _thread = threading.Thread(target=_loop, daemon=True, name="market-data-sync-scheduler")
+        _thread.start()
+        logger.info("[scheduler] 已启动")
+
+
+def stop_scheduler() -> None:
+    global _running
+    with _lock:
+        _running = False
+        logger.info("[scheduler] 已停止")
+
+
+# ── 默认任务 seed ────────────────────────────────────────────────────
+
+DEFAULT_SCHEDULES: list[tuple[str, dict, str]] = [
+    (
+        "schedule_stocks_basic",
+        {
+            "name": "开盘前同步股票信息",
+            "time": "09:00",
+            "condition": "trading_day",
+            "job": "stock_basic",
+            "enabled": True,
+        },
+        "每个交易日 09:00 拉取全市场股票基础信息 + 股本快照",
+    ),
+    (
+        "schedule_kline_index",
+        {
+            "name": "盘后指数日K线",
+            "time": "15:30",
+            "condition": "trading_day",
+            "job": "kline_index",
+            "enabled": True,
+        },
+        "每个交易日 15:30 拉取六大指数日K线",
+    ),
+    (
+        "schedule_kline_daily",
+        {
+            "name": "盘后日K线增量",
+            "time": "15:40",
+            "condition": "trading_day",
+            "job": "kline_daily",
+            "enabled": True,
+        },
+        "每个交易日 15:40 增量同步全市场日K线（多源交叉校验）",
+    ),
+    (
+        "schedule_kline_5min",
+        {
+            "name": "盘后5分钟K线",
+            "time": "16:00",
+            "condition": "trading_day",
+            "job": "kline_5min",
+            "enabled": True,
+        },
+        "每个交易日 16:00 拉取 5 分钟 K 线（数据量大，可选启用）",
+    ),
+    (
+        "schedule_moneyflow",
+        {
+            "name": "盘后资金流",
+            "time": "16:30",
+            "condition": "trading_day",
+            "job": "moneyflow",
+            "enabled": True,
+        },
+        "每个交易日 16:30 拉取个股资金流",
+    ),
+    (
+        "schedule_industry_sector",
+        {
+            "name": "周一行业映射",
+            "time": "09:30",
+            "condition": "trading_day",
+            "job": "industry_sector",
+            "enabled": True,
+        },
+        "每个交易日 09:30 全量更新股票-行业映射（开销大，可改为 weekly）",
+    ),
+    (
+        "schedule_market_regime",
+        {
+            "name": "盘后市场情绪",
+            "time": "16:15",
+            "condition": "trading_day",
+            "job": "market_regime",
+            "enabled": True,
+        },
+        "每个交易日 16:15 聚合市场情绪（基于本地日K线）",
+    ),
+    (
+        "schedule_share_snapshot",
+        {
+            "name": "盘后股本快照",
+            "time": "16:20",
+            "condition": "trading_day",
+            "job": "share_snapshot",
+            "enabled": True,
+        },
+        "每个交易日 16:20 刷新个股总股本/流通股本快照",
+    ),
+]
+
+
+def seed_schedule_configs() -> None:
+    """写入默认计划任务配置到 config 表。"""
+    existing = {r["key"]: r for r in db_ops.fetch_configs_by_category("schedule")}
+    for key, obj, desc in DEFAULT_SCHEDULES:
+        full = {
+            **obj,
+            "lastRun": "",
+            "lastStatus": "",
+            "lastMessage": "",
+        }
+        if key in existing:
+            # 保留 lastRun / lastStatus 等
+            try:
+                old = json.loads(existing[key].get("value", "") or "{}")
+                for k in ("lastRun", "lastStatus", "lastMessage"):
+                    if k in old:
+                        full[k] = old[k]
+            except Exception:
+                pass
+        db_ops.upsert_config(
+            key, json.dumps(full, ensure_ascii=False), category="schedule", description=desc,
+        )
+
+
+# ── 注册 job（延迟导入避免循环依赖）────────────────────────────────────
+
+
+def register_sync_jobs() -> None:
+    """把 8 个 sync 任务注册成 job。
+
+    注：job 函数不接受任何参数，trigger_source 默认为 'schedule'。
+    """
+    from app.tasks import get_task
+
+    for dataset_id in [
+        "stock_basic", "kline_daily", "kline_index", "kline_5min",
+        "moneyflow", "industry_sector", "share_snapshot", "market_regime",
+    ]:
+
+        def _make_job(did=dataset_id):
+            def _job() -> dict:
+                task = get_task(did)
+                return task.run(trigger_source="schedule")
+            _job.__name__ = f"job_{did}"
+            return _job
+
+        register_job(dataset_id)(_make_job())

app/core/sync/__init__.py

@@ -0,0 +1,24 @@
+"""sync 子包入口。"""
+from app.core.sync.registry import (
+    mark_sync_running,
+    mark_sync_progress,
+    mark_sync_success,
+    mark_sync_failed,
+    mark_sync_blocked,
+    get_registry_status,
+    recover_interrupted_syncs,
+    seed_sync_registry,
+    SYNC_DEFINITIONS,
+)
+
+__all__ = [
+    "mark_sync_running",
+    "mark_sync_progress",
+    "mark_sync_success",
+    "mark_sync_failed",
+    "mark_sync_blocked",
+    "get_registry_status",
+    "recover_interrupted_syncs",
+    "seed_sync_registry",
+    "SYNC_DEFINITIONS",
+]

app/core/sync/base.py

@@ -0,0 +1,141 @@
+"""SyncTask 基类。
+
+提供统一的：
+- 状态自动更新（running → success / failed）
+- 进度回调
+- 异常捕获 → 写 dataset_registry
+
+子类只需实现 _run() 即可。
+"""
+from __future__ import annotations
+
+import time
+import traceback
+from datetime import datetime, time as dtime
+from typing import Any, Optional
+
+from app.core.sync.registry import (
+    mark_sync_failed,
+    mark_sync_progress,
+    mark_sync_running,
+    mark_sync_success,
+)
+from app.core.utils.logging import get_logger
+
+logger = get_logger("sync.task")
+
+
+def _is_market_closed() -> bool:
+    """是否超过 15:30（盘后才完整，避免盘中部分数据）。"""
+    return datetime.now().time() >= dtime(15, 30)
+
+
+def _effective_sync_end() -> str:
+    """若盘后则今天，否则昨天。"""
+    today = datetime.now()
+    if today.time() >= dtime(15, 30):
+        return today.strftime("%Y-%m-%d")
+    # 昨天
+    from datetime import timedelta
+    return (today - timedelta(days=1)).strftime("%Y-%m-%d")
+
+
+class SyncTask:
+    """同步任务基类。"""
+
+    dataset_id: str = ""  # 子类必须设置
+
+    def __init__(self, dataset_id: Optional[str] = None):
+        if dataset_id:
+            self.dataset_id = dataset_id
+        if not self.dataset_id:
+            raise ValueError(f"{self.__class__.__name__} must set .dataset_id")
+
+    # ── 公开入口 ─────────────────────────────────────────
+
+    def run(self, *, trigger_source: str = "manual", **kwargs) -> dict[str, Any]:
+        """统一入口：自动包 mark_running / mark_success / mark_failed。"""
+        # 1. 确保 health check 跑过（CLI / 手动触发场景下 schedule 后台不会先跑）
+        self._ensure_health_checked()
+
+        # 2. 启动
+        mark_sync_running(
+            self.dataset_id,
+            trigger_source=trigger_source,
+            message=f"开始同步 {self.dataset_id}...",
+        )
+        t0 = time.time()
+        try:
+            result = self._run(trigger_source=trigger_source, **kwargs)
+            elapsed = round(time.time() - t0, 1)
+            status = result.get("status", "ok")
+            message = result.get("message", "")
+            if status == "ok":
+                mark_sync_success(self.dataset_id, message=message)
+            elif status == "blocked":
+                mark_sync_failed(
+                    self.dataset_id,
+                    message=message,
+                    error=result.get("error", message),
+                    status="blocked",
+                )
+            else:  # warning / partial
+                mark_sync_failed(
+                    self.dataset_id,
+                    message=message,
+                    error=result.get("error", message),
+                    status="warning",
+                )
+            result["elapsed_sec"] = elapsed
+            logger.info(f"[{self.dataset_id}] 完成 {status}: {message} ({elapsed}s)")
+            return result
+        except Exception as e:
+            elapsed = round(time.time() - t0, 1)
+            tb = traceback.format_exc(limit=2)
+            err_msg = f"{type(e).__name__}: {e}"
+            mark_sync_failed(self.dataset_id, message=err_msg, error=tb)
+            logger.error(f"[{self.dataset_id}] 失败: {err_msg}\n{tb}")
+            return {
+                "status": "error",
+                "message": err_msg,
+                "error": err_msg,
+                "elapsed_sec": elapsed,
+            }
+
+    def _ensure_health_checked(self) -> None:
+        """若内存里 health_results 为空，跑一次。"""
+        try:
+            from app.core.datasource.registry import get_health_status, run_health_check
+            if not get_health_status():
+                run_health_check()
+        except Exception as e:
+            logger.warning(f"health check 启动失败（不影响任务继续）: {e}")
+
+    # ── 子类实现 ─────────────────────────────────────────
+
+    def _run(self, *, trigger_source: str = "manual", **kwargs) -> dict[str, Any]:
+        """子类实现：实际同步逻辑。
+
+        返回 dict：
+          {
+            "status": "ok" | "warning" | "blocked",
+            "message": "...",
+            "ok": int, "fail": int, "skip": int,   # 可选统计
+            ...
+          }
+        任何异常会被外层捕获并标记 failed。
+        """
+        raise NotImplementedError
+
+    # ── 进度更新辅助 ─────────────────────────────────────
+
+    def _progress(self, *, message: str | None = None,
+                  current: int | None = None, total: int | None = None,
+                  current_step: str | None = None) -> None:
+        mark_sync_progress(
+            self.dataset_id,
+            message=message,
+            progress_current=current,
+            progress_total=total,
+            current_step=current_step,
+        )

app/core/sync/registry.py

@@ -0,0 +1,249 @@
+"""sync registry 状态机封装。
+
+模式照搬 dashboard/api/services/dataset_registry.py：
+- dataset_id 是唯一键
+- mark_sync_running / mark_sync_progress / mark_sync_success / mark_sync_failed / mark_sync_blocked
+- recover_interrupted_syncs 启动时把 status='running' 的任务改为 failed
+- seed_sync_registry 把所有同步任务定义 seed 到 dataset_registry 表
+"""
+from __future__ import annotations
+
+import json
+from datetime import datetime
+from typing import Any
+
+from app.core.db import ops as db_ops
+from app.core.utils.logging import get_logger
+
+logger = get_logger("sync.registry")
+
+
+# ── 同步任务定义（写入 dataset_registry 表）──────────────────────────────
+
+
+SYNC_DEFINITIONS: list[dict[str, Any]] = [
+    {
+        "dataset_id": "stock_basic",
+        "name": "股票基础信息（含最新股本）",
+        "description": "全市场 A 股代码 / 名称 / 交易所 / 上市状态 + 股本快照（雪球）",
+        "storage_uri": "MySQL market_data_sync_db.stocks",
+        "storage_layer": "mysql",
+        "management_role": "基础字典",
+        "source": "Baostock query_stock_basic + 雪球 quote_detail",
+        "sync_script": "app.tasks.task_stocks_basic:run",
+        "dependency_ids": [],
+        "sort_order": 10,
+    },
+    {
+        "dataset_id": "kline_daily",
+        "name": "原始日K线数据",
+        "description": "全市场 A 股日频 OHLCV，多源交叉校验（雪球/新浪/Baostock）",
+        "storage_uri": "MySQL market_data_sync_db.kline_stock",
+        "storage_layer": "mysql",
+        "management_role": "原始源",
+        "source": "雪球主源 + 新浪/Baostock 复核",
+        "sync_script": "app.tasks.task_kline_daily:run",
+        "dependency_ids": ["stock_basic"],
+        "sort_order": 20,
+    },
+    {
+        "dataset_id": "kline_index",
+        "name": "六大指数日线",
+        "description": "上证 / 深证 / 创业板 / 沪深300 / 中证500 / 中证1000",
+        "storage_uri": "MySQL market_data_sync_db.kline_index + indices",
+        "storage_layer": "mysql",
+        "management_role": "原始源",
+        "source": "新浪指数日K",
+        "sync_script": "app.tasks.task_kline_index:run",
+        "dependency_ids": [],
+        "sort_order": 30,
+    },
+    {
+        "dataset_id": "kline_5min",
+        "name": "5 分钟 K 线",
+        "description": "全市场 A 股 5 分钟 K 线（mairui 源）",
+        "storage_uri": "MySQL market_data_sync_db.kline_5min",
+        "storage_layer": "mysql",
+        "management_role": "原始源",
+        "source": "mairui stockMin",
+        "sync_script": "app.tasks.task_kline_5min:run",
+        "dependency_ids": ["stock_basic"],
+        "sort_order": 40,
+    },
+    {
+        "dataset_id": "moneyflow",
+        "name": "资金流",
+        "description": "个股资金流（主力/大/中/小单净额），mairui 源",
+        "storage_uri": "MySQL market_data_sync_db.moneyflow",
+        "storage_layer": "mysql",
+        "management_role": "原始源",
+        "source": "mairui hsstock/history/transaction",
+        "sync_script": "app.tasks.task_moneyflow:run",
+        "dependency_ids": ["stock_basic"],
+        "sort_order": 50,
+    },
+    {
+        "dataset_id": "industry_sector",
+        "name": "股票-行业映射",
+        "description": "股票-行业映射 + 行业字典（Baostock）",
+        "storage_uri": "MySQL market_data_sync_db.industry + sectors + stock_sector_map",
+        "storage_layer": "mysql",
+        "management_role": "基础字典",
+        "source": "Baostock query_stock_industry",
+        "sync_script": "app.tasks.task_industry_sector:run",
+        "dependency_ids": ["stock_basic"],
+        "sort_order": 60,
+    },
+    {
+        "dataset_id": "sector_features",
+        "name": "行业聚合特征",
+        "description": "由 kline_stock + industry 衍生：行业日收益 / 行业指数 close / EMA10-20-200 / score。纯本地计算，无外部 API。",
+        "storage_uri": "MySQL market_data_sync_db.sector_indices + sector_features_daily",
+        "storage_layer": "mysql",
+        "management_role": "衍生源",
+        "source": "本地计算（kline_stock + industry）",
+        "sync_script": "app.tasks.task_sector_features:run",
+        "dependency_ids": ["kline_daily", "industry_sector"],
+        "sort_order": 65,
+    },
+    {
+        "dataset_id": "share_snapshot",
+        "name": "股本快照",
+        "description": "全市场 A 股最新股本（雪球 quote_detail，单位亿股）",
+        "storage_uri": "MySQL market_data_sync_db.stocks.total_share/float_share + share 表",
+        "storage_layer": "mysql",
+        "management_role": "基础字典",
+        "source": "雪球 quote_detail",
+        "sync_script": "app.tasks.task_share_snapshot:run",
+        "dependency_ids": ["stock_basic"],
+        "sort_order": 70,
+    },
+    {
+        "dataset_id": "market_regime",
+        "name": "市场情绪 (Market Regime)",
+        "description": "基于本地日K线聚合的 advance_ratio / 恐慌标记（衍生源）",
+        "storage_uri": "MySQL market_data_sync_db.market_regime_daily",
+        "storage_layer": "mysql",
+        "management_role": "衍生源",
+        "source": "本地日K线聚合（无外部接口）",
+        "sync_script": "app.tasks.task_market_regime:run",
+        "dependency_ids": ["kline_daily"],
+        "sort_order": 80,
+    },
+]
+
+
+def seed_sync_registry() -> None:
+    """把 SYNC_DEFINITIONS 写入 dataset_registry 表。"""
+    rows = []
+    for d in SYNC_DEFINITIONS:
+        rows.append({
+            **d,
+            "dependency_ids": json.dumps(d.get("dependency_ids", []), ensure_ascii=False),
+            "enabled": 1,
+        })
+    db_ops.upsert_dataset_registry_rows(rows)
+
+
+# ── 状态机 API ────────────────────────────────────────────────────────
+
+
+def _now_ts() -> str:
+    return datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+
+
+def mark_sync_running(
+    dataset_id: str,
+    *,
+    trigger_source: str,
+    message: str = "",
+    progress_current: int = 0,
+    progress_total: int = 0,
+    current_step: str = "",
+) -> None:
+    db_ops.update_dataset_registry_state(
+        dataset_id,
+        status="running",
+        trigger_source=trigger_source,
+        started_at=_now_ts(),
+        finished_at=None,
+        message=message,
+        last_error=None,
+        needs_resync=0,
+        progress_current=progress_current,
+        progress_total=progress_total,
+        current_step=current_step,
+        updated_at=_now_ts(),
+    )
+
+
+def mark_sync_progress(
+    dataset_id: str,
+    *,
+    message: str | None = None,
+    progress_current: int | None = None,
+    progress_total: int | None = None,
+    current_step: str | None = None,
+) -> None:
+    payload: dict[str, Any] = {"updated_at": _now_ts()}
+    if message is not None:
+        payload["message"] = message
+    if progress_current is not None:
+        payload["progress_current"] = progress_current
+    if progress_total is not None:
+        payload["progress_total"] = progress_total
+    if current_step is not None:
+        payload["current_step"] = current_step
+    db_ops.update_dataset_registry_state(dataset_id, **payload)
+
+
+def mark_sync_success(dataset_id: str, *, message: str = "") -> None:
+    now = _now_ts()
+    db_ops.update_dataset_registry_state(
+        dataset_id,
+        status="success",
+        finished_at=now,
+        last_success_at=now,
+        message=message,
+        last_error=None,
+        needs_resync=0,
+        current_step="",
+        progress_current=0,
+        progress_total=0,
+        updated_at=now,
+    )
+
+
+def mark_sync_failed(
+    dataset_id: str,
+    *,
+    message: str = "",
+    error: str = "",
+    status: str = "failed",
+) -> None:
+    now = _now_ts()
+    final_message = message or error or "同步失败"
+    db_ops.update_dataset_registry_state(
+        dataset_id,
+        status=status,
+        finished_at=now,
+        last_failure_at=now,
+        message=final_message,
+        last_error=error or final_message,
+        needs_resync=1,
+        current_step="",
+        updated_at=now,
+    )
+
+
+def mark_sync_blocked(dataset_id: str, *, message: str) -> None:
+    mark_sync_failed(dataset_id, message=message, error=message, status="blocked")
+
+
+def recover_interrupted_syncs() -> int:
+    """启动时把 status='running' 的卡死任务标记为 failed + needs_resync=1。"""
+    return db_ops.recover_interrupted_dataset_registry()
+
+
+def get_registry_status() -> list[dict[str, Any]]:
+    return db_ops.fetch_dataset_registry_rows()

app/core/utils/__init__.py

@@ -0,0 +1,4 @@
+"""通用工具。"""
+from app.core.utils.logging import get_logger, setup_logging
+
+__all__ = ["get_logger", "setup_logging"]

app/core/utils/logging.py

@@ -0,0 +1,60 @@
+"""统一日志：同时输出到控制台 + 文件。"""
+from __future__ import annotations
+
+import logging
+import sys
+from logging.handlers import RotatingFileHandler
+from pathlib import Path
+
+from app.core.config import settings
+
+_INITIALIZED = False
+
+
+def _build_formatter() -> logging.Formatter:
+    return logging.Formatter(
+        "[%(asctime)s] %(levelname)-5s %(name)s | %(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S",
+    )
+
+
+def setup_logging(level: str | None = None) -> None:
+    global _INITIALIZED
+    if _INITIALIZED:
+        return
+
+    lvl = (level or settings.log_level or "INFO").upper()
+    root = logging.getLogger()
+    root.setLevel(lvl)
+    # 清掉 uvicorn / sqlalchemy 默认 handler，避免重复输出
+    for h in list(root.handlers):
+        root.removeHandler(h)
+
+    fmt = _build_formatter()
+
+    console = logging.StreamHandler(sys.stdout)
+    console.setFormatter(fmt)
+    root.addHandler(console)
+
+    log_path: Path = settings.log_path
+    file_handler = RotatingFileHandler(
+        log_path / "market_data_sync.log",
+        maxBytes=20 * 1024 * 1024,
+        backupCount=5,
+        encoding="utf-8",
+    )
+    file_handler.setFormatter(fmt)
+    root.addHandler(file_handler)
+
+    # 抑制第三方库过度刷屏
+    logging.getLogger("baostock").setLevel(logging.WARNING)
+    logging.getLogger("urllib3").setLevel(logging.WARNING)
+    logging.getLogger("sqlalchemy.engine").setLevel(logging.WARNING)
+
+    _INITIALIZED = True
+
+
+def get_logger(name: str) -> logging.Logger:
+    if not _INITIALIZED:
+        setup_logging()
+    return logging.getLogger(name)

app/entrypoints/__init__.py

@@ -0,0 +1,5 @@
+"""进程入口点。
+
+- `cli`    — 手动触发/状态查询命令行（`python -m app.entrypoints.cli ...`）
+- `worker` — 纯调度模式，无 Web 界面（`python -m app.entrypoints.worker`）
+"""

app/entrypoints/cli.py

@@ -0,0 +1,132 @@
+"""CLI 入口：手动触发 / 状态查询。"""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+_PROJECT_ROOT = Path(__file__).resolve().parent.parent
+if str(_PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(_PROJECT_ROOT))
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="market_data_sync CLI",
+    )
+    sub = parser.add_subparsers(dest="cmd", required=True)
+
+    # sync <task_id>
+    p_sync = sub.add_parser("sync", help="手动执行一次同步任务")
+    p_sync.add_argument("task_id", help="dataset_id, 例如 kline_daily")
+    p_sync.add_argument("--codes", help="指定股票代码，逗号分隔", default=None)
+    p_sync.add_argument("--start", help="起始日期 YYYY-MM-DD", default=None)
+    p_sync.add_argument("--end", help="结束日期 YYYY-MM-DD", default=None)
+    p_sync.add_argument("--workers", type=int, default=10, help="并发数")
+
+    sub.add_parser("list", help="列出所有同步任务")
+    sub.add_parser("status", help="查看整体状态（调度器 / 数据源 / 同步任务）")
+    sub.add_parser("datasources", help="查看数据源健康度")
+    sub.add_parser("schedule", help="查看计划任务")
+    sub.add_parser("reseed", help="重新 seed 默认计划任务到 config 表")
+
+    args = parser.parse_args()
+
+    if args.cmd == "sync":
+        from app.core.utils.logging import setup_logging
+        setup_logging()
+        from app.tasks import get_task
+        from app.core.datasource.registry import build_default_registry
+        from app.core.sync.registry import seed_sync_registry
+        build_default_registry()
+        seed_sync_registry()  # 确保 dataset_registry 有这条任务定义
+        try:
+            task = get_task(args.task_id)
+        except KeyError as e:
+            print(f"错误: {e}")
+            sys.exit(1)
+        kwargs = {}
+        if args.codes:
+            kwargs["codes"] = [c.strip() for c in args.codes.split(",") if c.strip()]
+        if args.start:
+            kwargs["start"] = args.start
+        if args.end:
+            kwargs["end"] = args.end
+        if args.workers:
+            kwargs["max_workers"] = args.workers
+        result = task.run(trigger_source="cli", **kwargs)
+        print(json.dumps(result, ensure_ascii=False, indent=2))
+        sys.exit(0 if result.get("status") in ("ok", "warning") else 2)
+
+    if args.cmd == "list":
+        from app.tasks import TASKS
+        print("可用同步任务：")
+        for tid, cls in TASKS.items():
+            doc = (cls.__doc__ or cls.__name__).split("\n")[0]
+            print(f"  {tid:25s}  {doc}")
+        return
+
+    if args.cmd == "status":
+        from app.core.utils.logging import setup_logging
+        setup_logging()
+        from app.core.scheduler import get_scheduler_status
+        from app.core.datasource.registry import build_default_registry, get_health_status
+        from app.core.sync import get_registry_status
+        build_default_registry()
+        print("\n=== 调度器 ===")
+        s = get_scheduler_status()
+        print(json.dumps(s, ensure_ascii=False, indent=2, default=str))
+        print("\n=== 数据源健康度 ===")
+        for r in get_health_status():
+            mark = "✓" if r.get("success") else "✗"
+            print(f"  {mark} {r.get('name', r.get('key')):30s}  {r.get('message', '')}")
+        print("\n=== 同步任务状态 ===")
+        for r in get_registry_status():
+            print(f"  {r['dataset_id']:25s}  status={r.get('status'):10s}  "
+                  f"lastRun={r.get('last_success_at') or r.get('last_failure_at') or '-':20s}  "
+                  f"msg={r.get('message', '')[:60]}")
+        return
+
+    if args.cmd == "datasources":
+        from app.core.utils.logging import setup_logging
+        setup_logging()
+        from app.core.datasource.registry import build_default_registry, get_health_status, run_health_check
+        build_default_registry()
+        print("正在跑连通性测试...")
+        run_health_check()
+        for r in get_health_status():
+            mark = "✓" if r.get("success") else "✗"
+            print(f"  {mark} {r.get('name', r.get('key')):30s}  provides={r.get('provides', [])}  {r.get('message', '')}")
+        return
+
+    if args.cmd == "schedule":
+        from app.core.utils.logging import setup_logging
+        setup_logging()
+        from app.core.scheduler import get_scheduler_status, seed_schedule_configs
+        seed_schedule_configs()
+        s = get_scheduler_status()
+        print(f"调度器运行中: {s['running']}")
+        for j in s["jobs"]:
+            enabled = "✓" if j.get("enabled") else "✗"
+            trigger = j.get("time") or f"every {j.get('intervalSeconds')}s"
+            print(f"  {enabled} {j.get('name', j['key']):30s}  {trigger:15s}  "
+                  f"condition={j.get('condition', 'always'):12s}  "
+                  f"last={j.get('lastStatus', '-')}")
+        return
+
+    if args.cmd == "reseed":
+        from app.core.utils.logging import setup_logging
+        setup_logging()
+        from app.core.scheduler.scheduler import seed_schedule_configs
+        from app.core.sync.registry import seed_sync_registry
+        from app.core.datasource.registry import seed_datasource_configs
+        seed_datasource_configs()
+        seed_sync_registry()
+        seed_schedule_configs()
+        print("已重新 seed：datasources + sync_registry + schedules")
+        return
+
+
+if __name__ == "__main__":
+    main()

app/entrypoints/worker.py

@@ -0,0 +1,73 @@
+"""纯 worker 模式（无 web）— 启动调度器 + 健康监控。
+
+用法：python -m app.worker
+"""
+from __future__ import annotations
+
+import json
+import sys
+import time
+from pathlib import Path
+
+_PROJECT_ROOT = Path(__file__).resolve().parent.parent
+if str(_PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(_PROJECT_ROOT))
+
+from app.core.config import settings
+from app.core.db import ops as db_ops
+from app.core.utils.logging import get_logger, setup_logging
+
+setup_logging()
+logger = get_logger("worker")
+
+
+def main():
+    logger.info("[worker] market_data_sync worker 启动")
+
+    # 1. 注册数据源 + seed config
+    from app.core.datasource.registry import build_default_registry, seed_datasource_configs
+    build_default_registry()
+    seed_datasource_configs()
+
+    # 2. seed sync registry + 恢复卡死任务
+    from app.core.sync.registry import recover_interrupted_syncs, seed_sync_registry
+    seed_sync_registry()
+    n = recover_interrupted_syncs()
+    if n > 0:
+        logger.warning(f"[worker] 恢复了 {n} 个中断的同步任务")
+
+    # 3. seed 节假日
+    if settings.holidays_list:
+        db_ops.upsert_config(
+            "trading_calendar_holidays",
+            json.dumps(settings.holidays_list, ensure_ascii=False),
+            category="general",
+            description="A 股休市日",
+        )
+
+    # 4. seed 默认计划任务
+    if settings.scheduler_auto_seed:
+        from app.core.scheduler.scheduler import seed_schedule_configs
+        seed_schedule_configs()
+
+    # 5. 注册 sync job + 启动调度器 + 健康监控
+    from app.core.scheduler.scheduler import register_sync_jobs, start_scheduler
+    from app.core.datasource.registry import start_health_monitor
+    register_sync_jobs()
+    start_scheduler()
+    start_health_monitor()
+
+    logger.info("[worker] 已启动调度器 + 健康监控，按 Ctrl+C 退出")
+
+    try:
+        while True:
+            time.sleep(60)
+    except KeyboardInterrupt:
+        logger.info("[worker] 收到 SIGINT，退出")
+        from app.core.scheduler.scheduler import stop_scheduler
+        stop_scheduler()
+        sys.exit(0)
+
+
+if __name__ == "__main__":
+    main()

app/sources/__init__.py

@@ -0,0 +1,5 @@
+"""数据源实现。
+
+每个模块导出一个 `DataSource` 子类，由
+`app.core.datasource.registry.build_default_registry()` 在启动时统一注册。
+"""

app/sources/baostock.py

@@ -0,0 +1,159 @@
+"""Baostock 数据源。
+
+提供：
+  - 全市场股票基础信息（query_stock_basic）
+  - 日 K 线（query_history_k_data_plus，免费，但 QPS 限制需要信号量）
+  - 股票-行业映射（query_stock_industry）
+
+免费、无 token，但需 bs.login() / bs.logout() 维护会话。
+"""
+from __future__ import annotations
+
+import threading
+from typing import Any, Optional
+
+import pandas as pd
+
+from app.core.datasource.base import DataSource
+from app.core.datasource.utils import (
+    code6_to_baostock,
+    normalize_kline,
+)
+
+
+class BaostockSource(DataSource):
+    key = "datasource_baostock"
+    name = "Baostock"
+    provides = ["kline_daily", "stock_basic", "industry"]
+    requires_credential = False
+
+    # 进程内只 login 一次
+    _login_lock = threading.Lock()
+    _logged_in = False
+    _semaphore = threading.BoundedSemaphore(3)  # QPS 限流
+
+    def _ensure_login(self) -> None:
+        if self._logged_in:
+            return
+        with self._login_lock:
+            if self._logged_in:
+                return
+            import baostock as bs
+            bs.login()
+            self._logged_in = True
+
+    def is_available(self) -> tuple[bool, str]:
+        try:
+            self._ensure_login()
+            return True, "ok"
+        except Exception as e:
+            return False, f"Baostock 登录失败: {e}"
+
+    def health_check(self) -> dict[str, Any]:
+        """连通性测试：复用 _ensure_login 避免重复 login 阻塞。"""
+        try:
+            self._ensure_login()
+            import baostock as bs
+            rs = bs.query_stock_basic(code="sh.600036")
+            if rs.error_code != "0":
+                return {"success": False, "message": f"Baostock 查询失败: {rs.error_msg}"}
+            rows = []
+            while rs.next():
+                rows.append(rs.get_row_data())
+            return {"success": True, "message": f"连接成功，查询到 {len(rows)} 条股票信息"}
+        except Exception as e:
+            return {"success": False, "message": f"Baostock 连接失败: {e}"}
+
+    def fetch_stock_basic(self) -> list[dict[str, Any]]:
+        """全市场 A 股基础信息。
+
+        baostock query_stock_basic 返回字段（按当前接口）：
+          code, code_name, ipoDate, outDate, type, status
+        按字段名索引，防止 Baostock 改顺序。
+        """
+        import baostock as bs
+        self._ensure_login()
+        try:
+            rs = bs.query_stock_basic()
+            if rs.error_code != "0":
+                raise RuntimeError(f"Baostock query_stock_basic 失败: {rs.error_msg}")
+            fields = list(rs.fields)
+            idx = {name: i for i, name in enumerate(fields)}
+            rows = []
+            while rs.next():
+                row = rs.get_row_data()
+                bs_code = row[idx["code"]]
+                name = row[idx["code_name"]]
+                stype = row[idx["type"]]
+                status = row[idx["status"]]
+                ipo = row[idx.get("ipoDate", -1)] if "ipoDate" in idx else ""
+                if stype != "1" or status != "1":
+                    continue
+                exchange = bs_code.split(".")[0].upper()
+                if exchange not in ("SH", "SZ", "BJ"):
+                    continue
+                code6 = bs_code.split(".")[-1]
+                rows.append({
+                    "code": f"{exchange}{code6}",
+                    "name": name,
+                    "exchange": exchange,
+                    "list_date": ipo,
+                    "listing_status": "st" if (name.startswith("ST") or name.startswith("*ST")) else "normal",
+                })
+            return rows
+        except Exception:
+            return []
+
+    def fetch_kline_daily(self, code6: str, start: str, end: str) -> pd.DataFrame:
+        import baostock as bs
+        self._ensure_login()
+        bs_code = code6_to_baostock(code6)
+        with self._semaphore:
+            try:
+                rs = bs.query_history_k_data_plus(
+                    bs_code,
+                    "date,open,high,low,close,volume",
+                    start_date=start, end_date=end,
+                    frequency="d", adjustflag="2",  # 前复权
+                )
+                if rs.error_code != "0":
+                    return pd.DataFrame()
+                rows = []
+                while rs.next():
+                    rows.append(rs.get_row_data())
+                if not rows:
+                    return pd.DataFrame()
+                df = pd.DataFrame(rows, columns=["trade_date", "open", "high", "low", "close", "volume"])
+                return normalize_kline(df)
+            except Exception:
+                return pd.DataFrame()
+
+    def fetch_industry_map(self) -> list[dict[str, Any]]:
+        """全市场股票-行业映射。"""
+        import baostock as bs
+        self._ensure_login()
+        try:
+            rs = bs.query_stock_industry(code="", date="")
+            if rs.error_code != "0":
+                raise RuntimeError(f"query_stock_industry 失败: {rs.error_msg}")
+            fields = list(rs.fields)  # 实际字段顺序：['updateDate', 'code', 'code_name', 'industry', 'industryClassification']
+            idx = {name: i for i, name in enumerate(fields)}
+            rows = []
+            while rs.next():
+                row = rs.get_row_data()
+                industry = row[idx.get("industry", 3)] if "industry" in idx else ""
+                if not industry:
+                    continue
+                bs_code = row[idx.get("code", 1)] if "code" in idx else ""
+                code6 = bs_code.split(".")[-1]
+                classification = row[idx.get("industryClassification", 4)] if "industryClassification" in idx else ""
+                update = row[idx.get("updateDate", 0)] if "updateDate" in idx else ""
+                rows.append({
+                    "code": code6,
+                    "industry_name": industry.strip(),
+                    "industry_classification": (classification or "").strip(),
+                    "update_date": update or "",
+                })
+            return rows
+        except Exception:
+            return []

app/sources/mairui.py

@@ -0,0 +1,379 @@
+"""麦蕊智数（mairui）数据源。
+
+提供：
+  - 日 K 线（`hsstock/history/{code}.{ex}/d/n/{licence}`）
+  - 5 分钟 K 线（`hsstock/history/{code}.{ex}/5/n/{licence}`）
+  - 指数日 K 线（`hsindex/history/{code}.{ex}/d/{licence}`）
+  - 资金流向（`hsstock/history/transaction/{code}.{ex}/{licence}`）
+
+限速：1分钟300次（默认保守到 5 RPS = 1分钟300次）
+凭证：licence（无需登录态，从环境变量 MAIRUI_LICENCE 读取）
+"""
+from __future__ import annotations
+
+import json
+import os
+import threading
+import time
+import urllib.request
+from typing import Any, Optional
+
+import pandas as pd
+
+from app.core.datasource.base import DataSource
+from app.core.datasource.utils import (
+    code6_to_mairui,
+    filter_date_range,
+    normalize_kline,
+)
+
+
+class MairuiSource(DataSource):
+    key = "datasource_mairui"
+    name = "麦蕊智数（mairui.club）"
+    provides = ["kline_daily", "kline_5min", "index_daily", "stock_basic", "moneyflow"]
+    requires_credential = True
+    credential_key = "MAIRUI_LICENCE"
+
+    _HEADERS = {
+        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)",
+        "Accept": "application/json",
+    }
+    _BASE_URL = "https://api.mairuiapi.com"
+
+    # mairui 免费 licence 1 分钟 300 次 = 5 RPS（**单 licence**硬上限）。
+    # 5 workers × 1 RPS/worker = 5 RPS 总 = 刚好踩满上限，避免风控。
+    # 用 thread-local 计时：每个 worker 独立计自己的 last_ts，
+    # 避免 N 个 worker 串行抢一把锁退化成 1 worker。
+    _RPS_LIMIT = 1.0
+    _rps_local = threading.local()
+
+    @classmethod
+    def _wait_rps(cls):
+        last = getattr(cls._rps_local, "last_ts", 0.0)
+        now = time.time()
+        elapsed = now - last
+        min_interval = 1.0 / cls._RPS_LIMIT
+        if elapsed < min_interval:
+            time.sleep(min_interval - elapsed)
+            now = time.time()
+        cls._rps_local.last_ts = now
+
+    def _get_licence(self) -> Optional[str]:
+        return os.environ.get(self.credential_key, "").strip() or None
+
+    def is_available(self) -> tuple[bool, str]:
+        if not self._get_licence():
+            return False, f"未配置 {self.credential_key}"
+        return True, "ok"
+
+    def health_check(self) -> dict[str, Any]:
+        lic = self._get_licence()
+        if not lic:
+            return {"success": False, "message": f"{self.credential_key} 未配置"}
+        try:
+            # 拿 1 条日线数据作为连通性 + 凭证测试
+            url = f"{self._BASE_URL}/hsstock/history/600519.SH/d/n/{lic}?st=20260609&et=20260609"
+            req = urllib.request.Request(url, headers=self._HEADERS)
+            with urllib.request.urlopen(req, timeout=15) as resp:
+                raw = resp.read().decode("utf-8", errors="replace")
+            data = json.loads(raw)
+            if isinstance(data, list) and data:
+                return {"success": True, "message": f"连接成功，获取到 {len(data)} 条 K 线"}
+            if isinstance(data, dict) and data.get("error"):
+                return {"success": False, "message": f"麦蕊返回: {data['error']}"}
+            return {"success": False, "message": f"麦蕊返回空/异常: {raw[:200]}"}
+        except Exception as e:
+            return {"success": False, "message": f"麦蕊连接失败: {e}"}
+
+    def _fetch(self, path: str, params: dict[str, Any], retry: int = 2) -> Any:
+        """通用 GET，含限速 + 重试。"""
+        self._wait_rps()
+        qs = "&".join(f"{k}={v}" for k, v in params.items() if v)
+        url = f"{self._BASE_URL}{path}"
+        if qs:
+            url = f"{url}?{qs}"
+        last_err = None
+        for attempt in range(retry + 1):
+            try:
+                req = urllib.request.Request(url, headers=self._HEADERS)
+                with urllib.request.urlopen(req, timeout=20) as resp:
+                    # 先按 gbk 试（中文乱码返回），再 fallback utf-8
+                    raw_bytes = resp.read()
+                    # 用 latin-1 永不失败地把 bytes 转成字符串（每字节 1 字符）
+                    raw = raw_bytes.decode("latin-1")
+                # mairui 风控时返回 "接收数据异常，请稍后再试"（gbk 编码）
+                if "请稍后再试" in raw or "codec can't decode" in raw or raw.startswith("'utf-8'") or "Error -3 while decompressing" in raw:
+                    raise RuntimeError(f"mairui 返回风控提示: {raw[:80]}")
+                # 试 JSON parse；如果是 gbk 编码的中文提示，要先解码
+                if raw.startswith("'") and raw.endswith("'"):
+                    # gbk 编码的 Python repr 字符串，如 "'接收数据异常，请稍后再试'"
+                    try:
+                        decoded = raw[1:-1].encode("latin-1").decode("gbk")
+                        if "请稍后再试" in decoded:
+                            raise RuntimeError(f"mairui 返回风控: {decoded}")
+                    except Exception:
+                        pass
+                return json.loads(raw)
+            except Exception as e:
+                last_err = e
+                if attempt < retry:
+                    time.sleep(1.0 * (attempt + 1))  # 风控重试退避长一点
+        from app.core.utils.logging import get_logger
+        get_logger("mairui").debug(f"mairui fetch {url} failed: {last_err}")
+        return []
+
+    # ── 股票列表 ───────────────────────────────────────────
+
+    def fetch_stock_list(self) -> list[dict]:
+        """全市场沪深 A 股基础列表。
+
+        API: GET /hslt/list/{licence}
+        返回字段：dm (代码.交易所，如 "000001.SZ"), mc (名称), jys (交易所 SZ/SH)
+        """
+        lic = self._get_licence()
+        if not lic:
+            return []
+        data = self._fetch(f"/hslt/list/{lic}", {})
+        if not isinstance(data, list):
+            return []
+        rows = []
+        for item in data:
+            try:
+                dm = str(item.get("dm", "")).strip()
+                mc = str(item.get("mc", "")).strip()
+                # Mairui 在 2 字简称中间填了空格（如 "万 科Ａ"），去掉多余空格
+                mc = "".join(mc.split())
+                jys = str(item.get("jys", "")).strip().upper()
+                # dm 格式: "000001.SZ" → code6="000001", exchange="SZ"
+                if "." in dm:
+                    code6, exch = dm.split(".", 1)
+                else:
+                    code6, exch = dm, jys
+                if not code6 or len(code6) != 6 or not code6.isdigit():
+                    continue
+                exchange = exch.upper() or jys
+                if exchange not in ("SH", "SZ"):
+                    continue
+                rows.append({
+                    "code": f"{exchange}{code6}",
+                    "name": mc,
+                    "exchange": exchange,
+                    "list_date": "",  # hslt/list 不返回 IPO 日期
+                    "listing_status": "st" if (mc.startswith("ST") or mc.startswith("*ST")) else "normal",
+                })
+            except Exception:
+                continue
+        return rows
+
+    # ── K 线 fetch ─────────────────────────────────────────
+
+    def fetch_kline_daily(self, code6: str, start: str, end: str) -> pd.DataFrame:
+        lic = self._get_licence()
+        if not lic:
+            return pd.DataFrame()
+        symbol = code6_to_mairui(code6)
+        path = f"/hsstock/history/{symbol}/d/n/{lic}"
+        params = {
+            "st": (start or "").replace("-", ""),
+            "et": (end or "").replace("-", ""),
+        }
+        data = self._fetch(path, params)
+        if not isinstance(data, list):
+            return pd.DataFrame()
+        records = []
+        for item in data:
+            try:
+                # t 字段：日线 "2026-06-09 00:00:00"，取日期部分
+                t = item.get("t", "")
+                d = t.split(" ")[0] if isinstance(t, str) else ""
+                if not d:
+                    continue
+                records.append({
+                    "trade_date": d,
+                    "open": float(item["o"]),
+                    "high": float(item["h"]),
+                    "low": float(item["l"]),
+                    "close": float(item["c"]),
+                    "volume": float(item.get("v") or 0),
+                })
+            except (KeyError, ValueError, TypeError):
+                continue
+        return filter_date_range(normalize_kline(pd.DataFrame(records)), start, end)
+
+    def fetch_kline_5min(self, code6: str, start: str, end: str) -> pd.DataFrame:
+        """5 分钟 K 线。返回标准列：bar_time, open, high, low, close, volume, amount。
+
+        mairui 用 level=5（数字，不是 "5m"）。
+        """
+        lic = self._get_licence()
+        if not lic:
+            return pd.DataFrame()
+        symbol = code6_to_mairui(code6)
+        path = f"/hsstock/history/{symbol}/5/n/{lic}"
+        params = {
+            "st": (start or "").replace("-", ""),
+            "et": (end or "").replace("-", ""),
+        }
+        data = self._fetch(path, params)
+        if not isinstance(data, list):
+            return pd.DataFrame()
+        records = []
+        for item in data:
+            try:
+                t = item.get("t", "")
+                # 分钟级时间格式 "2026-06-09 14:50:00" → 替换空格为 T 让 pandas 识别
+                bar_time = t.replace(" ", "T") if isinstance(t, str) else None
+                if not bar_time:
+                    continue
+                records.append({
+                    "bar_time": bar_time,
+                    "open": float(item["o"]),
+                    "high": float(item["h"]),
+                    "low": float(item["l"]),
+                    "close": float(item["c"]),
+                    "volume": float(item.get("v") or 0),
+                    "amount": float(item.get("a") or 0),
+                })
+            except (KeyError, ValueError, TypeError):
+                continue
+        if not records:
+            return pd.DataFrame()
+        df = pd.DataFrame(records)
+        df["bar_time"] = pd.to_datetime(df["bar_time"], errors="coerce")
+        df = df.dropna(subset=["bar_time"]).sort_values("bar_time").reset_index(drop=True)
+        # 过滤日期范围（按日期部分，不含时分）
+        if start:
+            df = df[df["bar_time"] >= pd.Timestamp(start)]
+        if end:
+            # end 包含整天
+            df = df[df["bar_time"] < pd.Timestamp(end) + pd.Timedelta(days=1)]
+        return df
+
+    def fetch_index_daily(self, index_code: str, start: str, end: str) -> pd.DataFrame:
+        """指数日 K。index_code 用 mairui 格式（如 '000300.SH'）。"""
+        lic = self._get_licence()
+        if not lic:
+            return pd.DataFrame()
+        path = f"/hsindex/history/{index_code}/d/{lic}"
+        params = {
+            "st": (start or "").replace("-", ""),
+            "et": (end or "").replace("-", ""),
+        }
+        data = self._fetch(path, params)
+        if not isinstance(data, list):
+            return pd.DataFrame()
+        records = []
+        for item in data:
+            try:
+                t = item.get("t", "")
+                d = t.split(" ")[0] if isinstance(t, str) else ""
+                if not d:
+                    continue
+                records.append({
+                    "trade_date": d,
+                    "open": float(item["o"]),
+                    "high": float(item["h"]),
+                    "low": float(item["l"]),
+                    "close": float(item["c"]),
+                    "volume": float(item.get("v") or 0),
+                })
+            except (KeyError, ValueError, TypeError):
+                continue
+        return filter_date_range(normalize_kline(pd.DataFrame(records)), start, end)
+
+    # ── 资金流向 ───────────────────────────────────────────
+
+    def fetch_moneyflow(self, code6: str, start: str, end: str) -> pd.DataFrame:
+        """个股资金流向（主力/大/中/小单 净额）。
+
+        API: GET /hsstock/history/transaction/{code}.{ex}/{licence}?st=YYYYMMDD&et=YYYYMMDD
+        返回字段（按买卖方向 × 单型 4×4 矩阵 + 主买/主卖/被动买/被动卖）：
+          zmbtdcje 主买特大单成交额
+          zmbddcje 主买大单成交额
+          zmbzdcje 主买中单成交额
+          zmbxdcje 主买小单成交额
+          zmstdcje 主卖特大单成交额
+          zmsddcje 主卖大单成交额
+          zmszdcje 主卖中单成交额
+          zmsxdcje 主卖小单成交额
+          bdmbtdcje 被动买特大单成交额
+          bdmbddcje 被动买大单成交额
+          bdmbzdcje 被动买中单成交额
+          bdmbxdcje 被动买小单成交额
+          bdmstdcje 被动卖特大单成交额
+          bdmsddcje 被动卖大单成交额
+          bdmszdcje 被动卖中单成交额
+          bdmsxdcje 被动卖小单成交额
+          ... 以及对应的成交量/笔数字段（zmbtdcjl 等），本接口暂只取成交额
+
+        单型口径（mairui 文档）：
+          特大单：成交额 ≥ 100 万 或 成交量 ≥ 5000 手
+          大单  ：成交额 ≥ 20 万  或 成交量 ≥ 1000 手
+          中单  ：成交额 ≥ 4 万   或 成交量 ≥ 200 手
+          小单  ：其他
+
+        输出字段：trade_date, main_net_inflow, large_net_inflow, medium_net_inflow, small_net_inflow
+        净额口径（与 akshare stock_individual_fund_flow 保持一致）：
+          主力净流入 = Σ主买四型 - Σ主卖四型       (zmb{t,d,z,x} - zms{t,d,z,x})
+          大单净额   = 主买大   - 主卖大           (zmbddcje - zmsddcje)
+          中单净额   = 主买中   - 主卖中           (zmbzdcje - zmszdcje)
+          小单净额   = 主买小   - 主卖小           (zmbxdcje - zmsxdcje)
+        注：不能加被动买/卖 —— 主动买 ≡ 被动卖、主动卖 ≡ 被动买
+        （同一笔成交记在两边），加起来恒等 0。
+        """
+        lic = self._get_licence()
+        if not lic:
+            return pd.DataFrame()
+        symbol = code6_to_mairui(code6)
+        path = f"/hsstock/history/transaction/{symbol}/{lic}"
+        params = {
+            "st": (start or "").replace("-", ""),
+            "et": (end or "").replace("-", ""),
+        }
+        data = self._fetch(path, params)
+        if not isinstance(data, list):
+            return pd.DataFrame()
+
+        def f(item, k) -> float:
+            try:
+                v = item.get(k)
+                return float(v) if v not in (None, "", "-") else 0.0
+            except (TypeError, ValueError):
+                return 0.0
+
+        records = []
+        for item in data:
+            t = item.get("t", "")
+            d = t.split(" ")[0] if isinstance(t, str) else ""
+            if not d:
+                continue
+            # 主买四型 / 主卖四型
+            main_buy = (
+                f(item, "zmbtdcje") + f(item, "zmbddcje")
+                + f(item, "zmbzdcje") + f(item, "zmbxdcje")
+            )
+            main_sell = (
+                f(item, "zmstdcje") + f(item, "zmsddcje")
+                + f(item, "zmszdcje") + f(item, "zmsxdcje")
+            )
+            # 大/中/小 净额：只看主动方（不包含被动方，否则恒等 0）
+            large_net = f(item, "zmbddcje") - f(item, "zmsddcje")
+            medium_net = f(item, "zmbzdcje") - f(item, "zmszdcje")
+            small_net = f(item, "zmbxdcje") - f(item, "zmsxdcje")
+            records.append({
+                "trade_date": d,
+                "main_net_inflow": round(main_buy - main_sell, 2),
+                "large_net_inflow": round(large_net, 2),
+                "medium_net_inflow": round(medium_net, 2),
+                "small_net_inflow": round(small_net, 2),
+            })
+        if not records:
+            return pd.DataFrame()
+        df = pd.DataFrame(records)
+        # 日期过滤
+        if start:
+            df = df[df["trade_date"] >= start]
+        if end:
+            df = df[df["trade_date"] <= end]
+        return df.sort_values("trade_date").reset_index(drop=True)

app/sources/sina.py

@@ -0,0 +1,137 @@
+"""新浪财经数据源。
+
+提供：
+  - 日 K 线（主源，免费，无 token）
+  - 指数日 K 线
+
+接口：https://money.finance.sina.com.cn/quotes_service/api/json_v2.php/CN_MarketData.getKLineData
+"""
+from __future__ import annotations
+
+import json
+import threading
+import time
+import urllib.request
+from typing import Any
+
+import pandas as pd
+
+from app.core.datasource.base import DataSource
+from app.core.datasource.utils import (
+    code6_to_sina,
+    filter_date_range,
+    normalize_kline,
+)
+
+
+class SinaSource(DataSource):
+    key = "datasource_xinlang"
+    name = "新浪财经"
+    provides = ["kline_daily", "index_daily"]
+    requires_credential = False
+
+    _HEADERS = {
+        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)",
+        "Referer": "https://finance.sina.com.cn",
+    }
+    _BASE_URL = (
+        "https://money.finance.sina.com.cn/quotes_service/api/json_v2.php"
+        "/CN_MarketData.getKLineData"
+    )
+
+    # RPS 限流（线程安全）
+    # 设为 3 防止触发新浪反爬（单 IP 频率过高会被临时封禁）
+    _RPS_LIMIT = 3.0
+    _rate_lock = threading.Lock()
+    _last_request_ts = 0.0
+
+    @classmethod
+    def _wait_rps(cls):
+        """全局 RPS 限流：所有线程共享同一个速率限制。"""
+        with cls._rate_lock:
+            now = time.time()
+            elapsed = now - cls._last_request_ts
+            min_interval = 1.0 / cls._RPS_LIMIT
+            if elapsed < min_interval:
+                time.sleep(min_interval - elapsed)
+            cls._last_request_ts = time.time()
+
+    def is_available(self) -> tuple[bool, str]:
+        return True, "ok"
+
+    def health_check(self) -> dict[str, Any]:
+        try:
+            url = (
+                f"{self._BASE_URL}?symbol=sh600036&scale=240&ma=no&datalen=5"
+            )
+            req = urllib.request.Request(url, headers=self._HEADERS)
+            with urllib.request.urlopen(req, timeout=15) as resp:
+                data = json.loads(resp.read().decode("utf-8"))
+            if not data or not isinstance(data, list):
+                return {"success": False, "message": "新浪财经返回数据格式异常"}
+            return {"success": True, "message": f"连接成功，获取到 {len(data)} 条 K 线数据"}
+        except Exception as e:
+            return {"success": False, "message": f"新浪财经连接失败: {e}"}
+
+    def _fetch(self, symbol: str, datalen: int = 5000, retry: int = 2) -> list:
+        self._wait_rps()
+        url = f"{self._BASE_URL}?symbol={symbol}&scale=240&ma=no&datalen={int(max(datalen, 260))}"
+        last_err = None
+        for attempt in range(retry + 1):
+            try:
+                req = urllib.request.Request(url, headers=self._HEADERS)
+                with urllib.request.urlopen(req, timeout=20) as resp:
+                    raw = resp.read().decode("gbk", errors="replace")
+                    return json.loads(raw)
+            except Exception as e:
+                last_err = e
+                if attempt < retry:
+                    time.sleep(0.5 * (attempt + 1))  # 短暂退避后重试
+                continue
+        if last_err:
+            from app.core.utils.logging import get_logger
+            get_logger("sina").debug(f"sina fetch {symbol} failed after {retry+1} attempts: {last_err}")
+        return []
+
+    def fetch_kline_daily(self, code6: str, start: str, end: str) -> pd.DataFrame:
+        symbol = code6_to_sina(code6)
+        data = self._fetch(symbol)
+        if not data or not isinstance(data, list):
+            return pd.DataFrame()
+        records = []
+        for item in data:
+            try:
+                records.append({
+                    "trade_date": item["day"],
+                    "open": item["open"],
+                    "high": item["high"],
+                    "low": item["low"],
+                    "close": item["close"],
+                    "volume": item["volume"],
+                })
+            except KeyError:
+                continue
+        return filter_date_range(normalize_kline(pd.DataFrame(records)), start, end)
+
+    def fetch_index_daily(self, index_code: str, start: str, end: str) -> pd.DataFrame:
+        if index_code.startswith(("5", "6", "9")):
+            symbol = f"sh{index_code}"
+        else:
+            symbol = f"sz{index_code}"
+        data = self._fetch(symbol)
+        if not data or not isinstance(data, list):
+            return pd.DataFrame()
+        records = []
+        for item in data:
+            try:
+                records.append({
+                    "trade_date": item["day"],
+                    "open": float(item["open"]),
+                    "high": float(item["high"]),
+                    "low": float(item["low"]),
+                    "close": float(item["close"]),
+                    "volume": float(item["volume"]),
+                })
+            except (KeyError, ValueError, TypeError):
+                continue
+        return filter_date_range(normalize_kline(pd.DataFrame(records)), start, end)

app/sources/xueqiu.py

@@ -0,0 +1,203 @@
+"""雪球数据源（pysnowball，非官方 SDK）。
+
+提供：
+  - 日 K 线复核（需要 token，免费但有限速）
+  - 最新股本快照（quote_detail，单位原始股，需 ÷1e8 转为亿股）
+
+没有 token 时降级：is_available() 返回 False。"""
+from __future__ import annotations
+
+import logging
+import os
+import time
+from datetime import datetime
+from typing import Any, Optional
+
+import pandas as pd
+
+from app.core.datasource.base import DataSource
+from app.core.datasource.utils import code6_to_xueqiu
+
+logger = logging.getLogger("sync.xueqiu")
+
+
+class XueqiuSource(DataSource):
+    key = "datasource_xueqiu"
+    name = "雪球"
+    provides = ["kline_daily", "share"]
+    requires_credential = True
+    credential_key = "XUEQIU_TOKEN"
+
+    # 进程内只 set_token 一次
+    _token_lock = __import__("threading").Lock()
+    _token_set = False
+
+    # RPS 限流（线程安全）
+    # 雪球 token 限速 10 RPS（用户实测），用满 10 才能在合理时间内跑完全市场
+    _RPS_LIMIT = 10.0
+    _rate_lock = __import__("threading").Lock()
+    _last_request = 0.0
+
+    def _read_token(self) -> str:
+        return os.environ.get(self.credential_key, "").strip()
+
+    def _import_ball(self):
+        """延迟 + 显式 import。失败时打 warning（之前全被 except 吞了）。"""
+        try:
+            import pysnowball as ball
+            return ball
+        except ImportError as e:
+            logger.error(
+                "pysnowball 未安装，雪球数据源不可用。pip install pysnowball。错误: %s", e,
+            )
+            raise
+
+    def _set_token_once(self) -> None:
+        if self._token_set:
+            return
+        with self._token_lock:
+            if self._token_set:
+                return
+            token = self._read_token()
+            if not token:
+                return
+            try:
+                ball = self._import_ball()
+                ball.set_token(token)
+                self._token_set = True
+            except ImportError:
+                pass  # _import_ball 已经打日志了
+
+    def _wait_rps(self) -> None:
+        with self._rate_lock:
+            now = time.time()
+            elapsed = now - self._last_request
+            min_interval = 1.0 / self._RPS_LIMIT
+            if elapsed < min_interval:
+                time.sleep(min_interval - elapsed)
+            self._last_request = time.time()
+
+    def is_available(self) -> tuple[bool, str]:
+        token = self._read_token()
+        if not token:
+            return False, f"未配置 {self.credential_key}"
+        # 顺便检查 pysnowball 是否装了
+        try:
+            import pysnowball  # noqa: F401
+        except ImportError:
+            return False, "pysnowball 未安装，pip install pysnowball"
+        return True, "ok"
+
+    def health_check(self) -> dict[str, Any]:
+        if not self._read_token():
+            return {"success": False, "message": f"未配置 {self.credential_key}"}
+        try:
+            ball = self._import_ball()
+            self._set_token_once()
+            result = ball.quote_detail("SH600036")
+            if result is None or result.get("error_code") != 0:
+                return {
+                    "success": False,
+                    "message": f"雪球返回异常: error_code={result.get('error_code') if result else 'None'} "
+                    f"desc={result.get('error_description') if result else 'None'}",
+                }
+            quote = (result or {}).get("data", {}).get("quote", {})
+            if not quote.get("total_shares") or not quote.get("float_shares"):
+                return {"success": False, "message": "雪球返回成功，但缺少 total_shares/float_shares 字段"}
+            return {"success": True, "message": "雪球连接成功，可读取股本快照"}
+        except Exception as e:
+            logger.exception("雪球 health_check 异常: %s", e)
+            return {"success": False, "message": f"雪球连接失败: {e}"}
+
+    def fetch_kline_daily(self, code6: str, start: str, end: str) -> pd.DataFrame:
+        if not self._read_token():
+            return pd.DataFrame()
+        try:
+            ball = self._import_ball()
+            self._set_token_once()
+            self._wait_rps()
+
+            symbol = code6_to_xueqiu(code6)
+            start_dt = datetime.strptime(start, "%Y-%m-%d")
+            end_dt = datetime.strptime(end, "%Y-%m-%d")
+            count = min(max((end_dt - start_dt).days + 60, 10), 5000)
+
+            # 加重试：雪球风控偶尔返回空/错误
+            result = None
+            for attempt in range(3):
+                result = ball.kline(symbol, period="day", count=count)
+                if result and result.get("error_code") == 0:
+                    break
+                time.sleep(0.3 * (attempt + 1))
+            if not result or result.get("error_code") != 0:
+                logger.warning(
+                    "[kline %s] 雪球 kline 失败: error_code=%s desc=%s",
+                    code6,
+                    result.get("error_code") if result else "None",
+                    result.get("error_description") if result else "None",
+                )
+                return pd.DataFrame()
+            data = result.get("data", {})
+            columns = data.get("column", [])
+            items = data.get("item", [])
+            if not columns or not items:
+                return pd.DataFrame()
+            needed = {"timestamp": 0, "volume": 1, "open": 2, "high": 3, "low": 4, "close": 5}
+            records = []
+            for row in items:
+                try:
+                    ts = row[needed["timestamp"]] / 1000
+                    records.append({
+                        "trade_date": datetime.fromtimestamp(ts).strftime("%Y-%m-%d"),
+                        "open": float(row[needed["open"]]),
+                        "high": float(row[needed["high"]]),
+                        "low": float(row[needed["low"]]),
+                        "close": float(row[needed["close"]]),
+                        "volume": float(row[needed["volume"]]),
+                    })
+                except (IndexError, ValueError, TypeError, OSError):
+                    continue
+            from app.core.datasource.utils import filter_date_range, normalize_kline
+            return filter_date_range(normalize_kline(pd.DataFrame(records)), start, end)
+        except Exception as e:
+            logger.exception("[kline %s] 雪球 kline 异常: %s", code6, e)
+            return pd.DataFrame()
+
+    def fetch_share_snapshot(self, code6: str) -> Optional[dict[str, Any]]:
+        """单只股票的最新股本快照。
+
+        返回：
+          {total_share, float_share, trade_date}  —— 单位：亿股
+        """
+        if not self._read_token():
+            return None
+        try:
+            ball = self._import_ball()
+            self._set_token_once()
+            self._wait_rps()
+
+            symbol = code6_to_xueqiu(code6)
+            result = ball.quote_detail(symbol)
+            if not result or result.get("error_code") != 0:
+                logger.warning(
+                    "[share %s] 雪球 quote_detail 失败: error_code=%s desc=%s",
+                    code6,
+                    result.get("error_code") if result else "None",
+                    result.get("error_description") if result else "None",
+                )
+                return None
+            quote = result.get("data", {}).get("quote", {})
+            today_str = datetime.now().strftime("%Y-%m-%d")
+            total = round(float(quote.get("total_shares") or 0) / 1e8, 4)
+            flt = round(float(quote.get("float_shares") or 0) / 1e8, 4)
+            if total <= 0 or flt <= 0:
+                logger.warning("[share %s] 雪球返回成功但 total/float 为 0", code6)
+                return None
+            return {
+                "trade_date": today_str,
+                "total_share": total,
+                "float_share": flt,
+            }
+        except Exception as e:
+            logger.exception("[share %s] 雪球 quote_detail 异常: %s", code6, e)
+            return None

app/tasks/__init__.py

@@ -0,0 +1,39 @@
+"""同步任务注册表。
+
+所有 `SyncTask` 子类在此集中注册，外部通过 `get_task(dataset_id)` 或
+遍历 `TASKS` 字典使用。
+"""
+from app.tasks.task_industry_sector import SyncIndustrySector
+from app.tasks.task_kline_5min import SyncKline5Min
+from app.tasks.task_kline_daily import SyncKlineDaily
+from app.tasks.task_kline_index import SyncKlineIndex
+from app.tasks.task_market_regime import SyncMarketRegime
+from app.tasks.task_moneyflow import SyncMoneyflow
+from app.tasks.task_sector_features import SyncSectorFeatures
+from app.tasks.task_share_snapshot import SyncShareSnapshot
+from app.tasks.task_stocks_basic import SyncStocksBasic
+
+TASKS: dict[str, type] = {
+    cls.dataset_id: cls
+    for cls in (
+        SyncStocksBasic,
+        SyncKlineDaily,
+        SyncKlineIndex,
+        SyncKline5Min,
+        SyncMoneyflow,
+        SyncIndustrySector,
+        SyncSectorFeatures,
+        SyncShareSnapshot,
+        SyncMarketRegime,
+    )
+}
+
+
+def get_task(dataset_id: str):
+    """通过 dataset_id 取同步任务类实例，找不到抛 KeyError。"""
+    if dataset_id not in TASKS:
+        raise KeyError(f"未知的同步任务: {dataset_id!r}，可选: {sorted(TASKS)}")
+    return TASKS[dataset_id]()
+
+
+__all__ = ["TASKS", "get_task"]

app/tasks/task_industry_sector.py

@@ -0,0 +1,122 @@
+"""同步任务：股票-行业映射（Baostock）。
+
+数据流：Baostock query_stock_industry() → 写 industry / sectors / stock_sector_map 三张表。"""
+from __future__ import annotations
+
+import time
+from typing import Any
+
+from app.core.db import ops as db_ops
+from app.core.datasource.base import registry as ds_registry
+from app.core.datasource.registry import is_source_ready
+from app.core.sync.base import SyncTask
+from app.core.sync.registry import mark_sync_blocked
+from app.core.utils.logging import get_logger
+
+logger = get_logger("sync.industry_sector")
+
+
+def _build_sector_key(taxonomy: str, sector_name: str) -> str:
+    tax = (taxonomy or "").strip() or "default"
+    name = (sector_name or "").strip()
+    return f"{tax}::{name}"
+
+
+class SyncIndustrySector(SyncTask):
+    dataset_id = "industry_sector"
+
+    def _run(self, *, trigger_source: str = "manual", **kwargs) -> dict[str, Any]:
+        bs = ds_registry.get("datasource_baostock")
+        if bs is None:
+            return {"status": "error", "message": "Baostock 数据源未注册"}
+        ok, reason = is_source_ready(bs.key)
+        if not ok:
+            mark_sync_blocked(self.dataset_id, message=f"Baostock 未就绪: {reason}")
+            return {"status": "blocked", "message": f"Baostock 未就绪: {reason}"}
+
+        self._progress(message="拉取股票-行业映射...")
+        t0 = time.time()
+        try:
+            raw_rows = bs.fetch_industry_map()
+        except Exception as e:
+            return {"status": "error", "message": f"拉取失败: {e}"}
+
+        if not raw_rows:
+            return {"status": "warning", "message": "Baostock 未返回行业数据"}
+
+        # 合并：sectors + stock_sector_map + industry
+        sectors_rows: dict[str, dict] = {}
+        stock_sector_rows: list[dict] = []
+        industry_rows: list[dict] = []
+
+        for r in raw_rows:
+            code6 = str(r["code"]).zfill(6)
+            name = (r.get("industry_name") or "").strip()
+            if not name:
+                continue
+            classification = (r.get("industry_classification") or "").strip()
+            sector_key = _build_sector_key(classification, name)
+
+            sectors_rows[sector_key] = {
+                "sector_key": sector_key,
+                "sector_name": name,
+                "taxonomy": classification,
+                "level": "",
+                "source": "baostock_query_stock_industry",
+                "enabled": 1,
+            }
+            stock_sector_rows.append({"stock_code": code6, "sector_key": sector_key})
+            industry_rows.append({
+                "code": code6,
+                "industry_name": name,
+                "industry_classification": classification,
+                "update_date": r.get("update_date", ""),
+            })
+
+        # 写库（全量替换）
+        try:
+            db_ops.replace_all_sectors(list(sectors_rows.values()))
+            db_ops.replace_all_stock_sector_map(stock_sector_rows)
+            db_ops.replace_all_industries(industry_rows)
+        except Exception as e:
+            return {"status": "error", "message": f"写库失败: {e}"}
+
+        # 回填 stocks.industry 字段（让 stocks 表也能直接看到）
+        # 使用批量 UPDATE ... CASE WHEN 方式一次性完成
+        try:
+            from app.core.db.connection import get_mysql
+            from app.core.db.schema import ensure_all_tables
+            ensure_all_tables(get_mysql())
+            # 按 industry_name 分组，对每组用一个 IN 查询批量更新
+            by_industry: dict[str, list[str]] = {}
+            for r in industry_rows:
+                name = r["industry_name"]
+                code = r["code"]
+                if name not in by_industry:
+                    by_industry[name] = []
+                by_industry[name].append(code)
+            with get_mysql().cursor() as cur:
+                for ind_name, codes in by_industry.items():
+                    # 构造 IN 子句：code IN (SHxxx, SZxxx, BJxxx)
+                    placeholders = []
+                    params = [ind_name]
+                    for c in codes:
+                        for prefix in ("SH", "SZ", "BJ"):
+                            placeholders.append("%s")
+                            params.append(f"{prefix}{c}")
+                    in_clause = ", ".join(placeholders)
+                    sql = f"UPDATE stocks SET industry = %s WHERE code IN ({in_clause})"
+                    cur.execute(sql, params)
+                get_mysql().commit()
+        except Exception as e:
+            logger.warning(f"回填 stocks.industry 失败（不影响主任务）: {e}")
+
+        elapsed = round(time.time() - t0, 1)
+        msg = f"行业 {len(sectors_rows)}个 映射{len(stock_sector_rows)}条, {elapsed}s"
+        return {
+            "status": "ok",
+            "message": msg,
+            "sectors": len(sectors_rows),
+            "stock_sector_map": len(stock_sector_rows),
+            "elapsed_sec": elapsed,
+        }

app/tasks/task_kline_5min.py

@@ -0,0 +1,203 @@
+"""同步任务：5 分钟 K 线（mairui）。
+
+设计要点：
+  1) 启动时**一次 SQL** 拿全表 {stock_code: max(bar_time)} 快照
+  2) 内存里给每只股票算 (start, end, windows_needed)
+       - DB 没数据 → start = mairui 历史深度起点 (2023-06-14)
+       - DB 有数据 → start = max(2023-06-14, max(bar_time) - 2 天)
+       - 兜底：start >= end → 跳过这只
+  3) 输出"计划"（全量/增量分类、预估请求数、预估耗时）再开始执行
+  4) 5 worker × 1 RPS/worker = 5 RPS 总（守住 mairui 上限）
+  5) 窗口大小 1 年（mairui 一次最多 ~11600 条）
+
+akshare 时代写法：start 写死 2020-01-01 + 4 天窗口，137 小时跑完全市场。
+本版：增量时通常只补最近 1-2 天，~30 分钟。
+"""
+from __future__ import annotations
+
+import os
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from datetime import datetime, timedelta
+from typing import Any, Optional
+
+from app.core.db import ops as db_ops
+from app.core.datasource.base import registry as ds_registry
+from app.core.datasource.registry import is_source_ready
+from app.core.datasource.utils import is_a_share_code, to_code6
+from app.core.sync.base import SyncTask
+from app.core.sync.registry import mark_sync_blocked
+from app.core.utils.logging import get_logger
+
+logger = get_logger("sync.kline_5min")
+
+# mairui 5min K 实测历史深度起点（实测 2023-06-14 才有数据，更早就 0）
+MAIRUI_5MIN_FLOOR = datetime(2023, 6, 14)
+# 单次请求窗口大小：1 年（mairui 单次最多 ~11640 条 5min）
+WINDOW_DAYS = 365
+# 增量时往前多取的天数（防交易日历边界漏当天）
+INCREMENT_OVERLAP_DAYS = 2
+
+
+class SyncKline5Min(SyncTask):
+    dataset_id = "kline_5min"
+
+    def _plan(
+        self,
+        stock_codes: list[str],
+        end: datetime,
+        snapshots: dict[str, Optional[datetime]],
+    ) -> list[tuple[str, datetime, datetime]]:
+        """为每只股票算 (start, end) — 已排除 start >= end 的"无需同步"。
+
+        Returns: [(code6, start, end), ...]
+        """
+        plans = []
+        for code6 in stock_codes:
+            latest = snapshots.get(code6)
+            if latest is None:
+                # DB 里没数据 → 全量从 mairui 历史深度起点
+                start = MAIRUI_5MIN_FLOOR
+            else:
+                # DB 有数据 → 增量：从 max(bar_time) - overlap 到 end
+                start = max(MAIRUI_5MIN_FLOOR, latest - timedelta(days=INCREMENT_OVERLAP_DAYS))
+            if start < end:
+                plans.append((code6, start, end))
+        return plans
+
+    def _run(
+        self,
+        *,
+        trigger_source: str = "manual",
+        codes: list[str] | None = None,
+        max_workers: int = 5,
+        **kwargs,
+    ) -> dict[str, Any]:
+        primary = ds_registry.get("datasource_mairui")
+        if primary is None:
+            return {"status": "error", "message": "5min 数据源 mairui 未注册"}
+        ok, reason = is_source_ready(primary.key)
+        if not ok:
+            mark_sync_blocked(self.dataset_id, message=f"{primary.key} 未就绪: {reason}")
+            return {"status": "blocked", "message": f"{primary.key} 未就绪: {reason}"}
+
+        if codes:
+            stock_codes = [to_code6(c) for c in codes]
+        else:
+            stock_codes = [c for c in db_ops.iter_stock_codes(active_only=True) if is_a_share_code(c)]
+        limit_raw = os.environ.get("MARKET_DATA_STOCK_LIMIT", "").strip()
+        if limit_raw.isdigit() and int(limit_raw) > 0:
+            stock_codes = stock_codes[: int(limit_raw)]
+
+        if not stock_codes:
+            return {"status": "error", "message": "无股票代码"}
+
+        # ── 1) 一次 SQL 拿全表快照 ──
+        logger.info(f"[5min] 读取 DB 快照 ({len(stock_codes)} 只待查)…")
+        snapshots = db_ops.get_kline_5min_snapshots()
+        logger.info(f"[5min] DB 里已有 {len(snapshots)} 只的 kline_5min 数据")
+
+        # ── 2) 内存算计划 ──
+        end = datetime.now()
+        plans = self._plan(stock_codes, end, snapshots)
+        if not plans:
+            msg = f"5min K线 — 全部 {len(stock_codes)} 只都已最新（无需同步）"
+            logger.info(msg)
+            return {"status": "ok", "message": msg, "ok": 0, "fail": 0, "rows": 0, "elapsed_sec": 0.0}
+
+        # 分类统计
+        full_sync = [p for p in plans if snapshots.get(p[0]) is None]
+        incr_sync = [p for p in plans if snapshots.get(p[0]) is not None]
+        total_windows = sum(
+            max(1, (p[2] - p[1]).days // WINDOW_DAYS + 1)
+            for p in plans
+        )
+        # mairui 5 RPS 上限（5 worker × 1 RPS）
+        est_sec = total_windows / 5.0
+        logger.info(
+            f"[5min] 计划: 总 {len(plans)} 只 (全量 {len(full_sync)} / 增量 {len(incr_sync)})，"
+            f"总请求 {total_windows}，按 5 RPS 预估 {est_sec:.0f}s ({est_sec/60:.1f}min)"
+        )
+        self._progress(
+            message=f"计划: {len(plans)} 只 (全量 {len(full_sync)}/增量 {len(incr_sync)}) "
+                    f"约 {total_windows} 个请求, 预估 {est_sec/60:.1f}min",
+            current=0, total=len(plans), current_step="planning",
+        )
+
+        # ── 3) 执行 ──
+        t0 = time.time()
+        ok_cnt = fail_cnt = 0
+        rows_total = 0
+        with ThreadPoolExecutor(max_workers=max_workers) as pool:
+            futures = {pool.submit(self._sync_one, primary, p[0], p[1], p[2]): p[0] for p in plans}
+            for i, future in enumerate(as_completed(futures), 1):
+                c6 = futures[future]
+                try:
+                    res = future.result()
+                    if res["status"] == "ok":
+                        ok_cnt += 1
+                        rows_total += res["rows"]
+                    else:
+                        fail_cnt += 1
+                        if res.get("error"):
+                            logger.warning(f"[5min {c6}] {res['error']}")
+                except Exception as e:
+                    fail_cnt += 1
+                    logger.warning(f"[5min {c6}] {e}")
+                if i % 20 == 0 or i == len(plans):
+                    self._progress(
+                        message=f"5min 进度 {i}/{len(plans)} OK:{ok_cnt} FAIL:{fail_cnt}",
+                        current=i, total=len(plans), current_step=c6,
+                    )
+        elapsed = round(time.time() - t0, 1)
+        msg = (
+            f"5min K线 {ok_cnt}成 {fail_cnt}败 共{rows_total}行, "
+            f"全量{len(full_sync)}/增量{len(incr_sync)}, {elapsed}s"
+        )
+        return {
+            "status": "ok" if fail_cnt == 0 else "warning",
+            "message": msg,
+            "ok": ok_cnt, "fail": fail_cnt, "rows": rows_total,
+            "full_sync": len(full_sync), "incr_sync": len(incr_sync),
+            "total_requests": total_windows, "elapsed_sec": elapsed,
+        }
+
+    def _sync_one(self, primary, code6: str, start: datetime, end: datetime) -> dict:
+        all_rows: list[dict] = []
+        try:
+            cur = start
+            while cur < end:
+                w_end = min(cur + timedelta(days=WINDOW_DAYS), end)
+                df = primary.fetch_kline_5min(
+                    code6,
+                    cur.strftime("%Y-%m-%d"),
+                    w_end.strftime("%Y-%m-%d"),
+                )
+                if df is not None and not df.empty:
+                    for _, r in df.iterrows():
+                        bar_time = r["bar_time"]
+                        all_rows.append({
+                            "stock_code": code6,
+                            "bar_time": bar_time.strftime("%Y-%m-%d %H:%M:%S")
+                                      if hasattr(bar_time, "strftime") else str(bar_time),
+                            "open": float(r.get("open") or 0),
+                            "high": float(r.get("high") or 0),
+                            "low": float(r.get("low") or 0),
+                            "close": float(r.get("close") or 0),
+                            "volume": float(r.get("volume") or 0),
+                            "amount": float(r.get("amount") or 0),
+                            "turnover_rate": float(r.get("turnover_rate") or 0),
+                        })
+                cur = w_end + timedelta(days=1)
+        except Exception as e:
+            return {"status": "fail", "error": str(e)}
+
+        if not all_rows:
+            return {"status": "fail", "error": "no data"}
+        try:
+            CHUNK = 5000
+            for i in range(0, len(all_rows), CHUNK):
+                db_ops.upsert_kline_5min(all_rows[i: i + CHUNK])
+        except Exception as e:
+            return {"status": "fail", "error": f"db write: {e}"}
+        return {"status": "ok", "rows": len(all_rows)}

app/tasks/task_kline_daily.py

@@ -0,0 +1,251 @@
+"""同步任务：全市场日 K 线。
+
+数据流：
+  1. 选主源（按 priority 选第一个 is_available 的源）
+  2. 增量判断：读 kline_stock.MAX(trade_date) 决定每只股票 start
+  3. **Producer-Consumer 模式**：
+     - N 个 fetcher 线程并发拉 K 线，把 (code, rows) 塞到 Queue
+     - 1 个 writer 线程攒满 BATCH_SIZE 就 executemany + commit 一次
+  4. 更新 stocks.kline_synced_at
+
+设计要点：
+  - 避免每只股票 commit（5000 只 = 5000 次 commit 太慢）
+  - 减少并发数（避免新浪限流 / MySQL 连接数爆）
+  - fetcher 间用 Queue 解耦，写库串行避免锁竞争
+"""
+from __future__ import annotations
+
+import os
+import queue
+import threading
+import time
+from datetime import datetime, timedelta
+from typing import Any
+
+import pandas as pd
+
+from app.core.db import ops as db_ops
+from app.core.datasource.base import registry as ds_registry
+from app.core.datasource.utils import is_a_share_code, to_code6
+from app.core.sync.base import SyncTask, _effective_sync_end
+from app.core.utils.logging import get_logger
+
+logger = get_logger("sync.kline_daily")
+
+# 主源优先级：mairui（5 RPS，免费 + 稳定 + 单接口含5min）→ 雪球 → Baostock → 新浪
+# 经验：sina 在单日窗口 (start=end=某天) 会返回空，雪球更可靠
+PRIMARY_PRIORITY = ["datasource_xueqiu", "datasource_mairui", "datasource_xinlang"]
+DEFAULT_START = "2018-01-01"
+# 写库批量：攒够 BATCH_SIZE 行就 executemany 一次 + commit
+BATCH_SIZE = 2000
+# fetcher 并发：实测单线程最稳（5000 只 × 0.2s = 17 分钟，足够）
+# 雪球 token 限速 10 RPS，单线程 0.1s/只 已经打满。多 worker 会触发风控 hang
+DEFAULT_FETCH_WORKERS = 1
+
+
+class SyncKlineDaily(SyncTask):
+    dataset_id = "kline_daily"
+
+    def _run(
+        self,
+        *,
+        trigger_source: str = "manual",
+        start: str | None = None,
+        end: str | None = None,
+        codes: list[str] | None = None,
+        incremental: bool = True,
+        max_workers: int = DEFAULT_FETCH_WORKERS,
+        **kwargs,
+    ) -> dict[str, Any]:
+        # 1. 选主源
+        primary = self._pick_primary()
+        if primary is None:
+            return {"status": "error", "message": "所有 K 线数据源均不可用"}
+
+        self._progress(
+            message=f"使用 {primary.name} 同步日K线 (fetcher={max_workers}, batch={BATCH_SIZE})...",
+            current_step=primary.key,
+        )
+
+        # 2. 拉股票列表
+        if codes:
+            stock_codes = [to_code6(c) for c in codes]
+        else:
+            stock_codes = [
+                c for c in db_ops.iter_stock_codes(active_only=True)
+                if is_a_share_code(c)
+            ]
+        limit_raw = os.environ.get("MARKET_DATA_STOCK_LIMIT", "").strip()
+        if limit_raw.isdigit() and int(limit_raw) > 0:
+            stock_codes = stock_codes[: int(limit_raw)]
+
+        if not stock_codes:
+            return {"status": "error", "message": "无股票代码（stocks 表为空？）"}
+
+        # 3. 增量判断
+        _end = end or _effective_sync_end()
+        end_date_obj = datetime.strptime(_end, "%Y-%m-%d").date()
+        jobs: list[tuple[str, str]] = []
+        if incremental and not start:
+            for c6 in stock_codes:
+                last = db_ops.get_stock_kline_max_date(c6)
+                if last is None:
+                    fetch_start = DEFAULT_START
+                elif datetime.strptime(last, "%Y-%m-%d").date() >= end_date_obj:
+                    continue
+                else:
+                    fetch_start = (datetime.strptime(last, "%Y-%m-%d").date() - timedelta(days=5)).strftime("%Y-%m-%d")
+                jobs.append((c6, fetch_start))
+        else:
+            fetch_start = start or DEFAULT_START
+            jobs = [(c, fetch_start) for c in stock_codes]
+
+        total = len(jobs)
+        skipped = len(stock_codes) - total
+        self._progress(
+            message=f"开始同步 {total} 只股票 (跳过 {skipped} 只已最新)",
+            current=0, total=total,
+        )
+
+        # 4. 并发 fetch + 攒 batch 写库（线程安全）
+        # mairui 等外部 API 无并发问题；DB 写串行加锁
+        from concurrent.futures import ThreadPoolExecutor, as_completed
+        t0 = time.time()
+        batch_lock = threading.Lock()
+        batch_rows: list[dict] = []
+        pending_codes: dict[str, str] = {}
+        ok_lock = threading.Lock()
+        ok_cnt = [0]
+        fail_cnt = [0]
+        rows_cnt = [0]
+
+        def _flush():
+            with batch_lock:
+                if not batch_rows:
+                    return
+                rows_to_write = list(batch_rows)
+                codes_to_update = dict(pending_codes)
+                batch_rows.clear()
+                pending_codes.clear()
+            try:
+                db_ops.upsert_kline_stock(rows_to_write)
+            except Exception as e:
+                logger.error(f"batch upsert 失败 ({len(rows_to_write)} 行): {e}")
+                return
+            for c6, latest in codes_to_update.items():
+                try:
+                    db_ops.update_stock_kline_synced_at(c6, latest)
+                except Exception as e:
+                    logger.warning(f"update synced_at 失败 {c6}: {e}")
+
+        def _fetch_one(code6: str, fetch_start: str) -> tuple[str, list[dict] | None, str | None]:
+            try:
+                df = primary.fetch_kline_daily(code6, fetch_start, _end)
+            except Exception as e:
+                return (code6, None, str(e)[:120])
+            if df is None or df.empty:
+                return (code6, None, "no data")
+            rows = []
+            for _, r in df.iterrows():
+                td = r["trade_date"]
+                rows.append({
+                    "stock_code": code6,
+                    "trade_date": td.strftime("%Y-%m-%d") if hasattr(td, "strftime") else str(td)[:10],
+                    "open": float(r["open"]),
+                    "high": float(r["high"]),
+                    "low": float(r["low"]),
+                    "close": float(r["close"]),
+                    "volume": float(r["volume"]),
+                })
+            latest = df["trade_date"].max()
+            latest = latest.strftime("%Y-%m-%d") if hasattr(latest, "strftime") else str(latest)[:10]
+            return (code6, rows, latest)
+
+        total = len(jobs)
+        with ThreadPoolExecutor(max_workers=max_workers) as pool:
+            futures = {pool.submit(_fetch_one, c6, fs): c6 for c6, fs in jobs}
+            done_cnt = 0
+            for future in as_completed(futures):
+                done_cnt += 1
+                code6, rows, latest = future.result()
+                if rows is None:
+                    with ok_lock:
+                        fail_cnt[0] += 1
+                    logger.warning(f"[kline {code6}] {latest}")
+                    continue
+                with batch_lock:
+                    batch_rows.extend(rows)
+                    pending_codes[code6] = latest
+                    cur_size = len(batch_rows)
+                with ok_lock:
+                    ok_cnt[0] += 1
+                    rows_cnt[0] += len(rows)
+
+                if cur_size >= BATCH_SIZE:
+                    _flush()
+
+                if done_cnt % 50 == 0 or done_cnt == total:
+                    elapsed = time.time() - t0
+                    rate = done_cnt / elapsed if elapsed > 0 else 0
+                    self._progress(
+                        message=f"进度 {done_cnt}/{total} OK:{ok_cnt[0]} FAIL:{fail_cnt[0]} {rate:.1f}只/秒",
+                        current=done_cnt, total=total, current_step=code6,
+                    )
+
+        # 收尾 flush
+        _flush()
+
+        elapsed = round(time.time() - t0, 1)
+        msg = f"日K线 {ok_cnt[0]}成 {fail_cnt[0]}败 {skipped}跳 共{rows_cnt[0]}行, {elapsed}s"
+        return {
+            "status": "ok" if fail_cnt[0] == 0 else "warning",
+            "message": msg,
+            "ok": ok_cnt[0],
+            "fail": fail_cnt[0],
+            "skip": skipped,
+            "rows": rows_cnt[0],
+            "elapsed_sec": elapsed,
+            "primary_source": primary.key,
+        }
+
+    def _pick_primary(self):
+        from app.core.datasource.registry import is_source_ready, run_health_check, get_health_status
+        if not get_health_status():
+            run_health_check()
+        for key in PRIMARY_PRIORITY:
+            ok, _ = is_source_ready(key)
+            if ok:
+                return ds_registry.get(key)
+        return None
+
+    def _sync_one(self, primary, code6: str, start: str, end: str) -> dict:
+        """保留这个方法以兼容外部调用（已不用，但单只测试可能用到）"""
+        try:
+            df = primary.fetch_kline_daily(code6, start, end)
+        except Exception as e:
+            return {"status": "fail", "error": str(e)}
+        if df is None or df.empty:
+            return {"status": "fail", "error": "no data"}
+        rows = []
+        for _, r in df.iterrows():
+            td = r["trade_date"]
+            rows.append({
+                "stock_code": code6,
+                "trade_date": td.strftime("%Y-%m-%d") if hasattr(td, "strftime") else str(td)[:10],
+                "open": float(r["open"]),
+                "high": float(r["high"]),
+                "low": float(r["low"]),
+                "close": float(r["close"]),
+                "volume": float(r["volume"]),
+            })
+        try:
+            db_ops.upsert_kline_stock(rows)
+        except Exception as e:
+            return {"status": "fail", "error": f"db write: {e}"}
+        if rows:
+            latest = max(r["trade_date"] for r in rows)
+            try:
+                db_ops.update_stock_kline_synced_at(code6, latest)
+            except Exception:
+                pass
+        return {"status": "ok", "rows": len(rows)}

app/tasks/task_kline_index.py

@@ -0,0 +1,109 @@
+"""同步任务：六大指数日 K 线。
+
+数据流：新浪指数日K接口 → 写 kline_index + indices 表。"""
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Any
+
+from app.core.db import ops as db_ops
+from app.core.datasource.base import registry as ds_registry
+from app.core.datasource.registry import is_source_ready
+from app.core.sync.base import SyncTask, _is_market_closed
+from app.core.sync.registry import mark_sync_blocked
+from app.core.utils.logging import get_logger
+
+logger = get_logger("sync.kline_index")
+
+MAJOR_INDICES = {
+    "000001": {"name": "上证指数", "market": "CN", "category": "broad_market"},
+    "399001": {"name": "深证成指", "market": "CN", "category": "broad_market"},
+    "399006": {"name": "创业板指", "market": "CN", "category": "broad_market"},
+    "000300": {"name": "沪深300", "market": "CN", "category": "broad_market"},
+    "000905": {"name": "中证500", "market": "CN", "category": "broad_market"},
+    "000852": {"name": "中证1000", "market": "CN", "category": "broad_market"},
+}
+
+
+class SyncKlineIndex(SyncTask):
+    dataset_id = "kline_index"
+
+    def _run(self, *, trigger_source: str = "manual", **kwargs) -> dict[str, Any]:
+        # 优先 mairui（指数无单日空窗问题），sina 作为 fallback
+        primary = ds_registry.get("datasource_mairui")
+        use_mairui = primary is not None and is_source_ready(primary.key)[0]
+        if not use_mairui:
+            primary = ds_registry.get("datasource_xinlang")
+        if primary is None:
+            return {"status": "error", "message": "指数数据源均不可用（mairui + 新浪）"}
+        ok, reason = is_source_ready(primary.key)
+        if not ok:
+            mark_sync_blocked(self.dataset_id, message=f"{primary.key} 未就绪: {reason}")
+            return {"status": "blocked", "message": f"{primary.key} 未就绪: {reason}"}
+
+        self._progress(message=f"开始同步 {len(MAJOR_INDICES)} 个指数...")
+        total = len(MAJOR_INDICES)
+        ok_cnt = fail_cnt = 0
+        results = {}
+
+        for i, (code, info) in enumerate(MAJOR_INDICES.items(), 1):
+            try:
+                # mairui 需带交易所后缀（如 000300.SH），sina 用纯6位
+                if use_mairui:
+                    # 000xxx 上证 → .SH；399xxx 深证 → .SZ
+                    if code.startswith("399"):
+                        suffix = ".SZ"
+                    else:
+                        suffix = ".SH"
+                    fetch_code = f"{code}{suffix}"
+                else:
+                    fetch_code = code
+                df = primary.fetch_index_daily(fetch_code, start="1990-01-01", end="2099-12-31")
+                if df is None or df.empty:
+                    raise RuntimeError("返回空数据")
+                # 盘中未收盘则去掉当天
+                if not _is_market_closed():
+                    today = datetime.now().strftime("%Y-%m-%d")
+                    df = df[df["trade_date"].dt.strftime("%Y-%m-%d") < today]
+
+                rows = [
+                    {
+                        "index_code": code,
+                        "trade_date": r["trade_date"].strftime("%Y-%m-%d"),
+                        "open": float(r["open"]),
+                        "high": float(r["high"]),
+                        "low": float(r["low"]),
+                        "close": float(r["close"]),
+                        "volume": float(r["volume"]),
+                    }
+                    for _, r in df.iterrows()
+                ]
+                db_ops.upsert_kline_index(rows)
+                db_ops.upsert_index(
+                    index_code=code, index_name=info["name"],
+                    market=info["market"], category=info["category"],
+                    source="mairui_index_daily" if use_mairui else "sina_index_daily", enabled=True,
+                )
+                ok_cnt += 1
+                results[code] = {
+                    "rows": len(rows),
+                    "date_min": rows[0]["trade_date"] if rows else "",
+                    "date_max": rows[-1]["trade_date"] if rows else "",
+                }
+            except Exception as e:
+                fail_cnt += 1
+                results[code] = {"error": str(e)}
+                logger.warning(f"[index {code}] 失败: {e}")
+
+            self._progress(
+                message=f"指数 {i}/{total} 完成 ({ok_cnt}成 {fail_cnt}败)",
+                current=i, total=total, current_step=code,
+            )
+
+        msg = f"六大指数 {ok_cnt}成 {fail_cnt}败"
+        return {
+            "status": "ok" if fail_cnt == 0 else "warning",
+            "message": msg,
+            "ok": ok_cnt, "fail": fail_cnt,
+            "details": results,
+        }

app/tasks/task_market_regime.py

@@ -0,0 +1,113 @@
+"""同步任务：市场情绪（market_regime_daily，衍生源）。
+
+基于本地 kline_stock 数据聚合：advancers / decliners / turnover / advance_ratio。"""
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Any
+
+import numpy as np
+import pandas as pd
+
+from app.core.db import ops as db_ops
+from app.core.sync.base import SyncTask
+from app.core.utils.logging import get_logger
+
+logger = get_logger("sync.market_regime")
+
+DEFAULT_START = "2018-01-01"
+EXTREME_PANIC_ADV = 0.2
+EXTREME_PANIC_TURN = 1.5
+
+
+def _is_extreme_panic(row: pd.Series) -> int:
+    ar = pd.to_numeric(row.get("advance_ratio"), errors="coerce")
+    tr = pd.to_numeric(row.get("turnover_ratio_5d"), errors="coerce")
+    if pd.isna(ar) or pd.isna(tr):
+        return 0
+    return int(ar < EXTREME_PANIC_ADV and tr > EXTREME_PANIC_TURN)
+
+
+class SyncMarketRegime(SyncTask):
+    dataset_id = "market_regime"
+
+    def _run(self, *, trigger_source: str = "manual", start: str = DEFAULT_START, **kwargs) -> dict[str, Any]:
+        end = datetime.now().strftime("%Y-%m-%d")
+        self._progress(message=f"构建 market_regime {start} ~ {end}...")
+
+        # 1. 取所有非退市股票代码
+        codes = list(db_ops.iter_stock_codes(active_only=True))
+        if not codes:
+            return {"status": "error", "message": "stocks 表为空"}
+
+        # 2. 拉所有 K 线（按股票）- 这里用 SQL GROUP BY 一次性算 daily turnover
+        from app.core.db.connection import get_mysql
+
+        sql = """
+            SELECT
+                stock_code,
+                trade_date,
+                `close`,
+                volume
+            FROM kline_stock
+            WHERE trade_date BETWEEN %s AND %s
+            ORDER BY stock_code, trade_date
+        """
+        with get_mysql().cursor() as cur:
+            cur.execute(sql, (start, end))
+            rows = cur.fetchall()
+        if not rows:
+            return {"status": "warning", "message": f"kline_stock 在 {start} ~ {end} 区间无数据"}
+
+        df = pd.DataFrame(rows)
+        df["trade_date"] = pd.to_datetime(df["trade_date"], errors="coerce")
+        df["close"] = pd.to_numeric(df["close"], errors="coerce")
+        df["volume"] = pd.to_numeric(df["volume"], errors="coerce").fillna(0.0)
+        df = df.dropna(subset=["trade_date", "close"]).sort_values(["stock_code", "trade_date"])
+        df["prev_close"] = df.groupby("stock_code")["close"].shift(1)
+        df = df.dropna(subset=["prev_close"])
+        df["advancers"] = (df["close"] > df["prev_close"]).astype(int)
+        df["decliners"] = (df["close"] < df["prev_close"]).astype(int)
+        df["turnover"] = df["close"] * df["volume"]
+
+        daily = df.groupby("trade_date", as_index=False)[["advancers", "decliners", "turnover"]].sum()
+        total = daily["advancers"] + daily["decliners"]
+        daily["advance_ratio"] = np.where(total > 0, daily["advancers"] / total, np.nan)
+        daily["turnover_avg_5d"] = daily["turnover"].rolling(5, min_periods=5).mean()
+        daily["turnover_ratio_5d"] = np.where(
+            daily["turnover_avg_5d"] > 0,
+            daily["turnover"] / daily["turnover_avg_5d"],
+            np.nan,
+        )
+        daily["source"] = "local_kline_proxy"
+        daily["is_extreme_panic"] = daily.apply(_is_extreme_panic, axis=1)
+        daily["trade_date"] = daily["trade_date"].dt.strftime("%Y-%m-%d")
+
+        rows_out = [
+            {
+                "trade_date": r["trade_date"],
+                "advancers": float(r["advancers"]),
+                "decliners": float(r["decliners"]),
+                "advance_ratio": float(r["advance_ratio"]) if pd.notna(r["advance_ratio"]) else 0,
+                "turnover": float(r["turnover"]),
+                "turnover_avg_5d": float(r["turnover_avg_5d"]) if pd.notna(r["turnover_avg_5d"]) else 0,
+                "turnover_ratio_5d": float(r["turnover_ratio_5d"]) if pd.notna(r["turnover_ratio_5d"]) else 0,
+                "source": r["source"],
+                "is_extreme_panic": int(r["is_extreme_panic"]),
+            }
+            for _, r in daily.iterrows()
+        ]
+
+        try:
+            db_ops.upsert_market_regime_rows(rows_out)
+        except Exception as e:
+            return {"status": "error", "message": f"写库失败: {e}"}
+
+        msg = f"market_regime {len(rows_out)} 天 ({rows_out[0]['trade_date']} ~ {rows_out[-1]['trade_date']})"
+        return {
+            "status": "ok",
+            "message": msg,
+            "rows": len(rows_out),
+            "date_min": rows_out[0]["trade_date"],
+            "date_max": rows_out[-1]["trade_date"],
+        }

app/tasks/task_moneyflow.py

@@ -0,0 +1,117 @@
+"""同步任务：个股资金流（mairui）。
+
+数据源：mairui `hsstock/history/transaction/{code}.{ex}/{licence}`
+口径：主力 / 大 / 中 / 小单 净额（与 akshare stock_individual_fund_flow 一致）
+
+akshare 已移除（项目级决策：永远不用），mairui 替代为唯一源。
+"""
+from __future__ import annotations
+
+import os
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from typing import Any
+
+from app.core.db import ops as db_ops
+from app.core.datasource.base import registry as ds_registry
+from app.core.datasource.registry import is_source_ready
+from app.core.datasource.utils import is_a_share_code, to_code6
+from app.core.sync.base import SyncTask
+from app.core.sync.registry import mark_sync_blocked
+from app.core.utils.logging import get_logger
+
+logger = get_logger("sync.moneyflow")
+
+
+class SyncMoneyflow(SyncTask):
+    dataset_id = "moneyflow"
+
+    def _run(
+        self,
+        *,
+        trigger_source: str = "manual",
+        codes: list[str] | None = None,
+        max_workers: int = 5,
+        **kwargs,
+    ) -> dict[str, Any]:
+        mr = ds_registry.get("datasource_mairui")
+        if mr is None:
+            return {"status": "error", "message": "mairui 数据源未注册"}
+        ok, reason = is_source_ready(mr.key)
+        if not ok:
+            mark_sync_blocked(self.dataset_id, message=f"mairui 未就绪: {reason}")
+            return {"status": "blocked", "message": f"mairui 未就绪: {reason}"}
+
+        if codes:
+            stock_codes = [to_code6(c) for c in codes]
+        else:
+            stock_codes = [c for c in db_ops.iter_stock_codes(active_only=True) if is_a_share_code(c)]
+        limit_raw = os.environ.get("MARKET_DATA_STOCK_LIMIT", "").strip()
+        if limit_raw.isdigit() and int(limit_raw) > 0:
+            stock_codes = stock_codes[: int(limit_raw)]
+
+        if not stock_codes:
+            return {"status": "error", "message": "无股票代码"}
+
+        total = len(stock_codes)
+        ok_cnt = fail_cnt = 0
+        rows_total = 0
+        t0 = time.time()
+        with ThreadPoolExecutor(max_workers=max_workers) as pool:
+            futures = {pool.submit(self._sync_one, mr, c6): c6 for c6 in stock_codes}
+            for i, future in enumerate(as_completed(futures), 1):
+                c6 = futures[future]
+                try:
+                    res = future.result()
+                    if res["status"] == "ok":
+                        ok_cnt += 1
+                        rows_total += res["rows"]
+                    else:
+                        fail_cnt += 1
+                        if res.get("error"):
+                            logger.warning(f"[moneyflow {c6}] {res['error']}")
+                except Exception as e:
+                    fail_cnt += 1
+                    logger.warning(f"[moneyflow {c6}] {e}")
+                if i % 20 == 0 or i == total:
+                    self._progress(
+                        message=f"moneyflow 进度 {i}/{total} OK:{ok_cnt} FAIL:{fail_cnt}",
+                        current=i, total=total, current_step=c6,
+                    )
+        elapsed = round(time.time() - t0, 1)
+        msg = f"资金流 {ok_cnt}成 {fail_cnt}败 共{rows_total}行, {elapsed}s"
+        return {
+            "status": "ok" if fail_cnt == 0 else "warning",
+            "message": msg,
+            "ok": ok_cnt, "fail": fail_cnt,
+            "rows": rows_total, "elapsed_sec": elapsed,
+        }
+
+    def _sync_one(self, mr, code6: str) -> dict:
+        # 取最近 30 天（mairui 的 transaction 接口历史深度有限，30 天足够覆盖日常）
+        from datetime import datetime, timedelta
+        end = datetime.now()
+        start = end - timedelta(days=30)
+        try:
+            df = mr.fetch_moneyflow(
+                code6,
+                start.strftime("%Y-%m-%d"),
+                end.strftime("%Y-%m-%d"),
+            )
+            if df is None or df.empty:
+                return {"status": "fail", "error": "no data"}
+            rows = [
+                {
+                    "stock_code": code6,
+                    "trade_date": str(r["trade_date"]),
+                    "main_net_inflow": float(r.get("main_net_inflow") or 0),
+                    "large_net_inflow": float(r.get("large_net_inflow") or 0),
+                    "medium_net_inflow": float(r.get("medium_net_inflow") or 0),
+                    "small_net_inflow": float(r.get("small_net_inflow") or 0),
+                }
+                for _, r in df.iterrows()
+            ]
+            db_ops.upsert_moneyflow(rows)
+            return {"status": "ok", "rows": len(rows)}
+        except Exception as e:
+            return {"status": "fail", "error": str(e)}

app/tasks/task_sector_features.py

@@ -0,0 +1,184 @@
+"""同步任务：行业聚合特征（衍生计算）。
+
+输入：kline_stock（日 K 线）+ industry（股票→行业映射）
+输出：
+  - sector_indices         ：行业日线（trade_date × sector_name × close × sector_amplitude）
+  - sector_features_daily  ：行业特征（+ sector_ret + ema10/20/200 + score）
+
+算法（参考 dashboard/api/services/sync/sector.py 重写，持久层改为 MySQL）：
+  1) 每只股票日涨跌幅 pct_chg = (close/prev_close - 1) * 100
+  2) 按行业聚合 sector_ret = 该行业所有股票当日 pct_chg 的均值
+  3) 行业指数 close = 100 × ∏(1 + sector_ret/100) （基点 100，复合收益）
+  4) 行业振幅 sector_amplitude = 该行业所有股票当日 (high-low)/close 的均值
+  5) EMA10/20/200  = close 的指数移动平均（adjust=False）
+  6) score: 0/1/2 — close>ema200 + 1，ema10>ema20 + 1
+
+无外部 API 调用，纯本地计算；增量逻辑以"全量重算"实现（数据量小 ~75K 行）。
+"""
+from __future__ import annotations
+
+import time
+from datetime import datetime, timedelta
+from typing import Any
+
+import pandas as pd
+from sqlalchemy import create_engine
+
+from app.core.config import settings
+from app.core.db import ops as db_ops
+from app.core.db.connection import get_mysql
+from app.core.sync.base import SyncTask
+from app.core.utils.logging import get_logger
+
+logger = get_logger("sync.sector_features")
+
+
+# 起算点：DB 里 kline_stock 最早一天 往前 5 年，或固定一个默认起点
+# （不用写死 2021-04-24，让数据自己决定；下面自动算）
+DEFAULT_START_YEARS_BACK = 5
+
+# 输出基点（行业指数 close 从多少开始）
+INDEX_BASE = 100.0
+
+
+class SyncSectorFeatures(SyncTask):
+    dataset_id = "sector_features"
+
+    def _run(
+        self,
+        *,
+        trigger_source: str = "manual",
+        codes: list[str] | None = None,
+        max_workers: int = 1,  # 本任务纯本地计算，单线程足够
+        **kwargs,
+    ) -> dict[str, Any]:
+        t0 = time.time()
+        logger.info("[sector] 启动行业聚合特征计算")
+
+        # ── 1) 拉 kline_stock（日线）只取需要的列 ──
+        # 用 SQLAlchemy engine（pymysql 裸连接 + pd.read_sql 偶发只读 1 行的 bug）
+        engine = create_engine(settings.mysql_url())
+        df_kline = pd.read_sql(
+            "SELECT stock_code, trade_date, open, high, low, `close`, volume "
+            "FROM kline_stock ORDER BY stock_code, trade_date",
+            engine,
+        )
+        if df_kline.empty:
+            return {"status": "error", "message": "kline_stock 为空"}
+        df_kline["stock_code"] = df_kline["stock_code"].astype(str).str.zfill(6)
+        df_kline["trade_date"] = pd.to_datetime(df_kline["trade_date"], errors="coerce")
+        logger.info(f"[sector] kline_stock: {len(df_kline):,} 行, {df_kline['stock_code'].nunique()} 只, "
+                    f"{df_kline['trade_date'].min().date()} ~ {df_kline['trade_date'].max().date()}")
+
+        # ── 2) 拉 industry（股票→行业映射）──
+        df_ind = pd.read_sql(
+            "SELECT code, industry_name FROM industry WHERE industry_name IS NOT NULL",
+            engine,
+        )
+        if df_ind.empty:
+            return {"status": "error", "message": "industry 表为空"}
+        df_ind["code"] = df_ind["code"].astype(str).str.zfill(6)
+        df_ind = df_ind.drop_duplicates(subset=["code"], keep="first")
+        logger.info(f"[sector] industry 映射: {len(df_ind):,} 行, {df_ind['industry_name'].nunique()} 个行业")
+
+        # ── 3) 算每只股票日涨跌幅 + 振幅 ──
+        df = df_kline.merge(df_ind, left_on="stock_code", right_on="code", how="inner")
+        df = df.sort_values(["stock_code", "trade_date"])
+        df["prev_close"] = df.groupby("stock_code")["close"].shift(1)
+        df["stock_amplitude"] = (df["high"] - df["low"]) / df["close"]
+        df = df.dropna(subset=["prev_close", "industry_name"])
+        df["pct_chg"] = (df["close"] / df["prev_close"] - 1.0) * 100.0
+        logger.info(f"[sector] 合并后: {len(df):,} 行, 行业 {df['industry_name'].nunique()} 个")
+
+        # ── 4) 按行业 + 日期聚合 ──
+        sector_daily = (
+            df.groupby(["industry_name", "trade_date"], as_index=False).agg(
+                sector_ret=("pct_chg", "mean"),
+                sector_amplitude=("stock_amplitude", "mean"),
+            )
+            .rename(columns={"industry_name": "sector_name"})
+            .sort_values(["sector_name", "trade_date"])
+            .reset_index(drop=True)
+        )
+        logger.info(f"[sector] 行业日聚合: {len(sector_daily):,} 行")
+
+        # ── 5) 算行业指数 close（基点 100，复合）──
+        sector_index = self._build_index(sector_daily)
+
+        # ── 6) EMA + score ──
+        sector_index = self._calc_ema(sector_index)
+        sector_index["score"] = sector_index.apply(
+            lambda r: self._calc_score(r["close"], r["ema10"], r["ema20"], r["ema200"]),
+            axis=1,
+        )
+
+        # 整理列名
+        out_cols = ["trade_date", "sector_name", "sector_ret", "sector_amplitude",
+                    "close", "ema10", "ema20", "ema200", "score"]
+        sector_index = sector_index[out_cols]
+        sector_index["trade_date"] = sector_index["trade_date"].dt.strftime("%Y-%m-%d")
+        # NaN → None（让 MySQL 接受 NULL）
+        sector_index = sector_index.where(pd.notnull(sector_index), None)
+
+        # ── 7) 写入两张表 ──
+        # 7a. sector_indices (4 列)
+        si_rows = sector_index[["trade_date", "sector_name", "close", "sector_amplitude"]].to_dict("records")
+        db_ops.replace_all_sector_indices(si_rows)
+        # 7b. sector_features_daily (9 列)
+        sf_rows = sector_index.to_dict("records")
+        db_ops.replace_all_sector_features(sf_rows)
+
+        elapsed = round(time.time() - t0, 1)
+        msg = (
+            f"行业聚合 {sector_index['sector_name'].nunique()} 个行业 "
+            f"× {sector_index['trade_date'].nunique()} 天, "
+            f"共 {len(sector_index):,} 行, {elapsed}s"
+        )
+        logger.info(f"[sector] {msg}")
+        return {
+            "status": "ok",
+            "message": msg,
+            "sectors": int(sector_index["sector_name"].nunique()),
+            "days": int(sector_index["trade_date"].nunique()),
+            "rows": int(len(sector_index)),
+            "elapsed_sec": elapsed,
+        }
+
+    @staticmethod
+    def _build_index(sector_daily: pd.DataFrame) -> pd.DataFrame:
+        """行业指数 close = INDEX_BASE × ∏(1 + sector_ret/100)"""
+        out = []
+        for sector_name, group in sector_daily.groupby("sector_name", sort=False):
+            g = group.sort_values("trade_date").copy()
+            close_vals = []
+            cur_base = INDEX_BASE
+            for ret in g["sector_ret"].to_numpy(dtype=float):
+                if pd.isna(ret):
+                    close_vals.append(cur_base)
+                else:
+                    cur_base = cur_base * (1 + float(ret) / 100.0)
+                    close_vals.append(cur_base)
+            g["close"] = close_vals
+            out.append(g)
+        return pd.concat(out, ignore_index=True)
+
+    @staticmethod
+    def _calc_ema(sector_index: pd.DataFrame) -> pd.DataFrame:
+        """每个行业分别算 EMA10/20/200"""
+        out = []
+        for sector_name, group in sector_index.groupby("sector_name", sort=False):
+            g = group.sort_values("trade_date").copy()
+            g["ema10"] = g["close"].ewm(span=10, adjust=False).mean()
+            g["ema20"] = g["close"].ewm(span=20, adjust=False).mean()
+            g["ema200"] = g["close"].ewm(span=200, adjust=False).mean()
+            out.append(g)
+        return pd.concat(out, ignore_index=True)
+
+    @staticmethod
+    def _calc_score(close, ema10, ema20, ema200) -> int:
+        score = 0
+        if pd.notna(close) and pd.notna(ema200) and close > ema200:
+            score += 1
+        if pd.notna(ema10) and pd.notna(ema20) and ema10 > ema20:
+            score += 1
+        return int(score)

app/tasks/task_share_snapshot.py

@@ -0,0 +1,133 @@
+"""同步任务：股本快照（雪球 quote_detail）。
+
+写入两张表：
+  - stocks.total_share / float_share / share_updated_at（基础信息表）
+  - share（独立股本时序表）
+"""
+from __future__ import annotations
+
+import os
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from typing import Any
+
+from app.core.config import settings
+from app.core.db import ops as db_ops
+from app.core.datasource.base import registry as ds_registry
+from app.core.datasource.registry import is_source_ready
+from app.core.datasource.utils import is_a_share_code, to_code6
+from app.core.sync.base import SyncTask
+from app.core.sync.registry import mark_sync_blocked
+from app.core.utils.logging import get_logger
+
+logger = get_logger("sync.share")
+
+
+class SyncShareSnapshot(SyncTask):
+    dataset_id = "share_snapshot"
+
+    def _run(
+        self,
+        *,
+        trigger_source: str = "manual",
+        codes: list[str] | None = None,
+        max_workers: int = 10,
+        **kwargs,
+    ) -> dict[str, Any]:
+        if not settings.xueqiu_token:
+            mark_sync_blocked(self.dataset_id, message="未配置 XUEQIU_TOKEN")
+            return {"status": "blocked", "message": "未配置 XUEQIU_TOKEN"}
+
+        xq = ds_registry.get("datasource_xueqiu")
+        if xq is None:
+            return {"status": "error", "message": "雪球数据源未注册"}
+        ok, reason = is_source_ready(xq.key)
+        if not ok:
+            mark_sync_blocked(self.dataset_id, message=f"雪球未就绪: {reason}")
+            return {"status": "blocked", "message": f"雪球未就绪: {reason}"}
+
+        if codes:
+            stock_codes = [to_code6(c) for c in codes]
+        else:
+            stock_codes = [c for c in db_ops.iter_stock_codes(active_only=True) if is_a_share_code(c)]
+        limit_raw = os.environ.get("MARKET_DATA_STOCK_LIMIT", "").strip()
+        if limit_raw.isdigit() and int(limit_raw) > 0:
+            stock_codes = stock_codes[: int(limit_raw)]
+
+        if not stock_codes:
+            return {"status": "error", "message": "无股票代码"}
+
+        # 跳过今日已刷过的
+        today = time.strftime("%Y-%m-%d")
+        existing = {s["code"]: s for s in db_ops.fetch_all_stocks()}
+        todo = []
+        for c6 in stock_codes:
+            old = existing.get(c6, {})
+            if str(old.get("share_updated_at", "") or "") == today and old.get("total_share", 0) > 0:
+                continue
+            todo.append(c6)
+        skip = len(stock_codes) - len(todo)
+        if not todo:
+            return {"status": "ok", "message": f"全部 {skip} 只已最新，跳过"}
+
+        total = len(todo)
+        ok_cnt = fail_cnt = 0
+        rows_to_share_table: list[dict] = []
+        t0 = time.time()
+        with ThreadPoolExecutor(max_workers=max_workers) as pool:
+            futures = {pool.submit(xq.fetch_share_snapshot, c6): c6 for c6 in todo}
+            for i, future in enumerate(as_completed(futures), 1):
+                c6 = futures[future]
+                try:
+                    r = future.result()
+                except Exception as e:
+                    fail_cnt += 1
+                    logger.warning(f"[share {c6}] 异常: {e}")
+                    continue
+                if r is None:
+                    fail_cnt += 1
+                    continue
+                # 写 stocks
+                hermes = self._to_hermes(c6)
+                db_ops.update_stock_share_snapshot(
+                    code=hermes,
+                    total_share=r["total_share"],
+                    float_share=r["float_share"],
+                    trade_date=r.get("trade_date", today),
+                )
+                # 也写 share 表
+                rows_to_share_table.append({
+                    "stock_code": c6,
+                    "trade_date": r.get("trade_date", today),
+                    "total_share": r["total_share"],
+                    "float_share": r["float_share"],
+                })
+                ok_cnt += 1
+                if i % 50 == 0 or i == total:
+                    self._progress(
+                        message=f"股本 {i}/{total} OK:{ok_cnt} FAIL:{fail_cnt}",
+                        current=i, total=total, current_step=c6,
+                    )
+        # 批量写 share 表
+        if rows_to_share_table:
+            try:
+                db_ops.upsert_share(rows_to_share_table)
+            except Exception as e:
+                logger.warning(f"写 share 表失败: {e}")
+
+        elapsed = round(time.time() - t0, 1)
+        msg = f"股本 {ok_cnt}成 {fail_cnt}败 {skip}跳, {elapsed}s"
+        return {
+            "status": "ok" if fail_cnt == 0 else "warning",
+            "message": msg,
+            "ok": ok_cnt, "fail": fail_cnt, "skip": skip,
+            "elapsed_sec": elapsed,
+        }
+
+    @staticmethod
+    def _to_hermes(code6: str) -> str:
+        if code6.startswith(("5", "6", "9")):
+            return f"SH{code6}"
+        if code6.startswith(("4", "8")):
+            return f"BJ{code6}"
+        return f"SZ{code6}"

app/tasks/task_stocks_basic.py

@@ -0,0 +1,221 @@
+"""同步任务：全市场股票基础信息 + 股本快照。
+
+数据流：
+  麦蕊智数 hslt/list（主）→ Baostock.query_stock_basic()（降级）→ 解析 → upsert stocks
+  可选：雪球 quote_detail 刷新股本（需要 XUEQIU_TOKEN）
+"""
+from __future__ import annotations
+
+import re
+import threading
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from typing import Any
+
+from app.core.config import settings
+from app.core.db import ops as db_ops
+from app.core.datasource.base import registry as ds_registry
+from app.core.sync.base import SyncTask
+from app.core.sync.registry import mark_sync_blocked
+from app.core.utils.logging import get_logger
+
+logger = get_logger("sync.stocks_basic")
+
+
+def _derive_listing_status(name: str) -> str:
+    n = name.strip()
+    if n.startswith("*ST") or n.startswith("ST"):
+        return "st"
+    return "normal"
+
+
+class SyncStocksBasic(SyncTask):
+    dataset_id = "stock_basic"
+
+    def _run(self, *, trigger_source: str = "manual", with_share: bool = True, **kwargs) -> dict[str, Any]:
+        self._progress(message="选择股票基础信息数据源...")
+
+        rows: list[dict] = []
+        source_name = ""
+
+        # 1. 优先使用麦蕊智数
+        mairui = ds_registry.get("datasource_mairui")
+        if mairui is not None:
+            ok, msg = mairui.is_available()
+            if ok:
+                self._progress(message="使用麦蕊智数获取股票列表...")
+                rows = mairui.fetch_stock_list()
+                if rows:
+                    source_name = "麦蕊智数"
+                    self._progress(message=f"麦蕊智数返回 {len(rows)} 只股票")
+
+        # 2. 无数据则报错
+        if not rows:
+            return {"status": "error", "message": "麦蕊智数未返回股票数据"}
+
+        self._progress(
+            message=f"获取到 {len(rows)} 只 A 股，开始写入...",
+            current=0, total=len(rows),
+        )
+
+        new_cnt = upd_cnt = unchanged = 0
+        existing = {s["code"]: s for s in db_ops.fetch_all_stocks()}
+
+        # 准备批量写入（性能：executemany 一次写 500 条，比单条快 10-20 倍）
+        bulk_rows = []
+        for info in rows:
+            code = info["code"]
+            old = existing.get(code)
+            if old is None:
+                new_cnt += 1
+            elif (old.get("name") != info["name"]
+                  or old.get("exchange") != info["exchange"]
+                  or old.get("listing_status", "normal") != info["listing_status"]):
+                upd_cnt += 1
+            else:
+                unchanged += 1
+            bulk_rows.append({
+                "code": code,
+                "name": info["name"],
+                "exchange": info["exchange"],
+                "listing_status": info["listing_status"],
+            })
+
+        # 分块 executemany 写入 + 进度更新
+        CHUNK = 500
+        for i in range(0, len(bulk_rows), CHUNK):
+            db_ops.upsert_stocks_bulk(bulk_rows[i: i + CHUNK])
+            done = min(i + CHUNK, len(bulk_rows))
+            if done % 500 == 0 or done == len(bulk_rows):
+                self._progress(
+                    message=f"写入数据库 {done}/{len(bulk_rows)}...",
+                    current=done, total=len(bulk_rows),
+                    current_step=bulk_rows[done - 1]["code"],
+                )
+
+        # 退市检测
+        delisted = 0
+        api_codes = {r["code"] for r in rows}
+        for code, old in existing.items():
+            if code not in api_codes and old.get("listing_status") != "delisted":
+                db_ops.upsert_stock(
+                    code=code,
+                    name=old.get("name", ""),
+                    exchange=old.get("exchange", ""),
+                    list_date=old.get("list_date", ""),
+                    listing_status="delisted",
+                    industry=old.get("industry", ""),
+                )
+                delisted += 1
+
+        msg = f"股票基础信息 {new_cnt}新 {upd_cnt}更 {unchanged}无变化 {delisted}退市"
+        self._progress(message=msg, current=len(rows), total=len(rows))
+
+        # 股本快照（可选 + 需要 token）
+        share_msg = ""
+        if with_share and settings.xueqiu_token:
+            self._progress(message="股票基础信息完成，开始刷新股本快照...")
+            xq = ds_registry.get("datasource_xueqiu")
+            if xq is not None:
+                try:
+                    share_ok, share_skip, share_fail = self._refresh_shares(
+                        xq, [r["code"] for r in rows]
+                    )
+                    share_msg = f"；股本 {share_ok}成 {share_skip}跳 {share_fail}败"
+                except Exception as e:
+                    share_msg = f"；股本刷新异常: {e}"
+        else:
+            if with_share and not settings.xueqiu_token:
+                share_msg = "；未配置 XUEQIU_TOKEN，跳过股本刷新"
+                mark_sync_blocked(self.dataset_id, message=msg + share_msg)
+
+        return {
+            "status": "ok" if not share_msg.startswith("；未配置") else "blocked",
+            "source": source_name,
+            "message": f"[{source_name}] {msg}{share_msg}",
+            "total": len(rows),
+            "new": new_cnt,
+            "updated": upd_cnt,
+            "delisted": delisted,
+            "unchanged": unchanged,
+        }
+
+    def _fetch_with_timeout(self, bs, *, timeout: int) -> list[dict]:
+        result: list = [None]
+        exc: list = [None]
+
+        def _do():
+            try:
+                result[0] = bs.fetch_stock_basic()
+            except Exception as e:
+                exc[0] = e
+
+        t = threading.Thread(target=_do, daemon=True)
+        t.start()
+        t.join(timeout)
+        if t.is_alive():
+            logger.warning(f"Baostock 拉取超时（{timeout}s）")
+            return []
+        if exc[0]:
+            raise exc[0]
+        return result[0] or []
+
+    def _refresh_shares(
+        self, xq, codes: list[str], max_workers: int = 10,
+    ) -> tuple[int, int, int]:
+        code6_list = []
+        for c in codes:
+            c6 = re.sub(r"^(SH|SZ|BJ)", "", c)
+            if c6.isdigit() and len(c6) == 6:
+                code6_list.append(c6)
+        code6_list = sorted(set(code6_list))
+
+        today = time.strftime("%Y-%m-%d")
+        existing = {s["code"]: s for s in db_ops.fetch_all_stocks()}
+        todo = []
+        for c6 in code6_list:
+            old = existing.get(c6, {})
+            if str(old.get("share_updated_at", "") or "") == today and old.get("total_share", 0) > 0:
+                continue
+            todo.append(c6)
+        if not todo:
+            return 0, len(code6_list), 0
+
+        ok = skip = fail = 0
+        with ThreadPoolExecutor(max_workers=max_workers) as pool:
+            futures = {pool.submit(xq.fetch_share_snapshot, c6): c6 for c6 in todo}
+            done = 0
+            for future in as_completed(futures):
+                c6 = futures[future]
+                done += 1
+                try:
+                    r = future.result()
+                except Exception as e:
+                    fail += 1
+                    logger.warning(f"[share {c6}] 异常: {e}")
+                    continue
+                if r is None:
+                    fail += 1
+                else:
+                    db_ops.update_stock_share_snapshot(
+                        code=self._to_hermes(c6),
+                        total_share=r["total_share"],
+                        float_share=r["float_share"],
+                        trade_date=r.get("trade_date", today),
+                    )
+                    ok += 1
+                if done % 100 == 0:
+                    self._progress(
+                        message=f"刷新股本 {done}/{len(todo)}...",
+                        current=done, total=len(todo), current_step=c6,
+                    )
+        skip = len(code6_list) - len(todo)
+        return ok, skip, fail
+
+    @staticmethod
+    def _to_hermes(code6: str) -> str:
+        if code6.startswith(("5", "6", "9")):
+            return f"SH{code6}"
+        if code6.startswith(("4", "8")):
+            return f"BJ{code6}"
+        return f"SZ{code6}"

bin/run_remaining.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+# 跑剩余三个任务：moneyflow → share_snapshot → market_regime
+# 跳过 kline_5min（全量回填 6 年太慢，下次有空再补）
+set -e
+cd "$(dirname "$0")/.."
+export PYTHONPATH="$(pwd)"
+mkdir -p logs
+
+# 先清理上次可能留下的卡死记录（recover_interrupted_syncs 启动时自动处理，但显式更稳）
+echo "=== 启动时间: $(date) ===" | tee logs/runall_remaining.log
+
+for TASK in moneyflow share_snapshot market_regime; do
+  echo "" | tee -a logs/runall_remaining.log
+  echo "=== [$TASK] 开始 $(date) ===" | tee -a logs/runall_remaining.log
+  T0=$(date +%s)
+  if .venv/bin/python -m app.entrypoints.cli sync "$TASK" 2>&1 | tee -a logs/runall_remaining.log; then
+    T1=$(date +%s)
+    echo "=== [$TASK] 完成，耗时 $((T1 - T0))s ===" | tee -a logs/runall_remaining.log
+  else
+    T1=$(date +%s)
+    echo "=== [$TASK] 失败，耗时 $((T1 - T0))s ===" | tee -a logs/runall_remaining.log
+    exit 1
+  fi
+done
+
+echo "" | tee -a logs/runall_remaining.log
+echo "=== 全部完成 $(date) ===" | tee -a logs/runall_remaining.log

bin/runall_once.py

@@ -0,0 +1,81 @@
+"""一次性跑完所有 sync task（按依赖顺序）。
+
+跳过 industry_sector（已 ok）。
+按顺序触发：stock_basic → kline_daily → kline_index → kline_5min → moneyflow
+          → share_snapshot → market_regime
+
+每个 task 跑完写日志 + 把 result 落到 summary.json。
+"""
+from __future__ import annotations
+
+import json
+import sys
+import time
+from pathlib import Path
+
+_PROJECT_ROOT = Path(__file__).resolve().parent.parent
+if str(_PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(_PROJECT_ROOT))
+
+from app.core.utils.logging import setup_logging, get_logger
+setup_logging()
+logger = get_logger("runall")
+
+from app.core.datasource.registry import build_default_registry
+from app.core.sync.registry import seed_sync_registry
+build_default_registry()
+seed_sync_registry()
+
+from app.tasks import get_task
+
+# 顺序执行，便于排查
+TASKS = [
+    "stock_basic",
+    "industry_sector",  # 已 ok，但再跑一次全量也 OK (用 SKIP 标志)
+    "kline_index",
+    "kline_daily",
+    "kline_5min",
+    "moneyflow",
+    "share_snapshot",
+    "market_regime",
+]
+
+# industry_sector 跑过一次（291s 全量）已 ok，跳过
+SKIP = {"industry_sector"}
+
+results = {}
+t_all = time.time()
+for tid in TASKS:
+    if tid in SKIP:
+        results[tid] = {"status": "skipped", "message": "已 ok"}
+        logger.info(f"[runall] {tid} 跳过（已 ok）")
+        continue
+    logger.info(f"[runall] >>> 开始 {tid}")
+    t0 = time.time()
+    try:
+        task = get_task(tid)
+        # 大表 task 限小批量股票，避免跑爆；其他 task 跑全量
+        kwargs = {"max_workers": 5}
+        if tid in {"kline_daily", "kline_5min", "moneyflow", "share_snapshot"}:
+            # 这些 task 支持 MARKET_DATA_STOCK_LIMIT 环境变量，但通过 cli 跑可 --codes 限
+            # 测全量费时，先跑全量
+            pass
+        r = task.run(trigger_source="runall", **kwargs)
+        elapsed = round(time.time() - t0, 1)
+        r["elapsed_sec"] = elapsed
+        results[tid] = r
+        logger.info(f"[runall] <<< {tid} 完成 status={r.get('status')} elapsed={elapsed}s msg={r.get('message')}")
+    except Exception as e:
+        elapsed = round(time.time() - t0, 1)
+        logger.exception(f"[runall] !!! {tid} 异常: {e}")
+        results[tid] = {"status": "error", "message": str(e), "elapsed_sec": elapsed}
+
+    # 每个 task 之间 sleep 5s 让健康监控跑
+    time.sleep(5)
+
+elapsed_total = round(time.time() - t_all, 1)
+summary = {"total_elapsed_sec": elapsed_total, "tasks": results}
+out = _PROJECT_ROOT / "logs" / "runall_summary.json"
+out.write_text(json.dumps(summary, ensure_ascii=False, indent=2))
+logger.info(f"[runall] 全部完成 total={elapsed_total}s summary={out}")
+print(json.dumps(summary, ensure_ascii=False, indent=2))

bin/structured_watch.sh

@@ -0,0 +1,72 @@
+#!/bin/bash
+# 结构化 watch 脚本（按 structured-watch-plan 规则）
+# 用法：structured_watch.sh <pgrep_pattern> <log_file> <output_file> [task_label]
+# 规则：1m×5 / 5m×2 / 10m×2 / 30m×2 / 60m×2 + 结束自停
+set -u
+
+PATTERN="${1:-app.entrypoints.cli}"
+LOG_FILE="${2:-/tmp/moneyflow_full.log}"
+OUT="${3:-/tmp/structured_watch.log}"
+LABEL="${4:-$PATTERN}"
+
+# 关键：pgrep -f 模式会**匹配到 watch 自己**（watch 命令行里有完整 PATTERN），
+# 任务结束后 watch 还认为"还活着"不会退出。
+# 解决：watch 用 `[p]ython` 风格 trick 排除自己（bash 进程没 'python' 字样），
+# 或者更直接：只匹配 python 进程。
+WATCH_BASENAME=$(basename "$0")
+# 构造一个绝不会匹配 watch 自己的模式：
+# 如果原 PATTERN 含 "python" 就保持，否则前缀 [p]ython 让 watch 自身不匹配
+case "$PATTERN" in
+  *python*) PGREP_PATTERN="$PATTERN" ;;
+  *)        PGREP_PATTERN="[p]ython.*$PATTERN" ;;
+esac
+
+# 把输出重定向到 OUT，system 通知时我能读到
+exec > "$OUT" 2>&1
+
+# schedule: 秒数 + 标签
+SCHEDULE_SEC=(60 60 60 60 60 300 300 600 600 1800 1800 3600 3600)
+SCHEDULE_LABELS=(
+  "[1m 1/5]" "[1m 2/5]" "[1m 3/5]" "[1m 4/5]" "[1m 5/5]"
+  "[5m 1/2]" "[5m 2/2]"
+  "[10m 1/2]" "[10m 2/2]"
+  "[30m 1/2]" "[30m 2/2]"
+  "[60m 1/2]" "[60m 2/2]"
+)
+
+echo "=== structured watch 启动 ==="
+echo "  task label: $LABEL"
+echo "  pgrep pattern (orig): $PATTERN"
+echo "  pgrep pattern (safe): $PGREP_PATTERN"
+echo "  log file: $LOG_FILE"
+echo "  启动时间: $(date)"
+echo
+
+for i in "${!SCHEDULE_SEC[@]}"; do
+  sleep "${SCHEDULE_SEC[$i]}"
+  LABEL_NOW="${SCHEDULE_LABELS[$i]}"
+  if ! pgrep -f "$PGREP_PATTERN" > /dev/null 2>&1; then
+    echo "=== checkpoint $LABEL_NOW: $(date) ==="
+    echo "  任务已退出（pgrep 不到 '$PGREP_PATTERN'），停止 schedule"
+    echo
+    echo "=== 最后 30 行日志 ==="
+    tail -30 "$LOG_FILE" 2>/dev/null
+    echo
+    echo "=== structured watch 自然结束：$(date) ==="
+    exit 0
+  fi
+  echo "=== checkpoint $LABEL_NOW: $(date) ==="
+  echo "  进程数: $(pgrep -af "$PGREP_PATTERN" | wc -l)"
+  echo "  --- log tail ---"
+  tail -10 "$LOG_FILE" 2>/dev/null
+  echo
+done
+
+echo "=== structured watch 跑完 13 个 checkpoint: $(date) ==="
+echo "  任务可能还在跑（pgrep 仍能找到），继续等待 pgrep 失效触发最终通知"
+# 13 个 checkpoint 后再每秒 pgrep 检查一次（轻量），任务一结束就退出
+while pgrep -f "$PGREP_PATTERN" > /dev/null 2>&1; do
+  sleep 30
+done
+echo "=== 任务最终退出: $(date) ==="
+tail -30 "$LOG_FILE" 2>/dev/null

docs/ARCHITECTURE.md

@@ -0,0 +1,202 @@
+# Market Data Sync — 架构说明
+
+> A 股市场数据定时同步框架。多个数据源 → 交易日感知调度器 → MySQL。
+
+## 1. 目录结构
+
+```
+market_data_sync/
+├── app/                          # 业务代码（唯一 Python 包）
+│   ├── __init__.py
+│   ├── core/                     # 核心抽象层（与业务无关的基础设施）
+│   │   ├── config.py             # 统一配置（pydantic-settings + .env）
+│   │   ├── datasource/           # 数据源抽象
+│   │   │   ├── base.py           # DataSource / FetchResult / registry
+│   │   │   └── registry.py       # 健康监控 + 自动降级 + seed
+│   │   ├── sync/                 # 同步任务抽象
+│   │   │   ├── base.py           # SyncTask 基类（自动状态机）
+│   │   │   └── registry.py       # dataset_registry 状态机封装
+│   │   ├── scheduler/            # 轻量调度器
+│   │   │   └── scheduler.py      # 交易日感知 + config 表持久化
+│   │   ├── db/                   # 数据库访问层
+│   │   │   ├── connection.py     # thread-local pymysql
+│   │   │   ├── schema.py         # 13 张表 DDL
+│   │   │   └── ops.py            # CRUD + 批量 upsert
+│   │   └── utils/
+│   │       └── logging.py        # RotatingFileHandler
+│   ├── sources/                  # 数据源实现（5 个）
+│   │   ├── sina.py               # 新浪（免费 K 线）
+│   │   ├── baostock.py           # Baostock（基础信息+行业）
+│   │   ├── mairui.py             # 麦蕊智数（需 MAIRUI_LICENCE）
+│   │   ├── akshare.py            # akshare/东方财富（5min+资金流）
+│   │   └── xueqiu.py             # 雪球（股本快照）
+│   ├── tasks/                    # 同步任务实现（8 个）
+│   │   ├── __init__.py           # TASKS 注册表 + get_task()
+│   │   ├── task_stocks_basic.py
+│   │   ├── task_kline_daily.py
+│   │   ├── task_kline_index.py
+│   │   ├── task_kline_5min.py
+│   │   ├── task_moneyflow.py
+│   │   ├── task_industry_sector.py
+│   │   ├── task_share_snapshot.py
+│   │   └── task_market_regime.py
+│   ├── api/                      # FastAPI 管理接口（端口 8100）
+│   │   ├── main.py               # 入口 + 生命周期钩子
+│   │   └── routes/
+│   │       ├── health.py
+│   │       ├── sync.py           # 手动触发 / 状态查询
+│   │       ├── schedule.py       # 计划任务
+│   │       └── datasources.py    # 数据源健康度
+│   └── entrypoints/              # 进程入口
+│       ├── cli.py                # CLI（python -m app.entrypoints.cli ...）
+│       └── worker.py             # 纯调度模式（python -m app.entrypoints.worker）
+├── bin/                          # 一次性脚本
+│   └── runall_once.py            # 一键串行跑全部任务
+├── scripts/                      # 旧 shell 脚本（保留作 fallback）
+├── tests/                        # 单元测试（pytest）
+├── docs/                         # 文档
+├── data/                         # 临时数据（一般为空）
+├── logs/                         # 运行日志（RotatingFileHandler）
+├── .env / .env.example           # 配置
+├── requirements.txt
+├── README.md
+└── start.sh                      # 统一启动入口
+```
+
+## 2. 模块依赖
+
+```
+            ┌────────────────────────────────────────┐
+            │            entrypoints/cli             │  手动/CLI
+            │            entrypoints/worker          │  调度模式
+            │                  │                     │
+            │                  ▼                     │
+            │  ┌─────────────────────────────┐       │
+            │  │        api (FastAPI)        │       │  HTTP 管理
+            │  └──────────────┬──────────────┘       │
+            │                 │                      │
+            ▼                 ▼                      │
+       ┌──────────┐    ┌─────────────┐               │
+       │  tasks   │◄───│   core      │               │
+       │  (8 个)  │    │  ├ config   │               │
+       └────┬─────┘    │  ├ db       │               │
+            │          │  ├ sync     │               │
+            │          │  ├ scheduler│               │
+            │          │  └ datasource│              │
+            ▼          └──────┬──────┘               │
+       ┌──────────┐           │                      │
+       │ sources  │◄──────────┘                      │
+       │  (5 个)  │                                  │
+       └──────────┘                                  │
+            │                                         │
+            ▼                                         │
+        数据源外部 API                                 │
+```
+
+依赖方向单向：上层 → 下层。`core` 是稳定抽象，`sources` / `tasks` 是可插拔实现。
+
+## 3. 核心抽象
+
+### 3.1 DataSource (`app/core/datasource/base.py`)
+
+```python
+class DataSource:
+    key: str                       # "datasource_xxx" 全局唯一
+    name: str                      # 显示名
+    provides: list[str]            # 能力列表 ["kline_daily", "moneyflow", ...]
+    requires_credential: bool
+
+    def is_available(self) -> tuple[bool, str]: ...
+    def health_check(self) -> dict: ...
+    def fetch_xxx(self, ...) -> pd.DataFrame: ...
+```
+
+- `registry` 是全局单例 dict[str, DataSource]
+- 启动时 `build_default_registry()` 把 5 个源注册进去
+- `pick_source(capability)` 按 `provides` 找最匹配的源
+
+### 3.2 SyncTask (`app/core/sync/base.py`)
+
+```python
+class SyncTask:
+    dataset_id: str                # 子类必须设置
+
+    def run(self, *, trigger_source="manual", **kwargs) -> dict:
+        mark_sync_running(...)
+        try:
+            result = self._run(...)
+            mark_sync_success/failed(...)
+        except Exception:
+            mark_sync_failed(...)
+```
+
+子类只需实现 `_run()`，状态机会自动包装。
+
+### 3.3 Scheduler (`app/core/scheduler/scheduler.py`)
+
+- 5s tick 扫描 config 表
+- 读取 `trading_calendar_holidays` 决定是否交易日
+- 时间匹配（HH:MM 精确触发）或 interval（间隔秒数）触发
+- 启动时 `recover_interrupted_syncs()` 把卡死的 `running` 任务改 `failed`
+
+## 4. 数据流（以 kline_daily 为例）
+
+```
+scheduler tick
+  └─ 时间到 (16:00)
+      └─ ScheduleJob.run()
+          └─ tasks.get_task("kline_daily")()
+              └─ SyncKlineDaily.run()
+                  ├─ mark_sync_running(...)            ← 状态机
+                  ├─ _ensure_health_checked()          ← 拉一次数据源健康
+                  ├─ pick_source("kline_daily")        ← 选雪球(主) > 新浪 > baostock
+                  ├─ 多线程 fetch (10 workers)         ← XUEQIU_TOKEN 限速
+                  ├─ 批量 upsert (CHUNK=500)           ← pymysql executemany
+                  └─ mark_sync_success(...)            ← 状态机
+```
+
+## 5. 添加新数据源
+
+1. 在 `app/sources/` 新建 `my_source.py`
+2. 继承 `DataSource`，设置 `key` / `name` / `provides` / `requires_credential`
+3. 实现 `is_available()` / `health_check()` / 业务方法
+4. 在 `app/core/datasource/registry.py` 的 `build_default_registry()` 注册
+
+## 6. 添加新同步任务
+
+1. 在 `app/tasks/` 新建 `task_xxx.py`
+2. 继承 `SyncTask`，设置 `dataset_id`
+3. 实现 `_run()`，返回 `{"status": "ok", "message": "..."}` 或 `{"status": "warning/blocked", ...}`
+4. 在 `app/tasks/__init__.py` 的 `TASKS` 字典中加一行
+5. （可选）在 `app/core/sync/registry.py` 的 `SYNC_DEFINITIONS` 加条目
+
+## 7. 数据库表
+
+13 张表在 `app/core/db/schema.py`：
+
+| 表 | 用途 |
+|---|---|
+| `stocks` | 股票基础信息 |
+| `kline_stock` | 日 K 线 |
+| `kline_index` | 指数日 K |
+| `kline_5min` | 5min K 线 |
+| `moneyflow` | 资金流 |
+| `industry` / `sectors` / `stock_sector_map` | 行业映射 |
+| `share` | 股本快照 |
+| `market_regime_daily` | 市场情绪 |
+| `dataset_registry` | 同步任务状态机 |
+| `config` | 通用配置（调度、节假日、token 等） |
+| `indices` | 指数字典 |
+
+## 8. 配置（.env）
+
+| 变量 | 必填 | 说明 |
+|---|---|---|
+| `MYSQL_HOST/PORT/USER/PASSWORD/DATABASE` | ✓ | 主库 |
+| `MYSQL_DATABASE_BUSINESS` |  | 业务库（默认 grid_seeker_model_base） |
+| `XUEQIU_TOKEN` |  | 雪球 token（K线主源 + 股本） |
+| `MAIRUI_LICENCE` |  | 麦蕊智数 licence |
+| `HOLIDAYS_LIST` |  | A 股休市日（逗号分隔 YYYY-MM-DD） |
+| `SCHEDULER_AUTO_SEED` |  | 启动时 seed 默认计划任务 |
+| `LOG_DIR` |  | 日志目录（默认 logs） |
+| `LOG_LEVEL` |  | 日志级别（默认 INFO） |

requirements.txt

@@ -0,0 +1,31 @@
+# Web/API
+fastapi>=0.110.0
+uvicorn[standard]>=0.30.0
+pydantic>=2.0.0
+pydantic-settings>=2.0.0
+
+# Data
+pandas>=2.2.0
+numpy>=1.26.0
+pyarrow>=15.0.0
+
+# Storage
+SQLAlchemy>=2.0.0
+PyMySQL>=1.0.0
+DBUtils>=3.0.0
+cryptography>=42.0.0
+
+# HTTP / Retry
+requests>=2.30.0
+tenacity>=8.0.0
+
+# Datasources — A 股免费源
+# akshare 永远不使用（项目级决策）
+baostock>=0.9.0
+# 雪球非官方 SDK：需要 XUEQIU_TOKEN（环境变量）才能拿到股本 / 日K 复核
+# 没装 → share / kline_daily 任务静默失败（ImportError 被 except 吞）
+pysnowball>=0.1.8
+
+# Logging / Config
+PyYAML>=6.0.0
+python-dotenv>=1.0.0

scripts/run_all_sync.sh

@@ -0,0 +1,73 @@
+#!/bin/bash
+# 串行跑所有 6 个同步任务，后台模式，写到 logs/
+# 用法：./run_all_sync.sh [STEP]
+#   STEP 为空 = 跑 1-6
+#   STEP=1 只跑 stock_basic，等等
+
+set -e
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
+cd "$PROJECT_DIR"
+
+mkdir -p logs
+LOG_DIR="$PROJECT_DIR/logs"
+TIMESTAMP=$(date +%Y%m%d_%H%M%S)
+RUN_LOG="$LOG_DIR/run_all_$TIMESTAMP.log"
+
+# 读 token
+if [ -f "$PROJECT_DIR/.env" ]; then
+    set -a
+    source "$PROJECT_DIR/.env"
+    set +a
+fi
+
+# 把 .env 里的 TRADING_HOLIDAYS 转成 -- 跳过
+# XUEQIU_TOKEN 在 .env 里
+
+source .venv/bin/activate
+export PYTHONPATH="$PROJECT_DIR"
+
+LOG_PREFIX="[$TIMESTAMP]"
+
+run_step() {
+    local step_name="$1"
+    local task_id="$2"
+    local extra_args="$3"
+    local step_log="$LOG_DIR/step_${step_name}_$TIMESTAMP.log"
+
+    echo "$LOG_PREFIX === Step $step_name: $task_id ===" | tee -a "$RUN_LOG"
+    echo "$LOG_PREFIX   log: $step_log" | tee -a "$RUN_LOG"
+
+    # 跑任务（前台模式，方便顺序执行 + 立即看到结果）
+    python -m app.entrypoints.cli sync "$task_id" $extra_args 2>&1 | tee "$step_log" | tail -10
+    local exit_code=${PIPESTATUS[0]}
+
+    if [ $exit_code -eq 0 ]; then
+        echo "$LOG_PREFIX ✓ Step $step_name OK" | tee -a "$RUN_LOG"
+    else
+        echo "$LOG_PREFIX ✗ Step $step_name FAIL (exit=$exit_code)" | tee -a "$RUN_LOG"
+        return $exit_code
+    fi
+}
+
+case "${1:-all}" in
+    1|stock_basic)        run_step "1_stock_basic" "stock_basic" ;;
+    2|industry_sector)    run_step "2_industry_sector" "industry_sector" ;;
+    3|kline_index)        run_step "3_kline_index" "kline_index" ;;
+    4|kline_daily)        run_step "4_kline_daily" "kline_daily" "--workers 20" ;;
+    5|share_snapshot)     run_step "5_share_snapshot" "share_snapshot" "--workers 10" ;;
+    6|market_regime)      run_step "6_market_regime" "market_regime" ;;
+    all)
+        run_step "1_stock_basic" "stock_basic" || exit 1
+        run_step "2_industry_sector" "industry_sector" || exit 1
+        run_step "3_kline_index" "kline_index" || exit 1
+        run_step "4_kline_daily" "kline_daily" "--workers 20" || exit 1
+        run_step "5_share_snapshot" "share_snapshot" "--workers 10" || exit 1
+        run_step "6_market_regime" "market_regime" || exit 1
+        echo "$LOG_PREFIX ✓ 全部 6 步完成" | tee -a "$RUN_LOG"
+        ;;
+    *)
+        echo "usage: $0 [1|2|3|4|5|6|all]"
+        exit 1
+        ;;
+esac

start.sh

@@ -0,0 +1,72 @@
+#!/bin/bash
+# market_data_sync 启动脚本
+# 用法：./start.sh         启动 API（默认端口 8100）
+#       ./start.sh worker  仅启动调度器 + 同步 worker（无 web）
+#       ./start.sh sync <task_id>  手动执行一次同步任务
+#       ./start.sh status  查看调度器 / 数据源 / 同步任务状态
+
+set -e
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+cd "$SCRIPT_DIR"
+
+# ---- venv ----
+VENV_DIR="$SCRIPT_DIR/.venv"
+if [ ! -d "$VENV_DIR" ]; then
+    echo "[market_data_sync] creating venv ..."
+    python3 -m venv "$VENV_DIR"
+fi
+source "$VENV_DIR/bin/activate"
+
+# ---- deps ----
+if [ ! -f "$VENV_DIR/.deps_installed" ] || [ requirements.txt -nt "$VENV_DIR/.deps_installed" ]; then
+    echo "[market_data_sync] installing deps ..."
+    pip install --upgrade pip -q
+    pip install -r requirements.txt -q
+    touch "$VENV_DIR/.deps_installed"
+fi
+
+# ---- .env ----
+if [ ! -f "$SCRIPT_DIR/.env" ]; then
+    echo "[market_data_sync] no .env found, copying .env.example"
+    cp "$SCRIPT_DIR/.env.example" "$SCRIPT_DIR/.env"
+    echo "[market_data_sync] ⚠️  please edit .env with your real MySQL credentials before continuing"
+fi
+
+# ---- export PYTHONPATH ----
+export PYTHONPATH="$SCRIPT_DIR${PYTHONPATH:+:$PYTHONPATH}"
+
+# ---- ensure logs dir ----
+mkdir -p "$SCRIPT_DIR/logs"
+
+# ---- dispatch ----
+MODE="${1:-api}"
+case "$MODE" in
+    api)
+        PORT="${API_PORT:-8100}"
+        HOST="${API_HOST:-0.0.0.0}"
+        echo "[market_data_sync] starting API on $HOST:$PORT"
+        exec uvicorn app.api.main:app --host "$HOST" --port "$PORT" --log-level info
+        ;;
+    worker)
+        echo "[market_data_sync] starting scheduler worker (no web)"
+        exec python -m app.entrypoints.worker
+        ;;
+    sync)
+        TASK_ID="${2:?usage: ./start.sh sync <task_id>}"
+        echo "[market_data_sync] manual sync: $TASK_ID"
+        exec python -m app.entrypoints.cli sync "$TASK_ID"
+        ;;
+    status)
+        exec python -m app.entrypoints.cli status
+        ;;
+    list)
+        exec python -m app.entrypoints.cli list
+        ;;
+    datasources)
+        exec python -m app.entrypoints.cli datasources
+        ;;
+    *)
+        echo "usage: $0 {api|worker|sync <task_id>|status|list|datasources}"
+        exit 1
+        ;;
+esac

tests/__init__.py

@@ -0,0 +1 @@
+"""tests 包标记。"""

tests/conftest.py

@@ -0,0 +1,10 @@
+"""pytest 配置：让测试可以直接 `import app`。
+
+只要把 tests/ 和 app/ 都放进 sys.path 即可。
+"""
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+if str(ROOT) not in sys.path:
+    sys.path.insert(0, str(ROOT))

tests/test_smoke.py

@@ -0,0 +1,93 @@
+"""冒烟测试：不依赖 MySQL/网络，只验证 import + 基本逻辑。
+
+跑：pytest tests/test_smoke.py -v
+"""
+import pytest
+
+
+def test_import_config():
+    from app.core.config import settings
+    assert settings is not None
+    assert hasattr(settings, "mysql_host")
+
+
+def test_import_data_source_base():
+    from app.core.datasource.base import DataSource, FetchResult, registry
+    assert DataSource is not None
+    assert FetchResult is not None
+    assert registry is not None
+    # registry 应该是 dict-like
+    assert hasattr(registry, "register")
+    assert hasattr(registry, "get")
+
+
+def test_import_sync_base():
+    from app.core.sync.base import SyncTask, _is_market_closed, _effective_sync_end
+    assert SyncTask is not None
+    assert isinstance(_is_market_closed(), bool)
+    assert isinstance(_effective_sync_end(), str)
+    # 同步任务基类不允许空 dataset_id
+    with pytest.raises(ValueError):
+        SyncTask()
+
+
+def test_tasks_registry():
+    from app.tasks import TASKS, get_task
+    assert isinstance(TASKS, dict)
+    # 注：每次新增 task 都要更新这里的数字
+    # 当前 9 个：stock_basic, kline_daily, kline_index, kline_5min, moneyflow,
+    #         industry_sector, sector_features, share_snapshot, market_regime
+    assert len(TASKS) == 9
+    # get_task 应该返回实例
+    task = get_task("kline_daily")
+    assert task.dataset_id == "kline_daily"
+
+
+def test_sync_definitions():
+    from app.core.sync.registry import SYNC_DEFINITIONS
+    assert len(SYNC_DEFINITIONS) == 9
+    # 所有 dataset_id 应唯一
+    ids = [d["dataset_id"] for d in SYNC_DEFINITIONS]
+    assert len(set(ids)) == 9
+
+
+def test_build_default_registry():
+    """不连接网络，只看是否能完成注册（部分源 may not available 不影响注册）。"""
+    from app.core.datasource.base import registry
+    from app.core.datasource.registry import build_default_registry
+    build_default_registry()
+    keys = [s.key for s in registry.all()]
+    assert "datasource_xinlang" in keys
+    assert "datasource_baostock" in keys
+    assert "datasource_mairui" in keys
+    # akshare 已移除（项目级决策：永远不用）
+    assert "datasource_akshare" not in keys
+    assert "datasource_xueqiu" in keys
+
+
+def test_data_source_abstract():
+    """DataSource 抽象方法未实现时不应能被直接实例化。"""
+    from app.core.datasource.base import DataSource
+    with pytest.raises(TypeError):
+        DataSource()  # 抽象方法未实现
+
+
+def test_api_health_route():
+    """FastAPI 加载 + /api/health 路由可访问。"""
+    from fastapi.testclient import TestClient
+    from app.api.main import app
+
+    client = TestClient(app)
+    resp = client.get("/api/health")
+    assert resp.status_code == 200
+    body = resp.json()
+    assert body.get("status") == "ok"
+
+
+def test_effective_sync_end_format():
+    from app.core.sync.base import _effective_sync_end
+    s = _effective_sync_end()
+    # YYYY-MM-DD 共 10 字符
+    assert len(s) == 10
+    assert s[4] == "-"
+    assert s[7] == "-"