From 05635b76b92f63189393ab54a6bcdfec2fd5d840 Mon Sep 17 00:00:00 2001 From: gao Date: Mon, 15 Jun 2026 15:36:09 +0800 Subject: [PATCH] =?UTF-8?q?chore:=20=E9=87=8D=E6=9E=84=E5=89=8D=E5=9F=BA?= =?UTF-8?q?=E7=BA=BF=20=E2=80=94=209=20=E4=B8=AA=20sync=20task=20=E5=85=A8?= =?UTF-8?q?=E9=83=A8=20ok=20+=20akshare=20=E7=A7=BB=E9=99=A4=20+=20mairui?= =?UTF-8?q?=20=E8=B5=84=E9=87=91=E6=B5=81=E6=8E=A5=E5=85=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 状态： - 9 个 sync task（stock_basic / kline_daily / kline_index / kline_5min / moneyflow / industry_sector / sector_features / share_snapshot / market_regime） - 数据源：baostock + mairui + 雪球(pysnowball) + 新浪（4 个） - 项目级约束：永远不用 akshare（已落实） - kline_5min 改用 DB 快照统一全量/增量逻辑 - 零后端 Chrome 扩展 xueqiu_sync（独立项目） --- .env.example | 38 ++ .gitignore | 27 ++ README.md | 77 ++++ app/__init__.py | 2 + app/api/__init__.py | 1 + app/api/main.py | 94 ++++ app/api/routes/__init__.py | 1 + app/api/routes/datasources.py | 47 ++ app/api/routes/health.py | 12 + app/api/routes/schedule.py | 68 +++ app/api/routes/sync.py | 82 ++++ app/core/config.py | 126 ++++++ app/core/datasource/__init__.py | 22 + app/core/datasource/base.py | 108 +++++ app/core/datasource/registry.py | 250 ++++++++++ app/core/datasource/utils.py | 115 +++++ app/core/db/__init__.py | 4 + app/core/db/connection.py | 104 +++++ app/core/db/ops.py | 728 ++++++++++++++++++++++++++++++ app/core/db/schema.py | 264 +++++++++++ app/core/scheduler/__init__.py | 20 + app/core/scheduler/scheduler.py | 392 ++++++++++++++++ app/core/sync/__init__.py | 24 + app/core/sync/base.py | 141 ++++++ app/core/sync/registry.py | 249 ++++++++++ app/core/utils/__init__.py | 4 + app/core/utils/logging.py | 60 +++ app/entrypoints/__init__.py | 5 + app/entrypoints/cli.py | 132 ++++++ app/entrypoints/worker.py | 73 +++ app/sources/__init__.py | 5 + app/sources/baostock.py | 159 +++++++ app/sources/mairui.py | 379 ++++++++++++++++ app/sources/sina.py | 137 ++++++ app/sources/xueqiu.py | 203 +++++++++ app/tasks/__init__.py | 39 ++ app/tasks/task_industry_sector.py | 122 +++++ app/tasks/task_kline_5min.py | 203 +++++++++ app/tasks/task_kline_daily.py | 251 ++++++++++ app/tasks/task_kline_index.py | 109 +++++ app/tasks/task_market_regime.py | 113 +++++ app/tasks/task_moneyflow.py | 117 +++++ app/tasks/task_sector_features.py | 184 ++++++++ app/tasks/task_share_snapshot.py | 133 ++++++ app/tasks/task_stocks_basic.py | 221 +++++++++ bin/run_remaining.sh | 27 ++ bin/runall_once.py | 81 ++++ bin/structured_watch.sh | 72 +++ docs/ARCHITECTURE.md | 202 +++++++++ requirements.txt | 31 ++ scripts/run_all_sync.sh | 73 +++ start.sh | 72 +++ tests/__init__.py | 1 + tests/conftest.py | 10 + tests/test_smoke.py | 93 ++++ 55 files changed, 6307 insertions(+) create mode 100644 .env.example create mode 100644 .gitignore create mode 100644 README.md create mode 100644 app/__init__.py create mode 100644 app/api/__init__.py create mode 100644 app/api/main.py create mode 100644 app/api/routes/__init__.py create mode 100644 app/api/routes/datasources.py create mode 100644 app/api/routes/health.py create mode 100644 app/api/routes/schedule.py create mode 100644 app/api/routes/sync.py create mode 100644 app/core/config.py create mode 100644 app/core/datasource/__init__.py create mode 100644 app/core/datasource/base.py create mode 100644 app/core/datasource/registry.py create mode 100644 app/core/datasource/utils.py create mode 100644 app/core/db/__init__.py create mode 100644 app/core/db/connection.py create mode 100644 app/core/db/ops.py create mode 100644 app/core/db/schema.py create mode 100644 app/core/scheduler/__init__.py create mode 100644 app/core/scheduler/scheduler.py create mode 100644 app/core/sync/__init__.py create mode 100644 app/core/sync/base.py create mode 100644 app/core/sync/registry.py create mode 100644 app/core/utils/__init__.py create mode 100644 app/core/utils/logging.py create mode 100644 app/entrypoints/__init__.py create mode 100644 app/entrypoints/cli.py create mode 100644 app/entrypoints/worker.py create mode 100644 app/sources/__init__.py create mode 100644 app/sources/baostock.py create mode 100644 app/sources/mairui.py create mode 100644 app/sources/sina.py create mode 100644 app/sources/xueqiu.py create mode 100644 app/tasks/__init__.py create mode 100644 app/tasks/task_industry_sector.py create mode 100644 app/tasks/task_kline_5min.py create mode 100644 app/tasks/task_kline_daily.py create mode 100644 app/tasks/task_kline_index.py create mode 100644 app/tasks/task_market_regime.py create mode 100644 app/tasks/task_moneyflow.py create mode 100644 app/tasks/task_sector_features.py create mode 100644 app/tasks/task_share_snapshot.py create mode 100644 app/tasks/task_stocks_basic.py create mode 100755 bin/run_remaining.sh create mode 100644 bin/runall_once.py create mode 100755 bin/structured_watch.sh create mode 100644 docs/ARCHITECTURE.md create mode 100644 requirements.txt create mode 100755 scripts/run_all_sync.sh create mode 100755 start.sh create mode 100644 tests/__init__.py create mode 100644 tests/conftest.py create mode 100644 tests/test_smoke.py diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..98da698 --- /dev/null +++ b/.env.example @@ -0,0 +1,38 @@ +# ============== MySQL ============== +# 主数据库（market_data_sync 写入的目标库） +MYSQL_HOST=127.0.0.1 +MYSQL_PORT=3306 +MYSQL_USER=root +MYSQL_PASSWORD= +MYSQL_DATABASE=market_data_sync_db + +# 简版库（对照测试，可选） +MYSQL_DATABASE_LITE=grid_seeker + +# 业务主库（保留 — 同步任务不再写入） +MYSQL_DATABASE_BUSINESS=grid_seeker_model_base + +# ============== 调度 ============== +SCHEDULER_TICK_SECONDS=5 +SCHEDULER_TIMEZONE=Asia/Shanghai +SCHEDULER_AUTO_SEED=true + +# ============== API 服务 ============== +API_HOST=0.0.0.0 +API_PORT=8100 + +# ============== 日志 ============== +LOG_DIR=logs +LOG_LEVEL=INFO + +# ============== 数据源开关 ============== +DS_BAOSTOCK_ENABLED=true +DS_SINA_ENABLED=true + +# ============== 雪球 ============== +# K 线主源 + 股本快照 +XUEQIU_TOKEN= + +# ============== 交易日历 ============== +# A 股休市日（逗号分隔 YYYY-MM-DD，pydantic 字段名是 trading_holidays） +TRADING_HOLIDAYS= diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..bb67e33 --- /dev/null +++ b/.gitignore @@ -0,0 +1,27 @@ +# Python +.venv/ +__pycache__/ +*.pyc +*.pyo +*.pyd +.pytest_cache/ +*.egg-info/ +build/ +dist/ + +# Logs & runtime data +logs/ +data/ +*.log + +# Environment / secrets +.env +.env.local + +# IDE +.vscode/ +.idea/ +.DS_Store + +# Market data caches +data_uat/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..a1ba5cc --- /dev/null +++ b/README.md @@ -0,0 +1,77 @@ +# market_data_sync + +A 股市场数据**定时同步与管理**框架。 + +把 `akshare` / `baostock` / 新浪 / 雪球 / 麦蕊智数 等多个数据源接入到 MySQL 库，提供： +- 后台线程调度器（time-based + interval-based，交易日感知） +- 多数据源 + 自动降级 + 健康度监控 +- `dataset_registry` 同步状态机（运行中 / 成功 / 失败 / 中断恢复） +- FastAPI 管理接口（手动触发、状态查询、失败重试） +- CLI 入口（手动跑任务、查看状态） + +> 📖 详细架构说明见 [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md) + +## 快速开始 + +```bash +# 1. 准备环境（首次） +cp .env.example .env # 编辑 .env 填写真实 MySQL 凭据 + XUEQIU_TOKEN +chmod +x start.sh +./start.sh # 自动创建 .venv + 装依赖 + 启动 API（http://localhost:8100） + +# 2. 常用命令 +./start.sh worker # 仅启动调度器（无 web） +./start.sh list # 列出所有同步任务 +./start.sh sync kline_daily # 手动触发一次（dataset_id 即可） +./start.sh status # 查看调度器/数据源/同步任务状态 +./start.sh datasources # 查看数据源健康度 +./start.sh reseed # 重新 seed 默认配置到 config 表 + +# 3. 一键全量同步（首次接入） +python bin/runall_once.py + +# 4. 跑测试 +.venv/bin/pytest tests/ -v +``` + +## 项目结构 + +``` +app/ +├── core/ # 核心抽象（config / db / sync / scheduler / datasource base） +├── sources/ # 数据源实现（5 个） +├── tasks/ # 同步任务实现（8 个） +├── api/ # FastAPI 管理接口 +└── entrypoints/ # 进程入口（cli + worker） +bin/ # 一次性脚本 +docs/ # 架构文档 +tests/ # 单元测试 +``` + +## 内置同步任务 + +| dataset_id | 触发时间 | 说明 | +|---|---|---| +| `stock_basic` | 交易日 09:00 | 全市场股票基础信息 + 股本 | +| `kline_daily` | 交易日 15:40 | 增量日 K 线（OHLCV，多源交叉） | +| `kline_index` | 交易日 15:50 | 六大指数日线 | +| `kline_5min` | 交易日 16:00 | 增量 5 分钟 K 线 | +| `moneyflow` | 交易日 16:30 | 资金流（主力/大/中/小单净额） | +| `industry_sector` | 周一 09:30 | 股票-行业映射（Baostock） | +| `share_snapshot` | 交易日 09:30 | 股本快照（雪球） | +| `market_regime` | 交易日 16:00 | 市场情绪（衍生源） | + +## 添加新数据源 + +1. 在 `app/sources/` 新建 `my_source.py`，继承 `DataSource` +2. 在 `app/core/datasource/registry.py` 的 `build_default_registry()` 注册 + +## 添加新同步任务 + +1. 在 `app/tasks/` 新建 `task_xxx.py`，继承 `SyncTask`，设置 `dataset_id` +2. 在 `app/tasks/__init__.py` 的 `TASKS` 字典注册 +3. （可选）在 `app/core/sync/registry.py` 的 `SYNC_DEFINITIONS` 加条目 + +## License + +私有 diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000..4f04de1 --- /dev/null +++ b/app/__init__.py @@ -0,0 +1,2 @@ +"""market_data_sync package.""" +__version__ = "0.1.0" diff --git a/app/api/__init__.py b/app/api/__init__.py new file mode 100644 index 0000000..3dacb8e --- /dev/null +++ b/app/api/__init__.py @@ -0,0 +1 @@ +"""API 子包入口。""" diff --git a/app/api/main.py b/app/api/main.py new file mode 100644 index 0000000..0d16bb5 --- /dev/null +++ b/app/api/main.py @@ -0,0 +1,94 @@ +"""FastAPI 入口 + 启动钩子。""" +from __future__ import annotations + +import json +import os +import sys +from pathlib import Path + +# 确保项目根在 sys.path +_PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent +if str(_PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(_PROJECT_ROOT)) + +from fastapi import FastAPI +from fastapi.middleware.cors import CORSMiddleware + +from app.core.config import settings +from app.core.utils.logging import get_logger, setup_logging +from app.api.routes import health, sync, datasources, schedule + +setup_logging() +logger = get_logger("api") + +app = FastAPI( + title="Market Data Sync API", + version="0.1.0", + description="股票市场数据定时同步与管理", +) + +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_methods=["*"], + allow_headers=["*"], +) + +app.include_router(health.router) +app.include_router(sync.router) +app.include_router(datasources.router) +app.include_router(schedule.router) + + +# ── 启动钩子 ────────────────────────────────────────────────────────── + + +@app.on_event("startup") +def _startup(): + logger.info("[startup] market_data_sync 启动") + + # 1. 注册数据源 + from app.core.datasource.registry import build_default_registry + from app.core.datasource.base import registry + build_default_registry() + logger.info(f"[startup] 已注册 {len(registry.all())} 个数据源") + + # 2. seed 数据源定义到 config 表 + from app.core.datasource.registry import seed_datasource_configs + seed_datasource_configs() + + # 3. seed 默认数据集注册 + from app.core.sync.registry import seed_sync_registry + seed_sync_registry() + + # 4. 启动时恢复卡死的 sync 任务 + from app.core.sync.registry import recover_interrupted_syncs + n = recover_interrupted_syncs() + if n > 0: + logger.warning(f"[startup] 恢复了 {n} 个中断的同步任务") + + # 5. seed 节假日 + from app.core.db import ops as db_ops + if settings.holidays_list: + db_ops.upsert_config( + "trading_calendar_holidays", + json.dumps(settings.holidays_list, ensure_ascii=False), + category="general", + description="A 股休市日", + ) + + # 6. seed 默认计划任务 + if settings.scheduler_auto_seed: + from app.core.scheduler.scheduler import seed_schedule_configs + seed_schedule_configs() + + # 7. 注册 sync job + 启动调度器 + from app.core.scheduler.scheduler import register_sync_jobs, start_scheduler + register_sync_jobs() + start_scheduler() + + # 8. 启动数据源健康监控 + from app.core.datasource.registry import start_health_monitor + start_health_monitor() + + logger.info("[startup] 完成") diff --git a/app/api/routes/__init__.py b/app/api/routes/__init__.py new file mode 100644 index 0000000..f406eb4 --- /dev/null +++ b/app/api/routes/__init__.py @@ -0,0 +1 @@ +"""routes 子包入口。""" diff --git a/app/api/routes/datasources.py b/app/api/routes/datasources.py new file mode 100644 index 0000000..1245256 --- /dev/null +++ b/app/api/routes/datasources.py @@ -0,0 +1,47 @@ +"""数据源管理：列出 / 测试 / 启用状态查询。""" +from __future__ import annotations + +from fastapi import APIRouter, HTTPException + +from app.core.datasource.base import registry as ds_registry +from app.core.datasource.registry import ( + get_health_status, + is_source_ready, + run_health_check, +) + +router = APIRouter(prefix="/api/datasources", tags=["datasources"]) + + +@router.get("") +def list_datasources(): + """列出所有注册数据源 + 状态。""" + items = [] + for src in ds_registry.all(): + ok, reason = is_source_ready(src.key) + health = next((h for h in get_health_status() if h["key"] == src.key), None) + items.append({ + "key": src.key, + "name": src.name, + "provides": src.provides, + "requiresCredential": src.requires_credential, + "credentialKey": src.credential_key, + "ready": ok, + "readyReason": reason, + "health": health, + }) + return {"datasources": items} + + +@router.post("/test/{source_key}") +def test_one(source_key: str): + if ds_registry.get(source_key) is None: + raise HTTPException(404, f"数据源 {source_key} 未注册") + result = run_health_check(source_key) + return {"result": result[0] if result else None} + + +@router.post("/test-all") +def test_all(): + results = run_health_check() + return {"results": results} diff --git a/app/api/routes/health.py b/app/api/routes/health.py new file mode 100644 index 0000000..3ee1123 --- /dev/null +++ b/app/api/routes/health.py @@ -0,0 +1,12 @@ +"""健康检查路由。""" +from __future__ import annotations + +from fastapi import APIRouter + +router = APIRouter(prefix="/api/health", tags=["health"]) + + +@router.get("") +def health_check(): + """服务存活检查。""" + return {"status": "ok", "service": "market_data_sync", "version": "0.1.0"} diff --git a/app/api/routes/schedule.py b/app/api/routes/schedule.py new file mode 100644 index 0000000..931749f --- /dev/null +++ b/app/api/routes/schedule.py @@ -0,0 +1,68 @@ +"""计划任务管理：列出 / 启用 / 禁用 / 改时间。""" +from __future__ import annotations + +import json + +from fastapi import APIRouter, HTTPException +from pydantic import BaseModel + +from app.core.db import ops as db_ops +from app.core.scheduler import ( + get_scheduler_status, + is_trading_day, + next_trading_day, + seed_schedule_configs, +) + +router = APIRouter(prefix="/api/schedule", tags=["schedule"]) + + +class ScheduleUpdate(BaseModel): + enabled: bool | None = None + time: str | None = None + intervalSeconds: int | None = None + condition: str | None = None + + +@router.get("") +def list_schedules(): + return {"scheduler": get_scheduler_status(), "isTradingDay": is_trading_day(), + "nextTradingDay": next_trading_day().isoformat()} + + +@router.post("/reseed") +def reseed(): + seed_schedule_configs() + return {"status": "ok"} + + +@router.patch("/{key}") +def update_schedule(key: str, body: ScheduleUpdate): + target = None + for r in db_ops.fetch_all_config(): + if r.get("key") == key and r.get("category") == "schedule": + target = r + break + if target is None: + raise HTTPException(404, f"未知 schedule key: {key}") + try: + obj = json.loads(target.get("value", "") or "{}") + except (json.JSONDecodeError, TypeError): + obj = {} + + if body.enabled is not None: + obj["enabled"] = body.enabled + if body.time is not None: + obj["time"] = body.time + if body.intervalSeconds is not None: + obj["intervalSeconds"] = body.intervalSeconds + if body.condition is not None: + if body.condition not in ("always", "trading_day"): + raise HTTPException(400, f"condition 仅支持 always / trading_day") + obj["condition"] = body.condition + + db_ops.upsert_config( + key, json.dumps(obj, ensure_ascii=False), + category="schedule", description=target.get("description", ""), + ) + return {"status": "ok", "key": key, "value": obj} diff --git a/app/api/routes/sync.py b/app/api/routes/sync.py new file mode 100644 index 0000000..818379b --- /dev/null +++ b/app/api/routes/sync.py @@ -0,0 +1,82 @@ +"""同步任务管理：手动触发 / 状态查询 / 失败重试。""" +from __future__ import annotations + +import threading +from typing import Any + +from fastapi import APIRouter, HTTPException, Query + +from app.core.sync import get_registry_status +from app.tasks import get_task, TASKS + +router = APIRouter(prefix="/api/sync", tags=["sync"]) + +# 防重入：同一 dataset_id 只能有一个 run 在跑 +_running_locks: dict[str, threading.Lock] = {} +_locks_guard = threading.Lock() + + +def _get_lock(dataset_id: str) -> threading.Lock: + with _locks_guard: + if dataset_id not in _running_locks: + _running_locks[dataset_id] = threading.Lock() + return _running_locks[dataset_id] + + +@router.get("/tasks") +def list_tasks(): + return {"tasks": list(TASKS.keys())} + + +@router.get("/registry") +def registry_status(): + return {"registry": get_registry_status()} + + +@router.get("/registry/{dataset_id}") +def registry_one(dataset_id: str): + rows = get_registry_status() + for r in rows: + if r["dataset_id"] == dataset_id: + return r + raise HTTPException(404, f"未知 dataset_id: {dataset_id}") + + +@router.post("/run/{dataset_id}") +def run_task(dataset_id: str, force: bool = Query(False)): + """同步触发一次。如果已在跑且 force=False 则返回 409。""" + if dataset_id not in TASKS: + raise HTTPException(404, f"未知 dataset_id: {dataset_id}，可选: {list(TASKS)}") + lk = _get_lock(dataset_id) + if not lk.acquire(blocking=False): + if not force: + raise HTTPException(409, f"{dataset_id} 正在运行中") + # force=True 时阻塞等 + lk.acquire() + + def _do(): + try: + task = get_task(dataset_id) + return task.run(trigger_source="manual") + finally: + lk.release() + + thread = threading.Thread(target=_do, daemon=True, name=f"manual-{dataset_id}") + thread.start() + return {"status": "accepted", "dataset_id": dataset_id, "trigger": "manual"} + + +@router.post("/reset/{dataset_id}") +def reset_status(dataset_id: str): + """清除 needs_resync / 错误状态，允许下次调度重试。""" + if dataset_id not in TASKS: + raise HTTPException(404, f"未知 dataset_id: {dataset_id}") + from app.core.db import ops as db_ops + db_ops.update_dataset_registry_state( + dataset_id, + status="idle", + needs_resync=0, + last_error=None, + message="手动重置", + ) + return {"status": "ok", "dataset_id": dataset_id} diff --git a/app/core/config.py b/app/core/config.py new file mode 100644 index 0000000..5b8a8b6 --- /dev/null +++ b/app/core/config.py @@ -0,0 +1,126 @@ +"""统一配置：从 .env / 环境变量读 MySQL、调度器、数据源开关等参数。""" +from __future__ import annotations + +import os +from pathlib import Path +from typing import Optional + +from pydantic_settings import BaseSettings, SettingsConfigDict + +try: + from dotenv import load_dotenv + _HAS_DOTENV = True +except ImportError: + _HAS_DOTENV = False + + +_PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent + +# 启动时自动把 .env 加载到 os.environ，使下游 `os.environ.get(...)` 代码 +# （如 app/sources/mairui.py / xueqiu.py）能拿到 credentials。 +if _HAS_DOTENV: + _env_file = _PROJECT_ROOT / ".env" + if _env_file.exists(): + load_dotenv(_env_file, override=False) + + +class Settings(BaseSettings): + """应用配置。所有字段都可以通过环境变量或 .env 文件覆盖。""" + + # MySQL + mysql_host: str = "127.0.0.1" + mysql_port: int = 3306 + mysql_user: str = "root" + mysql_password: str = "" + # 同步任务写入的目标库（原始市场数据） + mysql_database: str = "market_data_sync_db" + # 简版库（对照测试，可选） + mysql_database_lite: str = "grid_seeker" + # 业务主库（保留— 同步任务不再写入） + mysql_database_business: str = "grid_seeker_model_base" + + # Scheduler + scheduler_tick_seconds: int = 5 + scheduler_timezone: str = "Asia/Shanghai" + scheduler_auto_seed: bool = True + + # API + api_host: str = "0.0.0.0" + api_port: int = 8100 + + # Logging + log_level: str = "INFO" + log_dir: str = "./logs" + + # Datasources + ds_baostock_enabled: bool = True + ds_sina_enabled: bool = True + xueqiu_token: str = "" + + # Trading calendar + trading_holidays: str = "" + + model_config = SettingsConfigDict( + env_file=str(_PROJECT_ROOT / ".env"), + env_file_encoding="utf-8", + case_sensitive=False, + extra="ignore", + ) + + @property + def project_root(self) -> Path: + return _PROJECT_ROOT + + @property + def log_path(self) -> Path: + p = Path(self.log_dir) + if not p.is_absolute(): + p = _PROJECT_ROOT / p + p.mkdir(parents=True, exist_ok=True) + return p + + @property + def holidays_list(self) -> list[str]: + if not self.trading_holidays: + return [] + return [d.strip() for d in self.trading_holidays.split(",") if d.strip()] + + def mysql_url(self, database: Optional[str] = None) -> str: + """构造 SQLAlchemy URL。""" + db = database or self.mysql_database + return ( + f"mysql+pymysql://{self.mysql_user}:{self.mysql_password}" + f"@{self.mysql_host}:{self.mysql_port}/{db}?charset=utf8mb4" + ) + + def pymysql_connect_kwargs(self, database: Optional[str] = None) -> dict: + """构造 pymysql.connect 的关键字参数。 + + 内网自签名证书场景自动禁用 SSL。 + """ + db = database or self.mysql_database + return { + "host": self.mysql_host, + "port": self.mysql_port, + "user": self.mysql_user, + "password": self.mysql_password, + "database": db, + "charset": "utf8mb4", + "autocommit": False, + # 兼容自签名证书的内网 MySQL — 禁用 SSL + "ssl": None, + } + + +_settings: Optional[Settings] = None + + +def get_settings() -> Settings: + """全局单例。""" + global _settings + if _settings is None: + _settings = Settings() + return _settings + + +settings = get_settings() diff --git a/app/core/datasource/__init__.py b/app/core/datasource/__init__.py new file mode 100644 index 0000000..049de77 --- /dev/null +++ b/app/core/datasource/__init__.py @@ -0,0 +1,22 @@ +"""数据源层：DataSource 抽象 + 适配器 + 注册表 + 健康检查。""" +from app.core.datasource.base import DataSource, FetchResult, registry +from app.core.datasource.registry import ( + build_default_registry, + run_health_check, + get_health_status, + is_source_ready, + seed_datasource_configs, + pick_source, +) + +__all__ = [ + "DataSource", + "FetchResult", + "registry", + "build_default_registry", + "run_health_check", + "get_health_status", + "is_source_ready", + "seed_datasource_configs", + "pick_source", +] diff --git a/app/core/datasource/base.py b/app/core/datasource/base.py new file mode 100644 index 0000000..02f5d62 --- /dev/null +++ b/app/core/datasource/base.py @@ -0,0 +1,108 @@ +"""DataSource 抽象基类 + 全局注册器。 + +每个数据源实现一个 DataSource 子类，注册到全局 registry。 +fetch_* 方法返回 FetchResult，便于上层做交叉校验。""" +from __future__ import annotations + +import abc +from dataclasses import dataclass, field +from typing import Any, Callable, Optional + +import pandas as pd + + +@dataclass +class FetchResult: + """数据源拉取结果。""" + df: pd.DataFrame + source_key: str = "" + validated: bool = False + detail: str = "" + + @property + def empty(self) -> bool: + return self.df is None or self.df.empty + + +class DataSource(abc.ABC): + """数据源抽象。 + + 子类必须实现： + - key / name / provides / requires_credential + - is_available(): bool 是否就绪（凭证已配 + 通过连通性测试） + - health_check(): dict {success, message} + - fetch_* 方法：每个提供的数据类型一个 + """ + + key: str = "" + name: str = "" + provides: list[str] = [] # e.g. ["kline_daily", "stock_basic", "index_daily", "share", "industry", "kline_5min", "moneyflow"] + requires_credential: bool = False + credential_key: str = "" # e.g. "XUEQIU_TOKEN" + + @abc.abstractmethod + def is_available(self) -> tuple[bool, str]: + """返回 (是否就绪, 原因)。""" + ... + + @abc.abstractmethod + def health_check(self) -> dict[str, Any]: + """连通性测试，返回 {success: bool, message: str}。""" + ... + + # ── 通用可选方法（按 provides 类型实现）── + + def fetch_stock_basic(self) -> list[dict[str, Any]]: + """全市场股票基础信息。每条: {code, name, exchange, list_date, listing_status}""" + raise NotImplementedError + + def fetch_kline_daily(self, code6: str, start: str, end: str) -> pd.DataFrame: + """日 K 线。返回标准列：trade_date, open, high, low, close, volume""" + raise NotImplementedError + + def fetch_index_daily(self, index_code: str, start: str, end: str) -> pd.DataFrame: + """指数日 K。""" + raise NotImplementedError + + def fetch_share_snapshot(self, code6: str) -> Optional[dict[str, Any]]: + """单只股票的最新股本快照。返回 {total_share, float_share, trade_date}（单位：亿股）""" + raise NotImplementedError + + def fetch_industry_map(self) -> list[dict[str, Any]]: + """股票-行业映射。每条: {code, industry_name, industry_classification, update_date}""" + raise NotImplementedError + + def fetch_kline_5min(self, code6: str, start: str, end: str) -> pd.DataFrame: + """5 分钟 K 线。""" + raise NotImplementedError + + def fetch_moneyflow(self, code6: str, start: str, end: str) -> pd.DataFrame: + """资金流。每条: {trade_date, main_net_inflow, large_net_inflow, medium_net_inflow, small_net_inflow}""" + raise NotImplementedError + + +# ── 全局注册器（程序启动时往里塞）────────────────────────────────── + +class _GlobalRegistry: + def __init__(self): + self._sources: dict[str, DataSource] = {} + + def register(self, source: DataSource) -> None: + if not source.key: + raise ValueError(f"DataSource {source!r} missing .key") + self._sources[source.key] = source + + def get(self, key: str) -> Optional[DataSource]: + return self._sources.get(key) + + def all(self) -> list[DataSource]: + return list(self._sources.values()) + + def by_provides(self, capability: str) -> list[DataSource]: + return [s for s in self._sources.values() if capability in s.provides] + + def clear(self) -> None: + self._sources.clear() + + +registry = _GlobalRegistry() diff --git a/app/core/datasource/registry.py b/app/core/datasource/registry.py new file mode 100644 index 0000000..b620aa7 --- /dev/null +++ b/app/core/datasource/registry.py @@ -0,0 +1,250 @@ +"""数据源注册表 + 健康度监控。 + +模式照搬 dashboard/api/services/datasource_health.py，但适配新项目结构。 +""" +from __future__ import annotations + +import json +import os +import threading +import time +from datetime import datetime, timezone +from typing import Any, Optional + +from app.core.config import settings +from app.core.db import ops as db_ops +from app.core.datasource.base import DataSource, registry +from app.core.utils.logging import get_logger + +logger = get_logger("datasource_health") + + +# ── 数据源定义持久化到 config 表（datasource category）────────────────── + + +_DATASOURCE_SEED: list[tuple[str, dict, str]] = [ + ( + "datasource_xinlang", + { + "name": "新浪财经", + "provides": ["kline_daily", "index_daily"], + "requiresCredential": False, + "note": "免费 K 线主数据源", + }, + "日 K 线 + 指数 K 线主源", + ), + ( + "datasource_baostock", + { + "name": "Baostock", + "provides": ["kline_daily", "stock_basic", "industry"], + "requiresCredential": False, + "note": "免费，提供股票基础信息 + 行业映射 + K 线复核", + }, + "股票基础信息 + 行业映射 + 复核源", + ), + ( + "datasource_xueqiu", + { + "name": "雪球", + "provides": ["kline_daily", "share"], + "requiresCredential": True, + "credentialKey": "XUEQIU_TOKEN", + "note": "通过 pysnowball 补齐股本快照与 K 线复核", + }, + "股本快照（需 Token）+ 复核源", + ), + ( + "datasource_mairui", + { + "name": "麦蕊智数（mairui.club）", + "provides": ["kline_daily", "kline_5min", "index_daily"], + "requiresCredential": True, + "credentialKey": "MAIRUI_LICENCE", + "note": "K 线主源（日线 + 5min + 指数），免费 licence 1 分钟 300 次", + }, + "K 线主源（日线 + 5min + 指数）", + ), +] + + +def seed_datasource_configs() -> None: + """把数据源定义写入 config 表 category='datasource'。""" + for key, obj, desc in _DATASOURCE_SEED: + # 取已存在的 credentialValue（防止覆盖用户填的 token） + existing = next( + (r for r in db_ops.fetch_all_config() + if r.get("key") == key and r.get("category") == "datasource"), + None, + ) + if existing and not obj.get("credentialValue"): + try: + old = json.loads(existing.get("value", "") or "{}") + if old.get("credentialValue"): + obj["credentialValue"] = old["credentialValue"] + except Exception: + pass + db_ops.upsert_config( + key, json.dumps(obj, ensure_ascii=False), category="datasource", description=desc, + ) + + +def _read_datasource_configs() -> dict[str, dict]: + """从 config 表读出所有数据源定义。""" + result = {} + for r in db_ops.fetch_all_config(): + if r.get("category") != "datasource": + continue + try: + obj = json.loads(r.get("value", "") or "{}") + obj["_key"] = r["key"] + result[r["key"]] = obj + except (json.JSONDecodeError, TypeError): + continue + return result + + +# ── 健康监控状态（线程安全 dict）────────────────────────────────────── + + +_health_lock = threading.Lock() +_health_results: dict[str, dict[str, Any]] = {} +_health_started = False +_health_thread: Optional[threading.Thread] = None +_HEALTH_INTERVAL = 30 * 60 # 30 分钟 + + +def _now_iso() -> str: + return datetime.now(timezone.utc).isoformat() + + +def run_health_check(source_key: Optional[str] = None) -> list[dict[str, Any]]: + """跑一次连通性测试，更新内部缓存。 + + Args: + source_key: 跑单个；为 None 跑所有已注册的数据源。 + """ + results: list[dict[str, Any]] = [] + targets = [source_key] if source_key else [s.key for s in registry.all()] + + for key in targets: + src = registry.get(key) + if src is None: + results.append({ + "key": key, + "success": False, + "message": f"数据源 {key} 未注册", + "testedAt": _now_iso(), + }) + continue + try: + r = src.health_check() + results.append({ + "key": key, + "name": src.name, + "success": bool(r.get("success", False)), + "message": str(r.get("message", "")), + "testedAt": _now_iso(), + "provides": src.provides, + "requiresCredential": src.requires_credential, + }) + except Exception as e: + results.append({ + "key": key, + "name": getattr(src, "name", key), + "success": False, + "message": f"健康检查异常: {e}", + "testedAt": _now_iso(), + }) + + with _health_lock: + for r in results: + _health_results[r["key"]] = r + return results + + +def get_health_status(source_key: Optional[str] = None) -> list[dict[str, Any]]: + """读取最近一次健康检查结果（不重新跑）。""" + with _health_lock: + if source_key: + r = _health_results.get(source_key) + return [dict(r)] if r else [] + return [dict(_health_results[k]) for k in sorted(_health_results)] + + +def is_source_ready(source_key: str) -> tuple[bool, str]: + """综合判断：是否在 registry + 凭证已配 + 健康检查通过。""" + src = registry.get(source_key) + if src is None: + return False, f"数据源 {source_key} 未注册" + if src.requires_credential: + token = os.environ.get(src.credential_key, "").strip() + if not token: + return False, f"凭证 {src.credential_key} 未配置" + ok, reason = src.is_available() + if not ok: + return False, reason + with _health_lock: + last = _health_results.get(source_key) + if last is None: + return False, "健康检查尚未完成" + if not last.get("success"): + return False, f"连通性测试失败: {last.get('message', '未知错误')}" + return True, "就绪" + + +def pick_source(capability: str) -> Optional[DataSource]: + """从所有就绪数据源里挑一个能提供某能力的数据源，按注册顺序。""" + for s in registry.all(): + if capability in s.provides: + ok, _ = is_source_ready(s.key) + if ok: + return s + return None + + +# ── 后台线程定期跑健康检查 ────────────────────────────────────────── + + +def _health_loop() -> None: + while True: + try: + run_health_check() + logger.info("[datasource_health] 全数据源健康检查完成") + except Exception as e: + logger.error(f"[datasource_health] 健康检查异常: {e}") + time.sleep(_HEALTH_INTERVAL) + + +def start_health_monitor() -> None: + global _health_started, _health_thread + with _health_lock: + if _health_started: + return + _health_started = True + _health_thread = threading.Thread(target=_health_loop, daemon=True, name="datasource-health") + _health_thread.start() + + +# ── 启动时构建默认 registry ────────────────────────────────────────── + + +def build_default_registry() -> None: + """启动时调用：把默认 4 个数据源注册到全局 registry。""" + # 避免重复注册 + if registry.all(): + return + from app.sources.sina import SinaSource + from app.sources.baostock import BaostockSource + from app.sources.xueqiu import XueqiuSource + from app.sources.mairui import MairuiSource + + registry.register(SinaSource()) + registry.register(BaostockSource()) + registry.register(MairuiSource()) + if settings.xueqiu_token: + registry.register(XueqiuSource()) + else: + # 即使没 token 也注册，is_available() 会拦截 + registry.register(XueqiuSource()) + # akshare 已移除（项目级决策：永远不用） \ No newline at end of file diff --git a/app/core/datasource/utils.py b/app/core/datasource/utils.py new file mode 100644 index 0000000..eecfcd0 --- /dev/null +++ b/app/core/datasource/utils.py @@ -0,0 +1,115 @@ +"""数据源公共工具：代码转换、K 线标准化、日期过滤。 + +参考 dashboard/api/services/datasource/fetch_kline.py，但适配新项目。""" +from __future__ import annotations + +import re + +import pandas as pd + +KLINE_COLS = ["trade_date", "open", "high", "low", "close", "volume"] +KLINE_5MIN_COLS = ["bar_time", "open", "high", "low", "close", "volume", "amount", "turnover_rate"] + + +def to_code6(code: str) -> str: + """'SH600000' / 'sh.600000' / '600000' / 'SH600000.SH' → '600000'""" + c = str(code).strip().upper() + c = re.sub(r"^(SH|SZ|BJ)", "", c) + c = c.split(".")[0] + return c.zfill(6) + + +def code6_to_exchange(code6: str) -> str: + """6 位代码 → 交易所前缀。""" + if code6.startswith(("5", "6", "9")): + return "SH" + if code6.startswith(("4", "8")): + return "BJ" + return "SZ" + + +def code6_to_sina(code6: str) -> str: + """新浪财经 symbol：'sh600036' / 'sz000001'。""" + return f"{code6_to_exchange(code6).lower()}{code6}" + + +def code6_to_baostock(code6: str) -> str: + """Baostock 风格：'sh.600000'。""" + return f"{code6_to_exchange(code6).lower()}.{code6}" + + +def code6_to_xueqiu(code6: str) -> str: + """雪球 symbol：'SH600036'。""" + return f"{code6_to_exchange(code6)}{code6}" + + +def code6_to_mairui(code6: str) -> str: + """麦蕊智数 symbol：'600036.SH'。""" + return f"{code6}.{code6_to_exchange(code6)}" + + +def normalize_kline(df: pd.DataFrame) -> pd.DataFrame: + """标准化日 K 线 DataFrame。 + + 输入列名可能是 date / day / vol 等，输出固定为 KLINE_COLS。 + 含 OHLC 合法性校验（high >= max(o,l,c), low <= min(o,h,c)）。 + """ + if df is None or df.empty: + return pd.DataFrame(columns=KLINE_COLS) + df = df.rename(columns={"date": "trade_date", "day": "trade_date", "vol": "volume"}).copy() + missing = [c for c in KLINE_COLS if c not in df.columns] + if missing: + return pd.DataFrame(columns=KLINE_COLS) + df = df[KLINE_COLS] + df["trade_date"] = pd.to_datetime(df["trade_date"], errors="coerce").astype("datetime64[ns]") + for col in ["open", "high", "low", "close", "volume"]: + df[col] = pd.to_numeric(df[col], errors="coerce") + df = df.dropna(subset=["trade_date", "open", "high", "low", "close"]) + df["volume"] = df["volume"].fillna(0) + valid = ( + (df["open"] > 0) & (df["high"] > 0) & (df["low"] > 0) & (df["close"] > 0) + & (df["volume"] >= 0) + & (df["high"] >= df[["open", "low", "close"]].max(axis=1)) + & (df["low"] <= df[["open", "high", "close"]].min(axis=1)) + ) + df = df.loc[valid].sort_values("trade_date") + return df.drop_duplicates(subset=["trade_date"], keep="last").reset_index(drop=True) + + +def filter_date_range(df: pd.DataFrame, start: str, end: str) -> pd.DataFrame: + if df.empty: + return df + df = df[ + (df["trade_date"] >= pd.Timestamp(start)) + & (df["trade_date"] <= pd.Timestamp(end)) + ] + return df.reset_index(drop=True) + + +def normalize_5min(df: pd.DataFrame) -> pd.DataFrame: + """标准化 5 分钟 K 线。""" + if df is None or df.empty: + return pd.DataFrame(columns=KLINE_5MIN_COLS) + df = df.rename(columns={ + "时间": "bar_time", "time": "bar_time", "date": "bar_time", + "成交量": "volume", "成交额": "amount", "换手率": "turnover_rate", + }).copy() + if "bar_time" not in df.columns: + return pd.DataFrame(columns=KLINE_5MIN_COLS) + df["bar_time"] = pd.to_datetime(df["bar_time"], errors="coerce") + for col in ["open", "high", "low", "close", "volume", "amount", "turnover_rate"]: + if col in df.columns: + df[col] = pd.to_numeric(df[col], errors="coerce") + df = df.dropna(subset=["bar_time"]) + keep = ["bar_time"] + [c for c in KLINE_5MIN_COLS[1:] if c in df.columns] + df = df[keep].sort_values("bar_time").drop_duplicates(subset=["bar_time"], keep="last") + return df.reset_index(drop=True) + + +def is_a_share_code(code6: str) -> bool: + """判断 6 位代码是否是主板/创业板/科创板（排除北证 8 字头、可转债等）。""" + if not code6 or len(code6) != 6 or not code6.isdigit(): + return False + if code6.startswith(("4", "8")): + return False + return True diff --git a/app/core/db/__init__.py b/app/core/db/__init__.py new file mode 100644 index 0000000..6d7e5de --- /dev/null +++ b/app/core/db/__init__.py @@ -0,0 +1,4 @@ +"""DB 子包：连接管理 + schema + 业务 CRUD。""" +from app.core.db.connection import get_mysql, init_mysql_schema, close_mysql + +__all__ = ["get_mysql", "init_mysql_schema", "close_mysql"] diff --git a/app/core/db/connection.py b/app/core/db/connection.py new file mode 100644 index 0000000..560e30f --- /dev/null +++ b/app/core/db/connection.py @@ -0,0 +1,104 @@ +"""MySQL 连接管理（thread-local，模式照搬 dashboard，但适配 market_data_sync 项目）。 + +要点： +- pymysql 连接不是线程安全的，每个线程各持一份 +- 内网自签名证书场景：传 ssl_disabled=True 跳过 SSL 校验 +- DATE / DATETIME / TIMESTAMP 统一转字符串，避免下游类型处理 +- DictCursor：返回字典风格行，方便业务拼装 +""" +from __future__ import annotations + +import threading +from typing import Optional + +import pymysql +import pymysql.constants.FIELD_TYPE +from pymysql.converters import conversions as _pymysql_conv, convert_date, convert_datetime +from pymysql.cursors import DictCursor + +from app.core.config import settings + +_local = threading.local() + + +def _strdate(obj): + d = convert_date(obj) + return d.strftime("%Y-%m-%d") if d else None + + +def _strdatetime(obj): + d = convert_datetime(obj) + return d.strftime("%Y-%m-%d %H:%M:%S") if d else None + + +# 包装默认 converter，让 DATE / DATETIME / TIMESTAMP 一律出字符串 +_conv = _pymysql_conv.copy() +_conv[pymysql.constants.FIELD_TYPE.DATE] = _strdate +_conv[pymysql.constants.FIELD_TYPE.DATETIME] = _strdatetime +_conv[pymysql.constants.FIELD_TYPE.TIMESTAMP] = _strdatetime + + +def get_mysql(database: Optional[str] = None) -> pymysql.connections.Connection: + """取当前线程的 MySQL 连接。无则新建。 + + 关键修复：高并发场景下，连接可能被服务端超时断开 (wait_timeout)。 + 用 ping(reconnect=True) 让 pymysql 自动重连，避免 stale conn 导致 + 2013 Lost connection 错误。 + """ + attr = f"conn_{database or 'default'}" + conn = getattr(_local, attr, None) + if conn is not None: + try: + conn.ping(reconnect=True) + except Exception: + try: + conn.close() + except Exception: + pass + conn = None + if conn is None: + kwargs = dict( + host=settings.mysql_host, + port=settings.mysql_port, + user=settings.mysql_user, + password=settings.mysql_password, + database=database or settings.mysql_database, + charset="utf8mb4", + cursorclass=DictCursor, + autocommit=False, + conv=_conv, + ) + # 内网自签名证书：禁用 SSL（用户场景） + kwargs["ssl"] = None + # 高并发时设较短 read_timeout / write_timeout + kwargs["read_timeout"] = 30 + kwargs["write_timeout"] = 30 + conn = pymysql.connect(**kwargs) + setattr(_local, attr, conn) + return conn + + +def close_mysql() -> None: + """关闭当前线程的所有连接。""" + for attr in list(vars(_local).keys()): + conn = getattr(_local, attr, None) + if conn is not None: + try: + conn.close() + except Exception: + pass + try: + delattr(_local, attr) + except AttributeError: + pass + + +# ── Schema DDL 入口 ──────────────────────────────────────────────────────── + + +def init_mysql_schema() -> None: + """集中创建/补齐所有表。被 db/ops.py 在第一次访问时调用。""" + # 延后导入避免循环依赖 + from app.core.db.schema import ensure_all_tables + + ensure_all_tables(get_mysql()) diff --git a/app/core/db/ops.py b/app/core/db/ops.py new file mode 100644 index 0000000..18bb7ac --- /dev/null +++ b/app/core/db/ops.py @@ -0,0 +1,728 @@ +"""MySQL 业务 CRUD 函数集合（参考 dashboard/api/services/mysql_ops.py）。 + +本文件只包含 market_data_sync 同步任务需要的最小函数集，不涉及 dashboard 的 +accounts / positions / trade_record / scoring 等业务表。 + +设计要点： +- 每个 ensure_xxx_table() 调用会触发 init_mysql_schema()（幂等） +- 批量 upsert 走 executemany，效率高 +- 统一使用 DictCursor +""" +from __future__ import annotations + +import json +from contextlib import contextmanager +from datetime import datetime +from typing import Any, Iterable, Optional + +from app.core.db.connection import get_mysql, init_mysql_schema + + +# ── 内部 helpers ──────────────────────────────────────────────────────── + + +def _now_local_ts() -> str: + return datetime.now().strftime("%Y-%m-%d %H:%M:%S") + + +def _today_str() -> str: + return datetime.now().strftime("%Y-%m-%d") + + +def _date_or_none(val) -> str | None: + if val is None: + return None + s = str(val).strip() + return s if s else None + + +@contextmanager +def _cursor(): + """带 init_mysql_schema 的 cursor 上下文。""" + init_mysql_schema() + conn = get_mysql() + cur = conn.cursor() + try: + yield cur + conn.commit() + except Exception: + conn.rollback() + raise + finally: + cur.close() + + +# ── config 表 ─────────────────────────────────────────────────────────── + + +def fetch_all_config() -> list[dict[str, Any]]: + init_mysql_schema() + conn = get_mysql() + with conn.cursor() as cur: + cur.execute("SELECT `key`, `value`, category, description, updated_at FROM config") + return cur.fetchall() + + +def fetch_config_by_key(key: str) -> dict[str, Any] | None: + """按 key 查单条 config（比 fetch_all_config 高效）。""" + init_mysql_schema() + conn = get_mysql() + with conn.cursor() as cur: + cur.execute( + "SELECT `key`, `value`, category, description, updated_at FROM config WHERE `key` = %s", + (key,), + ) + return cur.fetchone() + + +def fetch_configs_by_category(category: str) -> list[dict[str, Any]]: + """按 category 查 config（比 fetch_all_config + 内存过滤高效）。""" + init_mysql_schema() + conn = get_mysql() + with conn.cursor() as cur: + cur.execute( + "SELECT `key`, `value`, category, description, updated_at FROM config WHERE category = %s", + (category,), + ) + return cur.fetchall() + + +def upsert_config(key: str, value: str, category: str = "general", description: str = "") -> None: + with _cursor() as cur: + cur.execute( + """ + INSERT INTO config (`key`, `value`, category, description) + VALUES (%s, %s, %s, %s) + ON DUPLICATE KEY UPDATE + `value` = VALUES(`value`), + category = VALUES(category), + description = VALUES(description) + """, + (key, value, category, description), + ) + + +def delete_config(key: str) -> None: + with _cursor() as cur: + cur.execute("DELETE FROM config WHERE `key` = %s", (key,)) + + +# ── dataset_registry ──────────────────────────────────────────────────── + + +def upsert_dataset_registry_rows(rows: list[dict[str, Any]]) -> None: + """批量 upsert 同步任务定义到 dataset_registry。""" + if not rows: + return + with _cursor() as cur: + for r in rows: + dep_json = r.get("dependency_ids") + if isinstance(dep_json, (list, dict)): + dep_json = json.dumps(dep_json, ensure_ascii=False) + cur.execute( + """ + INSERT INTO dataset_registry + (dataset_id, name, description, storage_uri, storage_layer, + management_role, source, sync_script, dependency_ids, + enabled, sort_order) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) + ON DUPLICATE KEY UPDATE + name = VALUES(name), + description = VALUES(description), + storage_uri = VALUES(storage_uri), + storage_layer = VALUES(storage_layer), + management_role = VALUES(management_role), + source = VALUES(source), + sync_script = VALUES(sync_script), + dependency_ids = VALUES(dependency_ids), + enabled = VALUES(enabled), + sort_order = VALUES(sort_order) + """, + ( + r.get("dataset_id", ""), + r.get("name", ""), + r.get("description", ""), + r.get("storage_uri", ""), + r.get("storage_layer", ""), + r.get("management_role", ""), + r.get("source", ""), + r.get("sync_script", ""), + dep_json or "[]", + r.get("enabled", 1), + r.get("sort_order", 0), + ), + ) + + +def fetch_dataset_registry_rows() -> list[dict[str, Any]]: + init_mysql_schema() + conn = get_mysql() + with conn.cursor() as cur: + cur.execute("SELECT * FROM dataset_registry ORDER BY sort_order, dataset_id") + return cur.fetchall() + + +def fetch_dataset_registry_map() -> dict[str, dict[str, Any]]: + rows = fetch_dataset_registry_rows() + return {r["dataset_id"]: r for r in rows} + + +def update_dataset_registry_state(dataset_id: str, **kwargs: Any) -> None: + """按需更新 dataset_registry 的状态字段。""" + allowed = { + "name", "description", "storage_uri", "storage_layer", "management_role", + "source", "sync_script", "dependency_ids", "enabled", "sort_order", + "status", "trigger_source", "started_at", "finished_at", + "last_success_at", "last_failure_at", "message", "last_error", + "needs_resync", "progress_current", "progress_total", "current_step", + "updated_at", + } + payload = {k: v for k, v in kwargs.items() if k in allowed} + if not payload: + return + if "updated_at" not in payload: + payload["updated_at"] = _now_local_ts() + set_clause = ", ".join(f"`{k}` = %s" for k in payload) + values = list(payload.values()) + [dataset_id] + with _cursor() as cur: + cur.execute( + f"UPDATE dataset_registry SET {set_clause} WHERE dataset_id = %s", + values, + ) + + +def recover_interrupted_dataset_registry() -> int: + """把状态卡在 running 的同步任务标记为 failed（启动时调用）。""" + with _cursor() as cur: + cur.execute( + """ + UPDATE dataset_registry + SET status = 'failed', + finished_at = %s, + last_failure_at = %s, + last_error = CONCAT('进程在状态为 running 时被中断，已自动恢复。', IFNULL(last_error, '')), + message = CONCAT('[recovered] ', IFNULL(message, '同步被中断')), + needs_resync = 1 + WHERE status = 'running' + """, + (_now_local_ts(), _now_local_ts()), + ) + return cur.rowcount + + +# ── stocks ────────────────────────────────────────────────────────────── + + +def upsert_stock( + code: str, + name: str, + exchange: str, + list_date: str = "", + listing_status: str = "normal", + industry: str = "", +) -> None: + """upsert 一只股票基础信息（单条）。 + + 注意：MySQL 9.7.0 在 ON DUPLICATE KEY UPDATE 阶段对 DATE 字段的 VALUES()/new.col + 求值存在 bug（会强制把空串塞进去，触发 1292 严格模式错误）。所以这里： + - list_date 字段在首次 INSERT 时写入；UPDATE 分支不更新（保持原值或 NULL） + - industry 字段也避开这个 bug + + 批量写入请用 upsert_stocks_bulk()，性能高 10-20 倍。 + """ + del list_date # 显式不接受 list_date 更新（旧值保留） + del industry # 同上 + with _cursor() as cur: + cur.execute( + """ + INSERT INTO stocks (code, name, exchange, listing_status) + VALUES (%s, %s, %s, %s) + ON DUPLICATE KEY UPDATE + name = VALUES(name), + exchange = VALUES(exchange), + listing_status = VALUES(listing_status) + """, + (code, name, exchange, listing_status), + ) + + +def upsert_stocks_bulk(rows: list[dict[str, Any]], chunk_size: int = 500) -> int: + """批量 upsert 股票基础信息。 + + 每条 row 需有: code, name, exchange, listing_status（可选 list_date / industry） + 性能：~5000 只股票从 ~50s 降到 ~3s。 + """ + if not rows: + return 0 + sql = ( + "INSERT INTO stocks (code, name, exchange, listing_status) " + "VALUES (%s, %s, %s, %s) " + "ON DUPLICATE KEY UPDATE " + " name = VALUES(name), " + " exchange = VALUES(exchange), " + " listing_status = VALUES(listing_status)" + ) + with _cursor() as cur: + total = 0 + for i in range(0, len(rows), chunk_size): + chunk = rows[i: i + chunk_size] + values = [ + (r.get("code", ""), r.get("name", ""), r.get("exchange", ""), r.get("listing_status", "normal")) + for r in chunk + ] + cur.executemany(sql, values) + total += len(chunk) + return total + + +def update_stock_share_snapshot(code: str, total_share: float, float_share: float, trade_date: str = "") -> None: + with _cursor() as cur: + cur.execute( + """ + UPDATE stocks + SET total_share = %s, float_share = %s, + share_updated_at = COALESCE(NULLIF(%s, ''), CURDATE()) + WHERE code = %s + """, + (float(total_share), float(float_share), trade_date or "", code), + ) + + +def update_stock_kline_synced_at(code: str, synced_at: str) -> None: + with _cursor() as cur: + cur.execute( + "UPDATE stocks SET kline_synced_at = %s WHERE code = %s", + (_date_or_none(synced_at), code), + ) + + +def fetch_all_stocks() -> list[dict[str, Any]]: + init_mysql_schema() + conn = get_mysql() + with conn.cursor() as cur: + cur.execute("SELECT * FROM stocks ORDER BY code") + return cur.fetchall() + + +def fetch_stock_by_code(code: str) -> Optional[dict[str, Any]]: + init_mysql_schema() + conn = get_mysql() + with conn.cursor() as cur: + cur.execute("SELECT * FROM stocks WHERE code = %s", (code,)) + row = cur.fetchone() + return dict(row) if row else None + + +def stock_count() -> int: + init_mysql_schema() + conn = get_mysql() + with conn.cursor() as cur: + cur.execute("SELECT COUNT(*) AS cnt FROM stocks") + row = cur.fetchone() + return int(row["cnt"]) if row else 0 + + +def iter_stock_codes(active_only: bool = True) -> Iterable[str]: + """yield 6 位股票代码（去除 SH/SZ/BJ 前缀）。""" + for s in fetch_all_stocks(): + if active_only and s.get("listing_status") == "delisted": + continue + code = str(s.get("code", "")).strip() + if not code: + continue + for prefix in ("SH", "SZ", "BJ"): + if code.startswith(prefix): + code = code[len(prefix):] + break + if code.isdigit() and len(code) == 6: + yield code + + +# ── indices ───────────────────────────────────────────────────────────── + + +def upsert_index( + index_code: str, + index_name: str, + market: str = "", + category: str = "", + source: str = "", + enabled: bool = True, +) -> None: + with _cursor() as cur: + cur.execute( + """ + INSERT INTO indices (index_code, index_name, market, category, source, enabled) + VALUES (%s, %s, %s, %s, %s, %s) + ON DUPLICATE KEY UPDATE + index_name = VALUES(index_name), + market = VALUES(market), + category = VALUES(category), + source = VALUES(source), + enabled = VALUES(enabled) + """, + (index_code, index_name, market, category, source, int(enabled)), + ) + + +# ── kline 通用批量 upsert ────────────────────────────────────────────── + + +def _bulk_upsert( + cur, + table: str, + columns: list[str], + rows: list[dict[str, Any]], + conflict_keys: list[str], + chunk_size: int = 1000, +) -> int: + """通用批量 INSERT ... ON DUPLICATE KEY UPDATE 工具。""" + if not rows: + return 0 + placeholders = ", ".join(["%s"] * len(columns)) + col_list = ", ".join(f"`{c}`" for c in columns) + update_clause = ", ".join( + f"`{c}` = VALUES(`{c}`)" + for c in columns + if c not in conflict_keys + ) + sql = ( + f"INSERT INTO {table} ({col_list}) VALUES ({placeholders}) " + f"ON DUPLICATE KEY UPDATE {update_clause}" + ) + total = 0 + for i in range(0, len(rows), chunk_size): + chunk = rows[i: i + chunk_size] + values = [ + tuple(_date_or_none(r.get(c)) if c.endswith("date") else r.get(c) for c in columns) + for r in chunk + ] + cur.executemany(sql, values) + total += len(chunk) + return total + + +def upsert_kline_stock(rows: list[dict[str, Any]]) -> int: + """rows: stock_code, trade_date, open, high, low, close, volume""" + with _cursor() as cur: + return _bulk_upsert( + cur, "kline_stock", + ["stock_code", "trade_date", "open", "high", "low", "close", "volume"], + rows, conflict_keys=["stock_code", "trade_date"], + ) + + +def upsert_kline_index(rows: list[dict[str, Any]]) -> int: + """rows: index_code, trade_date, open, high, low, close, volume""" + with _cursor() as cur: + return _bulk_upsert( + cur, "kline_index", + ["index_code", "trade_date", "open", "high", "low", "close", "volume"], + rows, conflict_keys=["index_code", "trade_date"], + ) + + +def upsert_kline_5min(rows: list[dict[str, Any]]) -> int: + """rows: stock_code, bar_time, open, high, low, close, volume, amount, turnover_rate""" + with _cursor() as cur: + return _bulk_upsert( + cur, "kline_5min", + ["stock_code", "bar_time", "open", "high", "low", "close", "volume", "amount", "turnover_rate"], + rows, conflict_keys=["stock_code", "bar_time"], + ) + + +def get_kline_5min_snapshots() -> dict[str, Optional[datetime]]: + """一次 SQL 拉 kline_5min 全表每只股票的最新 bar_time。 + + 返回：{stock_code (6位): max(bar_time) 或 None} + 用于 kline_5min 任务的"全量/增量"统一规划： + - DB 里有 → 增量（从 max - 2 天 到今天） + - DB 里没 → 全量（从 mairui 历史深度起点 到今天） + + DB 里 stock_code 形如 "000001.SZ"（mairui 写入格式），函数剥掉 + 交易所后缀统一为 6 位。bar_time 是 DATETIME 字段，pymysql DictCursor + 在某些连接参数下会返回 str，这里手动解析为 datetime（解析失败置 None）。 + """ + init_mysql_schema() + conn = get_mysql() + out: dict[str, Optional[datetime]] = {} + with conn.cursor() as cur: + cur.execute("SELECT stock_code, MAX(bar_time) AS latest FROM kline_5min GROUP BY stock_code") + for r in cur.fetchall(): + raw_code = str(r["stock_code"] or "").strip() + code6 = raw_code.split(".")[0] # "000001.SZ" → "000001" + if len(code6) != 6 or not code6.isdigit(): + continue + latest = r["latest"] + if isinstance(latest, datetime): + pass + elif isinstance(latest, str) and latest: + try: + latest = datetime.strptime(latest[:19], "%Y-%m-%d %H:%M:%S") + except ValueError: + latest = None + else: + latest = None + out[code6] = latest + return out + + +def upsert_moneyflow(rows: list[dict[str, Any]]) -> int: + """rows: stock_code, trade_date, main_net_inflow, large_net_inflow, medium_net_inflow, small_net_inflow""" + with _cursor() as cur: + return _bulk_upsert( + cur, "moneyflow", + ["stock_code", "trade_date", "main_net_inflow", "large_net_inflow", "medium_net_inflow", "small_net_inflow"], + rows, conflict_keys=["stock_code", "trade_date"], + ) + + +def upsert_share(rows: list[dict[str, Any]]) -> int: + """rows: stock_code, trade_date, total_share, float_share""" + with _cursor() as cur: + return _bulk_upsert( + cur, "share", + ["stock_code", "trade_date", "total_share", "float_share"], + rows, conflict_keys=["stock_code", "trade_date"], + ) + + +# ── 行业 / 概念板块 ───────────────────────────────────────────────────── + + +def replace_all_industries(rows: list[dict[str, Any]]) -> None: + """全量替换 industry 表。rows: code, industry_name, industry_classification, update_date""" + if not rows: + return + with _cursor() as cur: + cur.execute("DELETE FROM industry") + for r in rows: + cur.execute( + """ + INSERT INTO industry (code, industry_name, industry_classification, update_date) + VALUES (%s, %s, %s, %s) + ON DUPLICATE KEY UPDATE + industry_name = VALUES(industry_name), + industry_classification = VALUES(industry_classification), + update_date = VALUES(update_date) + """, + ( + str(r.get("code", "")).zfill(6), + r.get("industry_name") or None, + r.get("industry_classification") or None, + _date_or_none(r.get("update_date")), + ), + ) + + +def fetch_all_industries() -> list[dict[str, Any]]: + init_mysql_schema() + conn = get_mysql() + with conn.cursor() as cur: + cur.execute("SELECT code, industry_name, industry_classification, update_date FROM industry") + return cur.fetchall() + + +def replace_all_sectors(rows: list[dict[str, Any]]) -> None: + """rows: sector_key, sector_name, taxonomy, level, source, enabled""" + if not rows: + return + with _cursor() as cur: + cur.execute("DELETE FROM sectors") + for r in rows: + cur.execute( + """ + INSERT INTO sectors (sector_key, sector_name, taxonomy, level, source, enabled) + VALUES (%s, %s, %s, %s, %s, %s) + ON DUPLICATE KEY UPDATE + sector_name = VALUES(sector_name), + taxonomy = VALUES(taxonomy), + level = VALUES(level), + source = VALUES(source), + enabled = VALUES(enabled) + """, + ( + r.get("sector_key", ""), + r.get("sector_name", ""), + r.get("taxonomy", ""), + r.get("level", ""), + r.get("source", ""), + int(r.get("enabled", 1)), + ), + ) + + +def replace_all_stock_sector_map(rows: list[dict[str, Any]]) -> None: + """rows: stock_code, sector_key""" + if not rows: + return + with _cursor() as cur: + cur.execute("DELETE FROM stock_sector_map") + for r in rows: + cur.execute( + """ + INSERT INTO stock_sector_map (stock_code, sector_key) + VALUES (%s, %s) + ON DUPLICATE KEY UPDATE sector_key = VALUES(sector_key) + """, + (str(r.get("stock_code", "")).zfill(6), r.get("sector_key", "")), + ) + + +# ── 行业聚合（衍生）───────────────────────────────────────────────────── + + +def replace_all_sector_indices(rows: list[dict[str, Any]]) -> None: + """rows: trade_date, sector_name, close, sector_amplitude""" + if not rows: + return + with _cursor() as cur: + for r in rows: + cur.execute( + """ + INSERT INTO sector_indices (trade_date, sector_name, `close`, sector_amplitude) + VALUES (%s, %s, %s, %s) + ON DUPLICATE KEY UPDATE + `close` = VALUES(`close`), + sector_amplitude = VALUES(sector_amplitude) + """, + ( + _date_or_none(r.get("trade_date")), + r.get("sector_name", ""), + float(r.get("close") or 0), + float(r.get("sector_amplitude") or 0), + ), + ) + + +def replace_all_sector_features(rows: list[dict[str, Any]]) -> None: + """rows: trade_date, sector_name, sector_ret, sector_amplitude, close, ema10, ema20, ema200, score""" + if not rows: + return + with _cursor() as cur: + for r in rows: + cur.execute( + """ + INSERT INTO sector_features_daily + (trade_date, sector_name, sector_ret, sector_amplitude, `close`, ema10, ema20, ema200, score) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s) + ON DUPLICATE KEY UPDATE + sector_ret = VALUES(sector_ret), + sector_amplitude = VALUES(sector_amplitude), + `close` = VALUES(`close`), + ema10 = VALUES(ema10), + ema20 = VALUES(ema20), + ema200 = VALUES(ema200), + score = VALUES(score) + """, + ( + _date_or_none(r.get("trade_date")), + r.get("sector_name", ""), + float(r.get("sector_ret") or 0), + float(r.get("sector_amplitude") or 0), + float(r.get("close") or 0), + float(r.get("ema10") or 0), + float(r.get("ema20") or 0), + float(r.get("ema200") or 0), + int(r.get("score") or 0), + ), + ) + + +# ── 市场情绪（衍生）───────────────────────────────────────────────────── + + +def upsert_market_regime_rows(rows: list[dict[str, Any]]) -> None: + """rows: trade_date, advancers, decliners, advance_ratio, turnover, turnover_avg_5d, turnover_ratio_5d, source, is_extreme_panic""" + if not rows: + return + with _cursor() as cur: + for r in rows: + cur.execute( + """ + INSERT INTO market_regime_daily + (trade_date, advancers, decliners, advance_ratio, turnover, + turnover_avg_5d, turnover_ratio_5d, source, is_extreme_panic) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s) + ON DUPLICATE KEY UPDATE + advancers = VALUES(advancers), + decliners = VALUES(decliners), + advance_ratio = VALUES(advance_ratio), + turnover = VALUES(turnover), + turnover_avg_5d = VALUES(turnover_avg_5d), + turnover_ratio_5d = VALUES(turnover_ratio_5d), + source = VALUES(source), + is_extreme_panic = VALUES(is_extreme_panic) + """, + ( + _date_or_none(r.get("trade_date")), + float(r.get("advancers") or 0), + float(r.get("decliners") or 0), + float(r.get("advance_ratio") or 0), + float(r.get("turnover") or 0), + float(r.get("turnover_avg_5d") or 0), + float(r.get("turnover_ratio_5d") or 0), + r.get("source", ""), + int(r.get("is_extreme_panic") or 0), + ), + ) + + +# ── kline 查询（同步时用于判断增量起点）───────────────────────────────── + + +def get_stock_kline_max_date(code: str) -> Optional[str]: + init_mysql_schema() + conn = get_mysql() + with conn.cursor() as cur: + cur.execute( + "SELECT MAX(trade_date) AS d FROM kline_stock WHERE stock_code = %s", + (code,), + ) + row = cur.fetchone() + if row and row.get("d"): + return str(row["d"]) + return None + + +def get_index_kline_max_date(index_code: str) -> Optional[str]: + init_mysql_schema() + conn = get_mysql() + with conn.cursor() as cur: + cur.execute( + "SELECT MAX(trade_date) AS d FROM kline_index WHERE index_code = %s", + (index_code,), + ) + row = cur.fetchone() + if row and row.get("d"): + return str(row["d"]) + return None + + +def get_5min_max_bar_time(code: str) -> Optional[str]: + init_mysql_schema() + conn = get_mysql() + with conn.cursor() as cur: + cur.execute( + "SELECT MAX(bar_time) AS d FROM kline_5min WHERE stock_code = %s", + (code,), + ) + row = cur.fetchone() + if row and row.get("d"): + return str(row["d"]) + return None + + +def get_moneyflow_max_date(code: str) -> Optional[str]: + init_mysql_schema() + conn = get_mysql() + with conn.cursor() as cur: + cur.execute( + "SELECT MAX(trade_date) AS d FROM moneyflow WHERE stock_code = %s", + (code,), + ) + row = cur.fetchone() + if row and row.get("d"): + return str(row["d"]) + return None diff --git a/app/core/db/schema.py b/app/core/db/schema.py new file mode 100644 index 0000000..b4fb520 --- /dev/null +++ b/app/core/db/schema.py @@ -0,0 +1,264 @@ +"""Schema DDL — 集中所有 CREATE TABLE IF NOT EXISTS。 + +参考 dashboard/api/services/mysql_conn.py::init_mysql_schema()，但补齐 dashboard 缺的几张表 +（kline_stock / kline_5min / kline_index / moneyflow / share / sector_features_daily / sector_indices +/ stock_scores / registry），这些是 grid_seeker_model_base 生产库已存在但 mysql_conn.py 没建的表。 + +所有 DDL 写在同一个文件，方便审阅、diff 和补字段。""" +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + import pymysql.connections + + +# DDL 列表（顺序无关，全部 IF NOT EXISTS） +_DDL: list[str] = [ + # ─── config ───────────────────────────────────────────────────────── + """ + CREATE TABLE IF NOT EXISTS config ( + `key` VARCHAR(128) PRIMARY KEY, + `value` TEXT NOT NULL, + category VARCHAR(32) NOT NULL DEFAULT 'general', + description VARCHAR(255) DEFAULT '', + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 + """, + # ─── dataset_registry (同步状态机) ──────────────────────────────────── + """ + CREATE TABLE IF NOT EXISTS dataset_registry ( + dataset_id VARCHAR(64) PRIMARY KEY, + name VARCHAR(128) NOT NULL DEFAULT '', + description TEXT, + storage_uri VARCHAR(255) DEFAULT '', + storage_layer VARCHAR(32) DEFAULT '', + management_role VARCHAR(32) DEFAULT '', + source VARCHAR(255) DEFAULT '', + sync_script VARCHAR(255) DEFAULT '', + dependency_ids VARCHAR(4096) DEFAULT '[]', + enabled INT DEFAULT 1, + sort_order INT DEFAULT 0, + status VARCHAR(16) DEFAULT 'idle', + trigger_source VARCHAR(32) DEFAULT '', + started_at DATETIME DEFAULT NULL, + finished_at DATETIME DEFAULT NULL, + last_success_at DATETIME DEFAULT NULL, + last_failure_at DATETIME DEFAULT NULL, + message TEXT, + last_error TEXT, + needs_resync INT DEFAULT 0, + progress_current INT DEFAULT 0, + progress_total INT DEFAULT 0, + current_step VARCHAR(255) DEFAULT '', + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + INDEX idx_dataset_registry_sort_order (sort_order) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 + """, + # ─── stocks (基础信息 + 股本字段) ───────────────────────────────────── + """ + CREATE TABLE IF NOT EXISTS stocks ( + code VARCHAR(10) PRIMARY KEY, + name VARCHAR(32) NOT NULL DEFAULT '', + exchange VARCHAR(8) NOT NULL DEFAULT '', + list_date DATE DEFAULT NULL, + listing_status VARCHAR(16) NOT NULL DEFAULT 'normal', + industry VARCHAR(64) DEFAULT '', + total_share DOUBLE DEFAULT 0, + float_share DOUBLE DEFAULT 0, + share_updated_at DATE DEFAULT NULL, + kline_synced_at DATE DEFAULT NULL, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 + """, + # ─── indices (指数字典) ─────────────────────────────────────────────── + """ + CREATE TABLE IF NOT EXISTS indices ( + index_code VARCHAR(10) PRIMARY KEY, + index_name VARCHAR(64) NOT NULL DEFAULT '', + market VARCHAR(16) DEFAULT '', + category VARCHAR(16) DEFAULT '', + source VARCHAR(32) DEFAULT '', + enabled INT DEFAULT 1, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 + """, + # ─── kline_stock (日 K 线，按股票) ──────────────────────────────────── + """ + CREATE TABLE IF NOT EXISTS kline_stock ( + stock_code VARCHAR(10) NOT NULL, + trade_date DATE NOT NULL, + `open` DOUBLE NOT NULL, + `high` DOUBLE NOT NULL, + `low` DOUBLE NOT NULL, + `close` DOUBLE NOT NULL, + volume DOUBLE NOT NULL, + PRIMARY KEY (stock_code, trade_date), + INDEX idx_kline_stock_date (trade_date) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 + """, + # ─── kline_index (指数日 K 线) ──────────────────────────────────────── + """ + CREATE TABLE IF NOT EXISTS kline_index ( + index_code VARCHAR(10) NOT NULL, + trade_date DATE NOT NULL, + `open` DOUBLE NOT NULL, + `high` DOUBLE NOT NULL, + `low` DOUBLE NOT NULL, + `close` DOUBLE NOT NULL, + volume DOUBLE NOT NULL, + PRIMARY KEY (index_code, trade_date), + INDEX idx_kline_index_date (trade_date) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 + """, + # ─── kline_5min (5 分钟 K 线) ──────────────────────────────────────── + """ + CREATE TABLE IF NOT EXISTS kline_5min ( + stock_code VARCHAR(10) NOT NULL, + bar_time DATETIME NOT NULL, + `open` DOUBLE DEFAULT NULL, + `high` DOUBLE DEFAULT NULL, + `low` DOUBLE DEFAULT NULL, + `close` DOUBLE DEFAULT NULL, + volume DOUBLE DEFAULT NULL, + amount DOUBLE DEFAULT NULL, + turnover_rate DOUBLE DEFAULT NULL, + PRIMARY KEY (stock_code, bar_time), + INDEX idx_kline_5min_bar_time (bar_time) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 + """, + # ─── moneyflow (资金流，主力/大/中/小单净额) ────────────────────────── + """ + CREATE TABLE IF NOT EXISTS moneyflow ( + stock_code VARCHAR(10) NOT NULL, + trade_date DATE NOT NULL, + main_net_inflow DOUBLE DEFAULT 0, + large_net_inflow DOUBLE DEFAULT 0, + medium_net_inflow DOUBLE DEFAULT 0, + small_net_inflow DOUBLE DEFAULT 0, + PRIMARY KEY (stock_code, trade_date), + INDEX idx_moneyflow_date (trade_date) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 + """, + # ─── share (股本快照) ──────────────────────────────────────────────── + """ + CREATE TABLE IF NOT EXISTS share ( + stock_code VARCHAR(10) NOT NULL, + trade_date DATE NOT NULL, + total_share DOUBLE NOT NULL, + float_share DOUBLE NOT NULL, + PRIMARY KEY (stock_code, trade_date) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 + """, + # ─── sectors (行业字典) ────────────────────────────────────────────── + """ + CREATE TABLE IF NOT EXISTS sectors ( + sector_key VARCHAR(64) PRIMARY KEY, + sector_name VARCHAR(64) NOT NULL DEFAULT '', + taxonomy VARCHAR(64) DEFAULT '', + level VARCHAR(16) DEFAULT '', + source VARCHAR(32) DEFAULT '', + enabled INT DEFAULT 1, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + INDEX idx_sectors_sector_name (sector_name) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 + """, + # ─── stock_sector_map (股票→行业映射) ───────────────────────────────── + """ + CREATE TABLE IF NOT EXISTS stock_sector_map ( + stock_code VARCHAR(6) PRIMARY KEY, + sector_key VARCHAR(64) NOT NULL DEFAULT '', + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + INDEX idx_stock_sector_map_sector_key (sector_key) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 + """, + # ─── industry (股票-行业映射，datasource 实际写入用这张) ────────────── + """ + CREATE TABLE IF NOT EXISTS industry ( + code VARCHAR(6) PRIMARY KEY, + industry_name VARCHAR(64) DEFAULT '', + industry_classification VARCHAR(32) DEFAULT '', + update_date DATE DEFAULT NULL, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + INDEX idx_industry_industry_name (industry_name) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 + """, + # ─── sector_indices (行业聚合指数时序) ───────────────────────────────── + """ + CREATE TABLE IF NOT EXISTS sector_indices ( + trade_date DATE NOT NULL, + sector_name VARCHAR(64) NOT NULL, + `close` DOUBLE DEFAULT 0, + sector_amplitude DOUBLE DEFAULT 0, + PRIMARY KEY (trade_date, sector_name), + INDEX idx_sector_indices_sector_name (sector_name) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 + """, + # ─── sector_features_daily (行业特征衍生) ───────────────────────────── + """ + CREATE TABLE IF NOT EXISTS sector_features_daily ( + trade_date DATE NOT NULL, + sector_name VARCHAR(64) NOT NULL, + sector_ret DOUBLE DEFAULT 0, + sector_amplitude DOUBLE DEFAULT 0, + `close` DOUBLE DEFAULT 0, + ema10 DOUBLE DEFAULT 0, + ema20 DOUBLE DEFAULT 0, + ema200 DOUBLE DEFAULT 0, + score INT DEFAULT 0, + PRIMARY KEY (trade_date, sector_name), + INDEX idx_sector_features_sector_name (sector_name) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 + """, + # ─── market_regime_daily (市场情绪) ─────────────────────────────────── + """ + CREATE TABLE IF NOT EXISTS market_regime_daily ( + trade_date DATE PRIMARY KEY, + advancers DOUBLE DEFAULT 0, + decliners DOUBLE DEFAULT 0, + advance_ratio DOUBLE DEFAULT 0, + turnover DOUBLE DEFAULT 0, + turnover_avg_5d DOUBLE DEFAULT 0, + turnover_ratio_5d DOUBLE DEFAULT 0, + source VARCHAR(32) DEFAULT '', + is_extreme_panic INT DEFAULT 0, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + INDEX idx_market_regime_daily_date (trade_date) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 + """, + # ─── registry (模型注册，dashboard 业务无关但库里有) ───────────────── + """ + CREATE TABLE IF NOT EXISTS registry ( + model_name VARCHAR(64) NOT NULL, + role VARCHAR(32) NOT NULL, + version VARCHAR(16) DEFAULT NULL, + artifact_path VARCHAR(256) NOT NULL, + architecture VARCHAR(32) DEFAULT NULL, + enabled TINYINT(1) DEFAULT 1, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (model_name, role) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 + """, + # ─── stock_scores (评分快照，dashboard 业务无关但库里有) ───────────── + """ + CREATE TABLE IF NOT EXISTS stock_scores ( + `date` DATE NOT NULL, + stock_code VARCHAR(10) NOT NULL, + predicted_rounds DOUBLE DEFAULT 0, + stacking_probability DOUBLE DEFAULT 0, + meta_ranker_score DOUBLE DEFAULT NULL, + latest_close DOUBLE DEFAULT 0, + `rank` INT DEFAULT 0, + PRIMARY KEY (`date`, stock_code), + INDEX idx_stock_scores_date (`date`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 + """, +] + + +def ensure_all_tables(conn) -> None: + """遍历 _DDL 列表，逐条执行 CREATE TABLE IF NOT EXISTS。""" + with conn.cursor() as cur: + for ddl in _DDL: + cur.execute(ddl) + conn.commit() diff --git a/app/core/scheduler/__init__.py b/app/core/scheduler/__init__.py new file mode 100644 index 0000000..83d3d4d --- /dev/null +++ b/app/core/scheduler/__init__.py @@ -0,0 +1,20 @@ +"""scheduler 子包入口。""" +from app.core.scheduler.scheduler import ( + start_scheduler, + stop_scheduler, + get_scheduler_status, + seed_schedule_configs, + register_job, + is_trading_day, + next_trading_day, +) + +__all__ = [ + "start_scheduler", + "stop_scheduler", + "get_scheduler_status", + "seed_schedule_configs", + "register_job", + "is_trading_day", + "next_trading_day", +] diff --git a/app/core/scheduler/scheduler.py b/app/core/scheduler/scheduler.py new file mode 100644 index 0000000..c11bb41 --- /dev/null +++ b/app/core/scheduler/scheduler.py @@ -0,0 +1,392 @@ +"""轻量级定时任务调度器（照搬 dashboard/api/services/scheduler.py 模式）。 + +后台线程每 N 秒扫描 config 表中 category='schedule' 的任务，到达触发时间时 +执行对应的 job 函数。执行记录写回 config 表。 + +任务定义字段（存在 config 表，value 是 JSON）： + { + "name": "...", + "time": "15:40", # 触发时间（HH:MM），intervalSeconds=0 时必填 + "intervalSeconds": 0, # > 0 时按秒触发 + "condition": "trading_day", # always | trading_day + "job": "sync_kline_daily", # 已注册的 job 名 + "enabled": true, + "lastRun": "", "lastStatus": "", "lastMessage": "" + } +""" +from __future__ import annotations + +import json +import threading +import time +from datetime import date, datetime, timedelta +from typing import Any, Callable, Optional + +from app.core.config import settings +from app.core.db import ops as db_ops +from app.core.utils.logging import get_logger + +logger = get_logger("scheduler") + + +# ── 交易日判断 ───────────────────────────────────────────────────────── + +_KNOWN_HOLIDAYS: set[str] = set() + + +def _init_holidays() -> None: + """从 config 表读出休市日（key=trading_calendar_holidays, category=general）。""" + global _KNOWN_HOLIDAYS + r = db_ops.fetch_config_by_key("trading_calendar_holidays") + if r and r.get("category") == "general": + try: + holidays = json.loads(r.get("value", "") or "[]") + if isinstance(holidays, list): + _KNOWN_HOLIDAYS = {h for h in holidays if isinstance(h, str)} + except (json.JSONDecodeError, TypeError): + pass + + +def is_trading_day(d: Optional[date] = None) -> bool: + """判断 A 股交易日（简化：周一至周五 + 非 _KNOWN_HOLIDAYS）。""" + if d is None: + d = date.today() + if d.weekday() >= 5: + return False + if d.isoformat() in _KNOWN_HOLIDAYS: + return False + return True + + +def next_trading_day(after: Optional[date] = None) -> date: + d = after or date.today() + while not is_trading_day(d): + d += timedelta(days=1) + return d + + +# ── Job registry ────────────────────────────────────────────────────── + +_job_registry: dict[str, Callable[[], dict]] = {} + + +def register_job(name: str): + """装饰器：注册一个 job 函数。函数应返回 {status, message, ...}。""" + def decorator(fn): + _job_registry[name] = fn + return fn + return decorator + + +# ── Scheduler 状态 ──────────────────────────────────────────────────── + +_lock = threading.Lock() +_running = False +_thread: Optional[threading.Thread] = None + +# 配置缓存：避免每次 tick 全量查 config 表 +_config_cache_ts: float = 0.0 +_config_cache: list[dict] = [] +_CONFIG_CACHE_TTL = 10.0 # 缓存 10 秒，减少 DB 查询 + + +def get_scheduler_status() -> dict[str, Any]: + with _lock: + return { + "running": _running, + "jobs": _load_job_statuses(), + } + + +def _load_schedule_configs() -> list[dict]: + """从 config 表读出所有 category='schedule' 的任务定义（带缓存）。""" + global _config_cache_ts, _config_cache + now = time.time() + if now - _config_cache_ts < _CONFIG_CACHE_TTL and _config_cache: + return _config_cache + result = [] + for r in db_ops.fetch_configs_by_category("schedule"): + try: + obj = json.loads(r.get("value", "") or "{}") + obj["_key"] = r["key"] + result.append(obj) + except (json.JSONDecodeError, TypeError): + continue + _config_cache = result + _config_cache_ts = now + return result + + +def _load_job_statuses() -> list[dict]: + cfgs = _load_schedule_configs() + return [ + { + "key": c["_key"], + "name": c.get("name", ""), + "time": c.get("time", ""), + "intervalSeconds": c.get("intervalSeconds", 0), + "condition": c.get("condition", "always"), + "job": c.get("job", ""), + "enabled": c.get("enabled", True), + "lastRun": c.get("lastRun", ""), + "lastStatus": c.get("lastStatus", ""), + "lastMessage": c.get("lastMessage", ""), + } + for c in cfgs + ] + + +def update_job_status(key: str, status: str, message: str) -> None: + """更新 job 执行状态 — 使用 fetch_config_by_key 高效查找。""" + r = db_ops.fetch_config_by_key(key) + if r is None or r.get("category") != "schedule": + return + try: + obj = json.loads(r.get("value", "") or "{}") + except (json.JSONDecodeError, TypeError): + obj = {} + obj["lastRun"] = datetime.now().isoformat(timespec="seconds") + obj["lastStatus"] = status + obj["lastMessage"] = message + db_ops.upsert_config( + key, json.dumps(obj, ensure_ascii=False), + category="schedule", description=r.get("description", ""), + ) + # 更新后立即刷新缓存 + global _config_cache_ts + _config_cache_ts = 0.0 + + +# ── 调度循环 ────────────────────────────────────────────────────────── + +def _loop() -> None: + global _running + _init_holidays() + + last_fired: dict[str, str] = {} # key → date string + last_fired_ts: dict[str, float] = {} # key → timestamp + + while _running: + try: + now = datetime.now() + today_str = now.strftime("%Y-%m-%d") + current_time = now.strftime("%H:%M") + now_ts = time.time() + + for cfg in _load_schedule_configs(): + if not cfg.get("enabled", True): + continue + + key = cfg["_key"] + job_time = cfg.get("time", "") + condition = cfg.get("condition", "always") + job_name = cfg.get("job", "") + interval = int(cfg.get("intervalSeconds", 0) or 0) + persisted_last_run = str(cfg.get("lastRun", "") or "").strip() + + # 启动时记住今天已跑过的时间触发任务 + if not interval and persisted_last_run and key not in last_fired: + try: + persisted_date = datetime.fromisoformat(persisted_last_run).strftime("%Y-%m-%d") + except ValueError: + persisted_date = persisted_last_run[:10] + if persisted_date == today_str: + last_fired[key] = today_str + + if interval > 0: + if now_ts - last_fired_ts.get(key, 0) < interval: + continue + else: + if current_time < job_time: + continue + if last_fired.get(key) == today_str: + continue + + if condition == "trading_day" and not is_trading_day(): + continue + + # 触发 + fn = _job_registry.get(job_name) + if fn is None: + update_job_status(key, "error", f"未注册的 job: {job_name}") + continue + + logger.info(f"[scheduler] 触发 {cfg.get('name', key)} @ {current_time}") + try: + result = fn() or {} + status = result.get("status", "ok") + message = result.get("message", "") + update_job_status(key, status, message) + logger.info(f"[scheduler] 完成 {cfg.get('name', key)} → {status}: {message}") + except Exception as e: + err = f"{type(e).__name__}: {e}" + update_job_status(key, "error", err[:200]) + logger.error(f"[scheduler] 失败 {cfg.get('name', key)}: {err}") + + last_fired[key] = today_str + last_fired_ts[key] = now_ts + except Exception as e: + logger.error(f"[scheduler] loop 异常: {e}") + + time.sleep(max(5, settings.scheduler_tick_seconds)) + + +def start_scheduler() -> None: + global _running, _thread + with _lock: + if _running: + return + _running = True + _thread = threading.Thread(target=_loop, daemon=True, name="market-data-sync-scheduler") + _thread.start() + logger.info("[scheduler] 已启动") + + +def stop_scheduler() -> None: + global _running + with _lock: + _running = False + logger.info("[scheduler] 已停止") + + +# ── 默认任务 seed ──────────────────────────────────────────────────── + +DEFAULT_SCHEDULES: list[tuple[str, dict, str]] = [ + ( + "schedule_stocks_basic", + { + "name": "开盘前同步股票信息", + "time": "09:00", + "condition": "trading_day", + "job": "stock_basic", + "enabled": True, + }, + "每个交易日 09:00 拉取全市场股票基础信息 + 股本快照", + ), + ( + "schedule_kline_index", + { + "name": "盘后指数日K线", + "time": "15:30", + "condition": "trading_day", + "job": "kline_index", + "enabled": True, + }, + "每个交易日 15:30 拉取六大指数日K线", + ), + ( + "schedule_kline_daily", + { + "name": "盘后日K线增量", + "time": "15:40", + "condition": "trading_day", + "job": "kline_daily", + "enabled": True, + }, + "每个交易日 15:40 增量同步全市场日K线（多源交叉校验）", + ), + ( + "schedule_kline_5min", + { + "name": "盘后5分钟K线", + "time": "16:00", + "condition": "trading_day", + "job": "kline_5min", + "enabled": True, + }, + "每个交易日 16:00 拉取 5 分钟 K 线（数据量大，可选启用）", + ), + ( + "schedule_moneyflow", + { + "name": "盘后资金流", + "time": "16:30", + "condition": "trading_day", + "job": "moneyflow", + "enabled": True, + }, + "每个交易日 16:30 拉取个股资金流", + ), + ( + "schedule_industry_sector", + { + "name": "周一行业映射", + "time": "09:30", + "condition": "trading_day", + "job": "industry_sector", + "enabled": True, + }, + "每个交易日 09:30 全量更新股票-行业映射（开销大，可改为 weekly）", + ), + ( + "schedule_market_regime", + { + "name": "盘后市场情绪", + "time": "16:15", + "condition": "trading_day", + "job": "market_regime", + "enabled": True, + }, + "每个交易日 16:15 聚合市场情绪（基于本地日K线）", + ), + ( + "schedule_share_snapshot", + { + "name": "盘后股本快照", + "time": "16:20", + "condition": "trading_day", + "job": "share_snapshot", + "enabled": True, + }, + "每个交易日 16:20 刷新个股总股本/流通股本快照", + ), +] + + +def seed_schedule_configs() -> None: + """写入默认计划任务配置到 config 表。""" + existing = {r["key"]: r for r in db_ops.fetch_configs_by_category("schedule")} + for key, obj, desc in DEFAULT_SCHEDULES: + full = { + **obj, + "lastRun": "", + "lastStatus": "", + "lastMessage": "", + } + if key in existing: + # 保留 lastRun / lastStatus 等 + try: + old = json.loads(existing[key].get("value", "") or "{}") + for k in ("lastRun", "lastStatus", "lastMessage"): + if k in old: + full[k] = old[k] + except Exception: + pass + db_ops.upsert_config( + key, json.dumps(full, ensure_ascii=False), category="schedule", description=desc, + ) + + +# ── 注册 job（延迟导入避免循环依赖）──────────────────────────────────── + + +def register_sync_jobs() -> None: + """把 8 个 sync 任务注册成 job。 + + 注：job 函数不接受任何参数，trigger_source 默认为 'schedule'。 + """ + from app.tasks import get_task + + for dataset_id in [ + "stock_basic", "kline_daily", "kline_index", "kline_5min", + "moneyflow", "industry_sector", "share_snapshot", "market_regime", + ]: + + def _make_job(did=dataset_id): + def _job() -> dict: + task = get_task(did) + return task.run(trigger_source="schedule") + _job.__name__ = f"job_{did}" + return _job + + register_job(dataset_id)(_make_job()) diff --git a/app/core/sync/__init__.py b/app/core/sync/__init__.py new file mode 100644 index 0000000..1e96183 --- /dev/null +++ b/app/core/sync/__init__.py @@ -0,0 +1,24 @@ +"""sync 子包入口。""" +from app.core.sync.registry import ( + mark_sync_running, + mark_sync_progress, + mark_sync_success, + mark_sync_failed, + mark_sync_blocked, + get_registry_status, + recover_interrupted_syncs, + seed_sync_registry, + SYNC_DEFINITIONS, +) + +__all__ = [ + "mark_sync_running", + "mark_sync_progress", + "mark_sync_success", + "mark_sync_failed", + "mark_sync_blocked", + "get_registry_status", + "recover_interrupted_syncs", + "seed_sync_registry", + "SYNC_DEFINITIONS", +] diff --git a/app/core/sync/base.py b/app/core/sync/base.py new file mode 100644 index 0000000..8330bf6 --- /dev/null +++ b/app/core/sync/base.py @@ -0,0 +1,141 @@ +"""SyncTask 基类。 + +提供统一的： +- 状态自动更新（running → success / failed） +- 进度回调 +- 异常捕获 → 写 dataset_registry + +子类只需实现 _run() 即可。 +""" +from __future__ import annotations + +import time +import traceback +from datetime import datetime, time as dtime +from typing import Any, Optional + +from app.core.sync.registry import ( + mark_sync_failed, + mark_sync_progress, + mark_sync_running, + mark_sync_success, +) +from app.core.utils.logging import get_logger + +logger = get_logger("sync.task") + + +def _is_market_closed() -> bool: + """是否超过 15:30（盘后才完整，避免盘中部分数据）。""" + return datetime.now().time() >= dtime(15, 30) + + +def _effective_sync_end() -> str: + """若盘后则今天，否则昨天。""" + today = datetime.now() + if today.time() >= dtime(15, 30): + return today.strftime("%Y-%m-%d") + # 昨天 + from datetime import timedelta + return (today - timedelta(days=1)).strftime("%Y-%m-%d") + + +class SyncTask: + """同步任务基类。""" + + dataset_id: str = "" # 子类必须设置 + + def __init__(self, dataset_id: Optional[str] = None): + if dataset_id: + self.dataset_id = dataset_id + if not self.dataset_id: + raise ValueError(f"{self.__class__.__name__} must set .dataset_id") + + # ── 公开入口 ───────────────────────────────────────── + + def run(self, *, trigger_source: str = "manual", **kwargs) -> dict[str, Any]: + """统一入口：自动包 mark_running / mark_success / mark_failed。""" + # 1. 确保 health check 跑过（CLI / 手动触发场景下 schedule 后台不会先跑） + self._ensure_health_checked() + + # 2. 启动 + mark_sync_running( + self.dataset_id, + trigger_source=trigger_source, + message=f"开始同步 {self.dataset_id}...", + ) + t0 = time.time() + try: + result = self._run(trigger_source=trigger_source, **kwargs) + elapsed = round(time.time() - t0, 1) + status = result.get("status", "ok") + message = result.get("message", "") + if status == "ok": + mark_sync_success(self.dataset_id, message=message) + elif status == "blocked": + mark_sync_failed( + self.dataset_id, + message=message, + error=result.get("error", message), + status="blocked", + ) + else: # warning / partial + mark_sync_failed( + self.dataset_id, + message=message, + error=result.get("error", message), + status="warning", + ) + result["elapsed_sec"] = elapsed + logger.info(f"[{self.dataset_id}] 完成 {status}: {message} ({elapsed}s)") + return result + except Exception as e: + elapsed = round(time.time() - t0, 1) + tb = traceback.format_exc(limit=2) + err_msg = f"{type(e).__name__}: {e}" + mark_sync_failed(self.dataset_id, message=err_msg, error=tb) + logger.error(f"[{self.dataset_id}] 失败: {err_msg}\n{tb}") + return { + "status": "error", + "message": err_msg, + "error": err_msg, + "elapsed_sec": elapsed, + } + + def _ensure_health_checked(self) -> None: + """若内存里 health_results 为空，跑一次。""" + try: + from app.core.datasource.registry import get_health_status, run_health_check + if not get_health_status(): + run_health_check() + except Exception as e: + logger.warning(f"health check 启动失败（不影响任务继续）: {e}") + + # ── 子类实现 ───────────────────────────────────────── + + def _run(self, *, trigger_source: str = "manual", **kwargs) -> dict[str, Any]: + """子类实现：实际同步逻辑。 + + 返回 dict： + { + "status": "ok" | "warning" | "blocked", + "message": "...", + "ok": int, "fail": int, "skip": int, # 可选统计 + ... + } + 任何异常会被外层捕获并标记 failed。 + """ + raise NotImplementedError + + # ── 进度更新辅助 ───────────────────────────────────── + + def _progress(self, *, message: str | None = None, + current: int | None = None, total: int | None = None, + current_step: str | None = None) -> None: + mark_sync_progress( + self.dataset_id, + message=message, + progress_current=current, + progress_total=total, + current_step=current_step, + ) diff --git a/app/core/sync/registry.py b/app/core/sync/registry.py new file mode 100644 index 0000000..f252ac2 --- /dev/null +++ b/app/core/sync/registry.py @@ -0,0 +1,249 @@ +"""sync registry 状态机封装。 + +模式照搬 dashboard/api/services/dataset_registry.py： +- dataset_id 是唯一键 +- mark_sync_running / mark_sync_progress / mark_sync_success / mark_sync_failed / mark_sync_blocked +- recover_interrupted_syncs 启动时把 status='running' 的任务改为 failed +- seed_sync_registry 把所有同步任务定义 seed 到 dataset_registry 表 +""" +from __future__ import annotations + +import json +from datetime import datetime +from typing import Any + +from app.core.db import ops as db_ops +from app.core.utils.logging import get_logger + +logger = get_logger("sync.registry") + + +# ── 同步任务定义（写入 dataset_registry 表）────────────────────────────── + + +SYNC_DEFINITIONS: list[dict[str, Any]] = [ + { + "dataset_id": "stock_basic", + "name": "股票基础信息（含最新股本）", + "description": "全市场 A 股代码 / 名称 / 交易所 / 上市状态 + 股本快照（雪球）", + "storage_uri": "MySQL market_data_sync_db.stocks", + "storage_layer": "mysql", + "management_role": "基础字典", + "source": "Baostock query_stock_basic + 雪球 quote_detail", + "sync_script": "app.tasks.task_stocks_basic:run", + "dependency_ids": [], + "sort_order": 10, + }, + { + "dataset_id": "kline_daily", + "name": "原始日K线数据", + "description": "全市场 A 股日频 OHLCV，多源交叉校验（雪球/新浪/Baostock）", + "storage_uri": "MySQL market_data_sync_db.kline_stock", + "storage_layer": "mysql", + "management_role": "原始源", + "source": "雪球主源 + 新浪/Baostock 复核", + "sync_script": "app.tasks.task_kline_daily:run", + "dependency_ids": ["stock_basic"], + "sort_order": 20, + }, + { + "dataset_id": "kline_index", + "name": "六大指数日线", + "description": "上证 / 深证 / 创业板 / 沪深300 / 中证500 / 中证1000", + "storage_uri": "MySQL market_data_sync_db.kline_index + indices", + "storage_layer": "mysql", + "management_role": "原始源", + "source": "新浪指数日K", + "sync_script": "app.tasks.task_kline_index:run", + "dependency_ids": [], + "sort_order": 30, + }, + { + "dataset_id": "kline_5min", + "name": "5 分钟 K 线", + "description": "全市场 A 股 5 分钟 K 线（mairui 源）", + "storage_uri": "MySQL market_data_sync_db.kline_5min", + "storage_layer": "mysql", + "management_role": "原始源", + "source": "mairui stockMin", + "sync_script": "app.tasks.task_kline_5min:run", + "dependency_ids": ["stock_basic"], + "sort_order": 40, + }, + { + "dataset_id": "moneyflow", + "name": "资金流", + "description": "个股资金流（主力/大/中/小单净额），mairui 源", + "storage_uri": "MySQL market_data_sync_db.moneyflow", + "storage_layer": "mysql", + "management_role": "原始源", + "source": "mairui hsstock/history/transaction", + "sync_script": "app.tasks.task_moneyflow:run", + "dependency_ids": ["stock_basic"], + "sort_order": 50, + }, + { + "dataset_id": "industry_sector", + "name": "股票-行业映射", + "description": "股票-行业映射 + 行业字典（Baostock）", + "storage_uri": "MySQL market_data_sync_db.industry + sectors + stock_sector_map", + "storage_layer": "mysql", + "management_role": "基础字典", + "source": "Baostock query_stock_industry", + "sync_script": "app.tasks.task_industry_sector:run", + "dependency_ids": ["stock_basic"], + "sort_order": 60, + }, + { + "dataset_id": "sector_features", + "name": "行业聚合特征", + "description": "由 kline_stock + industry 衍生：行业日收益 / 行业指数 close / EMA10-20-200 / score。纯本地计算，无外部 API。", + "storage_uri": "MySQL market_data_sync_db.sector_indices + sector_features_daily", + "storage_layer": "mysql", + "management_role": "衍生源", + "source": "本地计算（kline_stock + industry）", + "sync_script": "app.tasks.task_sector_features:run", + "dependency_ids": ["kline_daily", "industry_sector"], + "sort_order": 65, + }, + { + "dataset_id": "share_snapshot", + "name": "股本快照", + "description": "全市场 A 股最新股本（雪球 quote_detail，单位亿股）", + "storage_uri": "MySQL market_data_sync_db.stocks.total_share/float_share + share 表", + "storage_layer": "mysql", + "management_role": "基础字典", + "source": "雪球 quote_detail", + "sync_script": "app.tasks.task_share_snapshot:run", + "dependency_ids": ["stock_basic"], + "sort_order": 70, + }, + { + "dataset_id": "market_regime", + "name": "市场情绪 (Market Regime)", + "description": "基于本地日K线聚合的 advance_ratio / 恐慌标记（衍生源）", + "storage_uri": "MySQL market_data_sync_db.market_regime_daily", + "storage_layer": "mysql", + "management_role": "衍生源", + "source": "本地日K线聚合（无外部接口）", + "sync_script": "app.tasks.task_market_regime:run", + "dependency_ids": ["kline_daily"], + "sort_order": 80, + }, +] + + +def seed_sync_registry() -> None: + """把 SYNC_DEFINITIONS 写入 dataset_registry 表。""" + rows = [] + for d in SYNC_DEFINITIONS: + rows.append({ + **d, + "dependency_ids": json.dumps(d.get("dependency_ids", []), ensure_ascii=False), + "enabled": 1, + }) + db_ops.upsert_dataset_registry_rows(rows) + + +# ── 状态机 API ──────────────────────────────────────────────────────── + + +def _now_ts() -> str: + return datetime.now().strftime("%Y-%m-%d %H:%M:%S") + + +def mark_sync_running( + dataset_id: str, + *, + trigger_source: str, + message: str = "", + progress_current: int = 0, + progress_total: int = 0, + current_step: str = "", +) -> None: + db_ops.update_dataset_registry_state( + dataset_id, + status="running", + trigger_source=trigger_source, + started_at=_now_ts(), + finished_at=None, + message=message, + last_error=None, + needs_resync=0, + progress_current=progress_current, + progress_total=progress_total, + current_step=current_step, + updated_at=_now_ts(), + ) + + +def mark_sync_progress( + dataset_id: str, + *, + message: str | None = None, + progress_current: int | None = None, + progress_total: int | None = None, + current_step: str | None = None, +) -> None: + payload: dict[str, Any] = {"updated_at": _now_ts()} + if message is not None: + payload["message"] = message + if progress_current is not None: + payload["progress_current"] = progress_current + if progress_total is not None: + payload["progress_total"] = progress_total + if current_step is not None: + payload["current_step"] = current_step + db_ops.update_dataset_registry_state(dataset_id, **payload) + + +def mark_sync_success(dataset_id: str, *, message: str = "") -> None: + now = _now_ts() + db_ops.update_dataset_registry_state( + dataset_id, + status="success", + finished_at=now, + last_success_at=now, + message=message, + last_error=None, + needs_resync=0, + current_step="", + progress_current=0, + progress_total=0, + updated_at=now, + ) + + +def mark_sync_failed( + dataset_id: str, + *, + message: str = "", + error: str = "", + status: str = "failed", +) -> None: + now = _now_ts() + final_message = message or error or "同步失败" + db_ops.update_dataset_registry_state( + dataset_id, + status=status, + finished_at=now, + last_failure_at=now, + message=final_message, + last_error=error or final_message, + needs_resync=1, + current_step="", + updated_at=now, + ) + + +def mark_sync_blocked(dataset_id: str, *, message: str) -> None: + mark_sync_failed(dataset_id, message=message, error=message, status="blocked") + + +def recover_interrupted_syncs() -> int: + """启动时把 status='running' 的卡死任务标记为 failed + needs_resync=1。""" + return db_ops.recover_interrupted_dataset_registry() + + +def get_registry_status() -> list[dict[str, Any]]: + return db_ops.fetch_dataset_registry_rows() diff --git a/app/core/utils/__init__.py b/app/core/utils/__init__.py new file mode 100644 index 0000000..6a3655b --- /dev/null +++ b/app/core/utils/__init__.py @@ -0,0 +1,4 @@ +"""通用工具。""" +from app.core.utils.logging import get_logger, setup_logging + +__all__ = ["get_logger", "setup_logging"] diff --git a/app/core/utils/logging.py b/app/core/utils/logging.py new file mode 100644 index 0000000..2e1f2de --- /dev/null +++ b/app/core/utils/logging.py @@ -0,0 +1,60 @@ +"""统一日志：同时输出到控制台 + 文件。""" +from __future__ import annotations + +import logging +import sys +from logging.handlers import RotatingFileHandler +from pathlib import Path + +from app.core.config import settings + +_INITIALIZED = False + + +def _build_formatter() -> logging.Formatter: + return logging.Formatter( + "[%(asctime)s] %(levelname)-5s %(name)s | %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + ) + + +def setup_logging(level: str | None = None) -> None: + global _INITIALIZED + if _INITIALIZED: + return + + lvl = (level or settings.log_level or "INFO").upper() + root = logging.getLogger() + root.setLevel(lvl) + # 清掉 uvicorn / sqlalchemy 默认 handler，避免重复输出 + for h in list(root.handlers): + root.removeHandler(h) + + fmt = _build_formatter() + + console = logging.StreamHandler(sys.stdout) + console.setFormatter(fmt) + root.addHandler(console) + + log_path: Path = settings.log_path + file_handler = RotatingFileHandler( + log_path / "market_data_sync.log", + maxBytes=20 * 1024 * 1024, + backupCount=5, + encoding="utf-8", + ) + file_handler.setFormatter(fmt) + root.addHandler(file_handler) + + # 抑制第三方库过度刷屏 + logging.getLogger("baostock").setLevel(logging.WARNING) + logging.getLogger("urllib3").setLevel(logging.WARNING) + logging.getLogger("sqlalchemy.engine").setLevel(logging.WARNING) + + _INITIALIZED = True + + +def get_logger(name: str) -> logging.Logger: + if not _INITIALIZED: + setup_logging() + return logging.getLogger(name) diff --git a/app/entrypoints/__init__.py b/app/entrypoints/__init__.py new file mode 100644 index 0000000..7436b75 --- /dev/null +++ b/app/entrypoints/__init__.py @@ -0,0 +1,5 @@ +"""进程入口点。 + +- `cli` — 手动触发/状态查询命令行（`python -m app.entrypoints.cli ...`） +- `worker` — 纯调度模式，无 Web 界面（`python -m app.entrypoints.worker`） +""" diff --git a/app/entrypoints/cli.py b/app/entrypoints/cli.py new file mode 100644 index 0000000..3ce1a6e --- /dev/null +++ b/app/entrypoints/cli.py @@ -0,0 +1,132 @@ +"""CLI 入口：手动触发 / 状态查询。""" +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +_PROJECT_ROOT = Path(__file__).resolve().parent.parent +if str(_PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(_PROJECT_ROOT)) + + +def main(): + parser = argparse.ArgumentParser( + description="market_data_sync CLI", + ) + sub = parser.add_subparsers(dest="cmd", required=True) + + # sync + p_sync = sub.add_parser("sync", help="手动执行一次同步任务") + p_sync.add_argument("task_id", help="dataset_id, 例如 kline_daily") + p_sync.add_argument("--codes", help="指定股票代码，逗号分隔", default=None) + p_sync.add_argument("--start", help="起始日期 YYYY-MM-DD", default=None) + p_sync.add_argument("--end", help="结束日期 YYYY-MM-DD", default=None) + p_sync.add_argument("--workers", type=int, default=10, help="并发数") + + sub.add_parser("list", help="列出所有同步任务") + sub.add_parser("status", help="查看整体状态（调度器 / 数据源 / 同步任务）") + sub.add_parser("datasources", help="查看数据源健康度") + sub.add_parser("schedule", help="查看计划任务") + sub.add_parser("reseed", help="重新 seed 默认计划任务到 config 表") + + args = parser.parse_args() + + if args.cmd == "sync": + from app.core.utils.logging import setup_logging + setup_logging() + from app.tasks import get_task + from app.core.datasource.registry import build_default_registry + from app.core.sync.registry import seed_sync_registry + build_default_registry() + seed_sync_registry() # 确保 dataset_registry 有这条任务定义 + try: + task = get_task(args.task_id) + except KeyError as e: + print(f"错误: {e}") + sys.exit(1) + kwargs = {} + if args.codes: + kwargs["codes"] = [c.strip() for c in args.codes.split(",") if c.strip()] + if args.start: + kwargs["start"] = args.start + if args.end: + kwargs["end"] = args.end + if args.workers: + kwargs["max_workers"] = args.workers + result = task.run(trigger_source="cli", **kwargs) + print(json.dumps(result, ensure_ascii=False, indent=2)) + sys.exit(0 if result.get("status") in ("ok", "warning") else 2) + + if args.cmd == "list": + from app.tasks import TASKS + print("可用同步任务：") + for tid, cls in TASKS.items(): + doc = (cls.__doc__ or cls.__name__).split("\n")[0] + print(f" {tid:25s} {doc}") + return + + if args.cmd == "status": + from app.core.utils.logging import setup_logging + setup_logging() + from app.core.scheduler import get_scheduler_status + from app.core.datasource.registry import build_default_registry, get_health_status + from app.core.sync import get_registry_status + build_default_registry() + print("\n=== 调度器 ===") + s = get_scheduler_status() + print(json.dumps(s, ensure_ascii=False, indent=2, default=str)) + print("\n=== 数据源健康度 ===") + for r in get_health_status(): + mark = "✓" if r.get("success") else "✗" + print(f" {mark} {r.get('name', r.get('key')):30s} {r.get('message', '')}") + print("\n=== 同步任务状态 ===") + for r in get_registry_status(): + print(f" {r['dataset_id']:25s} status={r.get('status'):10s} " + f"lastRun={r.get('last_success_at') or r.get('last_failure_at') or '-':20s} " + f"msg={r.get('message', '')[:60]}") + return + + if args.cmd == "datasources": + from app.core.utils.logging import setup_logging + setup_logging() + from app.core.datasource.registry import build_default_registry, get_health_status, run_health_check + build_default_registry() + print("正在跑连通性测试...") + run_health_check() + for r in get_health_status(): + mark = "✓" if r.get("success") else "✗" + print(f" {mark} {r.get('name', r.get('key')):30s} provides={r.get('provides', [])} {r.get('message', '')}") + return + + if args.cmd == "schedule": + from app.core.utils.logging import setup_logging + setup_logging() + from app.core.scheduler import get_scheduler_status, seed_schedule_configs + seed_schedule_configs() + s = get_scheduler_status() + print(f"调度器运行中: {s['running']}") + for j in s["jobs"]: + enabled = "✓" if j.get("enabled") else "✗" + trigger = j.get("time") or f"every {j.get('intervalSeconds')}s" + print(f" {enabled} {j.get('name', j['key']):30s} {trigger:15s} " + f"condition={j.get('condition', 'always'):12s} " + f"last={j.get('lastStatus', '-')}") + return + + if args.cmd == "reseed": + from app.core.utils.logging import setup_logging + setup_logging() + from app.core.scheduler.scheduler import seed_schedule_configs + from app.core.sync.registry import seed_sync_registry + from app.core.datasource.registry import seed_datasource_configs + seed_datasource_configs() + seed_sync_registry() + seed_schedule_configs() + print("已重新 seed：datasources + sync_registry + schedules") + return + + +if __name__ == "__main__": + main() diff --git a/app/entrypoints/worker.py b/app/entrypoints/worker.py new file mode 100644 index 0000000..004a855 --- /dev/null +++ b/app/entrypoints/worker.py @@ -0,0 +1,73 @@ +"""纯 worker 模式（无 web）— 启动调度器 + 健康监控。 + +用法：python -m app.worker +""" +from __future__ import annotations + +import json +import sys +import time +from pathlib import Path + +_PROJECT_ROOT = Path(__file__).resolve().parent.parent +if str(_PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(_PROJECT_ROOT)) + +from app.core.config import settings +from app.core.db import ops as db_ops +from app.core.utils.logging import get_logger, setup_logging + +setup_logging() +logger = get_logger("worker") + + +def main(): + logger.info("[worker] market_data_sync worker 启动") + + # 1. 注册数据源 + seed config + from app.core.datasource.registry import build_default_registry, seed_datasource_configs + build_default_registry() + seed_datasource_configs() + + # 2. seed sync registry + 恢复卡死任务 + from app.core.sync.registry import recover_interrupted_syncs, seed_sync_registry + seed_sync_registry() + n = recover_interrupted_syncs() + if n > 0: + logger.warning(f"[worker] 恢复了 {n} 个中断的同步任务") + + # 3. seed 节假日 + if settings.holidays_list: + db_ops.upsert_config( + "trading_calendar_holidays", + json.dumps(settings.holidays_list, ensure_ascii=False), + category="general", + description="A 股休市日", + ) + + # 4. seed 默认计划任务 + if settings.scheduler_auto_seed: + from app.core.scheduler.scheduler import seed_schedule_configs + seed_schedule_configs() + + # 5. 注册 sync job + 启动调度器 + 健康监控 + from app.core.scheduler.scheduler import register_sync_jobs, start_scheduler + from app.core.datasource.registry import start_health_monitor + register_sync_jobs() + start_scheduler() + start_health_monitor() + + logger.info("[worker] 已启动调度器 + 健康监控，按 Ctrl+C 退出") + + try: + while True: + time.sleep(60) + except KeyboardInterrupt: + logger.info("[worker] 收到 SIGINT，退出") + from app.core.scheduler.scheduler import stop_scheduler + stop_scheduler() + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/app/sources/__init__.py b/app/sources/__init__.py new file mode 100644 index 0000000..79e79c5 --- /dev/null +++ b/app/sources/__init__.py @@ -0,0 +1,5 @@ +"""数据源实现。 + +每个模块导出一个 `DataSource` 子类，由 +`app.core.datasource.registry.build_default_registry()` 在启动时统一注册。 +""" diff --git a/app/sources/baostock.py b/app/sources/baostock.py new file mode 100644 index 0000000..1a90d93 --- /dev/null +++ b/app/sources/baostock.py @@ -0,0 +1,159 @@ +"""Baostock 数据源。 + +提供： + - 全市场股票基础信息（query_stock_basic） + - 日 K 线（query_history_k_data_plus，免费，但 QPS 限制需要信号量） + - 股票-行业映射（query_stock_industry） + +免费、无 token，但需 bs.login() / bs.logout() 维护会话。 +""" +from __future__ import annotations + +import threading +from typing import Any, Optional + +import pandas as pd + +from app.core.datasource.base import DataSource +from app.core.datasource.utils import ( + code6_to_baostock, + normalize_kline, +) + + +class BaostockSource(DataSource): + key = "datasource_baostock" + name = "Baostock" + provides = ["kline_daily", "stock_basic", "industry"] + requires_credential = False + + # 进程内只 login 一次 + _login_lock = threading.Lock() + _logged_in = False + _semaphore = threading.BoundedSemaphore(3) # QPS 限流 + + def _ensure_login(self) -> None: + if self._logged_in: + return + with self._login_lock: + if self._logged_in: + return + import baostock as bs + bs.login() + self._logged_in = True + + def is_available(self) -> tuple[bool, str]: + try: + self._ensure_login() + return True, "ok" + except Exception as e: + return False, f"Baostock 登录失败: {e}" + + def health_check(self) -> dict[str, Any]: + """连通性测试：复用 _ensure_login 避免重复 login 阻塞。""" + try: + self._ensure_login() + import baostock as bs + rs = bs.query_stock_basic(code="sh.600036") + if rs.error_code != "0": + return {"success": False, "message": f"Baostock 查询失败: {rs.error_msg}"} + rows = [] + while rs.next(): + rows.append(rs.get_row_data()) + return {"success": True, "message": f"连接成功，查询到 {len(rows)} 条股票信息"} + except Exception as e: + return {"success": False, "message": f"Baostock 连接失败: {e}"} + + def fetch_stock_basic(self) -> list[dict[str, Any]]: + """全市场 A 股基础信息。 + + baostock query_stock_basic 返回字段（按当前接口）： + code, code_name, ipoDate, outDate, type, status + 按字段名索引，防止 Baostock 改顺序。 + """ + import baostock as bs + self._ensure_login() + try: + rs = bs.query_stock_basic() + if rs.error_code != "0": + raise RuntimeError(f"Baostock query_stock_basic 失败: {rs.error_msg}") + fields = list(rs.fields) + idx = {name: i for i, name in enumerate(fields)} + rows = [] + while rs.next(): + row = rs.get_row_data() + bs_code = row[idx["code"]] + name = row[idx["code_name"]] + stype = row[idx["type"]] + status = row[idx["status"]] + ipo = row[idx.get("ipoDate", -1)] if "ipoDate" in idx else "" + if stype != "1" or status != "1": + continue + exchange = bs_code.split(".")[0].upper() + if exchange not in ("SH", "SZ", "BJ"): + continue + code6 = bs_code.split(".")[-1] + rows.append({ + "code": f"{exchange}{code6}", + "name": name, + "exchange": exchange, + "list_date": ipo, + "listing_status": "st" if (name.startswith("ST") or name.startswith("*ST")) else "normal", + }) + return rows + except Exception: + return [] + + def fetch_kline_daily(self, code6: str, start: str, end: str) -> pd.DataFrame: + import baostock as bs + self._ensure_login() + bs_code = code6_to_baostock(code6) + with self._semaphore: + try: + rs = bs.query_history_k_data_plus( + bs_code, + "date,open,high,low,close,volume", + start_date=start, end_date=end, + frequency="d", adjustflag="2", # 前复权 + ) + if rs.error_code != "0": + return pd.DataFrame() + rows = [] + while rs.next(): + rows.append(rs.get_row_data()) + if not rows: + return pd.DataFrame() + df = pd.DataFrame(rows, columns=["trade_date", "open", "high", "low", "close", "volume"]) + return normalize_kline(df) + except Exception: + return pd.DataFrame() + + def fetch_industry_map(self) -> list[dict[str, Any]]: + """全市场股票-行业映射。""" + import baostock as bs + self._ensure_login() + try: + rs = bs.query_stock_industry(code="", date="") + if rs.error_code != "0": + raise RuntimeError(f"query_stock_industry 失败: {rs.error_msg}") + fields = list(rs.fields) # 实际字段顺序：['updateDate', 'code', 'code_name', 'industry', 'industryClassification'] + idx = {name: i for i, name in enumerate(fields)} + rows = [] + while rs.next(): + row = rs.get_row_data() + industry = row[idx.get("industry", 3)] if "industry" in idx else "" + if not industry: + continue + bs_code = row[idx.get("code", 1)] if "code" in idx else "" + code6 = bs_code.split(".")[-1] + classification = row[idx.get("industryClassification", 4)] if "industryClassification" in idx else "" + update = row[idx.get("updateDate", 0)] if "updateDate" in idx else "" + rows.append({ + "code": code6, + "industry_name": industry.strip(), + "industry_classification": (classification or "").strip(), + "update_date": update or "", + }) + return rows + except Exception: + return [] diff --git a/app/sources/mairui.py b/app/sources/mairui.py new file mode 100644 index 0000000..2c506da --- /dev/null +++ b/app/sources/mairui.py @@ -0,0 +1,379 @@ +"""麦蕊智数（mairui）数据源。 + +提供： + - 日 K 线（`hsstock/history/{code}.{ex}/d/n/{licence}`） + - 5 分钟 K 线（`hsstock/history/{code}.{ex}/5/n/{licence}`） + - 指数日 K 线（`hsindex/history/{code}.{ex}/d/{licence}`） + - 资金流向（`hsstock/history/transaction/{code}.{ex}/{licence}`） + +限速：1分钟300次（默认保守到 5 RPS = 1分钟300次） +凭证：licence（无需登录态，从环境变量 MAIRUI_LICENCE 读取） +""" +from __future__ import annotations + +import json +import os +import threading +import time +import urllib.request +from typing import Any, Optional + +import pandas as pd + +from app.core.datasource.base import DataSource +from app.core.datasource.utils import ( + code6_to_mairui, + filter_date_range, + normalize_kline, +) + + +class MairuiSource(DataSource): + key = "datasource_mairui" + name = "麦蕊智数（mairui.club）" + provides = ["kline_daily", "kline_5min", "index_daily", "stock_basic", "moneyflow"] + requires_credential = True + credential_key = "MAIRUI_LICENCE" + + _HEADERS = { + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)", + "Accept": "application/json", + } + _BASE_URL = "https://api.mairuiapi.com" + + # mairui 免费 licence 1 分钟 300 次 = 5 RPS（**单 licence**硬上限）。 + # 5 workers × 1 RPS/worker = 5 RPS 总 = 刚好踩满上限，避免风控。 + # 用 thread-local 计时：每个 worker 独立计自己的 last_ts， + # 避免 N 个 worker 串行抢一把锁退化成 1 worker。 + _RPS_LIMIT = 1.0 + _rps_local = threading.local() + + @classmethod + def _wait_rps(cls): + last = getattr(cls._rps_local, "last_ts", 0.0) + now = time.time() + elapsed = now - last + min_interval = 1.0 / cls._RPS_LIMIT + if elapsed < min_interval: + time.sleep(min_interval - elapsed) + now = time.time() + cls._rps_local.last_ts = now + + def _get_licence(self) -> Optional[str]: + return os.environ.get(self.credential_key, "").strip() or None + + def is_available(self) -> tuple[bool, str]: + if not self._get_licence(): + return False, f"未配置 {self.credential_key}" + return True, "ok" + + def health_check(self) -> dict[str, Any]: + lic = self._get_licence() + if not lic: + return {"success": False, "message": f"{self.credential_key} 未配置"} + try: + # 拿 1 条日线数据作为连通性 + 凭证测试 + url = f"{self._BASE_URL}/hsstock/history/600519.SH/d/n/{lic}?st=20260609&et=20260609" + req = urllib.request.Request(url, headers=self._HEADERS) + with urllib.request.urlopen(req, timeout=15) as resp: + raw = resp.read().decode("utf-8", errors="replace") + data = json.loads(raw) + if isinstance(data, list) and data: + return {"success": True, "message": f"连接成功，获取到 {len(data)} 条 K 线"} + if isinstance(data, dict) and data.get("error"): + return {"success": False, "message": f"麦蕊返回: {data['error']}"} + return {"success": False, "message": f"麦蕊返回空/异常: {raw[:200]}"} + except Exception as e: + return {"success": False, "message": f"麦蕊连接失败: {e}"} + + def _fetch(self, path: str, params: dict[str, Any], retry: int = 2) -> Any: + """通用 GET，含限速 + 重试。""" + self._wait_rps() + qs = "&".join(f"{k}={v}" for k, v in params.items() if v) + url = f"{self._BASE_URL}{path}" + if qs: + url = f"{url}?{qs}" + last_err = None + for attempt in range(retry + 1): + try: + req = urllib.request.Request(url, headers=self._HEADERS) + with urllib.request.urlopen(req, timeout=20) as resp: + # 先按 gbk 试（中文乱码返回），再 fallback utf-8 + raw_bytes = resp.read() + # 用 latin-1 永不失败地把 bytes 转成字符串（每字节 1 字符） + raw = raw_bytes.decode("latin-1") + # mairui 风控时返回 "接收数据异常，请稍后再试"（gbk 编码） + if "请稍后再试" in raw or "codec can't decode" in raw or raw.startswith("'utf-8'") or "Error -3 while decompressing" in raw: + raise RuntimeError(f"mairui 返回风控提示: {raw[:80]}") + # 试 JSON parse；如果是 gbk 编码的中文提示，要先解码 + if raw.startswith("'") and raw.endswith("'"): + # gbk 编码的 Python repr 字符串，如 "'接收数据异常，请稍后再试'" + try: + decoded = raw[1:-1].encode("latin-1").decode("gbk") + if "请稍后再试" in decoded: + raise RuntimeError(f"mairui 返回风控: {decoded}") + except Exception: + pass + return json.loads(raw) + except Exception as e: + last_err = e + if attempt < retry: + time.sleep(1.0 * (attempt + 1)) # 风控重试退避长一点 + from app.core.utils.logging import get_logger + get_logger("mairui").debug(f"mairui fetch {url} failed: {last_err}") + return [] + + # ── 股票列表 ─────────────────────────────────────────── + + def fetch_stock_list(self) -> list[dict]: + """全市场沪深 A 股基础列表。 + + API: GET /hslt/list/{licence} + 返回字段：dm (代码.交易所，如 "000001.SZ"), mc (名称), jys (交易所 SZ/SH) + """ + lic = self._get_licence() + if not lic: + return [] + data = self._fetch(f"/hslt/list/{lic}", {}) + if not isinstance(data, list): + return [] + rows = [] + for item in data: + try: + dm = str(item.get("dm", "")).strip() + mc = str(item.get("mc", "")).strip() + # Mairui 在 2 字简称中间填了空格（如 "万 科Ａ"），去掉多余空格 + mc = "".join(mc.split()) + jys = str(item.get("jys", "")).strip().upper() + # dm 格式: "000001.SZ" → code6="000001", exchange="SZ" + if "." in dm: + code6, exch = dm.split(".", 1) + else: + code6, exch = dm, jys + if not code6 or len(code6) != 6 or not code6.isdigit(): + continue + exchange = exch.upper() or jys + if exchange not in ("SH", "SZ"): + continue + rows.append({ + "code": f"{exchange}{code6}", + "name": mc, + "exchange": exchange, + "list_date": "", # hslt/list 不返回 IPO 日期 + "listing_status": "st" if (mc.startswith("ST") or mc.startswith("*ST")) else "normal", + }) + except Exception: + continue + return rows + + # ── K 线 fetch ───────────────────────────────────────── + + def fetch_kline_daily(self, code6: str, start: str, end: str) -> pd.DataFrame: + lic = self._get_licence() + if not lic: + return pd.DataFrame() + symbol = code6_to_mairui(code6) + path = f"/hsstock/history/{symbol}/d/n/{lic}" + params = { + "st": (start or "").replace("-", ""), + "et": (end or "").replace("-", ""), + } + data = self._fetch(path, params) + if not isinstance(data, list): + return pd.DataFrame() + records = [] + for item in data: + try: + # t 字段：日线 "2026-06-09 00:00:00"，取日期部分 + t = item.get("t", "") + d = t.split(" ")[0] if isinstance(t, str) else "" + if not d: + continue + records.append({ + "trade_date": d, + "open": float(item["o"]), + "high": float(item["h"]), + "low": float(item["l"]), + "close": float(item["c"]), + "volume": float(item.get("v") or 0), + }) + except (KeyError, ValueError, TypeError): + continue + return filter_date_range(normalize_kline(pd.DataFrame(records)), start, end) + + def fetch_kline_5min(self, code6: str, start: str, end: str) -> pd.DataFrame: + """5 分钟 K 线。返回标准列：bar_time, open, high, low, close, volume, amount。 + + mairui 用 level=5（数字，不是 "5m"）。 + """ + lic = self._get_licence() + if not lic: + return pd.DataFrame() + symbol = code6_to_mairui(code6) + path = f"/hsstock/history/{symbol}/5/n/{lic}" + params = { + "st": (start or "").replace("-", ""), + "et": (end or "").replace("-", ""), + } + data = self._fetch(path, params) + if not isinstance(data, list): + return pd.DataFrame() + records = [] + for item in data: + try: + t = item.get("t", "") + # 分钟级时间格式 "2026-06-09 14:50:00" → 替换空格为 T 让 pandas 识别 + bar_time = t.replace(" ", "T") if isinstance(t, str) else None + if not bar_time: + continue + records.append({ + "bar_time": bar_time, + "open": float(item["o"]), + "high": float(item["h"]), + "low": float(item["l"]), + "close": float(item["c"]), + "volume": float(item.get("v") or 0), + "amount": float(item.get("a") or 0), + }) + except (KeyError, ValueError, TypeError): + continue + if not records: + return pd.DataFrame() + df = pd.DataFrame(records) + df["bar_time"] = pd.to_datetime(df["bar_time"], errors="coerce") + df = df.dropna(subset=["bar_time"]).sort_values("bar_time").reset_index(drop=True) + # 过滤日期范围（按日期部分，不含时分） + if start: + df = df[df["bar_time"] >= pd.Timestamp(start)] + if end: + # end 包含整天 + df = df[df["bar_time"] < pd.Timestamp(end) + pd.Timedelta(days=1)] + return df + + def fetch_index_daily(self, index_code: str, start: str, end: str) -> pd.DataFrame: + """指数日 K。index_code 用 mairui 格式（如 '000300.SH'）。""" + lic = self._get_licence() + if not lic: + return pd.DataFrame() + path = f"/hsindex/history/{index_code}/d/{lic}" + params = { + "st": (start or "").replace("-", ""), + "et": (end or "").replace("-", ""), + } + data = self._fetch(path, params) + if not isinstance(data, list): + return pd.DataFrame() + records = [] + for item in data: + try: + t = item.get("t", "") + d = t.split(" ")[0] if isinstance(t, str) else "" + if not d: + continue + records.append({ + "trade_date": d, + "open": float(item["o"]), + "high": float(item["h"]), + "low": float(item["l"]), + "close": float(item["c"]), + "volume": float(item.get("v") or 0), + }) + except (KeyError, ValueError, TypeError): + continue + return filter_date_range(normalize_kline(pd.DataFrame(records)), start, end) + + # ── 资金流向 ─────────────────────────────────────────── + + def fetch_moneyflow(self, code6: str, start: str, end: str) -> pd.DataFrame: + """个股资金流向（主力/大/中/小单 净额）。 + + API: GET /hsstock/history/transaction/{code}.{ex}/{licence}?st=YYYYMMDD&et=YYYYMMDD + 返回字段（按买卖方向 × 单型 4×4 矩阵 + 主买/主卖/被动买/被动卖）： + zmbtdcje 主买特大单成交额 + zmbddcje 主买大单成交额 + zmbzdcje 主买中单成交额 + zmbxdcje 主买小单成交额 + zmstdcje 主卖特大单成交额 + zmsddcje 主卖大单成交额 + zmszdcje 主卖中单成交额 + zmsxdcje 主卖小单成交额 + bdmbtdcje 被动买特大单成交额 + bdmbddcje 被动买大单成交额 + bdmbzdcje 被动买中单成交额 + bdmbxdcje 被动买小单成交额 + bdmstdcje 被动卖特大单成交额 + bdmsddcje 被动卖大单成交额 + bdmszdcje 被动卖中单成交额 + bdmsxdcje 被动卖小单成交额 + ... 以及对应的成交量/笔数字段（zmbtdcjl 等），本接口暂只取成交额 + + 单型口径（mairui 文档）： + 特大单：成交额 ≥ 100 万 或 成交量 ≥ 5000 手 + 大单 ：成交额 ≥ 20 万 或 成交量 ≥ 1000 手 + 中单 ：成交额 ≥ 4 万 或 成交量 ≥ 200 手 + 小单 ：其他 + + 输出字段：trade_date, main_net_inflow, large_net_inflow, medium_net_inflow, small_net_inflow + 净额口径（与 akshare stock_individual_fund_flow 保持一致）： + 主力净流入 = Σ主买四型 - Σ主卖四型 (zmb{t,d,z,x} - zms{t,d,z,x}) + 大单净额 = 主买大 - 主卖大 (zmbddcje - zmsddcje) + 中单净额 = 主买中 - 主卖中 (zmbzdcje - zmszdcje) + 小单净额 = 主买小 - 主卖小 (zmbxdcje - zmsxdcje) + 注：不能加被动买/卖 —— 主动买 ≡ 被动卖、主动卖 ≡ 被动买 + （同一笔成交记在两边），加起来恒等 0。 + """ + lic = self._get_licence() + if not lic: + return pd.DataFrame() + symbol = code6_to_mairui(code6) + path = f"/hsstock/history/transaction/{symbol}/{lic}" + params = { + "st": (start or "").replace("-", ""), + "et": (end or "").replace("-", ""), + } + data = self._fetch(path, params) + if not isinstance(data, list): + return pd.DataFrame() + + def f(item, k) -> float: + try: + v = item.get(k) + return float(v) if v not in (None, "", "-") else 0.0 + except (TypeError, ValueError): + return 0.0 + + records = [] + for item in data: + t = item.get("t", "") + d = t.split(" ")[0] if isinstance(t, str) else "" + if not d: + continue + # 主买四型 / 主卖四型 + main_buy = ( + f(item, "zmbtdcje") + f(item, "zmbddcje") + + f(item, "zmbzdcje") + f(item, "zmbxdcje") + ) + main_sell = ( + f(item, "zmstdcje") + f(item, "zmsddcje") + + f(item, "zmszdcje") + f(item, "zmsxdcje") + ) + # 大/中/小 净额：只看主动方（不包含被动方，否则恒等 0） + large_net = f(item, "zmbddcje") - f(item, "zmsddcje") + medium_net = f(item, "zmbzdcje") - f(item, "zmszdcje") + small_net = f(item, "zmbxdcje") - f(item, "zmsxdcje") + records.append({ + "trade_date": d, + "main_net_inflow": round(main_buy - main_sell, 2), + "large_net_inflow": round(large_net, 2), + "medium_net_inflow": round(medium_net, 2), + "small_net_inflow": round(small_net, 2), + }) + if not records: + return pd.DataFrame() + df = pd.DataFrame(records) + # 日期过滤 + if start: + df = df[df["trade_date"] >= start] + if end: + df = df[df["trade_date"] <= end] + return df.sort_values("trade_date").reset_index(drop=True) \ No newline at end of file diff --git a/app/sources/sina.py b/app/sources/sina.py new file mode 100644 index 0000000..45c7697 --- /dev/null +++ b/app/sources/sina.py @@ -0,0 +1,137 @@ +"""新浪财经数据源。 + +提供： + - 日 K 线（主源，免费，无 token） + - 指数日 K 线 + +接口：https://money.finance.sina.com.cn/quotes_service/api/json_v2.php/CN_MarketData.getKLineData +""" +from __future__ import annotations + +import json +import threading +import time +import urllib.request +from typing import Any + +import pandas as pd + +from app.core.datasource.base import DataSource +from app.core.datasource.utils import ( + code6_to_sina, + filter_date_range, + normalize_kline, +) + + +class SinaSource(DataSource): + key = "datasource_xinlang" + name = "新浪财经" + provides = ["kline_daily", "index_daily"] + requires_credential = False + + _HEADERS = { + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)", + "Referer": "https://finance.sina.com.cn", + } + _BASE_URL = ( + "https://money.finance.sina.com.cn/quotes_service/api/json_v2.php" + "/CN_MarketData.getKLineData" + ) + + # RPS 限流（线程安全） + # 设为 3 防止触发新浪反爬（单 IP 频率过高会被临时封禁） + _RPS_LIMIT = 3.0 + _rate_lock = threading.Lock() + _last_request_ts = 0.0 + + @classmethod + def _wait_rps(cls): + """全局 RPS 限流：所有线程共享同一个速率限制。""" + with cls._rate_lock: + now = time.time() + elapsed = now - cls._last_request_ts + min_interval = 1.0 / cls._RPS_LIMIT + if elapsed < min_interval: + time.sleep(min_interval - elapsed) + cls._last_request_ts = time.time() + + def is_available(self) -> tuple[bool, str]: + return True, "ok" + + def health_check(self) -> dict[str, Any]: + try: + url = ( + f"{self._BASE_URL}?symbol=sh600036&scale=240&ma=no&datalen=5" + ) + req = urllib.request.Request(url, headers=self._HEADERS) + with urllib.request.urlopen(req, timeout=15) as resp: + data = json.loads(resp.read().decode("utf-8")) + if not data or not isinstance(data, list): + return {"success": False, "message": "新浪财经返回数据格式异常"} + return {"success": True, "message": f"连接成功，获取到 {len(data)} 条 K 线数据"} + except Exception as e: + return {"success": False, "message": f"新浪财经连接失败: {e}"} + + def _fetch(self, symbol: str, datalen: int = 5000, retry: int = 2) -> list: + self._wait_rps() + url = f"{self._BASE_URL}?symbol={symbol}&scale=240&ma=no&datalen={int(max(datalen, 260))}" + last_err = None + for attempt in range(retry + 1): + try: + req = urllib.request.Request(url, headers=self._HEADERS) + with urllib.request.urlopen(req, timeout=20) as resp: + raw = resp.read().decode("gbk", errors="replace") + return json.loads(raw) + except Exception as e: + last_err = e + if attempt < retry: + time.sleep(0.5 * (attempt + 1)) # 短暂退避后重试 + continue + if last_err: + from app.core.utils.logging import get_logger + get_logger("sina").debug(f"sina fetch {symbol} failed after {retry+1} attempts: {last_err}") + return [] + + def fetch_kline_daily(self, code6: str, start: str, end: str) -> pd.DataFrame: + symbol = code6_to_sina(code6) + data = self._fetch(symbol) + if not data or not isinstance(data, list): + return pd.DataFrame() + records = [] + for item in data: + try: + records.append({ + "trade_date": item["day"], + "open": item["open"], + "high": item["high"], + "low": item["low"], + "close": item["close"], + "volume": item["volume"], + }) + except KeyError: + continue + return filter_date_range(normalize_kline(pd.DataFrame(records)), start, end) + + def fetch_index_daily(self, index_code: str, start: str, end: str) -> pd.DataFrame: + if index_code.startswith(("5", "6", "9")): + symbol = f"sh{index_code}" + else: + symbol = f"sz{index_code}" + data = self._fetch(symbol) + if not data or not isinstance(data, list): + return pd.DataFrame() + records = [] + for item in data: + try: + records.append({ + "trade_date": item["day"], + "open": float(item["open"]), + "high": float(item["high"]), + "low": float(item["low"]), + "close": float(item["close"]), + "volume": float(item["volume"]), + }) + except (KeyError, ValueError, TypeError): + continue + return filter_date_range(normalize_kline(pd.DataFrame(records)), start, end) diff --git a/app/sources/xueqiu.py b/app/sources/xueqiu.py new file mode 100644 index 0000000..d6ecc53 --- /dev/null +++ b/app/sources/xueqiu.py @@ -0,0 +1,203 @@ +"""雪球数据源（pysnowball，非官方 SDK）。 + +提供： + - 日 K 线复核（需要 token，免费但有限速） + - 最新股本快照（quote_detail，单位原始股，需 ÷1e8 转为亿股） + +没有 token 时降级：is_available() 返回 False。""" +from __future__ import annotations + +import logging +import os +import time +from datetime import datetime +from typing import Any, Optional + +import pandas as pd + +from app.core.datasource.base import DataSource +from app.core.datasource.utils import code6_to_xueqiu + +logger = logging.getLogger("sync.xueqiu") + + +class XueqiuSource(DataSource): + key = "datasource_xueqiu" + name = "雪球" + provides = ["kline_daily", "share"] + requires_credential = True + credential_key = "XUEQIU_TOKEN" + + # 进程内只 set_token 一次 + _token_lock = __import__("threading").Lock() + _token_set = False + + # RPS 限流（线程安全） + # 雪球 token 限速 10 RPS（用户实测），用满 10 才能在合理时间内跑完全市场 + _RPS_LIMIT = 10.0 + _rate_lock = __import__("threading").Lock() + _last_request = 0.0 + + def _read_token(self) -> str: + return os.environ.get(self.credential_key, "").strip() + + def _import_ball(self): + """延迟 + 显式 import。失败时打 warning（之前全被 except 吞了）。""" + try: + import pysnowball as ball + return ball + except ImportError as e: + logger.error( + "pysnowball 未安装，雪球数据源不可用。pip install pysnowball。错误: %s", e, + ) + raise + + def _set_token_once(self) -> None: + if self._token_set: + return + with self._token_lock: + if self._token_set: + return + token = self._read_token() + if not token: + return + try: + ball = self._import_ball() + ball.set_token(token) + self._token_set = True + except ImportError: + pass # _import_ball 已经打日志了 + + def _wait_rps(self) -> None: + with self._rate_lock: + now = time.time() + elapsed = now - self._last_request + min_interval = 1.0 / self._RPS_LIMIT + if elapsed < min_interval: + time.sleep(min_interval - elapsed) + self._last_request = time.time() + + def is_available(self) -> tuple[bool, str]: + token = self._read_token() + if not token: + return False, f"未配置 {self.credential_key}" + # 顺便检查 pysnowball 是否装了 + try: + import pysnowball # noqa: F401 + except ImportError: + return False, "pysnowball 未安装，pip install pysnowball" + return True, "ok" + + def health_check(self) -> dict[str, Any]: + if not self._read_token(): + return {"success": False, "message": f"未配置 {self.credential_key}"} + try: + ball = self._import_ball() + self._set_token_once() + result = ball.quote_detail("SH600036") + if result is None or result.get("error_code") != 0: + return { + "success": False, + "message": f"雪球返回异常: error_code={result.get('error_code') if result else 'None'} " + f"desc={result.get('error_description') if result else 'None'}", + } + quote = (result or {}).get("data", {}).get("quote", {}) + if not quote.get("total_shares") or not quote.get("float_shares"): + return {"success": False, "message": "雪球返回成功，但缺少 total_shares/float_shares 字段"} + return {"success": True, "message": "雪球连接成功，可读取股本快照"} + except Exception as e: + logger.exception("雪球 health_check 异常: %s", e) + return {"success": False, "message": f"雪球连接失败: {e}"} + + def fetch_kline_daily(self, code6: str, start: str, end: str) -> pd.DataFrame: + if not self._read_token(): + return pd.DataFrame() + try: + ball = self._import_ball() + self._set_token_once() + self._wait_rps() + + symbol = code6_to_xueqiu(code6) + start_dt = datetime.strptime(start, "%Y-%m-%d") + end_dt = datetime.strptime(end, "%Y-%m-%d") + count = min(max((end_dt - start_dt).days + 60, 10), 5000) + + # 加重试：雪球风控偶尔返回空/错误 + result = None + for attempt in range(3): + result = ball.kline(symbol, period="day", count=count) + if result and result.get("error_code") == 0: + break + time.sleep(0.3 * (attempt + 1)) + if not result or result.get("error_code") != 0: + logger.warning( + "[kline %s] 雪球 kline 失败: error_code=%s desc=%s", + code6, + result.get("error_code") if result else "None", + result.get("error_description") if result else "None", + ) + return pd.DataFrame() + data = result.get("data", {}) + columns = data.get("column", []) + items = data.get("item", []) + if not columns or not items: + return pd.DataFrame() + needed = {"timestamp": 0, "volume": 1, "open": 2, "high": 3, "low": 4, "close": 5} + records = [] + for row in items: + try: + ts = row[needed["timestamp"]] / 1000 + records.append({ + "trade_date": datetime.fromtimestamp(ts).strftime("%Y-%m-%d"), + "open": float(row[needed["open"]]), + "high": float(row[needed["high"]]), + "low": float(row[needed["low"]]), + "close": float(row[needed["close"]]), + "volume": float(row[needed["volume"]]), + }) + except (IndexError, ValueError, TypeError, OSError): + continue + from app.core.datasource.utils import filter_date_range, normalize_kline + return filter_date_range(normalize_kline(pd.DataFrame(records)), start, end) + except Exception as e: + logger.exception("[kline %s] 雪球 kline 异常: %s", code6, e) + return pd.DataFrame() + + def fetch_share_snapshot(self, code6: str) -> Optional[dict[str, Any]]: + """单只股票的最新股本快照。 + + 返回： + {total_share, float_share, trade_date} —— 单位：亿股 + """ + if not self._read_token(): + return None + try: + ball = self._import_ball() + self._set_token_once() + self._wait_rps() + + symbol = code6_to_xueqiu(code6) + result = ball.quote_detail(symbol) + if not result or result.get("error_code") != 0: + logger.warning( + "[share %s] 雪球 quote_detail 失败: error_code=%s desc=%s", + code6, + result.get("error_code") if result else "None", + result.get("error_description") if result else "None", + ) + return None + quote = result.get("data", {}).get("quote", {}) + today_str = datetime.now().strftime("%Y-%m-%d") + total = round(float(quote.get("total_shares") or 0) / 1e8, 4) + flt = round(float(quote.get("float_shares") or 0) / 1e8, 4) + if total <= 0 or flt <= 0: + logger.warning("[share %s] 雪球返回成功但 total/float 为 0", code6) + return None + return { + "trade_date": today_str, + "total_share": total, + "float_share": flt, + } + except Exception as e: + logger.exception("[share %s] 雪球 quote_detail 异常: %s", code6, e) + return None diff --git a/app/tasks/__init__.py b/app/tasks/__init__.py new file mode 100644 index 0000000..ad5310a --- /dev/null +++ b/app/tasks/__init__.py @@ -0,0 +1,39 @@ +"""同步任务注册表。 + +所有 `SyncTask` 子类在此集中注册，外部通过 `get_task(dataset_id)` 或 +遍历 `TASKS` 字典使用。 +""" +from app.tasks.task_industry_sector import SyncIndustrySector +from app.tasks.task_kline_5min import SyncKline5Min +from app.tasks.task_kline_daily import SyncKlineDaily +from app.tasks.task_kline_index import SyncKlineIndex +from app.tasks.task_market_regime import SyncMarketRegime +from app.tasks.task_moneyflow import SyncMoneyflow +from app.tasks.task_sector_features import SyncSectorFeatures +from app.tasks.task_share_snapshot import SyncShareSnapshot +from app.tasks.task_stocks_basic import SyncStocksBasic + +TASKS: dict[str, type] = { + cls.dataset_id: cls + for cls in ( + SyncStocksBasic, + SyncKlineDaily, + SyncKlineIndex, + SyncKline5Min, + SyncMoneyflow, + SyncIndustrySector, + SyncSectorFeatures, + SyncShareSnapshot, + SyncMarketRegime, + ) +} + + +def get_task(dataset_id: str): + """通过 dataset_id 取同步任务类实例，找不到抛 KeyError。""" + if dataset_id not in TASKS: + raise KeyError(f"未知的同步任务: {dataset_id!r}，可选: {sorted(TASKS)}") + return TASKS[dataset_id]() + + +__all__ = ["TASKS", "get_task"] diff --git a/app/tasks/task_industry_sector.py b/app/tasks/task_industry_sector.py new file mode 100644 index 0000000..02d7c2f --- /dev/null +++ b/app/tasks/task_industry_sector.py @@ -0,0 +1,122 @@ +"""同步任务：股票-行业映射（Baostock）。 + +数据流：Baostock query_stock_industry() → 写 industry / sectors / stock_sector_map 三张表。""" +from __future__ import annotations + +import time +from typing import Any + +from app.core.db import ops as db_ops +from app.core.datasource.base import registry as ds_registry +from app.core.datasource.registry import is_source_ready +from app.core.sync.base import SyncTask +from app.core.sync.registry import mark_sync_blocked +from app.core.utils.logging import get_logger + +logger = get_logger("sync.industry_sector") + + +def _build_sector_key(taxonomy: str, sector_name: str) -> str: + tax = (taxonomy or "").strip() or "default" + name = (sector_name or "").strip() + return f"{tax}::{name}" + + +class SyncIndustrySector(SyncTask): + dataset_id = "industry_sector" + + def _run(self, *, trigger_source: str = "manual", **kwargs) -> dict[str, Any]: + bs = ds_registry.get("datasource_baostock") + if bs is None: + return {"status": "error", "message": "Baostock 数据源未注册"} + ok, reason = is_source_ready(bs.key) + if not ok: + mark_sync_blocked(self.dataset_id, message=f"Baostock 未就绪: {reason}") + return {"status": "blocked", "message": f"Baostock 未就绪: {reason}"} + + self._progress(message="拉取股票-行业映射...") + t0 = time.time() + try: + raw_rows = bs.fetch_industry_map() + except Exception as e: + return {"status": "error", "message": f"拉取失败: {e}"} + + if not raw_rows: + return {"status": "warning", "message": "Baostock 未返回行业数据"} + + # 合并：sectors + stock_sector_map + industry + sectors_rows: dict[str, dict] = {} + stock_sector_rows: list[dict] = [] + industry_rows: list[dict] = [] + + for r in raw_rows: + code6 = str(r["code"]).zfill(6) + name = (r.get("industry_name") or "").strip() + if not name: + continue + classification = (r.get("industry_classification") or "").strip() + sector_key = _build_sector_key(classification, name) + + sectors_rows[sector_key] = { + "sector_key": sector_key, + "sector_name": name, + "taxonomy": classification, + "level": "", + "source": "baostock_query_stock_industry", + "enabled": 1, + } + stock_sector_rows.append({"stock_code": code6, "sector_key": sector_key}) + industry_rows.append({ + "code": code6, + "industry_name": name, + "industry_classification": classification, + "update_date": r.get("update_date", ""), + }) + + # 写库（全量替换） + try: + db_ops.replace_all_sectors(list(sectors_rows.values())) + db_ops.replace_all_stock_sector_map(stock_sector_rows) + db_ops.replace_all_industries(industry_rows) + except Exception as e: + return {"status": "error", "message": f"写库失败: {e}"} + + # 回填 stocks.industry 字段（让 stocks 表也能直接看到） + # 使用批量 UPDATE ... CASE WHEN 方式一次性完成 + try: + from app.core.db.connection import get_mysql + from app.core.db.schema import ensure_all_tables + ensure_all_tables(get_mysql()) + # 按 industry_name 分组，对每组用一个 IN 查询批量更新 + by_industry: dict[str, list[str]] = {} + for r in industry_rows: + name = r["industry_name"] + code = r["code"] + if name not in by_industry: + by_industry[name] = [] + by_industry[name].append(code) + with get_mysql().cursor() as cur: + for ind_name, codes in by_industry.items(): + # 构造 IN 子句：code IN (SHxxx, SZxxx, BJxxx) + placeholders = [] + params = [ind_name] + for c in codes: + for prefix in ("SH", "SZ", "BJ"): + placeholders.append("%s") + params.append(f"{prefix}{c}") + in_clause = ", ".join(placeholders) + sql = f"UPDATE stocks SET industry = %s WHERE code IN ({in_clause})" + cur.execute(sql, params) + get_mysql().commit() + except Exception as e: + logger.warning(f"回填 stocks.industry 失败（不影响主任务）: {e}") + + elapsed = round(time.time() - t0, 1) + msg = f"行业 {len(sectors_rows)}个 映射{len(stock_sector_rows)}条, {elapsed}s" + return { + "status": "ok", + "message": msg, + "sectors": len(sectors_rows), + "stock_sector_map": len(stock_sector_rows), + "elapsed_sec": elapsed, + } diff --git a/app/tasks/task_kline_5min.py b/app/tasks/task_kline_5min.py new file mode 100644 index 0000000..47e1780 --- /dev/null +++ b/app/tasks/task_kline_5min.py @@ -0,0 +1,203 @@ +"""同步任务：5 分钟 K 线（mairui）。 + +设计要点： + 1) 启动时**一次 SQL** 拿全表 {stock_code: max(bar_time)} 快照 + 2) 内存里给每只股票算 (start, end, windows_needed) + - DB 没数据 → start = mairui 历史深度起点 (2023-06-14) + - DB 有数据 → start = max(2023-06-14, max(bar_time) - 2 天) + - 兜底：start >= end → 跳过这只 + 3) 输出"计划"（全量/增量分类、预估请求数、预估耗时）再开始执行 + 4) 5 worker × 1 RPS/worker = 5 RPS 总（守住 mairui 上限） + 5) 窗口大小 1 年（mairui 一次最多 ~11600 条） + +akshare 时代写法：start 写死 2020-01-01 + 4 天窗口，137 小时跑完全市场。 +本版：增量时通常只补最近 1-2 天，~30 分钟。 +""" +from __future__ import annotations + +import os +import time +from concurrent.futures import ThreadPoolExecutor, as_completed +from datetime import datetime, timedelta +from typing import Any, Optional + +from app.core.db import ops as db_ops +from app.core.datasource.base import registry as ds_registry +from app.core.datasource.registry import is_source_ready +from app.core.datasource.utils import is_a_share_code, to_code6 +from app.core.sync.base import SyncTask +from app.core.sync.registry import mark_sync_blocked +from app.core.utils.logging import get_logger + +logger = get_logger("sync.kline_5min") + +# mairui 5min K 实测历史深度起点（实测 2023-06-14 才有数据，更早就 0） +MAIRUI_5MIN_FLOOR = datetime(2023, 6, 14) +# 单次请求窗口大小：1 年（mairui 单次最多 ~11640 条 5min） +WINDOW_DAYS = 365 +# 增量时往前多取的天数（防交易日历边界漏当天） +INCREMENT_OVERLAP_DAYS = 2 + + +class SyncKline5Min(SyncTask): + dataset_id = "kline_5min" + + def _plan( + self, + stock_codes: list[str], + end: datetime, + snapshots: dict[str, Optional[datetime]], + ) -> list[tuple[str, datetime, datetime]]: + """为每只股票算 (start, end) — 已排除 start >= end 的"无需同步"。 + + Returns: [(code6, start, end), ...] + """ + plans = [] + for code6 in stock_codes: + latest = snapshots.get(code6) + if latest is None: + # DB 里没数据 → 全量从 mairui 历史深度起点 + start = MAIRUI_5MIN_FLOOR + else: + # DB 有数据 → 增量：从 max(bar_time) - overlap 到 end + start = max(MAIRUI_5MIN_FLOOR, latest - timedelta(days=INCREMENT_OVERLAP_DAYS)) + if start < end: + plans.append((code6, start, end)) + return plans + + def _run( + self, + *, + trigger_source: str = "manual", + codes: list[str] | None = None, + max_workers: int = 5, + **kwargs, + ) -> dict[str, Any]: + primary = ds_registry.get("datasource_mairui") + if primary is None: + return {"status": "error", "message": "5min 数据源 mairui 未注册"} + ok, reason = is_source_ready(primary.key) + if not ok: + mark_sync_blocked(self.dataset_id, message=f"{primary.key} 未就绪: {reason}") + return {"status": "blocked", "message": f"{primary.key} 未就绪: {reason}"} + + if codes: + stock_codes = [to_code6(c) for c in codes] + else: + stock_codes = [c for c in db_ops.iter_stock_codes(active_only=True) if is_a_share_code(c)] + limit_raw = os.environ.get("MARKET_DATA_STOCK_LIMIT", "").strip() + if limit_raw.isdigit() and int(limit_raw) > 0: + stock_codes = stock_codes[: int(limit_raw)] + + if not stock_codes: + return {"status": "error", "message": "无股票代码"} + + # ── 1) 一次 SQL 拿全表快照 ── + logger.info(f"[5min] 读取 DB 快照 ({len(stock_codes)} 只待查)…") + snapshots = db_ops.get_kline_5min_snapshots() + logger.info(f"[5min] DB 里已有 {len(snapshots)} 只的 kline_5min 数据") + + # ── 2) 内存算计划 ── + end = datetime.now() + plans = self._plan(stock_codes, end, snapshots) + if not plans: + msg = f"5min K线 — 全部 {len(stock_codes)} 只都已最新（无需同步）" + logger.info(msg) + return {"status": "ok", "message": msg, "ok": 0, "fail": 0, "rows": 0, "elapsed_sec": 0.0} + + # 分类统计 + full_sync = [p for p in plans if snapshots.get(p[0]) is None] + incr_sync = [p for p in plans if snapshots.get(p[0]) is not None] + total_windows = sum( + max(1, (p[2] - p[1]).days // WINDOW_DAYS + 1) + for p in plans + ) + # mairui 5 RPS 上限（5 worker × 1 RPS） + est_sec = total_windows / 5.0 + logger.info( + f"[5min] 计划: 总 {len(plans)} 只 (全量 {len(full_sync)} / 增量 {len(incr_sync)})，" + f"总请求 {total_windows}，按 5 RPS 预估 {est_sec:.0f}s ({est_sec/60:.1f}min)" + ) + self._progress( + message=f"计划: {len(plans)} 只 (全量 {len(full_sync)}/增量 {len(incr_sync)}) " + f"约 {total_windows} 个请求, 预估 {est_sec/60:.1f}min", + current=0, total=len(plans), current_step="planning", + ) + + # ── 3) 执行 ── + t0 = time.time() + ok_cnt = fail_cnt = 0 + rows_total = 0 + with ThreadPoolExecutor(max_workers=max_workers) as pool: + futures = {pool.submit(self._sync_one, primary, p[0], p[1], p[2]): p[0] for p in plans} + for i, future in enumerate(as_completed(futures), 1): + c6 = futures[future] + try: + res = future.result() + if res["status"] == "ok": + ok_cnt += 1 + rows_total += res["rows"] + else: + fail_cnt += 1 + if res.get("error"): + logger.warning(f"[5min {c6}] {res['error']}") + except Exception as e: + fail_cnt += 1 + logger.warning(f"[5min {c6}] {e}") + if i % 20 == 0 or i == len(plans): + self._progress( + message=f"5min 进度 {i}/{len(plans)} OK:{ok_cnt} FAIL:{fail_cnt}", + current=i, total=len(plans), current_step=c6, + ) + elapsed = round(time.time() - t0, 1) + msg = ( + f"5min K线 {ok_cnt}成 {fail_cnt}败 共{rows_total}行, " + f"全量{len(full_sync)}/增量{len(incr_sync)}, {elapsed}s" + ) + return { + "status": "ok" if fail_cnt == 0 else "warning", + "message": msg, + "ok": ok_cnt, "fail": fail_cnt, "rows": rows_total, + "full_sync": len(full_sync), "incr_sync": len(incr_sync), + "total_requests": total_windows, "elapsed_sec": elapsed, + } + + def _sync_one(self, primary, code6: str, start: datetime, end: datetime) -> dict: + all_rows: list[dict] = [] + try: + cur = start + while cur < end: + w_end = min(cur + timedelta(days=WINDOW_DAYS), end) + df = primary.fetch_kline_5min( + code6, + cur.strftime("%Y-%m-%d"), + w_end.strftime("%Y-%m-%d"), + ) + if df is not None and not df.empty: + for _, r in df.iterrows(): + bar_time = r["bar_time"] + all_rows.append({ + "stock_code": code6, + "bar_time": bar_time.strftime("%Y-%m-%d %H:%M:%S") + if hasattr(bar_time, "strftime") else str(bar_time), + "open": float(r.get("open") or 0), + "high": float(r.get("high") or 0), + "low": float(r.get("low") or 0), + "close": float(r.get("close") or 0), + "volume": float(r.get("volume") or 0), + "amount": float(r.get("amount") or 0), + "turnover_rate": float(r.get("turnover_rate") or 0), + }) + cur = w_end + timedelta(days=1) + except Exception as e: + return {"status": "fail", "error": str(e)} + + if not all_rows: + return {"status": "fail", "error": "no data"} + try: + CHUNK = 5000 + for i in range(0, len(all_rows), CHUNK): + db_ops.upsert_kline_5min(all_rows[i: i + CHUNK]) + except Exception as e: + return {"status": "fail", "error": f"db write: {e}"} + return {"status": "ok", "rows": len(all_rows)} diff --git a/app/tasks/task_kline_daily.py b/app/tasks/task_kline_daily.py new file mode 100644 index 0000000..4154a2c --- /dev/null +++ b/app/tasks/task_kline_daily.py @@ -0,0 +1,251 @@ +"""同步任务：全市场日 K 线。 + +数据流： + 1. 选主源（按 priority 选第一个 is_available 的源） + 2. 增量判断：读 kline_stock.MAX(trade_date) 决定每只股票 start + 3. **Producer-Consumer 模式**： + - N 个 fetcher 线程并发拉 K 线，把 (code, rows) 塞到 Queue + - 1 个 writer 线程攒满 BATCH_SIZE 就 executemany + commit 一次 + 4. 更新 stocks.kline_synced_at + +设计要点： + - 避免每只股票 commit（5000 只 = 5000 次 commit 太慢） + - 减少并发数（避免新浪限流 / MySQL 连接数爆） + - fetcher 间用 Queue 解耦，写库串行避免锁竞争 +""" +from __future__ import annotations + +import os +import queue +import threading +import time +from datetime import datetime, timedelta +from typing import Any + +import pandas as pd + +from app.core.db import ops as db_ops +from app.core.datasource.base import registry as ds_registry +from app.core.datasource.utils import is_a_share_code, to_code6 +from app.core.sync.base import SyncTask, _effective_sync_end +from app.core.utils.logging import get_logger + +logger = get_logger("sync.kline_daily") + +# 主源优先级：mairui（5 RPS，免费 + 稳定 + 单接口含5min）→ 雪球 → Baostock → 新浪 +# 经验：sina 在单日窗口 (start=end=某天) 会返回空，雪球更可靠 +PRIMARY_PRIORITY = ["datasource_xueqiu", "datasource_mairui", "datasource_xinlang"] +DEFAULT_START = "2018-01-01" +# 写库批量：攒够 BATCH_SIZE 行就 executemany 一次 + commit +BATCH_SIZE = 2000 +# fetcher 并发：实测单线程最稳（5000 只 × 0.2s = 17 分钟，足够） +# 雪球 token 限速 10 RPS，单线程 0.1s/只 已经打满。多 worker 会触发风控 hang +DEFAULT_FETCH_WORKERS = 1 + + +class SyncKlineDaily(SyncTask): + dataset_id = "kline_daily" + + def _run( + self, + *, + trigger_source: str = "manual", + start: str | None = None, + end: str | None = None, + codes: list[str] | None = None, + incremental: bool = True, + max_workers: int = DEFAULT_FETCH_WORKERS, + **kwargs, + ) -> dict[str, Any]: + # 1. 选主源 + primary = self._pick_primary() + if primary is None: + return {"status": "error", "message": "所有 K 线数据源均不可用"} + + self._progress( + message=f"使用 {primary.name} 同步日K线 (fetcher={max_workers}, batch={BATCH_SIZE})...", + current_step=primary.key, + ) + + # 2. 拉股票列表 + if codes: + stock_codes = [to_code6(c) for c in codes] + else: + stock_codes = [ + c for c in db_ops.iter_stock_codes(active_only=True) + if is_a_share_code(c) + ] + limit_raw = os.environ.get("MARKET_DATA_STOCK_LIMIT", "").strip() + if limit_raw.isdigit() and int(limit_raw) > 0: + stock_codes = stock_codes[: int(limit_raw)] + + if not stock_codes: + return {"status": "error", "message": "无股票代码（stocks 表为空？）"} + + # 3. 增量判断 + _end = end or _effective_sync_end() + end_date_obj = datetime.strptime(_end, "%Y-%m-%d").date() + jobs: list[tuple[str, str]] = [] + if incremental and not start: + for c6 in stock_codes: + last = db_ops.get_stock_kline_max_date(c6) + if last is None: + fetch_start = DEFAULT_START + elif datetime.strptime(last, "%Y-%m-%d").date() >= end_date_obj: + continue + else: + fetch_start = (datetime.strptime(last, "%Y-%m-%d").date() - timedelta(days=5)).strftime("%Y-%m-%d") + jobs.append((c6, fetch_start)) + else: + fetch_start = start or DEFAULT_START + jobs = [(c, fetch_start) for c in stock_codes] + + total = len(jobs) + skipped = len(stock_codes) - total + self._progress( + message=f"开始同步 {total} 只股票 (跳过 {skipped} 只已最新)", + current=0, total=total, + ) + + # 4. 并发 fetch + 攒 batch 写库（线程安全） + # mairui 等外部 API 无并发问题；DB 写串行加锁 + from concurrent.futures import ThreadPoolExecutor, as_completed + t0 = time.time() + batch_lock = threading.Lock() + batch_rows: list[dict] = [] + pending_codes: dict[str, str] = {} + ok_lock = threading.Lock() + ok_cnt = [0] + fail_cnt = [0] + rows_cnt = [0] + + def _flush(): + with batch_lock: + if not batch_rows: + return + rows_to_write = list(batch_rows) + codes_to_update = dict(pending_codes) + batch_rows.clear() + pending_codes.clear() + try: + db_ops.upsert_kline_stock(rows_to_write) + except Exception as e: + logger.error(f"batch upsert 失败 ({len(rows_to_write)} 行): {e}") + return + for c6, latest in codes_to_update.items(): + try: + db_ops.update_stock_kline_synced_at(c6, latest) + except Exception as e: + logger.warning(f"update synced_at 失败 {c6}: {e}") + + def _fetch_one(code6: str, fetch_start: str) -> tuple[str, list[dict] | None, str | None]: + try: + df = primary.fetch_kline_daily(code6, fetch_start, _end) + except Exception as e: + return (code6, None, str(e)[:120]) + if df is None or df.empty: + return (code6, None, "no data") + rows = [] + for _, r in df.iterrows(): + td = r["trade_date"] + rows.append({ + "stock_code": code6, + "trade_date": td.strftime("%Y-%m-%d") if hasattr(td, "strftime") else str(td)[:10], + "open": float(r["open"]), + "high": float(r["high"]), + "low": float(r["low"]), + "close": float(r["close"]), + "volume": float(r["volume"]), + }) + latest = df["trade_date"].max() + latest = latest.strftime("%Y-%m-%d") if hasattr(latest, "strftime") else str(latest)[:10] + return (code6, rows, latest) + + total = len(jobs) + with ThreadPoolExecutor(max_workers=max_workers) as pool: + futures = {pool.submit(_fetch_one, c6, fs): c6 for c6, fs in jobs} + done_cnt = 0 + for future in as_completed(futures): + done_cnt += 1 + code6, rows, latest = future.result() + if rows is None: + with ok_lock: + fail_cnt[0] += 1 + logger.warning(f"[kline {code6}] {latest}") + continue + with batch_lock: + batch_rows.extend(rows) + pending_codes[code6] = latest + cur_size = len(batch_rows) + with ok_lock: + ok_cnt[0] += 1 + rows_cnt[0] += len(rows) + + if cur_size >= BATCH_SIZE: + _flush() + + if done_cnt % 50 == 0 or done_cnt == total: + elapsed = time.time() - t0 + rate = done_cnt / elapsed if elapsed > 0 else 0 + self._progress( + message=f"进度 {done_cnt}/{total} OK:{ok_cnt[0]} FAIL:{fail_cnt[0]} {rate:.1f}只/秒", + current=done_cnt, total=total, current_step=code6, + ) + + # 收尾 flush + _flush() + + elapsed = round(time.time() - t0, 1) + msg = f"日K线 {ok_cnt[0]}成 {fail_cnt[0]}败 {skipped}跳 共{rows_cnt[0]}行, {elapsed}s" + return { + "status": "ok" if fail_cnt[0] == 0 else "warning", + "message": msg, + "ok": ok_cnt[0], + "fail": fail_cnt[0], + "skip": skipped, + "rows": rows_cnt[0], + "elapsed_sec": elapsed, + "primary_source": primary.key, + } + + def _pick_primary(self): + from app.core.datasource.registry import is_source_ready, run_health_check, get_health_status + if not get_health_status(): + run_health_check() + for key in PRIMARY_PRIORITY: + ok, _ = is_source_ready(key) + if ok: + return ds_registry.get(key) + return None + + def _sync_one(self, primary, code6: str, start: str, end: str) -> dict: + """保留这个方法以兼容外部调用（已不用，但单只测试可能用到）""" + try: + df = primary.fetch_kline_daily(code6, start, end) + except Exception as e: + return {"status": "fail", "error": str(e)} + if df is None or df.empty: + return {"status": "fail", "error": "no data"} + rows = [] + for _, r in df.iterrows(): + td = r["trade_date"] + rows.append({ + "stock_code": code6, + "trade_date": td.strftime("%Y-%m-%d") if hasattr(td, "strftime") else str(td)[:10], + "open": float(r["open"]), + "high": float(r["high"]), + "low": float(r["low"]), + "close": float(r["close"]), + "volume": float(r["volume"]), + }) + try: + db_ops.upsert_kline_stock(rows) + except Exception as e: + return {"status": "fail", "error": f"db write: {e}"} + if rows: + latest = max(r["trade_date"] for r in rows) + try: + db_ops.update_stock_kline_synced_at(code6, latest) + except Exception: + pass + return {"status": "ok", "rows": len(rows)} diff --git a/app/tasks/task_kline_index.py b/app/tasks/task_kline_index.py new file mode 100644 index 0000000..1adaa51 --- /dev/null +++ b/app/tasks/task_kline_index.py @@ -0,0 +1,109 @@ +"""同步任务：六大指数日 K 线。 + +数据流：新浪指数日K接口 → 写 kline_index + indices 表。""" +from __future__ import annotations + +from datetime import datetime +from typing import Any + +from app.core.db import ops as db_ops +from app.core.datasource.base import registry as ds_registry +from app.core.datasource.registry import is_source_ready +from app.core.sync.base import SyncTask, _is_market_closed +from app.core.sync.registry import mark_sync_blocked +from app.core.utils.logging import get_logger + +logger = get_logger("sync.kline_index") + +MAJOR_INDICES = { + "000001": {"name": "上证指数", "market": "CN", "category": "broad_market"}, + "399001": {"name": "深证成指", "market": "CN", "category": "broad_market"}, + "399006": {"name": "创业板指", "market": "CN", "category": "broad_market"}, + "000300": {"name": "沪深300", "market": "CN", "category": "broad_market"}, + "000905": {"name": "中证500", "market": "CN", "category": "broad_market"}, + "000852": {"name": "中证1000", "market": "CN", "category": "broad_market"}, +} + + +class SyncKlineIndex(SyncTask): + dataset_id = "kline_index" + + def _run(self, *, trigger_source: str = "manual", **kwargs) -> dict[str, Any]: + # 优先 mairui（指数无单日空窗问题），sina 作为 fallback + primary = ds_registry.get("datasource_mairui") + use_mairui = primary is not None and is_source_ready(primary.key)[0] + if not use_mairui: + primary = ds_registry.get("datasource_xinlang") + if primary is None: + return {"status": "error", "message": "指数数据源均不可用（mairui + 新浪）"} + ok, reason = is_source_ready(primary.key) + if not ok: + mark_sync_blocked(self.dataset_id, message=f"{primary.key} 未就绪: {reason}") + return {"status": "blocked", "message": f"{primary.key} 未就绪: {reason}"} + + self._progress(message=f"开始同步 {len(MAJOR_INDICES)} 个指数...") + total = len(MAJOR_INDICES) + ok_cnt = fail_cnt = 0 + results = {} + + for i, (code, info) in enumerate(MAJOR_INDICES.items(), 1): + try: + # mairui 需带交易所后缀（如 000300.SH），sina 用纯6位 + if use_mairui: + # 000xxx 上证 → .SH；399xxx 深证 → .SZ + if code.startswith("399"): + suffix = ".SZ" + else: + suffix = ".SH" + fetch_code = f"{code}{suffix}" + else: + fetch_code = code + df = primary.fetch_index_daily(fetch_code, start="1990-01-01", end="2099-12-31") + if df is None or df.empty: + raise RuntimeError("返回空数据") + # 盘中未收盘则去掉当天 + if not _is_market_closed(): + today = datetime.now().strftime("%Y-%m-%d") + df = df[df["trade_date"].dt.strftime("%Y-%m-%d") < today] + + rows = [ + { + "index_code": code, + "trade_date": r["trade_date"].strftime("%Y-%m-%d"), + "open": float(r["open"]), + "high": float(r["high"]), + "low": float(r["low"]), + "close": float(r["close"]), + "volume": float(r["volume"]), + } + for _, r in df.iterrows() + ] + db_ops.upsert_kline_index(rows) + db_ops.upsert_index( + index_code=code, index_name=info["name"], + market=info["market"], category=info["category"], + source="mairui_index_daily" if use_mairui else "sina_index_daily", enabled=True, + ) + ok_cnt += 1 + results[code] = { + "rows": len(rows), + "date_min": rows[0]["trade_date"] if rows else "", + "date_max": rows[-1]["trade_date"] if rows else "", + } + except Exception as e: + fail_cnt += 1 + results[code] = {"error": str(e)} + logger.warning(f"[index {code}] 失败: {e}") + + self._progress( + message=f"指数 {i}/{total} 完成 ({ok_cnt}成 {fail_cnt}败)", + current=i, total=total, current_step=code, + ) + + msg = f"六大指数 {ok_cnt}成 {fail_cnt}败" + return { + "status": "ok" if fail_cnt == 0 else "warning", + "message": msg, + "ok": ok_cnt, "fail": fail_cnt, + "details": results, + } diff --git a/app/tasks/task_market_regime.py b/app/tasks/task_market_regime.py new file mode 100644 index 0000000..ef1b281 --- /dev/null +++ b/app/tasks/task_market_regime.py @@ -0,0 +1,113 @@ +"""同步任务：市场情绪（market_regime_daily，衍生源）。 + +基于本地 kline_stock 数据聚合：advancers / decliners / turnover / advance_ratio。""" +from __future__ import annotations + +from datetime import datetime +from typing import Any + +import numpy as np +import pandas as pd + +from app.core.db import ops as db_ops +from app.core.sync.base import SyncTask +from app.core.utils.logging import get_logger + +logger = get_logger("sync.market_regime") + +DEFAULT_START = "2018-01-01" +EXTREME_PANIC_ADV = 0.2 +EXTREME_PANIC_TURN = 1.5 + + +def _is_extreme_panic(row: pd.Series) -> int: + ar = pd.to_numeric(row.get("advance_ratio"), errors="coerce") + tr = pd.to_numeric(row.get("turnover_ratio_5d"), errors="coerce") + if pd.isna(ar) or pd.isna(tr): + return 0 + return int(ar < EXTREME_PANIC_ADV and tr > EXTREME_PANIC_TURN) + + +class SyncMarketRegime(SyncTask): + dataset_id = "market_regime" + + def _run(self, *, trigger_source: str = "manual", start: str = DEFAULT_START, **kwargs) -> dict[str, Any]: + end = datetime.now().strftime("%Y-%m-%d") + self._progress(message=f"构建 market_regime {start} ~ {end}...") + + # 1. 取所有非退市股票代码 + codes = list(db_ops.iter_stock_codes(active_only=True)) + if not codes: + return {"status": "error", "message": "stocks 表为空"} + + # 2. 拉所有 K 线（按股票）- 这里用 SQL GROUP BY 一次性算 daily turnover + from app.core.db.connection import get_mysql + + sql = """ + SELECT + stock_code, + trade_date, + `close`, + volume + FROM kline_stock + WHERE trade_date BETWEEN %s AND %s + ORDER BY stock_code, trade_date + """ + with get_mysql().cursor() as cur: + cur.execute(sql, (start, end)) + rows = cur.fetchall() + if not rows: + return {"status": "warning", "message": f"kline_stock 在 {start} ~ {end} 区间无数据"} + + df = pd.DataFrame(rows) + df["trade_date"] = pd.to_datetime(df["trade_date"], errors="coerce") + df["close"] = pd.to_numeric(df["close"], errors="coerce") + df["volume"] = pd.to_numeric(df["volume"], errors="coerce").fillna(0.0) + df = df.dropna(subset=["trade_date", "close"]).sort_values(["stock_code", "trade_date"]) + df["prev_close"] = df.groupby("stock_code")["close"].shift(1) + df = df.dropna(subset=["prev_close"]) + df["advancers"] = (df["close"] > df["prev_close"]).astype(int) + df["decliners"] = (df["close"] < df["prev_close"]).astype(int) + df["turnover"] = df["close"] * df["volume"] + + daily = df.groupby("trade_date", as_index=False)[["advancers", "decliners", "turnover"]].sum() + total = daily["advancers"] + daily["decliners"] + daily["advance_ratio"] = np.where(total > 0, daily["advancers"] / total, np.nan) + daily["turnover_avg_5d"] = daily["turnover"].rolling(5, min_periods=5).mean() + daily["turnover_ratio_5d"] = np.where( + daily["turnover_avg_5d"] > 0, + daily["turnover"] / daily["turnover_avg_5d"], + np.nan, + ) + daily["source"] = "local_kline_proxy" + daily["is_extreme_panic"] = daily.apply(_is_extreme_panic, axis=1) + daily["trade_date"] = daily["trade_date"].dt.strftime("%Y-%m-%d") + + rows_out = [ + { + "trade_date": r["trade_date"], + "advancers": float(r["advancers"]), + "decliners": float(r["decliners"]), + "advance_ratio": float(r["advance_ratio"]) if pd.notna(r["advance_ratio"]) else 0, + "turnover": float(r["turnover"]), + "turnover_avg_5d": float(r["turnover_avg_5d"]) if pd.notna(r["turnover_avg_5d"]) else 0, + "turnover_ratio_5d": float(r["turnover_ratio_5d"]) if pd.notna(r["turnover_ratio_5d"]) else 0, + "source": r["source"], + "is_extreme_panic": int(r["is_extreme_panic"]), + } + for _, r in daily.iterrows() + ] + + try: + db_ops.upsert_market_regime_rows(rows_out) + except Exception as e: + return {"status": "error", "message": f"写库失败: {e}"} + + msg = f"market_regime {len(rows_out)} 天 ({rows_out[0]['trade_date']} ~ {rows_out[-1]['trade_date']})" + return { + "status": "ok", + "message": msg, + "rows": len(rows_out), + "date_min": rows_out[0]["trade_date"], + "date_max": rows_out[-1]["trade_date"], + } diff --git a/app/tasks/task_moneyflow.py b/app/tasks/task_moneyflow.py new file mode 100644 index 0000000..66d4bfd --- /dev/null +++ b/app/tasks/task_moneyflow.py @@ -0,0 +1,117 @@ +"""同步任务：个股资金流（mairui）。 + +数据源：mairui `hsstock/history/transaction/{code}.{ex}/{licence}` +口径：主力 / 大 / 中 / 小单 净额（与 akshare stock_individual_fund_flow 一致） + +akshare 已移除（项目级决策：永远不用），mairui 替代为唯一源。 +""" +from __future__ import annotations + +import os +import time +from concurrent.futures import ThreadPoolExecutor, as_completed +from typing import Any + +from app.core.db import ops as db_ops +from app.core.datasource.base import registry as ds_registry +from app.core.datasource.registry import is_source_ready +from app.core.datasource.utils import is_a_share_code, to_code6 +from app.core.sync.base import SyncTask +from app.core.sync.registry import mark_sync_blocked +from app.core.utils.logging import get_logger + +logger = get_logger("sync.moneyflow") + + +class SyncMoneyflow(SyncTask): + dataset_id = "moneyflow" + + def _run( + self, + *, + trigger_source: str = "manual", + codes: list[str] | None = None, + max_workers: int = 5, + **kwargs, + ) -> dict[str, Any]: + mr = ds_registry.get("datasource_mairui") + if mr is None: + return {"status": "error", "message": "mairui 数据源未注册"} + ok, reason = is_source_ready(mr.key) + if not ok: + mark_sync_blocked(self.dataset_id, message=f"mairui 未就绪: {reason}") + return {"status": "blocked", "message": f"mairui 未就绪: {reason}"} + + if codes: + stock_codes = [to_code6(c) for c in codes] + else: + stock_codes = [c for c in db_ops.iter_stock_codes(active_only=True) if is_a_share_code(c)] + limit_raw = os.environ.get("MARKET_DATA_STOCK_LIMIT", "").strip() + if limit_raw.isdigit() and int(limit_raw) > 0: + stock_codes = stock_codes[: int(limit_raw)] + + if not stock_codes: + return {"status": "error", "message": "无股票代码"} + + total = len(stock_codes) + ok_cnt = fail_cnt = 0 + rows_total = 0 + t0 = time.time() + with ThreadPoolExecutor(max_workers=max_workers) as pool: + futures = {pool.submit(self._sync_one, mr, c6): c6 for c6 in stock_codes} + for i, future in enumerate(as_completed(futures), 1): + c6 = futures[future] + try: + res = future.result() + if res["status"] == "ok": + ok_cnt += 1 + rows_total += res["rows"] + else: + fail_cnt += 1 + if res.get("error"): + logger.warning(f"[moneyflow {c6}] {res['error']}") + except Exception as e: + fail_cnt += 1 + logger.warning(f"[moneyflow {c6}] {e}") + if i % 20 == 0 or i == total: + self._progress( + message=f"moneyflow 进度 {i}/{total} OK:{ok_cnt} FAIL:{fail_cnt}", + current=i, total=total, current_step=c6, + ) + elapsed = round(time.time() - t0, 1) + msg = f"资金流 {ok_cnt}成 {fail_cnt}败 共{rows_total}行, {elapsed}s" + return { + "status": "ok" if fail_cnt == 0 else "warning", + "message": msg, + "ok": ok_cnt, "fail": fail_cnt, + "rows": rows_total, "elapsed_sec": elapsed, + } + + def _sync_one(self, mr, code6: str) -> dict: + # 取最近 30 天（mairui 的 transaction 接口历史深度有限，30 天足够覆盖日常） + from datetime import datetime, timedelta + end = datetime.now() + start = end - timedelta(days=30) + try: + df = mr.fetch_moneyflow( + code6, + start.strftime("%Y-%m-%d"), + end.strftime("%Y-%m-%d"), + ) + if df is None or df.empty: + return {"status": "fail", "error": "no data"} + rows = [ + { + "stock_code": code6, + "trade_date": str(r["trade_date"]), + "main_net_inflow": float(r.get("main_net_inflow") or 0), + "large_net_inflow": float(r.get("large_net_inflow") or 0), + "medium_net_inflow": float(r.get("medium_net_inflow") or 0), + "small_net_inflow": float(r.get("small_net_inflow") or 0), + } + for _, r in df.iterrows() + ] + db_ops.upsert_moneyflow(rows) + return {"status": "ok", "rows": len(rows)} + except Exception as e: + return {"status": "fail", "error": str(e)} diff --git a/app/tasks/task_sector_features.py b/app/tasks/task_sector_features.py new file mode 100644 index 0000000..a352b0f --- /dev/null +++ b/app/tasks/task_sector_features.py @@ -0,0 +1,184 @@ +"""同步任务：行业聚合特征（衍生计算）。 + +输入：kline_stock（日 K 线）+ industry（股票→行业映射） +输出： + - sector_indices ：行业日线（trade_date × sector_name × close × sector_amplitude） + - sector_features_daily ：行业特征（+ sector_ret + ema10/20/200 + score） + +算法（参考 dashboard/api/services/sync/sector.py 重写，持久层改为 MySQL）： + 1) 每只股票日涨跌幅 pct_chg = (close/prev_close - 1) * 100 + 2) 按行业聚合 sector_ret = 该行业所有股票当日 pct_chg 的均值 + 3) 行业指数 close = 100 × ∏(1 + sector_ret/100) （基点 100，复合收益） + 4) 行业振幅 sector_amplitude = 该行业所有股票当日 (high-low)/close 的均值 + 5) EMA10/20/200 = close 的指数移动平均（adjust=False） + 6) score: 0/1/2 — close>ema200 + 1，ema10>ema20 + 1 + +无外部 API 调用，纯本地计算；增量逻辑以"全量重算"实现（数据量小 ~75K 行）。 +""" +from __future__ import annotations + +import time +from datetime import datetime, timedelta +from typing import Any + +import pandas as pd +from sqlalchemy import create_engine + +from app.core.config import settings +from app.core.db import ops as db_ops +from app.core.db.connection import get_mysql +from app.core.sync.base import SyncTask +from app.core.utils.logging import get_logger + +logger = get_logger("sync.sector_features") + + +# 起算点：DB 里 kline_stock 最早一天 往前 5 年，或固定一个默认起点 +# （不用写死 2021-04-24，让数据自己决定；下面自动算） +DEFAULT_START_YEARS_BACK = 5 + +# 输出基点（行业指数 close 从多少开始） +INDEX_BASE = 100.0 + + +class SyncSectorFeatures(SyncTask): + dataset_id = "sector_features" + + def _run( + self, + *, + trigger_source: str = "manual", + codes: list[str] | None = None, + max_workers: int = 1, # 本任务纯本地计算，单线程足够 + **kwargs, + ) -> dict[str, Any]: + t0 = time.time() + logger.info("[sector] 启动行业聚合特征计算") + + # ── 1) 拉 kline_stock（日线）只取需要的列 ── + # 用 SQLAlchemy engine（pymysql 裸连接 + pd.read_sql 偶发只读 1 行的 bug） + engine = create_engine(settings.mysql_url()) + df_kline = pd.read_sql( + "SELECT stock_code, trade_date, open, high, low, `close`, volume " + "FROM kline_stock ORDER BY stock_code, trade_date", + engine, + ) + if df_kline.empty: + return {"status": "error", "message": "kline_stock 为空"} + df_kline["stock_code"] = df_kline["stock_code"].astype(str).str.zfill(6) + df_kline["trade_date"] = pd.to_datetime(df_kline["trade_date"], errors="coerce") + logger.info(f"[sector] kline_stock: {len(df_kline):,} 行, {df_kline['stock_code'].nunique()} 只, " + f"{df_kline['trade_date'].min().date()} ~ {df_kline['trade_date'].max().date()}") + + # ── 2) 拉 industry（股票→行业映射）── + df_ind = pd.read_sql( + "SELECT code, industry_name FROM industry WHERE industry_name IS NOT NULL", + engine, + ) + if df_ind.empty: + return {"status": "error", "message": "industry 表为空"} + df_ind["code"] = df_ind["code"].astype(str).str.zfill(6) + df_ind = df_ind.drop_duplicates(subset=["code"], keep="first") + logger.info(f"[sector] industry 映射: {len(df_ind):,} 行, {df_ind['industry_name'].nunique()} 个行业") + + # ── 3) 算每只股票日涨跌幅 + 振幅 ── + df = df_kline.merge(df_ind, left_on="stock_code", right_on="code", how="inner") + df = df.sort_values(["stock_code", "trade_date"]) + df["prev_close"] = df.groupby("stock_code")["close"].shift(1) + df["stock_amplitude"] = (df["high"] - df["low"]) / df["close"] + df = df.dropna(subset=["prev_close", "industry_name"]) + df["pct_chg"] = (df["close"] / df["prev_close"] - 1.0) * 100.0 + logger.info(f"[sector] 合并后: {len(df):,} 行, 行业 {df['industry_name'].nunique()} 个") + + # ── 4) 按行业 + 日期聚合 ── + sector_daily = ( + df.groupby(["industry_name", "trade_date"], as_index=False).agg( + sector_ret=("pct_chg", "mean"), + sector_amplitude=("stock_amplitude", "mean"), + ) + .rename(columns={"industry_name": "sector_name"}) + .sort_values(["sector_name", "trade_date"]) + .reset_index(drop=True) + ) + logger.info(f"[sector] 行业日聚合: {len(sector_daily):,} 行") + + # ── 5) 算行业指数 close（基点 100，复合）── + sector_index = self._build_index(sector_daily) + + # ── 6) EMA + score ── + sector_index = self._calc_ema(sector_index) + sector_index["score"] = sector_index.apply( + lambda r: self._calc_score(r["close"], r["ema10"], r["ema20"], r["ema200"]), + axis=1, + ) + + # 整理列名 + out_cols = ["trade_date", "sector_name", "sector_ret", "sector_amplitude", + "close", "ema10", "ema20", "ema200", "score"] + sector_index = sector_index[out_cols] + sector_index["trade_date"] = sector_index["trade_date"].dt.strftime("%Y-%m-%d") + # NaN → None（让 MySQL 接受 NULL） + sector_index = sector_index.where(pd.notnull(sector_index), None) + + # ── 7) 写入两张表 ── + # 7a. sector_indices (4 列) + si_rows = sector_index[["trade_date", "sector_name", "close", "sector_amplitude"]].to_dict("records") + db_ops.replace_all_sector_indices(si_rows) + # 7b. sector_features_daily (9 列) + sf_rows = sector_index.to_dict("records") + db_ops.replace_all_sector_features(sf_rows) + + elapsed = round(time.time() - t0, 1) + msg = ( + f"行业聚合 {sector_index['sector_name'].nunique()} 个行业 " + f"× {sector_index['trade_date'].nunique()} 天, " + f"共 {len(sector_index):,} 行, {elapsed}s" + ) + logger.info(f"[sector] {msg}") + return { + "status": "ok", + "message": msg, + "sectors": int(sector_index["sector_name"].nunique()), + "days": int(sector_index["trade_date"].nunique()), + "rows": int(len(sector_index)), + "elapsed_sec": elapsed, + } + + @staticmethod + def _build_index(sector_daily: pd.DataFrame) -> pd.DataFrame: + """行业指数 close = INDEX_BASE × ∏(1 + sector_ret/100)""" + out = [] + for sector_name, group in sector_daily.groupby("sector_name", sort=False): + g = group.sort_values("trade_date").copy() + close_vals = [] + cur_base = INDEX_BASE + for ret in g["sector_ret"].to_numpy(dtype=float): + if pd.isna(ret): + close_vals.append(cur_base) + else: + cur_base = cur_base * (1 + float(ret) / 100.0) + close_vals.append(cur_base) + g["close"] = close_vals + out.append(g) + return pd.concat(out, ignore_index=True) + + @staticmethod + def _calc_ema(sector_index: pd.DataFrame) -> pd.DataFrame: + """每个行业分别算 EMA10/20/200""" + out = [] + for sector_name, group in sector_index.groupby("sector_name", sort=False): + g = group.sort_values("trade_date").copy() + g["ema10"] = g["close"].ewm(span=10, adjust=False).mean() + g["ema20"] = g["close"].ewm(span=20, adjust=False).mean() + g["ema200"] = g["close"].ewm(span=200, adjust=False).mean() + out.append(g) + return pd.concat(out, ignore_index=True) + + @staticmethod + def _calc_score(close, ema10, ema20, ema200) -> int: + score = 0 + if pd.notna(close) and pd.notna(ema200) and close > ema200: + score += 1 + if pd.notna(ema10) and pd.notna(ema20) and ema10 > ema20: + score += 1 + return int(score) diff --git a/app/tasks/task_share_snapshot.py b/app/tasks/task_share_snapshot.py new file mode 100644 index 0000000..5250d4f --- /dev/null +++ b/app/tasks/task_share_snapshot.py @@ -0,0 +1,133 @@ +"""同步任务：股本快照（雪球 quote_detail）。 + +写入两张表： + - stocks.total_share / float_share / share_updated_at（基础信息表） + - share（独立股本时序表） +""" +from __future__ import annotations + +import os +import time +from concurrent.futures import ThreadPoolExecutor, as_completed +from typing import Any + +from app.core.config import settings +from app.core.db import ops as db_ops +from app.core.datasource.base import registry as ds_registry +from app.core.datasource.registry import is_source_ready +from app.core.datasource.utils import is_a_share_code, to_code6 +from app.core.sync.base import SyncTask +from app.core.sync.registry import mark_sync_blocked +from app.core.utils.logging import get_logger + +logger = get_logger("sync.share") + + +class SyncShareSnapshot(SyncTask): + dataset_id = "share_snapshot" + + def _run( + self, + *, + trigger_source: str = "manual", + codes: list[str] | None = None, + max_workers: int = 10, + **kwargs, + ) -> dict[str, Any]: + if not settings.xueqiu_token: + mark_sync_blocked(self.dataset_id, message="未配置 XUEQIU_TOKEN") + return {"status": "blocked", "message": "未配置 XUEQIU_TOKEN"} + + xq = ds_registry.get("datasource_xueqiu") + if xq is None: + return {"status": "error", "message": "雪球数据源未注册"} + ok, reason = is_source_ready(xq.key) + if not ok: + mark_sync_blocked(self.dataset_id, message=f"雪球未就绪: {reason}") + return {"status": "blocked", "message": f"雪球未就绪: {reason}"} + + if codes: + stock_codes = [to_code6(c) for c in codes] + else: + stock_codes = [c for c in db_ops.iter_stock_codes(active_only=True) if is_a_share_code(c)] + limit_raw = os.environ.get("MARKET_DATA_STOCK_LIMIT", "").strip() + if limit_raw.isdigit() and int(limit_raw) > 0: + stock_codes = stock_codes[: int(limit_raw)] + + if not stock_codes: + return {"status": "error", "message": "无股票代码"} + + # 跳过今日已刷过的 + today = time.strftime("%Y-%m-%d") + existing = {s["code"]: s for s in db_ops.fetch_all_stocks()} + todo = [] + for c6 in stock_codes: + old = existing.get(c6, {}) + if str(old.get("share_updated_at", "") or "") == today and old.get("total_share", 0) > 0: + continue + todo.append(c6) + skip = len(stock_codes) - len(todo) + if not todo: + return {"status": "ok", "message": f"全部 {skip} 只已最新，跳过"} + + total = len(todo) + ok_cnt = fail_cnt = 0 + rows_to_share_table: list[dict] = [] + t0 = time.time() + with ThreadPoolExecutor(max_workers=max_workers) as pool: + futures = {pool.submit(xq.fetch_share_snapshot, c6): c6 for c6 in todo} + for i, future in enumerate(as_completed(futures), 1): + c6 = futures[future] + try: + r = future.result() + except Exception as e: + fail_cnt += 1 + logger.warning(f"[share {c6}] 异常: {e}") + continue + if r is None: + fail_cnt += 1 + continue + # 写 stocks + hermes = self._to_hermes(c6) + db_ops.update_stock_share_snapshot( + code=hermes, + total_share=r["total_share"], + float_share=r["float_share"], + trade_date=r.get("trade_date", today), + ) + # 也写 share 表 + rows_to_share_table.append({ + "stock_code": c6, + "trade_date": r.get("trade_date", today), + "total_share": r["total_share"], + "float_share": r["float_share"], + }) + ok_cnt += 1 + if i % 50 == 0 or i == total: + self._progress( + message=f"股本 {i}/{total} OK:{ok_cnt} FAIL:{fail_cnt}", + current=i, total=total, current_step=c6, + ) + # 批量写 share 表 + if rows_to_share_table: + try: + db_ops.upsert_share(rows_to_share_table) + except Exception as e: + logger.warning(f"写 share 表失败: {e}") + + elapsed = round(time.time() - t0, 1) + msg = f"股本 {ok_cnt}成 {fail_cnt}败 {skip}跳, {elapsed}s" + return { + "status": "ok" if fail_cnt == 0 else "warning", + "message": msg, + "ok": ok_cnt, "fail": fail_cnt, "skip": skip, + "elapsed_sec": elapsed, + } + + @staticmethod + def _to_hermes(code6: str) -> str: + if code6.startswith(("5", "6", "9")): + return f"SH{code6}" + if code6.startswith(("4", "8")): + return f"BJ{code6}" + return f"SZ{code6}" diff --git a/app/tasks/task_stocks_basic.py b/app/tasks/task_stocks_basic.py new file mode 100644 index 0000000..2e242e1 --- /dev/null +++ b/app/tasks/task_stocks_basic.py @@ -0,0 +1,221 @@ +"""同步任务：全市场股票基础信息 + 股本快照。 + +数据流： + 麦蕊智数 hslt/list（主）→ Baostock.query_stock_basic()（降级）→ 解析 → upsert stocks + 可选：雪球 quote_detail 刷新股本（需要 XUEQIU_TOKEN） +""" +from __future__ import annotations + +import re +import threading +import time +from concurrent.futures import ThreadPoolExecutor, as_completed +from typing import Any + +from app.core.config import settings +from app.core.db import ops as db_ops +from app.core.datasource.base import registry as ds_registry +from app.core.sync.base import SyncTask +from app.core.sync.registry import mark_sync_blocked +from app.core.utils.logging import get_logger + +logger = get_logger("sync.stocks_basic") + + +def _derive_listing_status(name: str) -> str: + n = name.strip() + if n.startswith("*ST") or n.startswith("ST"): + return "st" + return "normal" + + +class SyncStocksBasic(SyncTask): + dataset_id = "stock_basic" + + def _run(self, *, trigger_source: str = "manual", with_share: bool = True, **kwargs) -> dict[str, Any]: + self._progress(message="选择股票基础信息数据源...") + + rows: list[dict] = [] + source_name = "" + + # 1. 优先使用麦蕊智数 + mairui = ds_registry.get("datasource_mairui") + if mairui is not None: + ok, msg = mairui.is_available() + if ok: + self._progress(message="使用麦蕊智数获取股票列表...") + rows = mairui.fetch_stock_list() + if rows: + source_name = "麦蕊智数" + self._progress(message=f"麦蕊智数返回 {len(rows)} 只股票") + + # 2. 无数据则报错 + if not rows: + return {"status": "error", "message": "麦蕊智数未返回股票数据"} + + self._progress( + message=f"获取到 {len(rows)} 只 A 股，开始写入...", + current=0, total=len(rows), + ) + + new_cnt = upd_cnt = unchanged = 0 + existing = {s["code"]: s for s in db_ops.fetch_all_stocks()} + + # 准备批量写入（性能：executemany 一次写 500 条，比单条快 10-20 倍） + bulk_rows = [] + for info in rows: + code = info["code"] + old = existing.get(code) + if old is None: + new_cnt += 1 + elif (old.get("name") != info["name"] + or old.get("exchange") != info["exchange"] + or old.get("listing_status", "normal") != info["listing_status"]): + upd_cnt += 1 + else: + unchanged += 1 + bulk_rows.append({ + "code": code, + "name": info["name"], + "exchange": info["exchange"], + "listing_status": info["listing_status"], + }) + + # 分块 executemany 写入 + 进度更新 + CHUNK = 500 + for i in range(0, len(bulk_rows), CHUNK): + db_ops.upsert_stocks_bulk(bulk_rows[i: i + CHUNK]) + done = min(i + CHUNK, len(bulk_rows)) + if done % 500 == 0 or done == len(bulk_rows): + self._progress( + message=f"写入数据库 {done}/{len(bulk_rows)}...", + current=done, total=len(bulk_rows), + current_step=bulk_rows[done - 1]["code"], + ) + + # 退市检测 + delisted = 0 + api_codes = {r["code"] for r in rows} + for code, old in existing.items(): + if code not in api_codes and old.get("listing_status") != "delisted": + db_ops.upsert_stock( + code=code, + name=old.get("name", ""), + exchange=old.get("exchange", ""), + list_date=old.get("list_date", ""), + listing_status="delisted", + industry=old.get("industry", ""), + ) + delisted += 1 + + msg = f"股票基础信息 {new_cnt}新 {upd_cnt}更 {unchanged}无变化 {delisted}退市" + self._progress(message=msg, current=len(rows), total=len(rows)) + + # 股本快照（可选 + 需要 token） + share_msg = "" + if with_share and settings.xueqiu_token: + self._progress(message="股票基础信息完成，开始刷新股本快照...") + xq = ds_registry.get("datasource_xueqiu") + if xq is not None: + try: + share_ok, share_skip, share_fail = self._refresh_shares( + xq, [r["code"] for r in rows] + ) + share_msg = f"；股本 {share_ok}成 {share_skip}跳 {share_fail}败" + except Exception as e: + share_msg = f"；股本刷新异常: {e}" + else: + if with_share and not settings.xueqiu_token: + share_msg = "；未配置 XUEQIU_TOKEN，跳过股本刷新" + mark_sync_blocked(self.dataset_id, message=msg + share_msg) + + return { + "status": "ok" if not share_msg.startswith("；未配置") else "blocked", + "source": source_name, + "message": f"[{source_name}] {msg}{share_msg}", + "total": len(rows), + "new": new_cnt, + "updated": upd_cnt, + "delisted": delisted, + "unchanged": unchanged, + } + + def _fetch_with_timeout(self, bs, *, timeout: int) -> list[dict]: + result: list = [None] + exc: list = [None] + + def _do(): + try: + result[0] = bs.fetch_stock_basic() + except Exception as e: + exc[0] = e + + t = threading.Thread(target=_do, daemon=True) + t.start() + t.join(timeout) + if t.is_alive(): + logger.warning(f"Baostock 拉取超时（{timeout}s）") + return [] + if exc[0]: + raise exc[0] + return result[0] or [] + + def _refresh_shares( + self, xq, codes: list[str], max_workers: int = 10, + ) -> tuple[int, int, int]: + code6_list = [] + for c in codes: + c6 = re.sub(r"^(SH|SZ|BJ)", "", c) + if c6.isdigit() and len(c6) == 6: + code6_list.append(c6) + code6_list = sorted(set(code6_list)) + + today = time.strftime("%Y-%m-%d") + existing = {s["code"]: s for s in db_ops.fetch_all_stocks()} + todo = [] + for c6 in code6_list: + old = existing.get(c6, {}) + if str(old.get("share_updated_at", "") or "") == today and old.get("total_share", 0) > 0: + continue + todo.append(c6) + if not todo: + return 0, len(code6_list), 0 + + ok = skip = fail = 0 + with ThreadPoolExecutor(max_workers=max_workers) as pool: + futures = {pool.submit(xq.fetch_share_snapshot, c6): c6 for c6 in todo} + done = 0 + for future in as_completed(futures): + c6 = futures[future] + done += 1 + try: + r = future.result() + except Exception as e: + fail += 1 + logger.warning(f"[share {c6}] 异常: {e}") + continue + if r is None: + fail += 1 + else: + db_ops.update_stock_share_snapshot( + code=self._to_hermes(c6), + total_share=r["total_share"], + float_share=r["float_share"], + trade_date=r.get("trade_date", today), + ) + ok += 1 + if done % 100 == 0: + self._progress( + message=f"刷新股本 {done}/{len(todo)}...", + current=done, total=len(todo), current_step=c6, + ) + skip = len(code6_list) - len(todo) + return ok, skip, fail + + @staticmethod + def _to_hermes(code6: str) -> str: + if code6.startswith(("5", "6", "9")): + return f"SH{code6}" + if code6.startswith(("4", "8")): + return f"BJ{code6}" + return f"SZ{code6}" diff --git a/bin/run_remaining.sh b/bin/run_remaining.sh new file mode 100755 index 0000000..23508b3 --- /dev/null +++ b/bin/run_remaining.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# 跑剩余三个任务：moneyflow → share_snapshot → market_regime +# 跳过 kline_5min（全量回填 6 年太慢，下次有空再补） +set -e +cd "$(dirname "$0")/.." +export PYTHONPATH="$(pwd)" +mkdir -p logs + +# 先清理上次可能留下的卡死记录（recover_interrupted_syncs 启动时自动处理，但显式更稳） +echo "=== 启动时间: $(date) ===" | tee logs/runall_remaining.log + +for TASK in moneyflow share_snapshot market_regime; do + echo "" | tee -a logs/runall_remaining.log + echo "=== [$TASK] 开始 $(date) ===" | tee -a logs/runall_remaining.log + T0=$(date +%s) + if .venv/bin/python -m app.entrypoints.cli sync "$TASK" 2>&1 | tee -a logs/runall_remaining.log; then + T1=$(date +%s) + echo "=== [$TASK] 完成，耗时 $((T1 - T0))s ===" | tee -a logs/runall_remaining.log + else + T1=$(date +%s) + echo "=== [$TASK] 失败，耗时 $((T1 - T0))s ===" | tee -a logs/runall_remaining.log + exit 1 + fi +done + +echo "" | tee -a logs/runall_remaining.log +echo "=== 全部完成 $(date) ===" | tee -a logs/runall_remaining.log diff --git a/bin/runall_once.py b/bin/runall_once.py new file mode 100644 index 0000000..60f1901 --- /dev/null +++ b/bin/runall_once.py @@ -0,0 +1,81 @@ +"""一次性跑完所有 sync task（按依赖顺序）。 + +跳过 industry_sector（已 ok）。 +按顺序触发：stock_basic → kline_daily → kline_index → kline_5min → moneyflow + → share_snapshot → market_regime + +每个 task 跑完写日志 + 把 result 落到 summary.json。 +""" +from __future__ import annotations + +import json +import sys +import time +from pathlib import Path + +_PROJECT_ROOT = Path(__file__).resolve().parent.parent +if str(_PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(_PROJECT_ROOT)) + +from app.core.utils.logging import setup_logging, get_logger +setup_logging() +logger = get_logger("runall") + +from app.core.datasource.registry import build_default_registry +from app.core.sync.registry import seed_sync_registry +build_default_registry() +seed_sync_registry() + +from app.tasks import get_task + +# 顺序执行，便于排查 +TASKS = [ + "stock_basic", + "industry_sector", # 已 ok，但再跑一次全量也 OK (用 SKIP 标志) + "kline_index", + "kline_daily", + "kline_5min", + "moneyflow", + "share_snapshot", + "market_regime", +] + +# industry_sector 跑过一次（291s 全量）已 ok，跳过 +SKIP = {"industry_sector"} + +results = {} +t_all = time.time() +for tid in TASKS: + if tid in SKIP: + results[tid] = {"status": "skipped", "message": "已 ok"} + logger.info(f"[runall] {tid} 跳过（已 ok）") + continue + logger.info(f"[runall] >>> 开始 {tid}") + t0 = time.time() + try: + task = get_task(tid) + # 大表 task 限小批量股票，避免跑爆；其他 task 跑全量 + kwargs = {"max_workers": 5} + if tid in {"kline_daily", "kline_5min", "moneyflow", "share_snapshot"}: + # 这些 task 支持 MARKET_DATA_STOCK_LIMIT 环境变量，但通过 cli 跑可 --codes 限 + # 测全量费时，先跑全量 + pass + r = task.run(trigger_source="runall", **kwargs) + elapsed = round(time.time() - t0, 1) + r["elapsed_sec"] = elapsed + results[tid] = r + logger.info(f"[runall] <<< {tid} 完成 status={r.get('status')} elapsed={elapsed}s msg={r.get('message')}") + except Exception as e: + elapsed = round(time.time() - t0, 1) + logger.exception(f"[runall] !!! {tid} 异常: {e}") + results[tid] = {"status": "error", "message": str(e), "elapsed_sec": elapsed} + + # 每个 task 之间 sleep 5s 让健康监控跑 + time.sleep(5) + +elapsed_total = round(time.time() - t_all, 1) +summary = {"total_elapsed_sec": elapsed_total, "tasks": results} +out = _PROJECT_ROOT / "logs" / "runall_summary.json" +out.write_text(json.dumps(summary, ensure_ascii=False, indent=2)) +logger.info(f"[runall] 全部完成 total={elapsed_total}s summary={out}") +print(json.dumps(summary, ensure_ascii=False, indent=2)) \ No newline at end of file diff --git a/bin/structured_watch.sh b/bin/structured_watch.sh new file mode 100755 index 0000000..cda1016 --- /dev/null +++ b/bin/structured_watch.sh @@ -0,0 +1,72 @@ +#!/bin/bash +# 结构化 watch 脚本（按 structured-watch-plan 规则） +# 用法：structured_watch.sh [task_label] +# 规则：1m×5 / 5m×2 / 10m×2 / 30m×2 / 60m×2 + 结束自停 +set -u + +PATTERN="${1:-app.entrypoints.cli}" +LOG_FILE="${2:-/tmp/moneyflow_full.log}" +OUT="${3:-/tmp/structured_watch.log}" +LABEL="${4:-$PATTERN}" + +# 关键：pgrep -f 模式会**匹配到 watch 自己**（watch 命令行里有完整 PATTERN）， +# 任务结束后 watch 还认为"还活着"不会退出。 +# 解决：watch 用 `[p]ython` 风格 trick 排除自己（bash 进程没 'python' 字样）， +# 或者更直接：只匹配 python 进程。 +WATCH_BASENAME=$(basename "$0") +# 构造一个绝不会匹配 watch 自己的模式： +# 如果原 PATTERN 含 "python" 就保持，否则前缀 [p]ython 让 watch 自身不匹配 +case "$PATTERN" in + *python*) PGREP_PATTERN="$PATTERN" ;; + *) PGREP_PATTERN="[p]ython.*$PATTERN" ;; +esac + +# 把输出重定向到 OUT，system 通知时我能读到 +exec > "$OUT" 2>&1 + +# schedule: 秒数 + 标签 +SCHEDULE_SEC=(60 60 60 60 60 300 300 600 600 1800 1800 3600 3600) +SCHEDULE_LABELS=( + "[1m 1/5]" "[1m 2/5]" "[1m 3/5]" "[1m 4/5]" "[1m 5/5]" + "[5m 1/2]" "[5m 2/2]" + "[10m 1/2]" "[10m 2/2]" + "[30m 1/2]" "[30m 2/2]" + "[60m 1/2]" "[60m 2/2]" +) + +echo "=== structured watch 启动 ===" +echo " task label: $LABEL" +echo " pgrep pattern (orig): $PATTERN" +echo " pgrep pattern (safe): $PGREP_PATTERN" +echo " log file: $LOG_FILE" +echo " 启动时间: $(date)" +echo + +for i in "${!SCHEDULE_SEC[@]}"; do + sleep "${SCHEDULE_SEC[$i]}" + LABEL_NOW="${SCHEDULE_LABELS[$i]}" + if ! pgrep -f "$PGREP_PATTERN" > /dev/null 2>&1; then + echo "=== checkpoint $LABEL_NOW: $(date) ===" + echo " 任务已退出（pgrep 不到 '$PGREP_PATTERN'），停止 schedule" + echo + echo "=== 最后 30 行日志 ===" + tail -30 "$LOG_FILE" 2>/dev/null + echo + echo "=== structured watch 自然结束：$(date) ===" + exit 0 + fi + echo "=== checkpoint $LABEL_NOW: $(date) ===" + echo " 进程数: $(pgrep -af "$PGREP_PATTERN" | wc -l)" + echo " --- log tail ---" + tail -10 "$LOG_FILE" 2>/dev/null + echo +done + +echo "=== structured watch 跑完 13 个 checkpoint: $(date) ===" +echo " 任务可能还在跑（pgrep 仍能找到），继续等待 pgrep 失效触发最终通知" +# 13 个 checkpoint 后再每秒 pgrep 检查一次（轻量），任务一结束就退出 +while pgrep -f "$PGREP_PATTERN" > /dev/null 2>&1; do + sleep 30 +done +echo "=== 任务最终退出: $(date) ===" +tail -30 "$LOG_FILE" 2>/dev/null diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md new file mode 100644 index 0000000..a837303 --- /dev/null +++ b/docs/ARCHITECTURE.md @@ -0,0 +1,202 @@ +# Market Data Sync — 架构说明 + +> A 股市场数据定时同步框架。多个数据源 → 交易日感知调度器 → MySQL。 + +## 1. 目录结构 + +``` +market_data_sync/ +├── app/ # 业务代码（唯一 Python 包） +│ ├── __init__.py +│ ├── core/ # 核心抽象层（与业务无关的基础设施） +│ │ ├── config.py # 统一配置（pydantic-settings + .env） +│ │ ├── datasource/ # 数据源抽象 +│ │ │ ├── base.py # DataSource / FetchResult / registry +│ │ │ └── registry.py # 健康监控 + 自动降级 + seed +│ │ ├── sync/ # 同步任务抽象 +│ │ │ ├── base.py # SyncTask 基类（自动状态机） +│ │ │ └── registry.py # dataset_registry 状态机封装 +│ │ ├── scheduler/ # 轻量调度器 +│ │ │ └── scheduler.py # 交易日感知 + config 表持久化 +│ │ ├── db/ # 数据库访问层 +│ │ │ ├── connection.py # thread-local pymysql +│ │ │ ├── schema.py # 13 张表 DDL +│ │ │ └── ops.py # CRUD + 批量 upsert +│ │ └── utils/ +│ │ └── logging.py # RotatingFileHandler +│ ├── sources/ # 数据源实现（5 个） +│ │ ├── sina.py # 新浪（免费 K 线） +│ │ ├── baostock.py # Baostock（基础信息+行业） +│ │ ├── mairui.py # 麦蕊智数（需 MAIRUI_LICENCE） +│ │ ├── akshare.py # akshare/东方财富（5min+资金流） +│ │ └── xueqiu.py # 雪球（股本快照） +│ ├── tasks/ # 同步任务实现（8 个） +│ │ ├── __init__.py # TASKS 注册表 + get_task() +│ │ ├── task_stocks_basic.py +│ │ ├── task_kline_daily.py +│ │ ├── task_kline_index.py +│ │ ├── task_kline_5min.py +│ │ ├── task_moneyflow.py +│ │ ├── task_industry_sector.py +│ │ ├── task_share_snapshot.py +│ │ └── task_market_regime.py +│ ├── api/ # FastAPI 管理接口（端口 8100） +│ │ ├── main.py # 入口 + 生命周期钩子 +│ │ └── routes/ +│ │ ├── health.py +│ │ ├── sync.py # 手动触发 / 状态查询 +│ │ ├── schedule.py # 计划任务 +│ │ └── datasources.py # 数据源健康度 +│ └── entrypoints/ # 进程入口 +│ ├── cli.py # CLI（python -m app.entrypoints.cli ...） +│ └── worker.py # 纯调度模式（python -m app.entrypoints.worker） +├── bin/ # 一次性脚本 +│ └── runall_once.py # 一键串行跑全部任务 +├── scripts/ # 旧 shell 脚本（保留作 fallback） +├── tests/ # 单元测试（pytest） +├── docs/ # 文档 +├── data/ # 临时数据（一般为空） +├── logs/ # 运行日志（RotatingFileHandler） +├── .env / .env.example # 配置 +├── requirements.txt +├── README.md +└── start.sh # 统一启动入口 +``` + +## 2. 模块依赖 + +``` + ┌────────────────────────────────────────┐ + │ entrypoints/cli │ 手动/CLI + │ entrypoints/worker │ 调度模式 + │ │ │ + │ ▼ │ + │ ┌─────────────────────────────┐ │ + │ │ api (FastAPI) │ │ HTTP 管理 + │ └──────────────┬──────────────┘ │ + │ │ │ + ▼ ▼ │ + ┌──────────┐ ┌─────────────┐ │ + │ tasks │◄───│ core │ │ + │ (8 个) │ │ ├ config │ │ + └────┬─────┘ │ ├ db │ │ + │ │ ├ sync │ │ + │ │ ├ scheduler│ │ + │ │ └ datasource│ │ + ▼ └──────┬──────┘ │ + ┌──────────┐ │ │ + │ sources │◄──────────┘ │ + │ (5 个) │ │ + └──────────┘ │ + │ │ + ▼ │ + 数据源外部 API │ +``` + +依赖方向单向：上层 → 下层。`core` 是稳定抽象，`sources` / `tasks` 是可插拔实现。 + +## 3. 核心抽象 + +### 3.1 DataSource (`app/core/datasource/base.py`) + +```python +class DataSource: + key: str # "datasource_xxx" 全局唯一 + name: str # 显示名 + provides: list[str] # 能力列表 ["kline_daily", "moneyflow", ...] + requires_credential: bool + + def is_available(self) -> tuple[bool, str]: ... + def health_check(self) -> dict: ... + def fetch_xxx(self, ...) -> pd.DataFrame: ... +``` + +- `registry` 是全局单例 dict[str, DataSource] +- 启动时 `build_default_registry()` 把 5 个源注册进去 +- `pick_source(capability)` 按 `provides` 找最匹配的源 + +### 3.2 SyncTask (`app/core/sync/base.py`) + +```python +class SyncTask: + dataset_id: str # 子类必须设置 + + def run(self, *, trigger_source="manual", **kwargs) -> dict: + mark_sync_running(...) + try: + result = self._run(...) + mark_sync_success/failed(...) + except Exception: + mark_sync_failed(...) +``` + +子类只需实现 `_run()`，状态机会自动包装。 + +### 3.3 Scheduler (`app/core/scheduler/scheduler.py`) + +- 5s tick 扫描 config 表 +- 读取 `trading_calendar_holidays` 决定是否交易日 +- 时间匹配（HH:MM 精确触发）或 interval（间隔秒数）触发 +- 启动时 `recover_interrupted_syncs()` 把卡死的 `running` 任务改 `failed` + +## 4. 数据流（以 kline_daily 为例） + +``` +scheduler tick + └─ 时间到 (16:00) + └─ ScheduleJob.run() + └─ tasks.get_task("kline_daily")() + └─ SyncKlineDaily.run() + ├─ mark_sync_running(...) ← 状态机 + ├─ _ensure_health_checked() ← 拉一次数据源健康 + ├─ pick_source("kline_daily") ← 选雪球(主) > 新浪 > baostock + ├─ 多线程 fetch (10 workers) ← XUEQIU_TOKEN 限速 + ├─ 批量 upsert (CHUNK=500) ← pymysql executemany + └─ mark_sync_success(...) ← 状态机 +``` + +## 5. 添加新数据源 + +1. 在 `app/sources/` 新建 `my_source.py` +2. 继承 `DataSource`，设置 `key` / `name` / `provides` / `requires_credential` +3. 实现 `is_available()` / `health_check()` / 业务方法 +4. 在 `app/core/datasource/registry.py` 的 `build_default_registry()` 注册 + +## 6. 添加新同步任务 + +1. 在 `app/tasks/` 新建 `task_xxx.py` +2. 继承 `SyncTask`，设置 `dataset_id` +3. 实现 `_run()`，返回 `{"status": "ok", "message": "..."}` 或 `{"status": "warning/blocked", ...}` +4. 在 `app/tasks/__init__.py` 的 `TASKS` 字典中加一行 +5. （可选）在 `app/core/sync/registry.py` 的 `SYNC_DEFINITIONS` 加条目 + +## 7. 数据库表 + +13 张表在 `app/core/db/schema.py`： + +| 表 | 用途 | +|---|---| +| `stocks` | 股票基础信息 | +| `kline_stock` | 日 K 线 | +| `kline_index` | 指数日 K | +| `kline_5min` | 5min K 线 | +| `moneyflow` | 资金流 | +| `industry` / `sectors` / `stock_sector_map` | 行业映射 | +| `share` | 股本快照 | +| `market_regime_daily` | 市场情绪 | +| `dataset_registry` | 同步任务状态机 | +| `config` | 通用配置（调度、节假日、token 等） | +| `indices` | 指数字典 | + +## 8. 配置（.env） + +| 变量 | 必填 | 说明 | +|---|---|---| +| `MYSQL_HOST/PORT/USER/PASSWORD/DATABASE` | ✓ | 主库 | +| `MYSQL_DATABASE_BUSINESS` | | 业务库（默认 grid_seeker_model_base） | +| `XUEQIU_TOKEN` | | 雪球 token（K线主源 + 股本） | +| `MAIRUI_LICENCE` | | 麦蕊智数 licence | +| `HOLIDAYS_LIST` | | A 股休市日（逗号分隔 YYYY-MM-DD） | +| `SCHEDULER_AUTO_SEED` | | 启动时 seed 默认计划任务 | +| `LOG_DIR` | | 日志目录（默认 logs） | +| `LOG_LEVEL` | | 日志级别（默认 INFO） | diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..0ef5b2c --- /dev/null +++ b/requirements.txt @@ -0,0 +1,31 @@ +# Web/API +fastapi>=0.110.0 +uvicorn[standard]>=0.30.0 +pydantic>=2.0.0 +pydantic-settings>=2.0.0 + +# Data +pandas>=2.2.0 +numpy>=1.26.0 +pyarrow>=15.0.0 + +# Storage +SQLAlchemy>=2.0.0 +PyMySQL>=1.0.0 +DBUtils>=3.0.0 +cryptography>=42.0.0 + +# HTTP / Retry +requests>=2.30.0 +tenacity>=8.0.0 + +# Datasources — A 股免费源 +# akshare 永远不使用（项目级决策） +baostock>=0.9.0 +# 雪球非官方 SDK：需要 XUEQIU_TOKEN（环境变量）才能拿到股本 / 日K 复核 +# 没装 → share / kline_daily 任务静默失败（ImportError 被 except 吞） +pysnowball>=0.1.8 + +# Logging / Config +PyYAML>=6.0.0 +python-dotenv>=1.0.0 \ No newline at end of file diff --git a/scripts/run_all_sync.sh b/scripts/run_all_sync.sh new file mode 100755 index 0000000..b7b1d4d --- /dev/null +++ b/scripts/run_all_sync.sh @@ -0,0 +1,73 @@ +#!/bin/bash +# 串行跑所有 6 个同步任务，后台模式，写到 logs/ +# 用法：./run_all_sync.sh [STEP] +# STEP 为空 = 跑 1-6 +# STEP=1 只跑 stock_basic，等等 + +set -e +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +cd "$PROJECT_DIR" + +mkdir -p logs +LOG_DIR="$PROJECT_DIR/logs" +TIMESTAMP=$(date +%Y%m%d_%H%M%S) +RUN_LOG="$LOG_DIR/run_all_$TIMESTAMP.log" + +# 读 token +if [ -f "$PROJECT_DIR/.env" ]; then + set -a + source "$PROJECT_DIR/.env" + set +a +fi + +# 把 .env 里的 TRADING_HOLIDAYS 转成 -- 跳过 +# XUEQIU_TOKEN 在 .env 里 + +source .venv/bin/activate +export PYTHONPATH="$PROJECT_DIR" + +LOG_PREFIX="[$TIMESTAMP]" + +run_step() { + local step_name="$1" + local task_id="$2" + local extra_args="$3" + local step_log="$LOG_DIR/step_${step_name}_$TIMESTAMP.log" + + echo "$LOG_PREFIX === Step $step_name: $task_id ===" | tee -a "$RUN_LOG" + echo "$LOG_PREFIX log: $step_log" | tee -a "$RUN_LOG" + + # 跑任务（前台模式，方便顺序执行 + 立即看到结果） + python -m app.entrypoints.cli sync "$task_id" $extra_args 2>&1 | tee "$step_log" | tail -10 + local exit_code=${PIPESTATUS[0]} + + if [ $exit_code -eq 0 ]; then + echo "$LOG_PREFIX ✓ Step $step_name OK" | tee -a "$RUN_LOG" + else + echo "$LOG_PREFIX ✗ Step $step_name FAIL (exit=$exit_code)" | tee -a "$RUN_LOG" + return $exit_code + fi +} + +case "${1:-all}" in + 1|stock_basic) run_step "1_stock_basic" "stock_basic" ;; + 2|industry_sector) run_step "2_industry_sector" "industry_sector" ;; + 3|kline_index) run_step "3_kline_index" "kline_index" ;; + 4|kline_daily) run_step "4_kline_daily" "kline_daily" "--workers 20" ;; + 5|share_snapshot) run_step "5_share_snapshot" "share_snapshot" "--workers 10" ;; + 6|market_regime) run_step "6_market_regime" "market_regime" ;; + all) + run_step "1_stock_basic" "stock_basic" || exit 1 + run_step "2_industry_sector" "industry_sector" || exit 1 + run_step "3_kline_index" "kline_index" || exit 1 + run_step "4_kline_daily" "kline_daily" "--workers 20" || exit 1 + run_step "5_share_snapshot" "share_snapshot" "--workers 10" || exit 1 + run_step "6_market_regime" "market_regime" || exit 1 + echo "$LOG_PREFIX ✓ 全部 6 步完成" | tee -a "$RUN_LOG" + ;; + *) + echo "usage: $0 [1|2|3|4|5|6|all]" + exit 1 + ;; +esac diff --git a/start.sh b/start.sh new file mode 100755 index 0000000..a5c5d87 --- /dev/null +++ b/start.sh @@ -0,0 +1,72 @@ +#!/bin/bash +# market_data_sync 启动脚本 +# 用法：./start.sh 启动 API（默认端口 8100） +# ./start.sh worker 仅启动调度器 + 同步 worker（无 web） +# ./start.sh sync 手动执行一次同步任务 +# ./start.sh status 查看调度器 / 数据源 / 同步任务状态 + +set -e +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +cd "$SCRIPT_DIR" + +# ---- venv ---- +VENV_DIR="$SCRIPT_DIR/.venv" +if [ ! -d "$VENV_DIR" ]; then + echo "[market_data_sync] creating venv ..." + python3 -m venv "$VENV_DIR" +fi +source "$VENV_DIR/bin/activate" + +# ---- deps ---- +if [ ! -f "$VENV_DIR/.deps_installed" ] || [ requirements.txt -nt "$VENV_DIR/.deps_installed" ]; then + echo "[market_data_sync] installing deps ..." + pip install --upgrade pip -q + pip install -r requirements.txt -q + touch "$VENV_DIR/.deps_installed" +fi + +# ---- .env ---- +if [ ! -f "$SCRIPT_DIR/.env" ]; then + echo "[market_data_sync] no .env found, copying .env.example" + cp "$SCRIPT_DIR/.env.example" "$SCRIPT_DIR/.env" + echo "[market_data_sync] ⚠️ please edit .env with your real MySQL credentials before continuing" +fi + +# ---- export PYTHONPATH ---- +export PYTHONPATH="$SCRIPT_DIR${PYTHONPATH:+:$PYTHONPATH}" + +# ---- ensure logs dir ---- +mkdir -p "$SCRIPT_DIR/logs" + +# ---- dispatch ---- +MODE="${1:-api}" +case "$MODE" in + api) + PORT="${API_PORT:-8100}" + HOST="${API_HOST:-0.0.0.0}" + echo "[market_data_sync] starting API on $HOST:$PORT" + exec uvicorn app.api.main:app --host "$HOST" --port "$PORT" --log-level info + ;; + worker) + echo "[market_data_sync] starting scheduler worker (no web)" + exec python -m app.entrypoints.worker + ;; + sync) + TASK_ID="${2:?usage: ./start.sh sync }" + echo "[market_data_sync] manual sync: $TASK_ID" + exec python -m app.entrypoints.cli sync "$TASK_ID" + ;; + status) + exec python -m app.entrypoints.cli status + ;; + list) + exec python -m app.entrypoints.cli list + ;; + datasources) + exec python -m app.entrypoints.cli datasources + ;; + *) + echo "usage: $0 {api|worker|sync |status|list|datasources}" + exit 1 + ;; +esac diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..40c1c1a --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +"""tests 包标记。""" diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..ad453af --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,10 @@ +"""pytest 配置：让测试可以直接 `import app`。 + +只要把 tests/ 和 app/ 都放进 sys.path 即可。 +""" +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) diff --git a/tests/test_smoke.py b/tests/test_smoke.py new file mode 100644 index 0000000..f1149b2 --- /dev/null +++ b/tests/test_smoke.py @@ -0,0 +1,93 @@ +"""冒烟测试：不依赖 MySQL/网络，只验证 import + 基本逻辑。 + +跑：pytest tests/test_smoke.py -v +""" +import pytest + + +def test_import_config(): + from app.core.config import settings + assert settings is not None + assert hasattr(settings, "mysql_host") + + +def test_import_data_source_base(): + from app.core.datasource.base import DataSource, FetchResult, registry + assert DataSource is not None + assert FetchResult is not None + assert registry is not None + # registry 应该是 dict-like + assert hasattr(registry, "register") + assert hasattr(registry, "get") + + +def test_import_sync_base(): + from app.core.sync.base import SyncTask, _is_market_closed, _effective_sync_end + assert SyncTask is not None + assert isinstance(_is_market_closed(), bool) + assert isinstance(_effective_sync_end(), str) + # 同步任务基类不允许空 dataset_id + with pytest.raises(ValueError): + SyncTask() + + +def test_tasks_registry(): + from app.tasks import TASKS, get_task + assert isinstance(TASKS, dict) + # 注：每次新增 task 都要更新这里的数字 + # 当前 9 个：stock_basic, kline_daily, kline_index, kline_5min, moneyflow, + # industry_sector, sector_features, share_snapshot, market_regime + assert len(TASKS) == 9 + # get_task 应该返回实例 + task = get_task("kline_daily") + assert task.dataset_id == "kline_daily" + + +def test_sync_definitions(): + from app.core.sync.registry import SYNC_DEFINITIONS + assert len(SYNC_DEFINITIONS) == 9 + # 所有 dataset_id 应唯一 + ids = [d["dataset_id"] for d in SYNC_DEFINITIONS] + assert len(set(ids)) == 9 + + +def test_build_default_registry(): + """不连接网络，只看是否能完成注册（部分源 may not available 不影响注册）。""" + from app.core.datasource.base import registry + from app.core.datasource.registry import build_default_registry + build_default_registry() + keys = [s.key for s in registry.all()] + assert "datasource_xinlang" in keys + assert "datasource_baostock" in keys + assert "datasource_mairui" in keys + # akshare 已移除（项目级决策：永远不用） + assert "datasource_akshare" not in keys + assert "datasource_xueqiu" in keys + + +def test_data_source_abstract(): + """DataSource 抽象方法未实现时不应能被直接实例化。""" + from app.core.datasource.base import DataSource + with pytest.raises(TypeError): + DataSource() # 抽象方法未实现 + + +def test_api_health_route(): + """FastAPI 加载 + /api/health 路由可访问。""" + from fastapi.testclient import TestClient + from app.api.main import app + + client = TestClient(app) + resp = client.get("/api/health") + assert resp.status_code == 200 + body = resp.json() + assert body.get("status") == "ok" + + +def test_effective_sync_end_format(): + from app.core.sync.base import _effective_sync_end + s = _effective_sync_end() + # YYYY-MM-DD 共 10 字符 + assert len(s) == 10 + assert s[4] == "-" + assert s[7] == "-"