kyugao/market_sync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: 重构前基线 — 9 个 sync task 全部 ok + akshare 移除 + mairui 资金流接入

Browse Source 
状态：
- 9 个 sync task（stock_basic / kline_daily / kline_index / kline_5min /
  moneyflow / industry_sector / sector_features / share_snapshot / market_regime）
- 数据源：baostock + mairui + 雪球(pysnowball) + 新浪（4 个）
- 项目级约束：永远不用 akshare（已落实）
- kline_5min 改用 DB 快照统一全量/增量逻辑
- 零后端 Chrome 扩展 xueqiu_sync（独立项目）
This commit is contained in:
gao
2026-06-15 15:36:09 +08:00
commit 05635b76b9
55 changed files with 6307 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/core/sync/__init__.py
+24
View File
@@ -0,0 +1,24 @@
"""sync 子包入口。"""
from app.core.sync.registry import (
mark_sync_running,
mark_sync_progress,
mark_sync_success,
mark_sync_failed,
mark_sync_blocked,
get_registry_status,
recover_interrupted_syncs,
seed_sync_registry,
SYNC_DEFINITIONS,
)
__all__ = [
"mark_sync_running",
"mark_sync_progress",
"mark_sync_success",
"mark_sync_failed",
"mark_sync_blocked",
"get_registry_status",
"recover_interrupted_syncs",
"seed_sync_registry",
"SYNC_DEFINITIONS",
]
app/core/sync/base.py
+141
View File
@@ -0,0 +1,141 @@
"""SyncTask 基类。
提供统一的：
- 状态自动更新（running → success / failed）
- 进度回调
- 异常捕获 → 写 dataset_registry
子类只需实现 _run() 即可。
"""
from __future__ import annotations
import time
import traceback
from datetime import datetime, time as dtime
from typing import Any, Optional
from app.core.sync.registry import (
mark_sync_failed,
mark_sync_progress,
mark_sync_running,
mark_sync_success,
)
from app.core.utils.logging import get_logger
logger = get_logger("sync.task")
def _is_market_closed() -> bool:
"""是否超过 15:30（盘后才完整，避免盘中部分数据）。"""
return datetime.now().time() >= dtime(15, 30)
def _effective_sync_end() -> str:
"""若盘后则今天，否则昨天。"""
today = datetime.now()
if today.time() >= dtime(15, 30):
return today.strftime("%Y-%m-%d")
# 昨天
from datetime import timedelta
return (today - timedelta(days=1)).strftime("%Y-%m-%d")
class SyncTask:
"""同步任务基类。"""
dataset_id: str = "" # 子类必须设置
def __init__(self, dataset_id: Optional[str] = None):
if dataset_id:
self.dataset_id = dataset_id
if not self.dataset_id:
raise ValueError(f"{self.__class__.__name__} must set .dataset_id")
# ── 公开入口 ─────────────────────────────────────────
def run(self, *, trigger_source: str = "manual", **kwargs) -> dict[str, Any]:
"""统一入口：自动包 mark_running / mark_success / mark_failed。"""
# 1. 确保 health check 跑过（CLI / 手动触发场景下 schedule 后台不会先跑）
self._ensure_health_checked()
# 2. 启动
mark_sync_running(
self.dataset_id,
trigger_source=trigger_source,
message=f"开始同步 {self.dataset_id}...",
)
t0 = time.time()
try:
result = self._run(trigger_source=trigger_source, **kwargs)
elapsed = round(time.time() - t0, 1)
status = result.get("status", "ok")
message = result.get("message", "")
if status == "ok":
mark_sync_success(self.dataset_id, message=message)
elif status == "blocked":
mark_sync_failed(
self.dataset_id,
message=message,
error=result.get("error", message),
status="blocked",
)
else: # warning / partial
mark_sync_failed(
self.dataset_id,
message=message,
error=result.get("error", message),
status="warning",
)
result["elapsed_sec"] = elapsed
logger.info(f"[{self.dataset_id}] 完成 {status}: {message} ({elapsed}s)")
return result
except Exception as e:
elapsed = round(time.time() - t0, 1)
tb = traceback.format_exc(limit=2)
err_msg = f"{type(e).__name__}: {e}"
mark_sync_failed(self.dataset_id, message=err_msg, error=tb)
logger.error(f"[{self.dataset_id}] 失败: {err_msg}\n{tb}")
return {
"status": "error",
"message": err_msg,
"error": err_msg,
"elapsed_sec": elapsed,
}
def _ensure_health_checked(self) -> None:
"""若内存里 health_results 为空，跑一次。"""
try:
from app.core.datasource.registry import get_health_status, run_health_check
if not get_health_status():
run_health_check()
except Exception as e:
logger.warning(f"health check 启动失败（不影响任务继续）: {e}")
# ── 子类实现 ─────────────────────────────────────────
def _run(self, *, trigger_source: str = "manual", **kwargs) -> dict[str, Any]:
"""子类实现：实际同步逻辑。
返回 dict：
{
"status": "ok" | "warning" | "blocked",
"message": "...",
"ok": int, "fail": int, "skip": int, # 可选统计
...
}
任何异常会被外层捕获并标记 failed。
"""
raise NotImplementedError
# ── 进度更新辅助 ─────────────────────────────────────
def _progress(self, *, message: str | None = None,
current: int | None = None, total: int | None = None,
current_step: str | None = None) -> None:
mark_sync_progress(
self.dataset_id,
message=message,
progress_current=current,
progress_total=total,
current_step=current_step,
)
app/core/sync/registry.py
+249
View File
@@ -0,0 +1,249 @@
"""sync registry 状态机封装。
模式照搬 dashboard/api/services/dataset_registry.py：
- dataset_id 是唯一键
- mark_sync_running / mark_sync_progress / mark_sync_success / mark_sync_failed / mark_sync_blocked
- recover_interrupted_syncs 启动时把 status='running' 的任务改为 failed
- seed_sync_registry 把所有同步任务定义 seed 到 dataset_registry 表
"""
from __future__ import annotations
import json
from datetime import datetime
from typing import Any
from app.core.db import ops as db_ops
from app.core.utils.logging import get_logger
logger = get_logger("sync.registry")
# ── 同步任务定义（写入 dataset_registry 表）──────────────────────────────
SYNC_DEFINITIONS: list[dict[str, Any]] = [
{
"dataset_id": "stock_basic",
"name": "股票基础信息（含最新股本）",
"description": "全市场 A 股代码 / 名称 / 交易所 / 上市状态 + 股本快照（雪球）",
"storage_uri": "MySQL market_data_sync_db.stocks",
"storage_layer": "mysql",
"management_role": "基础字典",
"source": "Baostock query_stock_basic + 雪球 quote_detail",
"sync_script": "app.tasks.task_stocks_basic:run",
"dependency_ids": [],
"sort_order": 10,
},
{
"dataset_id": "kline_daily",
"name": "原始日K线数据",
"description": "全市场 A 股日频 OHLCV，多源交叉校验（雪球/新浪/Baostock）",
"storage_uri": "MySQL market_data_sync_db.kline_stock",
"storage_layer": "mysql",
"management_role": "原始源",
"source": "雪球主源 + 新浪/Baostock 复核",
"sync_script": "app.tasks.task_kline_daily:run",
"dependency_ids": ["stock_basic"],
"sort_order": 20,
},
{
"dataset_id": "kline_index",
"name": "六大指数日线",
"description": "上证 / 深证 / 创业板 / 沪深300 / 中证500 / 中证1000",
"storage_uri": "MySQL market_data_sync_db.kline_index + indices",
"storage_layer": "mysql",
"management_role": "原始源",
"source": "新浪指数日K",
"sync_script": "app.tasks.task_kline_index:run",
"dependency_ids": [],
"sort_order": 30,
},
{
"dataset_id": "kline_5min",
"name": "5 分钟 K 线",
"description": "全市场 A 股 5 分钟 K 线（mairui 源）",
"storage_uri": "MySQL market_data_sync_db.kline_5min",
"storage_layer": "mysql",
"management_role": "原始源",
"source": "mairui stockMin",
"sync_script": "app.tasks.task_kline_5min:run",
"dependency_ids": ["stock_basic"],
"sort_order": 40,
},
{
"dataset_id": "moneyflow",
"name": "资金流",
"description": "个股资金流（主力/大/中/小单净额），mairui 源",
"storage_uri": "MySQL market_data_sync_db.moneyflow",
"storage_layer": "mysql",
"management_role": "原始源",
"source": "mairui hsstock/history/transaction",
"sync_script": "app.tasks.task_moneyflow:run",
"dependency_ids": ["stock_basic"],
"sort_order": 50,
},
{
"dataset_id": "industry_sector",
"name": "股票-行业映射",
"description": "股票-行业映射 + 行业字典（Baostock）",
"storage_uri": "MySQL market_data_sync_db.industry + sectors + stock_sector_map",
"storage_layer": "mysql",
"management_role": "基础字典",
"source": "Baostock query_stock_industry",
"sync_script": "app.tasks.task_industry_sector:run",
"dependency_ids": ["stock_basic"],
"sort_order": 60,
},
{
"dataset_id": "sector_features",
"name": "行业聚合特征",
"description": "由 kline_stock + industry 衍生：行业日收益 / 行业指数 close / EMA10-20-200 / score。纯本地计算，无外部 API。",
"storage_uri": "MySQL market_data_sync_db.sector_indices + sector_features_daily",
"storage_layer": "mysql",
"management_role": "衍生源",
"source": "本地计算（kline_stock + industry）",
"sync_script": "app.tasks.task_sector_features:run",
"dependency_ids": ["kline_daily", "industry_sector"],
"sort_order": 65,
},
{
"dataset_id": "share_snapshot",
"name": "股本快照",
"description": "全市场 A 股最新股本（雪球 quote_detail，单位亿股）",
"storage_uri": "MySQL market_data_sync_db.stocks.total_share/float_share + share 表",
"storage_layer": "mysql",
"management_role": "基础字典",
"source": "雪球 quote_detail",
"sync_script": "app.tasks.task_share_snapshot:run",
"dependency_ids": ["stock_basic"],
"sort_order": 70,
},
{
"dataset_id": "market_regime",
"name": "市场情绪 (Market Regime)",
"description": "基于本地日K线聚合的 advance_ratio / 恐慌标记（衍生源）",
"storage_uri": "MySQL market_data_sync_db.market_regime_daily",
"storage_layer": "mysql",
"management_role": "衍生源",
"source": "本地日K线聚合（无外部接口）",
"sync_script": "app.tasks.task_market_regime:run",
"dependency_ids": ["kline_daily"],
"sort_order": 80,
},
]
def seed_sync_registry() -> None:
"""把 SYNC_DEFINITIONS 写入 dataset_registry 表。"""
rows = []
for d in SYNC_DEFINITIONS:
rows.append({
**d,
"dependency_ids": json.dumps(d.get("dependency_ids", []), ensure_ascii=False),
"enabled": 1,
})
db_ops.upsert_dataset_registry_rows(rows)
# ── 状态机 API ────────────────────────────────────────────────────────
def _now_ts() -> str:
return datetime.now().strftime("%Y-%m-%d %H:%M:%S")
def mark_sync_running(
dataset_id: str,
*,
trigger_source: str,
message: str = "",
progress_current: int = 0,
progress_total: int = 0,
current_step: str = "",
) -> None:
db_ops.update_dataset_registry_state(
dataset_id,
status="running",
trigger_source=trigger_source,
started_at=_now_ts(),
finished_at=None,
message=message,
last_error=None,
needs_resync=0,
progress_current=progress_current,
progress_total=progress_total,
current_step=current_step,
updated_at=_now_ts(),
)
def mark_sync_progress(
dataset_id: str,
*,
message: str | None = None,
progress_current: int | None = None,
progress_total: int | None = None,
current_step: str | None = None,
) -> None:
payload: dict[str, Any] = {"updated_at": _now_ts()}
if message is not None:
payload["message"] = message
if progress_current is not None:
payload["progress_current"] = progress_current
if progress_total is not None:
payload["progress_total"] = progress_total
if current_step is not None:
payload["current_step"] = current_step
db_ops.update_dataset_registry_state(dataset_id, **payload)
def mark_sync_success(dataset_id: str, *, message: str = "") -> None:
now = _now_ts()
db_ops.update_dataset_registry_state(
dataset_id,
status="success",
finished_at=now,
last_success_at=now,
message=message,
last_error=None,
needs_resync=0,
current_step="",
progress_current=0,
progress_total=0,
updated_at=now,
)
def mark_sync_failed(
dataset_id: str,
*,
message: str = "",
error: str = "",
status: str = "failed",
) -> None:
now = _now_ts()
final_message = message or error or "同步失败"
db_ops.update_dataset_registry_state(
dataset_id,
status=status,
finished_at=now,
last_failure_at=now,
message=final_message,
last_error=error or final_message,
needs_resync=1,
current_step="",
updated_at=now,
)
def mark_sync_blocked(dataset_id: str, *, message: str) -> None:
mark_sync_failed(dataset_id, message=message, error=message, status="blocked")
def recover_interrupted_syncs() -> int:
"""启动时把 status='running' 的卡死任务标记为 failed + needs_resync=1。"""
return db_ops.recover_interrupted_dataset_registry()
def get_registry_status() -> list[dict[str, Any]]:
return db_ops.fetch_dataset_registry_rows()