gao
8f016f25df
- ORM migration: models/ops 重构 - deploy: 标准化 systemd timer/service 部署体系 - MCP server: 5 个只读工具(任务/状态/数据集) - daily check: 数据一致性巡检 - watch: task_watch 后台监控 - kline_daily: 麦蕊优先,时间门禁15:00 - 删除: task_industry_sector, task_sector_features
361 lines
16 KiB
Python
361 lines
16 KiB
Python
"""同步任务：5 分钟 K 线（mairui）。
|
||
|
||
设计要点：
|
||
1) 启动时**一次 SQL** 拿全表 {stock_code: max(bar_time)} 快照
|
||
2) 内存里给每只股票算 (start, end, windows_needed)
|
||
- DB 没数据 → start = mairui 历史深度起点 (2023-06-14)
|
||
- DB 有数据 → start = max(2023-06-14, max(bar_time) - 2 天)
|
||
- 兜底：start >= end → 跳过这只
|
||
3) 输出"计划"（全量/增量分类、预估请求数、预估耗时）再开始执行
|
||
4) 5 worker × 1 RPS/worker = 5 RPS 总（守住 mairui 上限）
|
||
5) 窗口大小 1 年（mairui 一次最多 ~11600 条）
|
||
|
||
akshare 时代写法：start 写死 2020-01-01 + 4 天窗口，137 小时跑完全市场。
|
||
本版：增量时通常只补最近 1-2 天，~30 分钟。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
import time
|
||
from concurrent.futures import ThreadPoolExecutor, as_completed, TimeoutError as FuturesTimeoutError
|
||
from datetime import datetime, timedelta, timezone
|
||
from typing import Any, Optional
|
||
|
||
from app.core.db import ops as db_ops
|
||
from app.core.datasource.base import registry as ds_registry
|
||
from app.core.datasource.registry import is_source_ready
|
||
from app.core.datasource.utils import is_a_share_code, to_code6, to_hermes
|
||
from app.core.sync.base import SyncTask
|
||
from app.core.sync.registry import mark_sync_blocked
|
||
from app.core.utils.logging import get_logger
|
||
|
||
logger = get_logger("sync.kline_5min")
|
||
|
||
# mairui 5min K 实测历史深度起点（实测 2023-06-14 才有数据，更早就 0）
|
||
MAIRUI_5MIN_FLOOR = datetime(2023, 6, 14, tzinfo=timezone.utc)
|
||
# 单次请求窗口大小：1 年（mairui 单次最多 ~11640 条 5min）
|
||
WINDOW_DAYS = 365
|
||
# 增量时往前多取的天数（防交易日历边界漏当天）
|
||
INCREMENT_OVERLAP_DAYS = 2
|
||
# 2026-07-08 教训: 数据源层有 20s timeout,task 层再硬兜底,
|
||
# 防 SDK 升级 / 网络层 bug 让单只股票卡住(7月7日 4.4只/秒 后突然 0 持续 24h)
|
||
# 7月9日 mairui/雪球 间歇性 "服务器连接失败" + Broken pipe,拉慢。给 120s 容忍。
|
||
FETCH_HARD_TIMEOUT = 120.0 # 5min K 一只拉多年,单只允许更久
|
||
|
||
|
||
class SyncKline5Min(SyncTask):
|
||
dataset_id = "kline_5min"
|
||
|
||
def _plan(
|
||
self,
|
||
stock_codes: list[str],
|
||
end: datetime,
|
||
snapshots: dict[str, Optional[datetime]],
|
||
) -> list[tuple[str, datetime, datetime]]:
|
||
"""为每只股票算 (start, end) — 已排除 start >= end 的"无需同步"。
|
||
|
||
Returns: [(code6, start, end), ...]
|
||
"""
|
||
plans = []
|
||
for code6 in stock_codes:
|
||
latest = snapshots.get(code6)
|
||
if latest is None:
|
||
# DB 里没数据 → 全量从 mairui 历史深度起点
|
||
start = MAIRUI_5MIN_FLOOR
|
||
else:
|
||
# DB 有数据 → 增量：从 max(bar_time) - overlap 到 end
|
||
start = max(MAIRUI_5MIN_FLOOR, latest - timedelta(days=INCREMENT_OVERLAP_DAYS))
|
||
if start < end:
|
||
plans.append((code6, start, end))
|
||
return plans
|
||
|
||
def _run(
|
||
self,
|
||
*,
|
||
trigger_source: str = "manual",
|
||
codes: list[str] | None = None,
|
||
max_workers: int = 5,
|
||
**kwargs,
|
||
) -> dict[str, Any]:
|
||
primary = ds_registry.get("datasource_mairui")
|
||
if primary is None:
|
||
return {"status": "error", "message": "5min 数据源 mairui 未注册"}
|
||
ok, reason = is_source_ready(primary.key)
|
||
if not ok:
|
||
mark_sync_blocked(self.dataset_id, message=f"{primary.key} 未就绪: {reason}")
|
||
return {"status": "blocked", "message": f"{primary.key} 未就绪: {reason}"}
|
||
|
||
if codes:
|
||
stock_codes = [to_code6(c) for c in codes]
|
||
else:
|
||
stock_codes = [c for c in db_ops.iter_stock_codes(active_only=True) if is_a_share_code(c)]
|
||
limit_raw = os.environ.get("MARKET_DATA_STOCK_LIMIT", "").strip()
|
||
if limit_raw.isdigit() and int(limit_raw) > 0:
|
||
stock_codes = stock_codes[: int(limit_raw)]
|
||
|
||
if not stock_codes:
|
||
return {"status": "error", "message": "无股票代码"}
|
||
|
||
# ── 1) 一次 SQL 拿全表快照 ──
|
||
logger.info(f"[5min] 读取 DB 快照 ({len(stock_codes)} 只待查)…")
|
||
snapshots = db_ops.get_kline_5min_snapshots()
|
||
logger.info(f"[5min] DB 里已有 {len(snapshots)} 只的 kline_5min 数据")
|
||
|
||
# ── 2) 内存算计划 ──
|
||
# PG TIMESTAMPTZ 是 timezone-aware，所以 end 也必须 tz-aware 否则比较会炸
|
||
# 2026-07-12 修复: end 改为上海时区最近交易日 15:00，避免 UTC 与上海时间混用
|
||
from app.core.datasource.utils import effective_market_date
|
||
end_date_str = effective_market_date()
|
||
end = datetime.strptime(f"{end_date_str} 15:00:00", "%Y-%m-%d %H:%M:%S").replace(
|
||
tzinfo=timezone(timedelta(hours=8))
|
||
)
|
||
plans = self._plan(stock_codes, end, snapshots)
|
||
if not plans:
|
||
msg = f"5min K线 — 全部 {len(stock_codes)} 只都已最新（无需同步）"
|
||
logger.info(msg)
|
||
return {"status": "ok", "message": msg, "ok": 0, "fail": 0, "rows": 0, "elapsed_sec": 0.0}
|
||
|
||
# 分类统计
|
||
full_sync = [p for p in plans if snapshots.get(p[0]) is None]
|
||
incr_sync = [p for p in plans if snapshots.get(p[0]) is not None]
|
||
total_windows = sum(
|
||
max(1, (p[2] - p[1]).days // WINDOW_DAYS + 1)
|
||
for p in plans
|
||
)
|
||
# mairui 5 RPS 上限（5 worker × 1 RPS）
|
||
est_sec = total_windows / 5.0
|
||
logger.info(
|
||
f"[5min] 计划: 总 {len(plans)} 只 (全量 {len(full_sync)} / 增量 {len(incr_sync)})，"
|
||
f"总请求 {total_windows}，按 5 RPS 预估 {est_sec:.0f}s ({est_sec/60:.1f}min)"
|
||
)
|
||
self._progress(
|
||
message=f"计划: {len(plans)} 只 (全量 {len(full_sync)}/增量 {len(incr_sync)}) "
|
||
f"约 {total_windows} 个请求, 预估 {est_sec/60:.1f}min",
|
||
current=0, total=len(plans), current_step="planning",
|
||
)
|
||
|
||
# ── 3) 执行 ──
|
||
t0 = time.time()
|
||
ok_cnt = fail_cnt = 0
|
||
rows_total = 0
|
||
# 不用 `with ThreadPoolExecutor(...) as pool:` — 它的 __exit__ 默认 wait=True,
|
||
# 一旦某个 worker 卡在 xueqiu IO,主线程会在 as_completed 触发 60s timeout 后
|
||
# 仍被 __exit__ 阻塞等 worker 退出 → 进程挂死(2026-07-09 13:21 教训)。
|
||
# 改成手动管理 + shutdown(wait=False),主线程能立刻退出。
|
||
pool = ThreadPoolExecutor(max_workers=max_workers)
|
||
futures = {pool.submit(self._sync_one, primary, p[0], p[1], p[2]): p[0] for p in plans}
|
||
try:
|
||
for i, future in enumerate(as_completed(futures, timeout=FETCH_HARD_TIMEOUT), 1):
|
||
c6 = futures[future]
|
||
try:
|
||
res = future.result(timeout=0.1) # 已被 as_completed 释放
|
||
if res["status"] == "ok":
|
||
ok_cnt += 1
|
||
rows_total += res["rows"]
|
||
else:
|
||
fail_cnt += 1
|
||
if res.get("error"):
|
||
logger.warning(f"[5min {c6}] {res['error']}")
|
||
except FuturesTimeoutError:
|
||
fail_cnt += 1
|
||
logger.warning(f"[5min {c6}] 内部 race timeout, 记 fail")
|
||
except Exception as e:
|
||
fail_cnt += 1
|
||
logger.warning(f"[5min {c6}] {e}")
|
||
if i % 20 == 0 or i == len(plans):
|
||
self._progress(
|
||
message=f"5min 进度 {i}/{len(plans)} OK:{ok_cnt} FAIL:{fail_cnt}",
|
||
current=i, total=len(plans), current_step=c6,
|
||
)
|
||
except FuturesTimeoutError:
|
||
stuck = [c6 for _, c6 in futures.items() if not _.done()]
|
||
logger.error(
|
||
"[5min] 所有 fetcher 卡死 (>%ss), %d 只股票未完成",
|
||
FETCH_HARD_TIMEOUT, len(stuck),
|
||
)
|
||
for f, c6 in futures.items():
|
||
try:
|
||
if not f.done() and hasattr(f, "cancel"):
|
||
f.cancel()
|
||
except Exception as e:
|
||
logger.warning(f"[5min] cancel future for {c6} 失败: {e}")
|
||
try:
|
||
pool.shutdown(wait=False)
|
||
except Exception as e:
|
||
logger.warning(f"[5min] pool.shutdown(wait=False) 失败: {e}")
|
||
|
||
# ── 补偿：冷却后重入队列 ──
|
||
# mairui 偶发全 hang（中间件重启/网络抖动），等 30s 再试一次。
|
||
# 如果还 hang 就放弃，下次调度增量重拉。
|
||
if stuck:
|
||
cooldown = 30
|
||
logger.warning(f"[5min] 冷却 {cooldown}s 后重试 {len(stuck)} 只…")
|
||
time.sleep(cooldown)
|
||
retry_ok = retry_fail = retry_rows = 0
|
||
retry_pool = ThreadPoolExecutor(max_workers=max_workers)
|
||
plan_dict = {p[0]: (p[1], p[2]) for p in plans}
|
||
retry_futs = {}
|
||
for c6 in stuck:
|
||
if c6 in plan_dict:
|
||
s, e = plan_dict[c6]
|
||
# 重试直接用雪球 fallback（Mairui 已全 hang）
|
||
retry_futs[retry_pool.submit(self._fallback_xueqiu_5m_and_write, c6, s, e)] = c6
|
||
if retry_futs:
|
||
for fut in as_completed(retry_futs, timeout=FETCH_HARD_TIMEOUT):
|
||
c6 = retry_futs[fut]
|
||
try:
|
||
res = fut.result(timeout=0.1)
|
||
if res["status"] == "ok":
|
||
retry_ok += 1
|
||
retry_rows += res["rows"]
|
||
else:
|
||
retry_fail += 1
|
||
except Exception:
|
||
retry_fail += 1
|
||
try:
|
||
retry_pool.shutdown(wait=False)
|
||
except Exception:
|
||
pass
|
||
ok_cnt += retry_ok
|
||
fail_cnt -= retry_ok # 从 fail 挪到 ok
|
||
rows_total += retry_rows
|
||
logger.warning(
|
||
f"[5min] 重试结果: {retry_ok}成 {retry_fail}败 "
|
||
f"共{retry_rows}行, 救回 {retry_ok} 只"
|
||
)
|
||
else:
|
||
pool.shutdown(wait=True)
|
||
elapsed = round(time.time() - t0, 1)
|
||
msg = (
|
||
f"5min K线 {ok_cnt}成 {fail_cnt}败 共{rows_total}行, "
|
||
f"全量{len(full_sync)}/增量{len(incr_sync)}, {elapsed}s"
|
||
)
|
||
return {
|
||
"status": "ok" if fail_cnt == 0 else "warning",
|
||
"message": msg,
|
||
"ok": ok_cnt, "fail": fail_cnt, "rows": rows_total,
|
||
"full_sync": len(full_sync), "incr_sync": len(incr_sync),
|
||
"total_requests": total_windows, "elapsed_sec": elapsed,
|
||
}
|
||
|
||
def _sync_one(self, primary, code6: str, start: datetime, end: datetime) -> dict:
|
||
all_rows: list[dict] = []
|
||
try:
|
||
cur = start
|
||
while cur < end:
|
||
w_end = min(cur + timedelta(days=WINDOW_DAYS), end)
|
||
df = primary.fetch_kline_5min(
|
||
code6,
|
||
cur.strftime("%Y-%m-%d"),
|
||
w_end.strftime("%Y-%m-%d"),
|
||
)
|
||
if df is not None and not df.empty:
|
||
hermes = to_hermes(code6)
|
||
for _, r in df.iterrows():
|
||
bar_time = r["bar_time"]
|
||
all_rows.append({
|
||
"stock_code": hermes,
|
||
"bar_time": bar_time.strftime("%Y-%m-%d %H:%M:%S")
|
||
if hasattr(bar_time, "strftime") else str(bar_time),
|
||
"open": float(r.get("open") or 0),
|
||
"high": float(r.get("high") or 0),
|
||
"low": float(r.get("low") or 0),
|
||
"close": float(r.get("close") or 0),
|
||
"volume": float(r.get("volume") or 0),
|
||
"amount": float(r.get("amount") or 0),
|
||
"turnover_rate": float(r.get("turnover_rate") or 0),
|
||
})
|
||
cur = w_end + timedelta(days=1)
|
||
except Exception as e:
|
||
logger.warning(f"[5min {code6}] mairui 失败, 尝试雪球 5m fallback: {e}")
|
||
all_rows = self._fallback_xueqiu_5m(code6, start, end)
|
||
|
||
if not all_rows:
|
||
# Mairui 无数据, 尝试雪球 fallback
|
||
logger.info(f"[5min {code6}] mairui 无数据, 尝试雪球 fallback")
|
||
all_rows = self._fallback_xueqiu_5m(code6, start, end)
|
||
|
||
if not all_rows:
|
||
return {"status": "fail", "error": "no data (mairui + xueqiu fallback)"}
|
||
try:
|
||
CHUNK = 5000
|
||
for i in range(0, len(all_rows), CHUNK):
|
||
db_ops.upsert_kline_5min(all_rows[i: i + CHUNK])
|
||
except Exception as e:
|
||
return {"status": "fail", "error": f"db write: {e}"}
|
||
return {"status": "ok", "rows": len(all_rows)}
|
||
|
||
@staticmethod
|
||
def _fallback_xueqiu_5m_and_write(code6: str, start: datetime, end: datetime) -> dict:
|
||
"""雪球 5m fallback + 直接写库，对齐 _sync_one 返回格式供 retry 逻辑使用。"""
|
||
rows = SyncKline5Min._fallback_xueqiu_5m(code6, start, end)
|
||
if not rows:
|
||
return {"status": "fail", "error": "no data via xueqiu"}
|
||
try:
|
||
CHUNK = 5000
|
||
for i in range(0, len(rows), CHUNK):
|
||
db_ops.upsert_kline_5min(rows[i: i + CHUNK])
|
||
except Exception as e:
|
||
return {"status": "fail", "error": f"db write: {e}"}
|
||
return {"status": "ok", "rows": len(rows)}
|
||
|
||
@staticmethod
|
||
def _fallback_xueqiu_5m(code6: str, start: datetime, end: datetime) -> list[dict]:
|
||
"""通过雪球 5m K线 fallback 拉取数据，对齐 _sync_one 返回格式。"""
|
||
import os
|
||
from datetime import datetime as dt_mod
|
||
from app.core.datasource.utils import code6_to_xueqiu, to_hermes
|
||
|
||
token_raw = os.environ.get("XUEQIU_TOKEN", "").strip()
|
||
if not token_raw:
|
||
# 从 .env 读
|
||
try:
|
||
with open("/home/gao/Development/quant_home/market_sync/.env") as f:
|
||
for line in f:
|
||
if line.startswith("XUEQIU_TOKEN="):
|
||
token_raw = line.strip().split("=", 1)[1]
|
||
break
|
||
except Exception:
|
||
pass
|
||
if not token_raw:
|
||
logger.warning("[xueqiu_5m] 无 XUEQIU_TOKEN, 跳过 fallback")
|
||
return []
|
||
|
||
import pysnowball as ball
|
||
ball.set_token(token_raw)
|
||
|
||
symbol = code6_to_xueqiu(code6)
|
||
# 计算需要多少根 5m bar (每天 48 根, 加 buffer)
|
||
days_needed = max((end - start).days + 2, 1)
|
||
count = days_needed * 48
|
||
|
||
try:
|
||
data = ball.kline(symbol, "5m", min(count, 5000))
|
||
except Exception as e:
|
||
logger.warning(f"[xueqiu_5m {code6}] 请求失败: {e}")
|
||
return []
|
||
|
||
items = data.get("data", {}).get("item", [])
|
||
if not items:
|
||
return []
|
||
|
||
columns = data["data"]["column"] # [timestamp, volume, open, high, low, close, ...]
|
||
idx = {c: i for i, c in enumerate(columns)}
|
||
hermes = to_hermes(code6)
|
||
rows = []
|
||
for item in items:
|
||
ts = item[idx["timestamp"]] / 1000
|
||
bar_time = dt_mod.fromtimestamp(ts).strftime("%Y-%m-%d %H:%M:%S")
|
||
rows.append({
|
||
"stock_code": hermes,
|
||
"bar_time": bar_time,
|
||
"open": float(item[idx["open"]]),
|
||
"high": float(item[idx["high"]]),
|
||
"low": float(item[idx["low"]]),
|
||
"close": float(item[idx["close"]]),
|
||
"volume": float(item[idx["volume"]]),
|
||
"amount": 0.0,
|
||
"turnover_rate": 0.0,
|
||
})
|
||
return rows
|