gao
8f016f25df
- ORM migration: models/ops 重构 - deploy: 标准化 systemd timer/service 部署体系 - MCP server: 5 个只读工具(任务/状态/数据集) - daily check: 数据一致性巡检 - watch: task_watch 后台监控 - kline_daily: 麦蕊优先,时间门禁15:00 - 删除: task_industry_sector, task_sector_features
201 lines
9.2 KiB
Python
201 lines
9.2 KiB
Python
"""同步任务：当天逐笔交易（mairui 源）。
|
||
|
||
数据源：mairui `hsrl/zbjy/{code6}/{licence}`
|
||
口径：单笔成交 — 时间 / 价 / 量 / 买卖方向
|
||
更新：每日 21:00（mairui 当天数据此时才发布；21:00 之前 API 返空或昨日数据）
|
||
|
||
设计要点：
|
||
1) 起步门控：21:00 之前（未传 force=True）→ 早返 warning
|
||
避免 0 行空跑（5000 只股票 × 1s 限速 ≈ 17 分钟浪费）
|
||
2) 5 worker × 1 RPS/worker = 5 RPS（守住 mairui 上限）
|
||
3) 不做"今日已同步"dedup —— mairui "当天"实际是"最近已发布交易日"，
|
||
trade_date 可能比 wall-clock 早一天；重跑靠 PK + ON CONFLICT 幂等
|
||
4) 单只股票日均 5w-10w tick，CHUNK=2000 upsert 防 driver 撑爆
|
||
5) 数据是"当天 only"，无 start/end 窗口；不存增量概念
|
||
6) 2026-07-13 加固: 加上外层 FETCH_HARD_TIMEOUT,避免 mairui 挂起时
|
||
as_completed 无 timeout 导致整个 task 永久卡住(systemd 2h 超时杀)。
|
||
改用手动 pool 管理 + shutdown(wait=False),与 kline_daily 对齐。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
import time
|
||
from concurrent.futures import ThreadPoolExecutor, as_completed, TimeoutError as FuturesTimeoutError
|
||
from datetime import datetime
|
||
from typing import Any
|
||
|
||
from app.core.db import ops as db_ops
|
||
from app.core.datasource.base import registry as ds_registry
|
||
from app.core.datasource.registry import is_source_ready
|
||
from app.core.datasource.utils import is_a_share_code, to_code6, to_hermes
|
||
from app.core.sync.base import SyncTask
|
||
from app.core.sync.registry import mark_sync_blocked
|
||
from app.core.utils.logging import get_logger
|
||
|
||
logger = get_logger("sync.tick_trade")
|
||
|
||
# mairui 文档：「更新：每日 21:00」—— 早于此时 mairui 仍返昨日/空数据
|
||
MAIRUI_TICK_TRADE_PUBLISH_HOUR = 21
|
||
MAIRUI_TICK_TRADE_PUBLISH_MIN = 0
|
||
# 21:00 之后再跑（留 5 分钟缓冲，等 mairui 完整入库）
|
||
TICK_TRADE_RUN_GATE = (MAIRUI_TICK_TRADE_PUBLISH_HOUR, MAIRUI_TICK_TRADE_PUBLISH_MIN + 5)
|
||
# 外层超时: mairui _fetch 有 20s timeout,但 as_completed 无 timeout 的话
|
||
# 若所有 worker 同时被 mairui 挂起(罕见),task 会永久卡住等 systemd 2h 杀。
|
||
# 120s 内无任何 future 完成 → 判全部失败退出。
|
||
FETCH_HARD_TIMEOUT = 120.0
|
||
|
||
|
||
def _passes_publish_gate(now: Optional[datetime] = None, *, force: bool = False) -> tuple[bool, str]:
|
||
"""判断当前时间是否已过 mairui 21:00 数据发布门控。
|
||
|
||
force=True 跳过门控（手动 / 测试用）。
|
||
"""
|
||
if force:
|
||
return True, "force=True 跳过门控"
|
||
n = now or datetime.now()
|
||
gate = n.replace(hour=TICK_TRADE_RUN_GATE[0], minute=TICK_TRADE_RUN_GATE[1], second=0, microsecond=0)
|
||
if n < gate:
|
||
return False, f"mairui 逐笔数据 {MAIRUI_TICK_TRADE_PUBLISH_HOUR:02d}:{MAIRUI_TICK_TRADE_PUBLISH_MIN:02d} 发布；当前 {n.strftime('%H:%M')} 早于门控，skip"
|
||
return True, "ok"
|
||
|
||
|
||
class SyncTickTrade(SyncTask):
|
||
dataset_id = "tick_trade"
|
||
|
||
def _run(
|
||
self,
|
||
*,
|
||
trigger_source: str = "manual",
|
||
codes: list[str] | None = None,
|
||
max_workers: int = 5,
|
||
force: bool = False,
|
||
**kwargs,
|
||
) -> dict[str, Any]:
|
||
mr = ds_registry.get("datasource_mairui")
|
||
if mr is None:
|
||
return {"status": "error", "message": "mairui 数据源未注册"}
|
||
ok, reason = is_source_ready(mr.key)
|
||
if not ok:
|
||
mark_sync_blocked(self.dataset_id, message=f"mairui 未就绪: {reason}")
|
||
return {"status": "blocked", "message": f"mairui 未就绪: {reason}"}
|
||
|
||
# ── 21:00 门控（force=True 跳过）──
|
||
gate_ok, gate_msg = _passes_publish_gate(force=force)
|
||
if not gate_ok:
|
||
logger.info(f"[tick_trade] 门控未过: {gate_msg}")
|
||
return {"status": "warning", "message": gate_msg, "ok": 0, "fail": 0, "rows": 0, "elapsed_sec": 0.0}
|
||
|
||
# ── 取股票列表 ──
|
||
if codes:
|
||
stock_codes = [to_code6(c) for c in codes]
|
||
else:
|
||
stock_codes = [c for c in db_ops.iter_stock_codes(active_only=True) if is_a_share_code(c)]
|
||
limit_raw = os.environ.get("MARKET_DATA_STOCK_LIMIT", "").strip()
|
||
if limit_raw.isdigit() and int(limit_raw) > 0:
|
||
stock_codes = stock_codes[: int(limit_raw)]
|
||
if not stock_codes:
|
||
return {"status": "error", "message": "无股票代码"}
|
||
|
||
# 注：不做"今日已同步"dedup。原因：
|
||
# - mairui 的"当天"实际是"最近一个已发布的交易日"（21:00 之前是昨日），
|
||
# 用 today_str 做去重会失效（即使刚跑完，trade_date 还是昨日）。
|
||
# - PK 含 trade_date，重跑是 ON CONFLICT DO UPDATE，安全且幂等。
|
||
# - 17 分钟全市场开销可接受；非交易日用 `MARKET_DATA_STOCK_LIMIT=0` 跳。
|
||
|
||
total = len(stock_codes)
|
||
if total == 0:
|
||
return {"status": "ok", "message": "无股票需要同步", "ok": 0, "fail": 0, "rows": 0, "elapsed_sec": 0.0}
|
||
|
||
# ── 预估耗时：5 RPS 上限 ──
|
||
est_sec = total / 5.0
|
||
logger.info(f"[tick_trade] 计划: {total} 只, 预估 {est_sec:.0f}s ({est_sec/60:.1f}min)")
|
||
self._progress(
|
||
message=f"计划: {total} 只, 预估 {est_sec/60:.1f}min @5RPS",
|
||
current=0, total=total, current_step="planning",
|
||
)
|
||
|
||
# ── 执行 ──
|
||
# 不用 `with ThreadPoolExecutor` — 与外层 as_completed timeout 配合,
|
||
# 超时后 `with` 的 shutdown(wait=True) 会阻塞等卡死线程 → 进程挂死(同 kline_daily 教训)。
|
||
t0 = time.time()
|
||
ok_cnt = fail_cnt = 0
|
||
rows_total = 0
|
||
pool = ThreadPoolExecutor(max_workers=max_workers)
|
||
futures = {pool.submit(self._sync_one, mr, c6): c6 for c6 in stock_codes}
|
||
try:
|
||
for i, future in enumerate(as_completed(futures, timeout=FETCH_HARD_TIMEOUT), 1):
|
||
c6 = futures[future]
|
||
try:
|
||
res = future.result(timeout=0.1)
|
||
if res["status"] == "ok":
|
||
ok_cnt += 1
|
||
rows_total += res["rows"]
|
||
else:
|
||
fail_cnt += 1
|
||
if res.get("error"):
|
||
logger.warning(f"[tick_trade {c6}] {res['error']}")
|
||
except FuturesTimeoutError:
|
||
fail_cnt += 1
|
||
logger.warning(f"[tick_trade {c6}] 内部 race timeout, 记 fail")
|
||
except Exception as e:
|
||
fail_cnt += 1
|
||
logger.warning(f"[tick_trade {c6}] {e}")
|
||
if i % 50 == 0 or i == total:
|
||
self._progress(
|
||
message=f"tick_trade 进度 {i}/{total} OK:{ok_cnt} FAIL:{fail_cnt} 行:{rows_total}",
|
||
current=i, total=total, current_step=c6,
|
||
)
|
||
except FuturesTimeoutError:
|
||
stuck = [(f, c6) for f, c6 in futures.items() if not f.done()]
|
||
logger.error(
|
||
"[tick_trade] 所有 fetcher 卡死 (>%ss), %d 只股票未完成, 标 fail 后退出",
|
||
FETCH_HARD_TIMEOUT, len(stuck),
|
||
)
|
||
fail_cnt += len(stuck)
|
||
for f, c6 in stuck:
|
||
try:
|
||
if hasattr(f, "cancel"):
|
||
f.cancel()
|
||
except Exception as e:
|
||
logger.warning(f"[tick_trade] cancel future for {c6} 失败: {e}")
|
||
try:
|
||
pool.shutdown(wait=False)
|
||
except Exception as e:
|
||
logger.warning(f"[tick_trade] pool.shutdown(wait=False) 失败: {e}")
|
||
else:
|
||
pool.shutdown(wait=True)
|
||
elapsed = round(time.time() - t0, 1)
|
||
msg = f"逐笔 {ok_cnt}成 {fail_cnt}败 共{rows_total}行, {elapsed}s"
|
||
return {
|
||
"status": "ok" if fail_cnt == 0 else "warning",
|
||
"message": msg,
|
||
"ok": ok_cnt, "fail": fail_cnt, "rows": rows_total, "elapsed_sec": elapsed,
|
||
}
|
||
|
||
def _sync_one(self, mr, code6: str) -> dict:
|
||
try:
|
||
df = mr.fetch_tick_trade(code6)
|
||
except Exception as e:
|
||
return {"status": "fail", "error": str(e)}
|
||
if df is None or df.empty:
|
||
return {"status": "fail", "error": "no data (mairui 21:00 前可能为空)"}
|
||
try:
|
||
rows = []
|
||
for _, r in df.iterrows():
|
||
# trade_time 来自 source 层,保证是 tz-aware Asia/Shanghai datetime
|
||
# (contract 见 app/sources/mairui.py:fetch_tick_trade docstring)
|
||
rows.append({
|
||
"stock_code": to_hermes(code6),
|
||
"trade_date": str(r["trade_date"]),
|
||
"trade_time": r["trade_time"],
|
||
"price": float(r.get("price") or 0),
|
||
"volume": float(r.get("volume") or 0),
|
||
"direction_code": int(r.get("direction_code") or 0),
|
||
"direction": str(r.get("direction") or ""),
|
||
"amount": float(r.get("amount") or 0),
|
||
})
|
||
db_ops.upsert_tick_trade(rows)
|
||
return {"status": "ok", "rows": len(rows)}
|
||
except Exception as e:
|
||
return {"status": "fail", "error": f"db write: {e}"}
|