chore: baseline — ORM migration + systemd deploy + MCP server + daily check

- ORM migration: models/ops 重构
- deploy: 标准化 systemd timer/service 部署体系
- MCP server: 5 个只读工具(任务/状态/数据集)
- daily check: 数据一致性巡检
- watch: task_watch 后台监控
- kline_daily: 麦蕊优先,时间门禁15:00
- 删除: task_industry_sector, task_sector_features
This commit is contained in:
gao
2026-07-21 16:37:00 +08:00
parent 91e83a3b0b
commit 8f016f25df
94 changed files with 4117 additions and 1176 deletions
+37 -5
View File
@@ -12,12 +12,15 @@
trade_date 可能比 wall-clock 早一天;重跑靠 PK + ON CONFLICT 幂等
4) 单只股票日均 5w-10w tickCHUNK=2000 upsert 防 driver 撑爆
5) 数据是"当天 only",无 start/end 窗口;不存增量概念
6) 2026-07-13 加固: 加上外层 FETCH_HARD_TIMEOUT,避免 mairui 挂起时
as_completed 无 timeout 导致整个 task 永久卡住(systemd 2h 超时杀)。
改用手动 pool 管理 + shutdown(wait=False),与 kline_daily 对齐。
"""
from __future__ import annotations
import os
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from concurrent.futures import ThreadPoolExecutor, as_completed, TimeoutError as FuturesTimeoutError
from datetime import datetime
from typing import Any
@@ -36,6 +39,10 @@ MAIRUI_TICK_TRADE_PUBLISH_HOUR = 21
MAIRUI_TICK_TRADE_PUBLISH_MIN = 0
# 21:00 之后再跑(留 5 分钟缓冲,等 mairui 完整入库)
TICK_TRADE_RUN_GATE = (MAIRUI_TICK_TRADE_PUBLISH_HOUR, MAIRUI_TICK_TRADE_PUBLISH_MIN + 5)
# 外层超时: mairui _fetch 有 20s timeout,但 as_completed 无 timeout 的话
# 若所有 worker 同时被 mairui 挂起(罕见),task 会永久卡住等 systemd 2h 杀。
# 120s 内无任何 future 完成 → 判全部失败退出。
FETCH_HARD_TIMEOUT = 120.0
def _passes_publish_gate(now: Optional[datetime] = None, *, force: bool = False) -> tuple[bool, str]:
@@ -108,15 +115,18 @@ class SyncTickTrade(SyncTask):
)
# ── 执行 ──
# 不用 `with ThreadPoolExecutor` — 与外层 as_completed timeout 配合,
# 超时后 `with` 的 shutdown(wait=True) 会阻塞等卡死线程 → 进程挂死(同 kline_daily 教训)。
t0 = time.time()
ok_cnt = fail_cnt = 0
rows_total = 0
with ThreadPoolExecutor(max_workers=max_workers) as pool:
futures = {pool.submit(self._sync_one, mr, c6): c6 for c6 in stock_codes}
for i, future in enumerate(as_completed(futures), 1):
pool = ThreadPoolExecutor(max_workers=max_workers)
futures = {pool.submit(self._sync_one, mr, c6): c6 for c6 in stock_codes}
try:
for i, future in enumerate(as_completed(futures, timeout=FETCH_HARD_TIMEOUT), 1):
c6 = futures[future]
try:
res = future.result()
res = future.result(timeout=0.1)
if res["status"] == "ok":
ok_cnt += 1
rows_total += res["rows"]
@@ -124,6 +134,9 @@ class SyncTickTrade(SyncTask):
fail_cnt += 1
if res.get("error"):
logger.warning(f"[tick_trade {c6}] {res['error']}")
except FuturesTimeoutError:
fail_cnt += 1
logger.warning(f"[tick_trade {c6}] 内部 race timeout, 记 fail")
except Exception as e:
fail_cnt += 1
logger.warning(f"[tick_trade {c6}] {e}")
@@ -132,6 +145,25 @@ class SyncTickTrade(SyncTask):
message=f"tick_trade 进度 {i}/{total} OK:{ok_cnt} FAIL:{fail_cnt} 行:{rows_total}",
current=i, total=total, current_step=c6,
)
except FuturesTimeoutError:
stuck = [(f, c6) for f, c6 in futures.items() if not f.done()]
logger.error(
"[tick_trade] 所有 fetcher 卡死 (>%ss), %d 只股票未完成, 标 fail 后退出",
FETCH_HARD_TIMEOUT, len(stuck),
)
fail_cnt += len(stuck)
for f, c6 in stuck:
try:
if hasattr(f, "cancel"):
f.cancel()
except Exception as e:
logger.warning(f"[tick_trade] cancel future for {c6} 失败: {e}")
try:
pool.shutdown(wait=False)
except Exception as e:
logger.warning(f"[tick_trade] pool.shutdown(wait=False) 失败: {e}")
else:
pool.shutdown(wait=True)
elapsed = round(time.time() - t0, 1)
msg = f"逐笔 {ok_cnt}{fail_cnt}败 共{rows_total}行, {elapsed}s"
return {