kyugao/market_sync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: baseline — ORM migration + systemd deploy + MCP server + daily check

Browse Source 
- ORM migration: models/ops 重构
- deploy: 标准化 systemd timer/service 部署体系
- MCP server: 5 个只读工具(任务/状态/数据集)
- daily check: 数据一致性巡检
- watch: task_watch 后台监控
- kline_daily: 麦蕊优先,时间门禁15:00
- 删除: task_industry_sector, task_sector_features
This commit is contained in:
gao
2026-07-21 16:37:00 +08:00
parent 91e83a3b0b
commit 8f016f25df
94 changed files with 4117 additions and 1176 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/tasks/task_kline_5min.py
+163 -8
View File
@@ -17,7 +17,7 @@ from __future__ import annotations
import os
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from concurrent.futures import ThreadPoolExecutor, as_completed, TimeoutError as FuturesTimeoutError
from datetime import datetime, timedelta, timezone
from typing import Any, Optional
@@ -37,6 +37,10 @@ MAIRUI_5MIN_FLOOR = datetime(2023, 6, 14, tzinfo=timezone.utc)
WINDOW_DAYS = 365
# 增量时往前多取的天数（防交易日历边界漏当天）
INCREMENT_OVERLAP_DAYS = 2
# 2026-07-08 教训: 数据源层有 20s timeout,task 层再硬兜底,
# 防 SDK 升级 / 网络层 bug 让单只股票卡住(7月7日 4.4只/秒 后突然 0 持续 24h)
# 7月9日 mairui/雪球 间歇性 "服务器连接失败" + Broken pipe,拉慢。给 120s 容忍。
FETCH_HARD_TIMEOUT = 120.0 # 5min K 一只拉多年,单只允许更久
class SyncKline5Min(SyncTask):
@@ -99,7 +103,12 @@ class SyncKline5Min(SyncTask):
# ── 2) 内存算计划 ──
# PG TIMESTAMPTZ 是 timezone-aware，所以 end 也必须 tz-aware 否则比较会炸
end = datetime.now(timezone.utc)
# 2026-07-12 修复: end 改为上海时区最近交易日 15:00，避免 UTC 与上海时间混用
from app.core.datasource.utils import effective_market_date
end_date_str = effective_market_date()
end = datetime.strptime(f"{end_date_str} 15:00:00", "%Y-%m-%d %H:%M:%S").replace(
tzinfo=timezone(timedelta(hours=8))
)
plans = self._plan(stock_codes, end, snapshots)
if not plans:
msg = f"5min K线 — 全部 {len(stock_codes)} 只都已最新（无需同步）"
@@ -129,12 +138,17 @@ class SyncKline5Min(SyncTask):
t0 = time.time()
ok_cnt = fail_cnt = 0
rows_total = 0
with ThreadPoolExecutor(max_workers=max_workers) as pool:
futures = {pool.submit(self._sync_one, primary, p[0], p[1], p[2]): p[0] for p in plans}
for i, future in enumerate(as_completed(futures), 1):
# 不用 `with ThreadPoolExecutor(...) as pool:` — 它的 __exit__ 默认 wait=True,
# 一旦某个 worker 卡在 xueqiu IO,主线程会在 as_completed 触发 60s timeout 后
# 仍被 __exit__ 阻塞等 worker 退出 → 进程挂死(2026-07-09 13:21 教训)。
# 改成手动管理 + shutdown(wait=False),主线程能立刻退出。
pool = ThreadPoolExecutor(max_workers=max_workers)
futures = {pool.submit(self._sync_one, primary, p[0], p[1], p[2]): p[0] for p in plans}
try:
for i, future in enumerate(as_completed(futures, timeout=FETCH_HARD_TIMEOUT), 1):
c6 = futures[future]
try:
res = future.result()
res = future.result(timeout=0.1) # 已被 as_completed 释放
if res["status"] == "ok":
ok_cnt += 1
rows_total += res["rows"]
@@ -142,6 +156,9 @@ class SyncKline5Min(SyncTask):
fail_cnt += 1
if res.get("error"):
logger.warning(f"[5min {c6}] {res['error']}")
except FuturesTimeoutError:
fail_cnt += 1
logger.warning(f"[5min {c6}] 内部 race timeout, 记 fail")
except Exception as e:
fail_cnt += 1
logger.warning(f"[5min {c6}] {e}")
@@ -150,6 +167,64 @@ class SyncKline5Min(SyncTask):
message=f"5min 进度 {i}/{len(plans)} OK:{ok_cnt} FAIL:{fail_cnt}",
current=i, total=len(plans), current_step=c6,
)
except FuturesTimeoutError:
stuck = [c6 for _, c6 in futures.items() if not _.done()]
logger.error(
"[5min] 所有 fetcher 卡死 (>%ss), %d 只股票未完成",
FETCH_HARD_TIMEOUT, len(stuck),
)
for f, c6 in futures.items():
try:
if not f.done() and hasattr(f, "cancel"):
f.cancel()
except Exception as e:
logger.warning(f"[5min] cancel future for {c6} 失败: {e}")
try:
pool.shutdown(wait=False)
except Exception as e:
logger.warning(f"[5min] pool.shutdown(wait=False) 失败: {e}")
# ── 补偿：冷却后重入队列 ──
# mairui 偶发全 hang（中间件重启/网络抖动），等 30s 再试一次。
# 如果还 hang 就放弃，下次调度增量重拉。
if stuck:
cooldown = 30
logger.warning(f"[5min] 冷却 {cooldown}s 后重试 {len(stuck)} 只…")
time.sleep(cooldown)
retry_ok = retry_fail = retry_rows = 0
retry_pool = ThreadPoolExecutor(max_workers=max_workers)
plan_dict = {p[0]: (p[1], p[2]) for p in plans}
retry_futs = {}
for c6 in stuck:
if c6 in plan_dict:
s, e = plan_dict[c6]
# 重试直接用雪球 fallback（Mairui 已全 hang）
retry_futs[retry_pool.submit(self._fallback_xueqiu_5m_and_write, c6, s, e)] = c6
if retry_futs:
for fut in as_completed(retry_futs, timeout=FETCH_HARD_TIMEOUT):
c6 = retry_futs[fut]
try:
res = fut.result(timeout=0.1)
if res["status"] == "ok":
retry_ok += 1
retry_rows += res["rows"]
else:
retry_fail += 1
except Exception:
retry_fail += 1
try:
retry_pool.shutdown(wait=False)
except Exception:
pass
ok_cnt += retry_ok
fail_cnt -= retry_ok # 从 fail 挪到 ok
rows_total += retry_rows
logger.warning(
f"[5min] 重试结果: {retry_ok}成 {retry_fail}败 "
f"共{retry_rows}行, 救回 {retry_ok} 只"
)
else:
pool.shutdown(wait=True)
elapsed = round(time.time() - t0, 1)
msg = (
f"5min K线 {ok_cnt}成 {fail_cnt}败 共{rows_total}行, "
@@ -192,10 +267,16 @@ class SyncKline5Min(SyncTask):
})
cur = w_end + timedelta(days=1)
except Exception as e:
return {"status": "fail", "error": str(e)}
logger.warning(f"[5min {code6}] mairui 失败, 尝试雪球 5m fallback: {e}")
all_rows = self._fallback_xueqiu_5m(code6, start, end)
if not all_rows:
return {"status": "fail", "error": "no data"}
# Mairui 无数据, 尝试雪球 fallback
logger.info(f"[5min {code6}] mairui 无数据, 尝试雪球 fallback")
all_rows = self._fallback_xueqiu_5m(code6, start, end)
if not all_rows:
return {"status": "fail", "error": "no data (mairui + xueqiu fallback)"}
try:
CHUNK = 5000
for i in range(0, len(all_rows), CHUNK):
@@ -203,3 +284,77 @@ class SyncKline5Min(SyncTask):
except Exception as e:
return {"status": "fail", "error": f"db write: {e}"}
return {"status": "ok", "rows": len(all_rows)}
@staticmethod
def _fallback_xueqiu_5m_and_write(code6: str, start: datetime, end: datetime) -> dict:
"""雪球 5m fallback + 直接写库，对齐 _sync_one 返回格式供 retry 逻辑使用。"""
rows = SyncKline5Min._fallback_xueqiu_5m(code6, start, end)
if not rows:
return {"status": "fail", "error": "no data via xueqiu"}
try:
CHUNK = 5000
for i in range(0, len(rows), CHUNK):
db_ops.upsert_kline_5min(rows[i: i + CHUNK])
except Exception as e:
return {"status": "fail", "error": f"db write: {e}"}
return {"status": "ok", "rows": len(rows)}
@staticmethod
def _fallback_xueqiu_5m(code6: str, start: datetime, end: datetime) -> list[dict]:
"""通过雪球 5m K线 fallback 拉取数据，对齐 _sync_one 返回格式。"""
import os
from datetime import datetime as dt_mod
from app.core.datasource.utils import code6_to_xueqiu, to_hermes
token_raw = os.environ.get("XUEQIU_TOKEN", "").strip()
if not token_raw:
# 从 .env 读
try:
with open("/home/gao/Development/quant_home/market_sync/.env") as f:
for line in f:
if line.startswith("XUEQIU_TOKEN="):
token_raw = line.strip().split("=", 1)[1]
break
except Exception:
pass
if not token_raw:
logger.warning("[xueqiu_5m] 无 XUEQIU_TOKEN, 跳过 fallback")
return []
import pysnowball as ball
ball.set_token(token_raw)
symbol = code6_to_xueqiu(code6)
# 计算需要多少根 5m bar (每天 48 根, 加 buffer)
days_needed = max((end - start).days + 2, 1)
count = days_needed * 48
try:
data = ball.kline(symbol, "5m", min(count, 5000))
except Exception as e:
logger.warning(f"[xueqiu_5m {code6}] 请求失败: {e}")
return []
items = data.get("data", {}).get("item", [])
if not items:
return []
columns = data["data"]["column"] # [timestamp, volume, open, high, low, close, ...]
idx = {c: i for i, c in enumerate(columns)}
hermes = to_hermes(code6)
rows = []
for item in items:
ts = item[idx["timestamp"]] / 1000
bar_time = dt_mod.fromtimestamp(ts).strftime("%Y-%m-%d %H:%M:%S")
rows.append({
"stock_code": hermes,
"bar_time": bar_time,
"open": float(item[idx["open"]]),
"high": float(item[idx["high"]]),
"low": float(item[idx["low"]]),
"close": float(item[idx["close"]]),
"volume": float(item[idx["volume"]]),
"amount": 0.0,
"turnover_rate": 0.0,
})
return rows