chore: baseline — ORM migration + systemd deploy + MCP server + daily check
- ORM migration: models/ops 重构 - deploy: 标准化 systemd timer/service 部署体系 - MCP server: 5 个只读工具(任务/状态/数据集) - daily check: 数据一致性巡检 - watch: task_watch 后台监控 - kline_daily: 麦蕊优先,时间门禁15:00 - 删除: task_industry_sector, task_sector_features
This commit is contained in:
gao
+152
-29
@@ -19,6 +19,7 @@ import os
|
||||
import queue
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed, TimeoutError as FuturesTimeoutError
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
@@ -32,15 +33,19 @@ from app.core.utils.logging import get_logger
|
||||
|
||||
logger = get_logger("sync.kline_daily")
|
||||
|
||||
# 主源优先级：mairui（5 RPS，免费 + 稳定 + 单接口含5min）→ 雪球 → Baostock → 新浪
|
||||
# 经验：sina 在单日窗口 (start=end=某天) 会返回空，雪球更可靠
|
||||
PRIMARY_PRIORITY = ["datasource_xueqiu", "datasource_mairui", "datasource_xinlang"]
|
||||
# 源优先级：麦蕊 → 雪球 → 新浪
|
||||
# 2026-07-21 改: 麦蕊15:10已有数据,雪球要15:40后才齐,麦蕊为主源提速
|
||||
PRIMARY_PRIORITY = ["datasource_mairui", "datasource_xueqiu", "datasource_xinlang"]
|
||||
DEFAULT_START = "2018-01-01"
|
||||
# 写库批量：攒够 BATCH_SIZE 行就 executemany 一次 + commit
|
||||
BATCH_SIZE = 2000
|
||||
# fetcher 并发：实测单线程最稳（5000 只 × 0.2s = 17 分钟，足够）
|
||||
# 雪球 token 限速 10 RPS，单线程 0.1s/只 已经打满。多 worker 会触发风控 hang
|
||||
DEFAULT_FETCH_WORKERS = 1
|
||||
# 2026-07-08 教训: 数据源层虽然加了 fetch timeout (20s), task 层也加硬上限兜底,
|
||||
# 防止数据源 SDK 升级后又被绕过。60s 对应 xueqiu/mairui 的 15-20s 上限 + DB upsert
|
||||
# / DataFrame 处理余量 + 7月9日 mairui 间歇慢请求的容忍窗口。
|
||||
FETCH_HARD_TIMEOUT = 180.0
|
||||
|
||||
|
||||
class SyncKlineDaily(SyncTask):
|
||||
@@ -57,14 +62,15 @@ class SyncKlineDaily(SyncTask):
|
||||
max_workers: int = DEFAULT_FETCH_WORKERS,
|
||||
**kwargs,
|
||||
) -> dict[str, Any]:
|
||||
# 1. 选主源
|
||||
primary = self._pick_primary()
|
||||
if primary is None:
|
||||
# 1. 获取所有可用源（按优先级），支持 per-stock fallback
|
||||
sources = self._get_available_sources()
|
||||
if not sources:
|
||||
return {"status": "error", "message": "所有 K 线数据源均不可用"}
|
||||
source_names = ", ".join(s.name for s in sources)
|
||||
|
||||
self._progress(
|
||||
message=f"使用 {primary.name} 同步日K线 (fetcher={max_workers}, batch={BATCH_SIZE})...",
|
||||
current_step=primary.key,
|
||||
message=f"使用 [{source_names}] 同步日K线 (fetcher={max_workers}, batch={BATCH_SIZE})...",
|
||||
current_step=sources[0].key,
|
||||
)
|
||||
|
||||
# 2. 拉股票列表
|
||||
@@ -108,7 +114,6 @@ class SyncKlineDaily(SyncTask):
|
||||
)
|
||||
|
||||
# 4. 并发 fetch + 攒 batch 写库（线程安全）
|
||||
# mairui 等外部 API 无并发问题；DB 写串行加锁
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
t0 = time.time()
|
||||
batch_lock = threading.Lock()
|
||||
@@ -116,6 +121,7 @@ class SyncKlineDaily(SyncTask):
|
||||
pending_codes: dict[str, str] = {}
|
||||
ok_lock = threading.Lock()
|
||||
ok_cnt = [0]
|
||||
fallback_ok_cnt = [0]
|
||||
fail_cnt = [0]
|
||||
rows_cnt = [0]
|
||||
|
||||
@@ -138,15 +144,7 @@ class SyncKlineDaily(SyncTask):
|
||||
except Exception as e:
|
||||
logger.warning(f"update synced_at 失败 {c6}: {e}")
|
||||
|
||||
def _fetch_one(code6: str, fetch_start: str) -> tuple[str, list[dict] | None, str | None]:
|
||||
try:
|
||||
df = primary.fetch_kline_daily(code6, fetch_start, _end)
|
||||
except Exception as e:
|
||||
return (code6, None, str(e)[:120])
|
||||
if df is None or df.empty:
|
||||
return (code6, None, "no data")
|
||||
# 写库用 hermes（带 SH/SZ 前缀）—— 2026-07-01 重构：历史上 kline_stock 用
|
||||
# 6位 code 与 stocks.code(SH600519) 格式不一致，跨表 JOIN 失败。现统一。
|
||||
def _rows_from_df(code6: str, df: pd.DataFrame) -> tuple[list[dict], str]:
|
||||
hermes = to_hermes(code6)
|
||||
rows = []
|
||||
for _, r in df.iterrows():
|
||||
@@ -162,20 +160,56 @@ class SyncKlineDaily(SyncTask):
|
||||
})
|
||||
latest = df["trade_date"].max()
|
||||
latest = latest.strftime("%Y-%m-%d") if hasattr(latest, "strftime") else str(latest)[:10]
|
||||
return (code6, rows, latest)
|
||||
return (rows, latest)
|
||||
|
||||
def _fetch_one(code6: str, fetch_start: str) -> tuple[str, list[dict] | None, str | None, bool]:
|
||||
errors = []
|
||||
for src in sources:
|
||||
try:
|
||||
df = src.fetch_kline_daily(code6, fetch_start, _end)
|
||||
except Exception as e:
|
||||
errors.append(f"{src.key}: {e}")
|
||||
continue
|
||||
if df is None or df.empty:
|
||||
errors.append(f"{src.key}: no data")
|
||||
continue
|
||||
rows, latest = _rows_from_df(code6, df)
|
||||
is_fallback = src is not sources[0]
|
||||
return (code6, rows, latest, is_fallback)
|
||||
return (code6, None, "; ".join(errors)[:200], False)
|
||||
|
||||
total = len(jobs)
|
||||
with ThreadPoolExecutor(max_workers=max_workers) as pool:
|
||||
futures = {pool.submit(_fetch_one, c6, fs): c6 for c6, fs in jobs}
|
||||
done_cnt = 0
|
||||
for future in as_completed(futures):
|
||||
# 不用 `with ThreadPoolExecutor(...) as pool:` — 它的 __exit__ 默认 wait=True,
|
||||
# 一旦某个 worker 卡在 xueqiu IO,主线程会在 as_completed 触发 timeout 后
|
||||
# 仍被 __exit__ 阻塞等 worker 退出 → 进程挂死(2026-07-09 13:21 教训)。
|
||||
# 改成手动管理 + shutdown(wait=False),主线程能立刻退出。
|
||||
pool = ThreadPoolExecutor(max_workers=max_workers)
|
||||
futures = {pool.submit(_fetch_one, c6, fs): c6 for c6, fs in jobs}
|
||||
done_cnt = 0
|
||||
try:
|
||||
for future in as_completed(futures, timeout=FETCH_HARD_TIMEOUT):
|
||||
done_cnt += 1
|
||||
code6, rows, latest = future.result()
|
||||
code6 = futures[future]
|
||||
try:
|
||||
code6_r, rows, latest, is_fallback = future.result(timeout=0.1)
|
||||
except FuturesTimeoutError:
|
||||
with ok_lock:
|
||||
fail_cnt[0] += 1
|
||||
logger.warning(f"[kline {code6}] 内部 race timeout, 记 fail")
|
||||
continue
|
||||
except Exception as e:
|
||||
with ok_lock:
|
||||
fail_cnt[0] += 1
|
||||
logger.warning(f"[kline {code6}] 异常: {e}")
|
||||
continue
|
||||
if rows is None:
|
||||
with ok_lock:
|
||||
fail_cnt[0] += 1
|
||||
logger.warning(f"[kline {code6}] {latest}")
|
||||
continue
|
||||
if is_fallback:
|
||||
with ok_lock:
|
||||
fallback_ok_cnt[0] += 1
|
||||
with batch_lock:
|
||||
batch_rows.extend(rows)
|
||||
pending_codes[code6] = latest
|
||||
@@ -194,32 +228,121 @@ class SyncKlineDaily(SyncTask):
|
||||
message=f"进度 {done_cnt}/{total} OK:{ok_cnt[0]} FAIL:{fail_cnt[0]} {rate:.1f}只/秒",
|
||||
current=done_cnt, total=total, current_step=code6,
|
||||
)
|
||||
except FuturesTimeoutError:
|
||||
# 30s 内没 future 完成 → 所有 worker 都被卡死(数据源层 timeout 兜底失败)
|
||||
# 取消 pending futures,标 fail。shutdown(wait=False) 让主线程立刻返回,
|
||||
# 不被卡住的 worker 拖死。
|
||||
# 2026-07-11 修复: 防御性 cancel,避免异常处理链里变量被遮蔽导致
|
||||
# AttributeError: 'str' object has no attribute 'cancel'。
|
||||
stuck = [(f, c6) for f, c6 in futures.items() if not f.done()]
|
||||
logger.error(
|
||||
"所有 fetcher 卡死 (>%ss), %d 只股票未完成, 标 fail 后退出主循环",
|
||||
FETCH_HARD_TIMEOUT, len(stuck),
|
||||
)
|
||||
with ok_lock:
|
||||
fail_cnt[0] += len(stuck)
|
||||
for f, c6 in stuck:
|
||||
try:
|
||||
if hasattr(f, "cancel"):
|
||||
f.cancel()
|
||||
except Exception as cancel_err:
|
||||
logger.warning(f"[kline] cancel future for {c6} 失败: {cancel_err}")
|
||||
try:
|
||||
pool.shutdown(wait=False)
|
||||
except Exception as shutdown_err:
|
||||
logger.warning(f"[kline] pool.shutdown(wait=False) 失败: {shutdown_err}")
|
||||
|
||||
# ── 补偿：冷却后重入队列 ──
|
||||
# 雪球/mairui 偶发全 hang（服务端限流/网络抖动），等 30s 再试一次。
|
||||
# 单线程拉取场景下，卡住通常意味着当前这只股票请求 hang 死；
|
||||
# 冷却后把未完成的股票重新排队重试，救回临时故障。
|
||||
if stuck:
|
||||
cooldown = 30
|
||||
logger.warning(f"[kline] 冷却 {cooldown}s 后重试 {len(stuck)} 只…")
|
||||
time.sleep(cooldown)
|
||||
retry_ok = 0
|
||||
retry_fb = 0
|
||||
retry_rows = 0
|
||||
retry_pool = ThreadPoolExecutor(max_workers=max_workers)
|
||||
jobs_dict = {c6: fs for c6, fs in jobs}
|
||||
retry_futs = {}
|
||||
for _, c6 in stuck:
|
||||
if c6 in jobs_dict:
|
||||
retry_futs[retry_pool.submit(_fetch_one, c6, jobs_dict[c6])] = c6
|
||||
if retry_futs:
|
||||
try:
|
||||
for fut in as_completed(retry_futs, timeout=FETCH_HARD_TIMEOUT):
|
||||
c6 = retry_futs[fut]
|
||||
try:
|
||||
code6_r, rows, latest, is_fallback = fut.result(timeout=0.1)
|
||||
except Exception:
|
||||
logger.warning(f"[kline {c6}] 重试异常")
|
||||
continue
|
||||
if rows is None:
|
||||
logger.warning(f"[kline {c6}] 重试仍失败: {latest}")
|
||||
continue
|
||||
with batch_lock:
|
||||
batch_rows.extend(rows)
|
||||
pending_codes[c6] = latest
|
||||
cur_size = len(batch_rows)
|
||||
with ok_lock:
|
||||
retry_ok += 1
|
||||
retry_rows += len(rows)
|
||||
if is_fallback:
|
||||
retry_fb += 1
|
||||
if cur_size >= BATCH_SIZE:
|
||||
_flush()
|
||||
except FuturesTimeoutError:
|
||||
logger.error(
|
||||
"[kline] 重试仍卡死 (>%ss), %d 只股票放弃",
|
||||
FETCH_HARD_TIMEOUT, sum(1 for f in retry_futs if not f.done()),
|
||||
)
|
||||
try:
|
||||
retry_pool.shutdown(wait=False)
|
||||
except Exception:
|
||||
pass
|
||||
with ok_lock:
|
||||
ok_cnt[0] += retry_ok
|
||||
fail_cnt[0] -= retry_ok
|
||||
fallback_ok_cnt[0] += retry_fb
|
||||
rows_cnt[0] += retry_rows
|
||||
logger.warning(
|
||||
f"[kline] 重试结果: {retry_ok}成 "
|
||||
f"({retry_fb}只备胎) 共{retry_rows}行, 救回 {retry_ok} 只"
|
||||
)
|
||||
else:
|
||||
pool.shutdown(wait=True)
|
||||
|
||||
# 收尾 flush
|
||||
_flush()
|
||||
|
||||
elapsed = round(time.time() - t0, 1)
|
||||
msg = f"日K线 {ok_cnt[0]}成 {fail_cnt[0]}败 {skipped}跳 共{rows_cnt[0]}行, {elapsed}s"
|
||||
fb = fallback_ok_cnt[0]
|
||||
fb_info = f" ({fb}只备胎)" if fb else ""
|
||||
msg = f"日K线 {ok_cnt[0]}成 {fail_cnt[0]}败 {skipped}跳 共{rows_cnt[0]}行{fb_info}, {elapsed}s"
|
||||
return {
|
||||
"status": "ok" if fail_cnt[0] == 0 else "warning",
|
||||
"message": msg,
|
||||
"ok": ok_cnt[0],
|
||||
"fail": fail_cnt[0],
|
||||
"fallback_ok": fb,
|
||||
"skip": skipped,
|
||||
"rows": rows_cnt[0],
|
||||
"elapsed_sec": elapsed,
|
||||
"primary_source": primary.key,
|
||||
"primary_source": sources[0].key if sources else "",
|
||||
"sources": [s.key for s in sources],
|
||||
}
|
||||
|
||||
def _pick_primary(self):
|
||||
def _get_available_sources(self):
|
||||
from app.core.datasource.registry import is_source_ready, run_health_check, get_health_status
|
||||
if not get_health_status():
|
||||
run_health_check()
|
||||
available = []
|
||||
for key in PRIMARY_PRIORITY:
|
||||
ok, _ = is_source_ready(key)
|
||||
if ok:
|
||||
return ds_registry.get(key)
|
||||
return None
|
||||
available.append(ds_registry.get(key))
|
||||
return available
|
||||
|
||||
def _sync_one(self, primary, code6: str, start: str, end: str) -> dict:
|
||||
"""保留这个方法以兼容外部调用（已不用，但单只测试可能用到）"""
|
||||
|
||||
Reference in New Issue
Block a user