refactor: PG-only 迁移 + 龙虎榜/tick/moneyflow 同步 + stock_code 统一 + mairui 编码修复
## 1. PG-only 重构
- 删 app/core/db/connection.py + schema.py (MySQL 路径)
- 新 app/core/db/{orm,models,pg_bootstrap}.py — SQLAlchemy 2.x ORM 一键建表
- 16 张业务表全在 market_data schema,原生 TIMESTAMPTZ / JSONB / Float / TEXT
- requirements.txt 删 PyMySQL 路径,加 psycopg2
## 2. 同步任务扩展(3 个新 task)
- **tick_trade** (mairui hsrl/zbjy):当天逐笔交易,21:00 发布
- **moneyflow** (mairui hsstock/history/transaction):个股资金流,21:30 发布
- **longhubang** (akshare):龙虎榜聚合层 + 席位层(2 张新表)
- 长虎榜放宽 akshare 政策:仅"无替代源 + 烟测通过"场景允许
- data_eastmoney 私有 API 不需要(akshare 烟测通过)
- 3-timer 设计:
- 15:30 market-sync.service (8 base tasks via runall_once)
- 21:05 market-sync-tick.service (tick_trade)
- 21:35 market-sync-moneyflow.service (moneyflow)
- 22:00 market-sync-lhb.service (longhubang,新加)
- bin/systemd/ 新增 tick / moneyflow / lhb 各 1 对 service+timer
- bin/market_sync_*_run.sh wrapper 脚本(不做法定节假日过滤,fail-open)
## 3. stock_code 统一为带 SH/SZ/BJ 前缀
- 历史 bug:stocks.code 用 SH600519,但 kline/moneyflow/tick_trade/kline_5min
/stock_sector_map/industry 6 张表用纯 6 位 600519,跨表 JOIN 全部 0 行
- 新增 to_hermes() 工具:6位 / 9位(mairui `000001.SZ` 格式)→ 统一 SH000001
- 5 个 task 改写:用 to_hermes(code6) 写入 stock_code
- 一次性迁移 6 张表存量 154M 行(CASE WHEN 探测 + 去重 + 加前缀)
- ORM: stock_sector_map.stock_code / industry.code String(6)→String(10)
## 4. Bug 修复
- **share table stock_code 格式**:之前写 6 位不带前缀,与 stocks 不一致
→ 修 task_share_snapshot + 一次性 UPDATE 63,417 行加前缀
- **share_snapshot warning 状态错填 last_error**:
→ 加 mark_sync_warning() 走专用路径,不写 last_failure_at / last_error
- **schedule config lastRun 不同步**:
→ 加 update_job_status_for_dataset(),SyncTask.run() 完成后自动镜像
→ cli/runall 触发的 task 也能更新 schedule config
## 5. mairui UTF-8 编码修复
- 历史 bug:mairui.py:_fetch 用 latin-1 兜底解码,把所有 UTF-8 中文名
double-encoded 写入 stocks.name(如 `歌华有线` 变成 `æ\xad\x8cå\x8d\x8e...`)
- 加 _decode_response():UTF-8 → GBK → latin-1 兜底
- 一次性修复 stocks.name 5,213 行:
- 4,370 行 (encode('latin-1').decode('utf-8') 反向解码)
- 616 行 (含 fullwidth A,宽松 printable 检查)
- 820 行 (mid-character 截断,重新从 mairui 拉)
## 6. 测试
- tests/test_smoke.py: TASKS 10→11, SYNC_DEFINITIONS 10→11
- tests/test_schema_models.py: 16→18 张表,新增 longhubang_daily/seat
- pytest 11/11 passed
## 验证
- 6 张表 0 残留无前缀行
- stocks JOIN kline_stock / kline_5min / moneyflow / tick_trade / stock_sector_map:88-100% 命中
- 5,213 stocks.name 全部正确 UTF-8 中文
- pytest 11/11 passed
This commit is contained in:
@@ -0,0 +1,168 @@
|
||||
"""同步任务:当天逐笔交易(mairui 源)。
|
||||
|
||||
数据源:mairui `hsrl/zbjy/{code6}/{licence}`
|
||||
口径:单笔成交 — 时间 / 价 / 量 / 买卖方向
|
||||
更新:每日 21:00(mairui 当天数据此时才发布;21:00 之前 API 返空或昨日数据)
|
||||
|
||||
设计要点:
|
||||
1) 起步门控:21:00 之前(未传 force=True)→ 早返 warning
|
||||
避免 0 行空跑(5000 只股票 × 1s 限速 ≈ 17 分钟浪费)
|
||||
2) 5 worker × 1 RPS/worker = 5 RPS(守住 mairui 上限)
|
||||
3) 不做"今日已同步"dedup —— mairui "当天"实际是"最近已发布交易日",
|
||||
trade_date 可能比 wall-clock 早一天;重跑靠 PK + ON CONFLICT 幂等
|
||||
4) 单只股票日均 5w-10w tick,CHUNK=2000 upsert 防 driver 撑爆
|
||||
5) 数据是"当天 only",无 start/end 窗口;不存增量概念
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from app.core.db import ops as db_ops
|
||||
from app.core.datasource.base import registry as ds_registry
|
||||
from app.core.datasource.registry import is_source_ready
|
||||
from app.core.datasource.utils import is_a_share_code, to_code6, to_hermes
|
||||
from app.core.sync.base import SyncTask
|
||||
from app.core.sync.registry import mark_sync_blocked
|
||||
from app.core.utils.logging import get_logger
|
||||
|
||||
logger = get_logger("sync.tick_trade")
|
||||
|
||||
# mairui 文档:「更新:每日 21:00」—— 早于此时 mairui 仍返昨日/空数据
|
||||
MAIRUI_TICK_TRADE_PUBLISH_HOUR = 21
|
||||
MAIRUI_TICK_TRADE_PUBLISH_MIN = 0
|
||||
# 21:00 之后再跑(留 5 分钟缓冲,等 mairui 完整入库)
|
||||
TICK_TRADE_RUN_GATE = (MAIRUI_TICK_TRADE_PUBLISH_HOUR, MAIRUI_TICK_TRADE_PUBLISH_MIN + 5)
|
||||
|
||||
|
||||
def _passes_publish_gate(now: Optional[datetime] = None, *, force: bool = False) -> tuple[bool, str]:
|
||||
"""判断当前时间是否已过 mairui 21:00 数据发布门控。
|
||||
|
||||
force=True 跳过门控(手动 / 测试用)。
|
||||
"""
|
||||
if force:
|
||||
return True, "force=True 跳过门控"
|
||||
n = now or datetime.now()
|
||||
gate = n.replace(hour=TICK_TRADE_RUN_GATE[0], minute=TICK_TRADE_RUN_GATE[1], second=0, microsecond=0)
|
||||
if n < gate:
|
||||
return False, f"mairui 逐笔数据 {MAIRUI_TICK_TRADE_PUBLISH_HOUR:02d}:{MAIRUI_TICK_TRADE_PUBLISH_MIN:02d} 发布;当前 {n.strftime('%H:%M')} 早于门控,skip"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
class SyncTickTrade(SyncTask):
|
||||
dataset_id = "tick_trade"
|
||||
|
||||
def _run(
|
||||
self,
|
||||
*,
|
||||
trigger_source: str = "manual",
|
||||
codes: list[str] | None = None,
|
||||
max_workers: int = 5,
|
||||
force: bool = False,
|
||||
**kwargs,
|
||||
) -> dict[str, Any]:
|
||||
mr = ds_registry.get("datasource_mairui")
|
||||
if mr is None:
|
||||
return {"status": "error", "message": "mairui 数据源未注册"}
|
||||
ok, reason = is_source_ready(mr.key)
|
||||
if not ok:
|
||||
mark_sync_blocked(self.dataset_id, message=f"mairui 未就绪: {reason}")
|
||||
return {"status": "blocked", "message": f"mairui 未就绪: {reason}"}
|
||||
|
||||
# ── 21:00 门控(force=True 跳过)──
|
||||
gate_ok, gate_msg = _passes_publish_gate(force=force)
|
||||
if not gate_ok:
|
||||
logger.info(f"[tick_trade] 门控未过: {gate_msg}")
|
||||
return {"status": "warning", "message": gate_msg, "ok": 0, "fail": 0, "rows": 0, "elapsed_sec": 0.0}
|
||||
|
||||
# ── 取股票列表 ──
|
||||
if codes:
|
||||
stock_codes = [to_code6(c) for c in codes]
|
||||
else:
|
||||
stock_codes = [c for c in db_ops.iter_stock_codes(active_only=True) if is_a_share_code(c)]
|
||||
limit_raw = os.environ.get("MARKET_DATA_STOCK_LIMIT", "").strip()
|
||||
if limit_raw.isdigit() and int(limit_raw) > 0:
|
||||
stock_codes = stock_codes[: int(limit_raw)]
|
||||
if not stock_codes:
|
||||
return {"status": "error", "message": "无股票代码"}
|
||||
|
||||
# 注:不做"今日已同步"dedup。原因:
|
||||
# - mairui 的"当天"实际是"最近一个已发布的交易日"(21:00 之前是昨日),
|
||||
# 用 today_str 做去重会失效(即使刚跑完,trade_date 还是昨日)。
|
||||
# - PK 含 trade_date,重跑是 ON CONFLICT DO UPDATE,安全且幂等。
|
||||
# - 17 分钟全市场开销可接受;非交易日用 `MARKET_DATA_STOCK_LIMIT=0` 跳。
|
||||
|
||||
total = len(stock_codes)
|
||||
if total == 0:
|
||||
return {"status": "ok", "message": "无股票需要同步", "ok": 0, "fail": 0, "rows": 0, "elapsed_sec": 0.0}
|
||||
|
||||
# ── 预估耗时:5 RPS 上限 ──
|
||||
est_sec = total / 5.0
|
||||
logger.info(f"[tick_trade] 计划: {total} 只, 预估 {est_sec:.0f}s ({est_sec/60:.1f}min)")
|
||||
self._progress(
|
||||
message=f"计划: {total} 只, 预估 {est_sec/60:.1f}min @5RPS",
|
||||
current=0, total=total, current_step="planning",
|
||||
)
|
||||
|
||||
# ── 执行 ──
|
||||
t0 = time.time()
|
||||
ok_cnt = fail_cnt = 0
|
||||
rows_total = 0
|
||||
with ThreadPoolExecutor(max_workers=max_workers) as pool:
|
||||
futures = {pool.submit(self._sync_one, mr, c6): c6 for c6 in stock_codes}
|
||||
for i, future in enumerate(as_completed(futures), 1):
|
||||
c6 = futures[future]
|
||||
try:
|
||||
res = future.result()
|
||||
if res["status"] == "ok":
|
||||
ok_cnt += 1
|
||||
rows_total += res["rows"]
|
||||
else:
|
||||
fail_cnt += 1
|
||||
if res.get("error"):
|
||||
logger.warning(f"[tick_trade {c6}] {res['error']}")
|
||||
except Exception as e:
|
||||
fail_cnt += 1
|
||||
logger.warning(f"[tick_trade {c6}] {e}")
|
||||
if i % 50 == 0 or i == total:
|
||||
self._progress(
|
||||
message=f"tick_trade 进度 {i}/{total} OK:{ok_cnt} FAIL:{fail_cnt} 行:{rows_total}",
|
||||
current=i, total=total, current_step=c6,
|
||||
)
|
||||
elapsed = round(time.time() - t0, 1)
|
||||
msg = f"逐笔 {ok_cnt}成 {fail_cnt}败 共{rows_total}行, {elapsed}s"
|
||||
return {
|
||||
"status": "ok" if fail_cnt == 0 else "warning",
|
||||
"message": msg,
|
||||
"ok": ok_cnt, "fail": fail_cnt, "rows": rows_total, "elapsed_sec": elapsed,
|
||||
}
|
||||
|
||||
def _sync_one(self, mr, code6: str) -> dict:
|
||||
try:
|
||||
df = mr.fetch_tick_trade(code6)
|
||||
except Exception as e:
|
||||
return {"status": "fail", "error": str(e)}
|
||||
if df is None or df.empty:
|
||||
return {"status": "fail", "error": "no data (mairui 21:00 前可能为空)"}
|
||||
try:
|
||||
rows = [
|
||||
{
|
||||
"stock_code": to_hermes(code6),
|
||||
"trade_date": str(r["trade_date"]),
|
||||
"trade_time": r["trade_time"].strftime("%Y-%m-%d %H:%M:%S")
|
||||
if hasattr(r["trade_time"], "strftime") else str(r["trade_time"]),
|
||||
"price": float(r.get("price") or 0),
|
||||
"volume": float(r.get("volume") or 0),
|
||||
"direction_code": int(r.get("direction_code") or 0),
|
||||
"direction": str(r.get("direction") or ""),
|
||||
"amount": float(r.get("amount") or 0),
|
||||
}
|
||||
for _, r in df.iterrows()
|
||||
]
|
||||
db_ops.upsert_tick_trade(rows)
|
||||
return {"status": "ok", "rows": len(rows)}
|
||||
except Exception as e:
|
||||
return {"status": "fail", "error": f"db write: {e}"}
|
||||
Reference in New Issue
Block a user