kyugao/market_sync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: QMT Bridge 设为主数据源，新增指数日K支持及数据源文档

Browse Source 
- QmtBridgeSource: provides 增加 index_daily，新增 fetch_index_daily 方法
- task_kline_index: 优先级改为 qmt_bridge → mairui → sina 三级降级
- task_kline_5min: 优先用 qmt_bridge，不可用时降级 mairui
- task_kline_daily: 优先级加入 qmt_bridge（首位）
- config: 新增 qmt_bridge_url 配置项
- registry: qmt_bridge 注册信息同步更新
- docs: 新增 DATASETS_AND_SOURCES.md，完整说明数据集与数据源依赖关系
- AGENTS.md: 同步更新指数数据源描述
This commit is contained in:
gao
2026-07-23 23:39:59 +08:00
parent 4516f6d75a
commit daef609e8b
9 changed files with 448 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/core/config.py
+3
View File
@@ -62,6 +62,9 @@ class Settings(BaseSettings):
pg_password: str = "market_sync"
pg_db_name: str = "market_data"
# QMT Bridge
qmt_bridge_url: str = "http://127.0.0.1:8610"
# Trading calendar
trading_holidays: str = ""
app/core/datasource/registry.py
+12
View File
@@ -23,6 +23,16 @@ logger = get_logger("datasource_health")
_DATASOURCE_SEED: list[tuple[str, dict, str]] = [
(
"datasource_qmt_bridge",
{
"name": "QMT Bridge（本地行情桥）",
"provides": ["kline_daily", "kline_5min", "index_daily"],
"requiresCredential": False,
"note": "本地 QMT HTTP 桥，日K + 5分钟K + 指数日K 主源",
},
"日 K + 5 分钟 K + 指数日 K 主源（替代 mairui）",
),
(
"datasource_xinlang",
{
@@ -248,10 +258,12 @@ def build_default_registry() -> None:
from app.sources.baostock import BaostockSource
from app.sources.xueqiu import XueqiuSource
from app.sources.mairui import MairuiSource
from app.sources.qmt_bridge import QmtBridgeSource
registry.register(SinaSource())
registry.register(BaostockSource())
registry.register(MairuiSource())
registry.register(QmtBridgeSource())
if settings.xueqiu_token:
registry.register(XueqiuSource())
else:
app/core/sync/registry.py
+1 -1
View File
@@ -53,7 +53,7 @@ SYNC_DEFINITIONS: list[dict[str, Any]] = [
"storage_uri": "PG market_data.kline_index + indices",
"storage_layer": "pg",
"management_role": "原始源",
"source": "新浪指数日K",
"source": "QMT Bridge（主）→ 麦蕊 → 新浪 逐级 fallback",
"sync_script": "app.tasks.task_kline_index:run",
"dependency_ids": [],
"sort_order": 30,
app/sources/qmt_bridge.py
+235
View File
@@ -0,0 +1,235 @@
"""QMT Bridge 数据源（本地行情桥 HTTP 接口）。
将 xtquant 行情 API 通过 HTTP 暴露的本地服务，提供日 K、5 分钟 K 线和指数日 K。
无需凭证（本地服务），超时短（局域网），不限速。
"""
from __future__ import annotations
import json
import urllib.request
from datetime import datetime
from typing import Any, Optional
import pandas as pd
from app.core.config import settings
from app.core.datasource.base import DataSource
from app.core.datasource.utils import code6_to_mairui, normalize_5min
from app.core.utils.logging import get_logger
logger = get_logger("qmt_bridge")
KLINE_FIELDS = "close,open,high,low,volume"
KLINE_5MIN_FIELDS = "close,open,high,low,volume,amount"
class QmtBridgeSource(DataSource):
key = "datasource_qmt_bridge"
name = "QMT Bridge（本地行情桥）"
provides = ["kline_daily", "kline_5min", "index_daily"]
requires_credential = False
credential_key = ""
def __init__(self) -> None:
self._base_url = (settings.qmt_bridge_url or "http://127.0.0.1:8610").rstrip("/")
# ── 健康检查 / 可用性 ─────────────────────────────────
def is_available(self) -> tuple[bool, str]:
try:
resp = self._get("/health")
if isinstance(resp, dict) and resp.get("status") == "ok":
return True, "就绪"
return False, f"bridge 返回异常: {resp}"
except Exception as e:
return False, f"bridge 不可达: {e}"
def health_check(self) -> dict[str, Any]:
try:
resp = self._get("/health")
if isinstance(resp, dict) and resp.get("status") == "ok":
return {"success": True, "message": f"bridge 连接正常"}
return {"success": False, "message": f"bridge 返回异常: {resp}"}
except Exception as e:
return {"success": False, "message": f"bridge 连接失败: {e}"}
# ── K 线 fetch ─────────────────────────────────────
def fetch_kline_daily(self, code6: str, start: str, end: str) -> pd.DataFrame:
"""日 K 线。
Args:
code6: 6 位股票代码（如 "600519"）
start: "YYYY-MM-DD"
end: "YYYY-MM-DD"
Returns:
DataFrame 列: trade_date, open, high, low, close, volume
"""
symbol = code6_to_mairui(code6)
params = {
"code": symbol,
"period": "1d",
"start": start.replace("-", ""),
"end": end.replace("-", ""),
"count": "-1",
"fields": KLINE_FIELDS,
}
data = self._get("/kline", params)
records = self._parse_kline_data(data, is_daily=True)
if not records:
return pd.DataFrame()
df = pd.DataFrame(records)
df["trade_date"] = pd.to_datetime(df["trade_date"])
return self._normalize_kline(df)
def fetch_kline_5min(self, code6: str, start: str, end: str) -> pd.DataFrame:
"""5 分钟 K 线。
Args:
code6: 6 位股票代码（如 "600519"）
start: "YYYY-MM-DD"
end: "YYYY-MM-DD"
Returns:
DataFrame 列: bar_time, open, high, low, close, volume, amount, turnover_rate
"""
symbol = code6_to_mairui(code6)
params = {
"code": symbol,
"period": "5m",
"start": start.replace("-", ""),
"end": end.replace("-", ""),
"count": "-1",
"fields": KLINE_5MIN_FIELDS,
}
data = self._get("/kline", params)
records = self._parse_kline_data(data, is_daily=False)
if not records:
return pd.DataFrame()
df = pd.DataFrame(records)
# 用 normalize_5min 标准化
return normalize_5min(df)
def fetch_index_daily(self, index_code: str, start: str, end: str) -> pd.DataFrame:
"""指数日 K 线。
Args:
index_code: 带交易所后缀的指数代码（如 "000300.SH"）
start: "YYYY-MM-DD"
end: "YYYY-MM-DD"
Returns:
DataFrame 列: trade_date, open, high, low, close, volume
"""
params = {
"code": index_code,
"period": "1d",
"start": start.replace("-", ""),
"end": end.replace("-", ""),
"count": "-1",
"fields": KLINE_FIELDS,
}
data = self._get("/kline", params)
records = self._parse_kline_data(data, is_daily=True)
if not records:
return pd.DataFrame()
df = pd.DataFrame(records)
df["trade_date"] = pd.to_datetime(df["trade_date"])
return self._normalize_kline(df)
# ── 内部方法 ───────────────────────────────────────
def _get(self, path: str, params: dict[str, str] | None = None) -> Any:
"""GET 请求 bridge，返回解析后的 JSON。"""
url = f"{self._base_url}{path}"
if params:
qs = "&".join(f"{k}={v}" for k, v in params.items() if v)
url = f"{url}?{qs}"
req = urllib.request.Request(url, headers={"Accept": "application/json"})
try:
with urllib.request.urlopen(req, timeout=30) as resp:
raw = resp.read().decode("utf-8")
return json.loads(raw)
except Exception as e:
logger.warning(f"[qmt_bridge] GET {path} 失败: {e}")
raise
def _parse_kline_data(
self, data: dict[str, Any], is_daily: bool
) -> list[dict[str, Any]]:
"""桥响应解析为统一记录列表。跳过全零记录（停牌/未交易）。"""
records = []
raw_list = data.get("data") if isinstance(data, dict) else None
if not isinstance(raw_list, list):
return records
for item in raw_list:
try:
t = str(item.get("time", "") or "")
if not t:
continue
# 跳过全零记录（停牌/未交易）
o = float(item.get("open") or 0)
h = float(item.get("high") or 0)
l = float(item.get("low") or 0)
c = float(item.get("close") or 0)
v = float(item.get("volume") or 0)
if o <= 0 and h <= 0 and l <= 0 and c <= 0 and v <= 0:
continue
if is_daily:
# 日 K: time = "YYYYMMDD"
if len(t) >= 8:
trade_date = f"{t[:4]}-{t[4:6]}-{t[6:8]}"
else:
continue
records.append({
"trade_date": trade_date,
"open": o,
"high": h,
"low": l,
"close": c,
"volume": v,
})
else:
# 5min K: time = "YYYYMMDDHHmmss"
if len(t) >= 14:
bar_time = f"{t[:4]}-{t[4:6]}-{t[6:8]} {t[8:10]}:{t[10:12]}:{t[12:14]}"
elif len(t) >= 8:
bar_time = f"{t[:4]}-{t[4:6]}-{t[6:8]} 00:00:00"
else:
continue
amt = float(item.get("amount") or 0)
records.append({
"bar_time": bar_time,
"open": o,
"high": h,
"low": l,
"close": c,
"volume": v,
"amount": amt,
"turnover_rate": 0.0, # bridge 不提供
})
except (KeyError, ValueError, TypeError):
continue
return records
@staticmethod
def _normalize_kline(df: pd.DataFrame) -> pd.DataFrame:
"""日 K 标准化：类型转换 + OHLC 校验。"""
if df.empty:
return df
for col in ["open", "high", "low", "close", "volume"]:
if col in df.columns:
df[col] = pd.to_numeric(df[col], errors="coerce")
df = df.dropna(subset=["trade_date", "open", "high", "low", "close"])
df["volume"] = df["volume"].fillna(0)
# OHLC 合理性检查
valid = (
(df["open"] > 0) & (df["high"] > 0) & (df["low"] > 0) & (df["close"] > 0)
& (df["volume"] >= 0)
& (df["high"] >= df[["open", "low", "close"]].max(axis=1) - 0.001)
& (df["low"] <= df[["open", "high", "close"]].min(axis=1) + 0.001)
)
df = df.loc[valid].sort_values("trade_date")
return df.drop_duplicates(subset=["trade_date"], keep="last").reset_index(drop=True)
app/tasks/task_kline_5min.py
+16 -4
View File
@@ -77,13 +77,25 @@ class SyncKline5Min(SyncTask):
max_workers: int = 5,
**kwargs,
) -> dict[str, Any]:
primary = ds_registry.get("datasource_mairui")
# QMT Bridge 优先，不可用时降级到 mairui
primary = ds_registry.get("datasource_qmt_bridge") or ds_registry.get("datasource_mairui")
if primary is None:
return {"status": "error", "message": "5min 数据源 mairui 未注册"}
return {"status": "error", "message": "5min 数据源 qmt_bridge/mairui 均未注册"}
ok, reason = is_source_ready(primary.key)
if not ok:
mark_sync_blocked(self.dataset_id, message=f"{primary.key} 未就绪: {reason}")
return {"status": "blocked", "message": f"{primary.key} 未就绪: {reason}"}
# 如果 bridge 不可用但 mairui 可用，尝试切换
fallback = ds_registry.get("datasource_mairui") if primary.key == "datasource_qmt_bridge" else None
if fallback:
fb_ok, fb_reason = is_source_ready(fallback.key)
if fb_ok:
logger.warning(f"[5min] {primary.key} 不可用 ({reason}), 降级到 {fallback.key}")
primary = fallback
else:
mark_sync_blocked(self.dataset_id, message=f"{primary.key}/{fallback.key} 均不可用")
return {"status": "blocked", "message": f"{primary.key}/{fallback.key} 均不可用"}
else:
mark_sync_blocked(self.dataset_id, message=f"{primary.key} 未就绪: {reason}")
return {"status": "blocked", "message": f"{primary.key} 未就绪: {reason}"}
if codes:
stock_codes = [to_code6(c) for c in codes]
app/tasks/task_kline_daily.py
+2 -3
View File
@@ -33,9 +33,8 @@ from app.core.utils.logging import get_logger
logger = get_logger("sync.kline_daily")
# 源优先级：麦蕊 → 雪球 → 新浪
# 2026-07-21 改: 麦蕊15:10已有数据,雪球要15:40后才齐,麦蕊为主源提速
PRIMARY_PRIORITY = ["datasource_mairui", "datasource_xueqiu", "datasource_xinlang"]
# 源优先级：QMT Bridge（本地） → 麦蕊 → 雪球 → 新浪
PRIMARY_PRIORITY = ["datasource_qmt_bridge", "datasource_mairui", "datasource_xueqiu", "datasource_xinlang"]
DEFAULT_START = "2018-01-01"
# 写库批量：攒够 BATCH_SIZE 行就 executemany 一次 + commit
BATCH_SIZE = 2000
app/tasks/task_kline_index.py
+26 -12
View File
@@ -1,6 +1,6 @@
"""同步任务：六大指数日 K 线。
数据流：新浪指数日K接口 → 写 kline_index + indices 表。"""
数据流：QMT Bridge（主）→ 麦蕊 → 新浪 逐级 fallback → 写 kline_index + indices 表。"""
from __future__ import annotations
from datetime import datetime
@@ -24,32 +24,46 @@ MAJOR_INDICES = {
"000852": {"name": "中证1000", "market": "CN", "category": "broad_market"},
}
# 数据源优先级：qmt_bridge（免限速免凭证）→ mairui → sina
SOURCE_PRIORITY = [
("datasource_qmt_bridge", True, "qmt_bridge_index_daily"),
("datasource_mairui", True, "mairui_index_daily"),
("datasource_xinlang", False, "sina_index_daily"),
]
class SyncKlineIndex(SyncTask):
dataset_id = "kline_index"
def _run(self, *, trigger_source: str = "manual", **kwargs) -> dict[str, Any]:
# 优先 mairui（指数无单日空窗问题），sina 作为 fallback
primary = ds_registry.get("datasource_mairui")
use_mairui = primary is not None and is_source_ready(primary.key)[0]
if not use_mairui:
primary = ds_registry.get("datasource_xinlang")
# 按优先级选择第一个就绪的数据源
primary = None
use_suffix = True
source_label = ""
for key, suffix, label in SOURCE_PRIORITY:
src = ds_registry.get(key)
if src is not None and is_source_ready(src.key)[0]:
primary = src
use_suffix = suffix
source_label = label
break
if primary is None:
return {"status": "error", "message": "指数数据源均不可用（mairui + 新浪）"}
return {"status": "error", "message": "指数数据源均不可用（qmt_bridge + mairui + 新浪）"}
ok, reason = is_source_ready(primary.key)
if not ok:
mark_sync_blocked(self.dataset_id, message=f"{primary.key} 未就绪: {reason}")
return {"status": "blocked", "message": f"{primary.key} 未就绪: {reason}"}
self._progress(message=f"开始同步 {len(MAJOR_INDICES)} 个指数...")
self._progress(message=f"开始同步 {len(MAJOR_INDICES)} 个指数，数据源: {source_label}...")
total = len(MAJOR_INDICES)
ok_cnt = fail_cnt = 0
results = {}
for i, (code, info) in enumerate(MAJOR_INDICES.items(), 1):
try:
# mairui 需带交易所后缀（如 000300.SH），sina 用纯6位
if use_mairui:
# qmt_bridge / mairui 需带交易所后缀（如 000300.SH），sina 用纯6位
if use_suffix:
# 000xxx 上证 → .SH；399xxx 深证 → .SZ
if code.startswith("399"):
suffix = ".SZ"
@@ -82,7 +96,7 @@ class SyncKlineIndex(SyncTask):
db_ops.upsert_index(
index_code=code, index_name=info["name"],
market=info["market"], category=info["category"],
source="mairui_index_daily" if use_mairui else "sina_index_daily", enabled=True,
source=source_label, enabled=True,
)
ok_cnt += 1
results[code] = {
@@ -100,7 +114,7 @@ class SyncKlineIndex(SyncTask):
current=i, total=total, current_step=code,
)
msg = f"六大指数 {ok_cnt}成 {fail_cnt}败"
msg = f"六大指数 {ok_cnt}成 {fail_cnt}败（{source_label}）"
return {
"status": "ok" if fail_cnt == 0 else "warning",
"message": msg,