gao
a8ea132924
medium effort code-review 暴露 8 个 finding,按 P0/P1/P2 优先级修复: P0 (数据正确性 / 跑得起来): #1 replace_all_node_categories 改 scope-aware delete 之前无条件 delete(NodeCategory) + 插入 runtime filter 后的子集, 导致 --type2 0 跑把其它 6 个 category 误删。改为只 delete 本批 category_key 集合,其它 type2 不动。 #2 replace_all_stock_node_map 对称化 之前空 rows 时 early-return 留下 stale,导致「categories 新 但 mappings 旧」不一致。改为 scope-aware delete 同 #1。 #3 stock_node systemd --workers 20 → 8 20 workers × 默认 10 RPS = 200 RPS,撞穿 mairui 钻石档 100 RPS 上限,触发风控。改为 8 × 10 = 80 RPS,留 20% buffer。 P1 (静默错): #4 is_a_share_code 接 hermes 格式 之前硬性要求 len(c)==6,iter_stock_codes 改返 hermes 时 5 个 task (moneyflow/share_snapshot/kline_5min/tick_trade/kline_daily) 会静默过滤成空 list。剥前缀再判断,兼容 'SH600519'。 #5 mairui tz 契约钉在 source 层 tz_localize 改为「已带 tz 就保留,没有再标 Asia/Shanghai」, 避免 mairui 改格式时抛 TypeError。task_tick_trade 删掉 strftime fallback + 冗长注释(契约已在 source 层 docstring)。 #6 CLI inspect.signature 过滤 kwargs --type2 / --backfill 加在共享 p_sync parser 上,任何 task 通过 **kwargs 静默吞掉。改为 dispatcher 按 task._run 签名过滤, 不支持的参数打 warning。 P2 (维护): #7 task_stock_node --type2 输入校验 之前空字符串 / 「concept」 / 99 都 silently collapse 成空 set, 走「无匹配叶子节点」warning 分支(被 daily_check 当正常)。 CLI 不再 silent-drop,任务层加 strict 校验,typo 返 status=error。 #8 code6_to_exchange 删 3 个死分支 4 个 code6_to_* wrapper 都先 _strip_hermes,3 个 hermes if 分支 走不到。删除后 function 简化,所有 caller 行为不变。 附带: - tests/test_smoke.py: 11 → 13 (含 mairui_ma_daily + stock_node) - tests/test_schema_models.py: 21 → 22 (含 kline_stock_ma_daily + node_categories/nodes/stock_node_map) - bin/market_sync_stock_node_run.sh 注释补充 RPS 计算 验证: - pytest tests/ 11 passed - is_a_share_code 8 个 hermes 边界 case 全过 - cli sync kline_daily --type2 2,3 → warning 已打印 - cli sync stock_node --type2 concept → status=error - cli sync stock_node --type2 99 → status=error - cli sync stock_node --type2 0 → 仅 0:0 行被刷新,其它 6 类不动 - fetch_tick_trade('600519') 返回 tz=Asia/Shanghai +08:00
552 lines
24 KiB
Python
552 lines
24 KiB
Python
"""麦蕊智数（mairui）数据源。
|
||
|
||
提供：
|
||
- 日 K 线（`hsstock/history/{code}.{ex}/d/n/{licence}`）
|
||
- 5 分钟 K 线（`hsstock/history/{code}.{ex}/5/n/{licence}`）
|
||
- 指数日 K 线（`hsindex/history/{code}.{ex}/d/{licence}`）
|
||
- 资金流向（`hsstock/history/transaction/{code}.{ex}/{licence}`）
|
||
- 当天逐笔交易（`hsrl/zbjy/{code6}/{licence}`）
|
||
|
||
限速：1分钟300次（默认保守到 5 RPS = 1分钟300次）
|
||
凭证：licence（无需登录态，从环境变量 MAIRUI_LICENCE 读取）
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import os
|
||
import threading
|
||
import time
|
||
import urllib.request
|
||
from typing import Any, Optional
|
||
|
||
import pandas as pd
|
||
|
||
from app.core.datasource.base import DataSource
|
||
from app.core.datasource.utils import (
|
||
code6_to_mairui,
|
||
filter_date_range,
|
||
normalize_kline,
|
||
)
|
||
|
||
|
||
class MairuiSource(DataSource):
|
||
key = "datasource_mairui"
|
||
name = "麦蕊智数（mairui.club）"
|
||
provides = ["kline_daily", "kline_5min", "index_daily", "stock_basic", "moneyflow", "tick_trade", "stock_node"]
|
||
requires_credential = True
|
||
credential_key = "MAIRUI_LICENCE"
|
||
|
||
_HEADERS = {
|
||
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)",
|
||
"Accept": "application/json",
|
||
}
|
||
_BASE_URL = "https://a.mairuiapi.com"
|
||
|
||
# mairui licence 限速（按 tier）：
|
||
# 免费版: 1 min/300 次 = 5 RPS
|
||
# 体验版: 1 min/1千次 = 16.67 RPS
|
||
# 包年版: 1 min/3千次 = 50 RPS
|
||
# 钻石版: 1 min/6千次 = 100 RPS ← 当前项目用
|
||
#
|
||
# 用法：5 workers × _RPS_LIMIT RPS/worker = 总 RPS。
|
||
# 默认 10.0 → 5 workers = 50 RPS（钻石 100 RPS 的一半，留 50% buffer 防风控）。
|
||
# 如需调：env `MAIRUI_RPS_LIMIT=20` → 100 RPS (踩满钻石);
|
||
# `MAIRUI_RPS_LIMIT=1` → 5 RPS (回退到免费档兼容)。
|
||
#
|
||
# 用 thread-local 计时：每个 worker 独立计自己的 last_ts，
|
||
# 避免 N 个 worker 串行抢一把锁退化成 1 worker。
|
||
_RPS_LIMIT = float(os.environ.get("MAIRUI_RPS_LIMIT", "10"))
|
||
_rps_local = threading.local()
|
||
|
||
@classmethod
|
||
def _wait_rps(cls):
|
||
last = getattr(cls._rps_local, "last_ts", 0.0)
|
||
now = time.time()
|
||
elapsed = now - last
|
||
min_interval = 1.0 / cls._RPS_LIMIT
|
||
if elapsed < min_interval:
|
||
time.sleep(min_interval - elapsed)
|
||
now = time.time()
|
||
cls._rps_local.last_ts = now
|
||
|
||
def _get_licence(self) -> Optional[str]:
|
||
return os.environ.get(self.credential_key, "").strip() or None
|
||
|
||
def is_available(self) -> tuple[bool, str]:
|
||
if not self._get_licence():
|
||
return False, f"未配置 {self.credential_key}"
|
||
return True, "ok"
|
||
|
||
def health_check(self) -> dict[str, Any]:
|
||
lic = self._get_licence()
|
||
if not lic:
|
||
return {"success": False, "message": f"{self.credential_key} 未配置"}
|
||
try:
|
||
# 拿 1 条日线数据作为连通性 + 凭证测试
|
||
url = f"{self._BASE_URL}/hsstock/history/600519.SH/d/n/{lic}?st=20260609&et=20260609"
|
||
req = urllib.request.Request(url, headers=self._HEADERS)
|
||
with urllib.request.urlopen(req, timeout=15) as resp:
|
||
raw = resp.read().decode("utf-8", errors="replace")
|
||
data = json.loads(raw)
|
||
if isinstance(data, list) and data:
|
||
return {"success": True, "message": f"连接成功，获取到 {len(data)} 条 K 线"}
|
||
if isinstance(data, dict) and data.get("error"):
|
||
return {"success": False, "message": f"麦蕊返回: {data['error']}"}
|
||
return {"success": False, "message": f"麦蕊返回空/异常: {raw[:200]}"}
|
||
except Exception as e:
|
||
return {"success": False, "message": f"麦蕊连接失败: {e}"}
|
||
|
||
def _fetch(self, path: str, params: dict[str, Any], retry: int = 2) -> Any:
|
||
"""通用 GET，含限速 + 重试。
|
||
|
||
编码探测顺序 (2026-07-01 修复):
|
||
1) UTF-8 — 正常 JSON 响应
|
||
2) GBK — mairui 风控时的中文错误消息 ("接收数据异常，请稍后再试")
|
||
3) latin-1 兜底 — 防御性 fallback (历史 Python repr'd 字符串)
|
||
|
||
历史 bug (2026-06-16 之前): 无条件 `raw_bytes.decode("latin-1")`，
|
||
把 UTF-8 中文名全部 double-encoded (Latin-1 → UTF-8) → 写入 DB
|
||
后 stock_name 看起来正常但 hex 是错的双倍长度字节。
|
||
修复后正常 JSON 走 UTF-8，错误消息走 GBK，Latin-1 仅作兜底。
|
||
"""
|
||
self._wait_rps()
|
||
qs = "&".join(f"{k}={v}" for k, v in params.items() if v)
|
||
url = f"{self._BASE_URL}{path}"
|
||
if qs:
|
||
url = f"{url}?{qs}"
|
||
last_err = None
|
||
for attempt in range(retry + 1):
|
||
try:
|
||
req = urllib.request.Request(url, headers=self._HEADERS)
|
||
with urllib.request.urlopen(req, timeout=20) as resp:
|
||
raw_bytes = resp.read()
|
||
# 编码探测: 优先 UTF-8, fallback GBK, 兜底 latin-1
|
||
raw = self._decode_response(raw_bytes)
|
||
# 风控提示 (GBK 编码的中文消息)
|
||
if "请稍后再试" in raw or "codec can't decode" in raw or raw.startswith("'utf-8'") or "Error -3 while decompressing" in raw:
|
||
raise RuntimeError(f"mairui 返回风控提示: {raw[:80]}")
|
||
# Python repr'd 字符串: "'接收数据异常，请稍后再试'" (GBK bytes 用 latin-1 解码后又被 Python repr)
|
||
if raw.startswith("'") and raw.endswith("'"):
|
||
try:
|
||
decoded = raw[1:-1].encode("latin-1").decode("gbk")
|
||
if "请稍后再试" in decoded:
|
||
raise RuntimeError(f"mairui 返回风控: {decoded}")
|
||
except Exception:
|
||
pass
|
||
return json.loads(raw)
|
||
except Exception as e:
|
||
last_err = e
|
||
if attempt < retry:
|
||
time.sleep(1.0 * (attempt + 1)) # 风控重试退避长一点
|
||
from app.core.utils.logging import get_logger
|
||
get_logger("mairui").debug(f"mairui fetch {url} failed: {last_err}")
|
||
return []
|
||
|
||
@staticmethod
|
||
def _decode_response(raw_bytes: bytes) -> str:
|
||
"""探测响应字节流的编码,优先 UTF-8,fallback GBK,latin-1 兜底。"""
|
||
# 1) UTF-8: 99% 的情况 (正常 JSON 响应)
|
||
try:
|
||
return raw_bytes.decode("utf-8")
|
||
except UnicodeDecodeError:
|
||
pass
|
||
# 2) GBK: mairui 风控时返 GBK 编码的中文错误消息
|
||
try:
|
||
return raw_bytes.decode("gbk")
|
||
except UnicodeDecodeError:
|
||
pass
|
||
# 3) latin-1 兜底: 永不失败,但可能产生错乱字符
|
||
return raw_bytes.decode("latin-1")
|
||
|
||
# ── 股票列表 ───────────────────────────────────────────
|
||
|
||
def fetch_stock_list(self) -> list[dict]:
|
||
"""全市场沪深 A 股基础列表。
|
||
|
||
API: GET /hslt/list/{licence}
|
||
返回字段：dm (代码.交易所，如 "000001.SZ"), mc (名称), jys (交易所 SZ/SH)
|
||
"""
|
||
lic = self._get_licence()
|
||
if not lic:
|
||
return []
|
||
data = self._fetch(f"/hslt/list/{lic}", {})
|
||
if not isinstance(data, list):
|
||
return []
|
||
rows = []
|
||
for item in data:
|
||
try:
|
||
dm = str(item.get("dm", "")).strip()
|
||
mc = str(item.get("mc", "")).strip()
|
||
# Mairui 在 2 字简称中间填了空格（如 "万 科Ａ"），去掉多余空格
|
||
mc = "".join(mc.split())
|
||
jys = str(item.get("jys", "")).strip().upper()
|
||
# dm 格式: "000001.SZ" → code6="000001", exchange="SZ"
|
||
if "." in dm:
|
||
code6, exch = dm.split(".", 1)
|
||
else:
|
||
code6, exch = dm, jys
|
||
if not code6 or len(code6) != 6 or not code6.isdigit():
|
||
continue
|
||
exchange = exch.upper() or jys
|
||
if exchange not in ("SH", "SZ"):
|
||
continue
|
||
rows.append({
|
||
"code": f"{exchange}{code6}",
|
||
"name": mc,
|
||
"exchange": exchange,
|
||
"list_date": "", # hslt/list 不返回 IPO 日期
|
||
"listing_status": "st" if (mc.startswith("ST") or mc.startswith("*ST")) else "normal",
|
||
})
|
||
except Exception:
|
||
continue
|
||
return rows
|
||
|
||
# ── K 线 fetch ─────────────────────────────────────────
|
||
|
||
def fetch_kline_daily(self, code6: str, start: str, end: str) -> pd.DataFrame:
|
||
lic = self._get_licence()
|
||
if not lic:
|
||
return pd.DataFrame()
|
||
symbol = code6_to_mairui(code6)
|
||
path = f"/hsstock/history/{symbol}/d/n/{lic}"
|
||
params = {
|
||
"st": (start or "").replace("-", ""),
|
||
"et": (end or "").replace("-", ""),
|
||
}
|
||
data = self._fetch(path, params)
|
||
if not isinstance(data, list):
|
||
return pd.DataFrame()
|
||
records = []
|
||
for item in data:
|
||
try:
|
||
# t 字段：日线 "2026-06-09 00:00:00"，取日期部分
|
||
t = item.get("t", "")
|
||
d = t.split(" ")[0] if isinstance(t, str) else ""
|
||
if not d:
|
||
continue
|
||
records.append({
|
||
"trade_date": d,
|
||
"open": float(item["o"]),
|
||
"high": float(item["h"]),
|
||
"low": float(item["l"]),
|
||
"close": float(item["c"]),
|
||
"volume": float(item.get("v") or 0),
|
||
})
|
||
except (KeyError, ValueError, TypeError):
|
||
continue
|
||
return filter_date_range(normalize_kline(pd.DataFrame(records)), start, end)
|
||
|
||
def fetch_kline_5min(self, code6: str, start: str, end: str) -> pd.DataFrame:
|
||
"""5 分钟 K 线。返回标准列：bar_time, open, high, low, close, volume, amount。
|
||
|
||
mairui 用 level=5（数字，不是 "5m"）。
|
||
"""
|
||
lic = self._get_licence()
|
||
if not lic:
|
||
return pd.DataFrame()
|
||
symbol = code6_to_mairui(code6)
|
||
path = f"/hsstock/history/{symbol}/5/n/{lic}"
|
||
params = {
|
||
"st": (start or "").replace("-", ""),
|
||
"et": (end or "").replace("-", ""),
|
||
}
|
||
data = self._fetch(path, params)
|
||
if not isinstance(data, list):
|
||
return pd.DataFrame()
|
||
records = []
|
||
for item in data:
|
||
try:
|
||
t = item.get("t", "")
|
||
# 分钟级时间格式 "2026-06-09 14:50:00" → 替换空格为 T 让 pandas 识别
|
||
bar_time = t.replace(" ", "T") if isinstance(t, str) else None
|
||
if not bar_time:
|
||
continue
|
||
records.append({
|
||
"bar_time": bar_time,
|
||
"open": float(item["o"]),
|
||
"high": float(item["h"]),
|
||
"low": float(item["l"]),
|
||
"close": float(item["c"]),
|
||
"volume": float(item.get("v") or 0),
|
||
"amount": float(item.get("a") or 0),
|
||
})
|
||
except (KeyError, ValueError, TypeError):
|
||
continue
|
||
if not records:
|
||
return pd.DataFrame()
|
||
df = pd.DataFrame(records)
|
||
df["bar_time"] = pd.to_datetime(df["bar_time"], errors="coerce")
|
||
df = df.dropna(subset=["bar_time"]).sort_values("bar_time").reset_index(drop=True)
|
||
# 过滤日期范围（按日期部分，不含时分）
|
||
if start:
|
||
df = df[df["bar_time"] >= pd.Timestamp(start)]
|
||
if end:
|
||
# end 包含整天
|
||
df = df[df["bar_time"] < pd.Timestamp(end) + pd.Timedelta(days=1)]
|
||
return df
|
||
|
||
def fetch_index_daily(self, index_code: str, start: str, end: str) -> pd.DataFrame:
|
||
"""指数日 K。index_code 用 mairui 格式（如 '000300.SH'）。"""
|
||
lic = self._get_licence()
|
||
if not lic:
|
||
return pd.DataFrame()
|
||
path = f"/hsindex/history/{index_code}/d/{lic}"
|
||
params = {
|
||
"st": (start or "").replace("-", ""),
|
||
"et": (end or "").replace("-", ""),
|
||
}
|
||
data = self._fetch(path, params)
|
||
if not isinstance(data, list):
|
||
return pd.DataFrame()
|
||
records = []
|
||
for item in data:
|
||
try:
|
||
t = item.get("t", "")
|
||
d = t.split(" ")[0] if isinstance(t, str) else ""
|
||
if not d:
|
||
continue
|
||
records.append({
|
||
"trade_date": d,
|
||
"open": float(item["o"]),
|
||
"high": float(item["h"]),
|
||
"low": float(item["l"]),
|
||
"close": float(item["c"]),
|
||
"volume": float(item.get("v") or 0),
|
||
})
|
||
except (KeyError, ValueError, TypeError):
|
||
continue
|
||
return filter_date_range(normalize_kline(pd.DataFrame(records)), start, end)
|
||
|
||
# ── 资金流向 ───────────────────────────────────────────
|
||
|
||
def fetch_moneyflow(self, code6: str, start: str, end: str) -> pd.DataFrame:
|
||
"""个股资金流向（主力/大/中/小单 净额）。
|
||
|
||
API: GET /hsstock/history/transaction/{code}.{ex}/{licence}?st=YYYYMMDD&et=YYYYMMDD
|
||
返回字段（按买卖方向 × 单型 4×4 矩阵 + 主买/主卖/被动买/被动卖）：
|
||
zmbtdcje 主买特大单成交额
|
||
zmbddcje 主买大单成交额
|
||
zmbzdcje 主买中单成交额
|
||
zmbxdcje 主买小单成交额
|
||
zmstdcje 主卖特大单成交额
|
||
zmsddcje 主卖大单成交额
|
||
zmszdcje 主卖中单成交额
|
||
zmsxdcje 主卖小单成交额
|
||
bdmbtdcje 被动买特大单成交额
|
||
bdmbddcje 被动买大单成交额
|
||
bdmbzdcje 被动买中单成交额
|
||
bdmbxdcje 被动买小单成交额
|
||
bdmstdcje 被动卖特大单成交额
|
||
bdmsddcje 被动卖大单成交额
|
||
bdmszdcje 被动卖中单成交额
|
||
bdmsxdcje 被动卖小单成交额
|
||
... 以及对应的成交量/笔数字段（zmbtdcjl 等），本接口暂只取成交额
|
||
|
||
单型口径（mairui 文档）：
|
||
特大单：成交额 ≥ 100 万 或 成交量 ≥ 5000 手
|
||
大单 ：成交额 ≥ 20 万 或 成交量 ≥ 1000 手
|
||
中单 ：成交额 ≥ 4 万 或 成交量 ≥ 200 手
|
||
小单 ：其他
|
||
|
||
输出字段：trade_date, main_net_inflow, large_net_inflow, medium_net_inflow, small_net_inflow
|
||
净额口径（与 akshare stock_individual_fund_flow 保持一致）：
|
||
主力净流入 = Σ主买四型 - Σ主卖四型 (zmb{t,d,z,x} - zms{t,d,z,x})
|
||
大单净额 = 主买大 - 主卖大 (zmbddcje - zmsddcje)
|
||
中单净额 = 主买中 - 主卖中 (zmbzdcje - zmszdcje)
|
||
小单净额 = 主买小 - 主卖小 (zmbxdcje - zmsxdcje)
|
||
注：不能加被动买/卖 —— 主动买 ≡ 被动卖、主动卖 ≡ 被动买
|
||
（同一笔成交记在两边），加起来恒等 0。
|
||
"""
|
||
lic = self._get_licence()
|
||
if not lic:
|
||
return pd.DataFrame()
|
||
symbol = code6_to_mairui(code6)
|
||
path = f"/hsstock/history/transaction/{symbol}/{lic}"
|
||
params = {
|
||
"st": (start or "").replace("-", ""),
|
||
"et": (end or "").replace("-", ""),
|
||
}
|
||
data = self._fetch(path, params)
|
||
if not isinstance(data, list):
|
||
return pd.DataFrame()
|
||
|
||
def f(item, k) -> float:
|
||
try:
|
||
v = item.get(k)
|
||
return float(v) if v not in (None, "", "-") else 0.0
|
||
except (TypeError, ValueError):
|
||
return 0.0
|
||
|
||
records = []
|
||
for item in data:
|
||
t = item.get("t", "")
|
||
d = t.split(" ")[0] if isinstance(t, str) else ""
|
||
if not d:
|
||
continue
|
||
# 主买四型 / 主卖四型
|
||
main_buy = (
|
||
f(item, "zmbtdcje") + f(item, "zmbddcje")
|
||
+ f(item, "zmbzdcje") + f(item, "zmbxdcje")
|
||
)
|
||
main_sell = (
|
||
f(item, "zmstdcje") + f(item, "zmsddcje")
|
||
+ f(item, "zmszdcje") + f(item, "zmsxdcje")
|
||
)
|
||
# 大/中/小 净额：只看主动方（不包含被动方，否则恒等 0）
|
||
large_net = f(item, "zmbddcje") - f(item, "zmsddcje")
|
||
medium_net = f(item, "zmbzdcje") - f(item, "zmszdcje")
|
||
small_net = f(item, "zmbxdcje") - f(item, "zmsxdcje")
|
||
records.append({
|
||
"trade_date": d,
|
||
"main_net_inflow": round(main_buy - main_sell, 2),
|
||
"large_net_inflow": round(large_net, 2),
|
||
"medium_net_inflow": round(medium_net, 2),
|
||
"small_net_inflow": round(small_net, 2),
|
||
})
|
||
if not records:
|
||
return pd.DataFrame()
|
||
df = pd.DataFrame(records)
|
||
# 日期过滤
|
||
if start:
|
||
df = df[df["trade_date"] >= start]
|
||
if end:
|
||
df = df[df["trade_date"] <= end]
|
||
return df.sort_values("trade_date").reset_index(drop=True)
|
||
|
||
# ── 当天逐笔交易 ───────────────────────────────────────
|
||
|
||
def fetch_tick_trade(self, code6: str) -> pd.DataFrame:
|
||
"""当天逐笔交易（mairui 不支持历史回溯，仅当天数据）。
|
||
|
||
API: GET /hsrl/zbjy/{code6}/{licence} （无 st/et 参数）
|
||
文档：https://mairui.club/hsdata → "当天逐笔交易"
|
||
更新：每日 21:00。
|
||
排序：按时间倒序。
|
||
限速：1 min/300 次（与其它接口共享 licence 配额）。
|
||
|
||
返回字段：
|
||
d string 数据归属日期（yyyy-MM-dd）
|
||
t string 时间（HH:mm:dd，文档笔误，应为 HH:mm:ss）
|
||
v number 成交量（股）
|
||
p number 成交价（元）
|
||
ts number 交易方向 0=中性盘 / 1=买入 / 2=卖出
|
||
|
||
派生字段：
|
||
trade_time = d + 'T' + t，tz=Asia/Shanghai（mairui 返回北京时间 UTC+8）
|
||
Source 层契约（2026-07-07 强化）: 保证返回的 trade_time
|
||
一定是 tz-aware datetime,Task 层无需再 strftime 转换。
|
||
SQLAlchemy 存到 TIMESTAMPTZ 时自动转 UTC。
|
||
direction = 0/1/2 → 'neutral'/'buy'/'sell'
|
||
amount = price * volume（元）
|
||
stock_code = 外部传入的 6 位 code（保持与其它表一致）
|
||
|
||
历史 bug（2026-07-02 修复）：之前 trade_time 是 naive datetime,被 PG 当 UTC 存,
|
||
导致查询时差 8 小时（如 SH600519 第一条 tick 实际是 09:15:08 BJT = 01:15:08 UTC,
|
||
之前显示为 09:15:08 UTC = 17:15:08 BJT,跟实际交易时段不符）。
|
||
2026-07-07 再加固: 若 mairui 某行 d/t 已带 tz-offset,不做 tz_localize(避免
|
||
"Already tz-aware" 报错),保持原 tz。
|
||
"""
|
||
lic = self._get_licence()
|
||
if not lic:
|
||
return pd.DataFrame()
|
||
path = f"/hsrl/zbjy/{code6}/{lic}"
|
||
data = self._fetch(path, {}) # 200/200 —— 不需要 start/end
|
||
if not isinstance(data, list) or not data:
|
||
return pd.DataFrame()
|
||
|
||
direction_map = {0: "neutral", 1: "buy", 2: "sell"}
|
||
records = []
|
||
for item in data:
|
||
try:
|
||
d = str(item.get("d", "") or "").strip()
|
||
t = str(item.get("t", "") or "").strip()
|
||
if not d or not t:
|
||
continue
|
||
# mairui `t` 形如 "14:53:21" —— 是北京时间(UTC+8)
|
||
iso = f"{d}T{t}"
|
||
ts_val = pd.to_datetime(iso, errors="coerce")
|
||
if pd.isna(ts_val):
|
||
continue
|
||
# Source 层保证 tz-aware: 已有 tz 就保留,没有就标 Asia/Shanghai。
|
||
if ts_val.tzinfo is None:
|
||
ts_val = ts_val.tz_localize("Asia/Shanghai")
|
||
price = float(item.get("p") or 0)
|
||
volume = float(item.get("v") or 0)
|
||
if price <= 0 or volume <= 0:
|
||
continue
|
||
ts_code = int(item.get("ts") or 0)
|
||
records.append({
|
||
"stock_code": code6,
|
||
"trade_date": d,
|
||
"trade_time": ts_val.to_pydatetime(),
|
||
"price": price,
|
||
"volume": volume,
|
||
"direction_code": ts_code,
|
||
"direction": direction_map.get(ts_code, ""),
|
||
"amount": round(price * volume, 2),
|
||
})
|
||
except (KeyError, ValueError, TypeError):
|
||
continue
|
||
if not records:
|
||
return pd.DataFrame()
|
||
df = pd.DataFrame(records)
|
||
# mairui 按时间倒序，统一升序便于入库
|
||
return df.sort_values("trade_time").reset_index(drop=True)
|
||
# ── 指数/行业/概念 树 (mairui /hszg) ───────────────────────────
|
||
|
||
def fetch_node_tree(self) -> pd.DataFrame:
|
||
"""mairui /hszg/list/{licence} 指数/行业/概念树。
|
||
|
||
API: GET https://a.mairuiapi.com/hszg/list/{licence}
|
||
文档: https://mairui.club/hsdata → 指数/行业/概念 → /hszg/list
|
||
更新: 每周六 03:05。
|
||
|
||
返回 DataFrame 列:
|
||
name, code, type1, type2, level, pcode, pname, isleaf
|
||
|
||
1464 节点覆盖:A 股(1131) + 港股(31) + 基金/债券/美股/外汇/期货/黄金 等。
|
||
type1 标识市场(0=A股),type2 标识子类(0=申万一级, 2=热门概念, 3=概念板块,
|
||
5=证监会行业, 7=指数成分 等),isleaf=1 是可直接喂给 /hszg/gg 的叶子节点。
|
||
"""
|
||
lic = self._get_licence()
|
||
if not lic:
|
||
return pd.DataFrame()
|
||
path = f"/hszg/list/{lic}"
|
||
data = self._fetch(path, {})
|
||
return pd.DataFrame(data) if isinstance(data, list) else pd.DataFrame()
|
||
|
||
def fetch_stock_nodes(self, code6: str) -> pd.DataFrame:
|
||
"""mairui /hszg/zg/{code6}/{licence} 股票→相关节点。
|
||
|
||
API: GET https://a.mairuiapi.com/hszg/zg/{code6}/{licence}
|
||
文档: https://mairui.club/hsdata → 指数/行业/概念 → /hszg/zg
|
||
更新: 每周六 11:00。
|
||
实测: SH600000 约 30 行(code + name, code 喂给 /hszg/gg 拿成分股)。
|
||
|
||
返回 DataFrame 列: code, name
|
||
"""
|
||
lic = self._get_licence()
|
||
if not lic:
|
||
return pd.DataFrame()
|
||
path = f"/hszg/zg/{code6}/{lic}"
|
||
data = self._fetch(path, {})
|
||
return pd.DataFrame(data) if isinstance(data, list) else pd.DataFrame()
|
||
|
||
def fetch_node_stocks(self, node_code: str) -> pd.DataFrame:
|
||
"""mairui /hszg/gg/{code}/{licence} 节点→成分股。
|
||
|
||
API: GET https://a.mairuiapi.com/hszg/gg/{code}/{licence}
|
||
文档: https://mairui.club/hsdata → 指数/行业/概念 → /hszg/gg
|
||
更新: 每周六 11:00。
|
||
实测: sw_sysh(申万银行)约 48 行,概念节点约 30 行,沪深300 约 300 行。
|
||
|
||
返回 DataFrame 列: dm(6 位代码), mc(名称), jys(交易所 sh/sz/bj)
|
||
任务层需 dm → hermes 转换。
|
||
"""
|
||
lic = self._get_licence()
|
||
if not lic:
|
||
return pd.DataFrame()
|
||
path = f"/hszg/gg/{node_code}/{lic}"
|
||
data = self._fetch(path, {})
|
||
return pd.DataFrame(data) if isinstance(data, list) else pd.DataFrame()
|