chore: 重构前基线 — 9 个 sync task 全部 ok + akshare 移除 + mairui 资金流接入
状态： - 9 个 sync task（stock_basic / kline_daily / kline_index / kline_5min / moneyflow / industry_sector / sector_features / share_snapshot / market_regime） - 数据源：baostock + mairui + 雪球(pysnowball) + 新浪（4 个） - 项目级约束：永远不用 akshare（已落实） - kline_5min 改用 DB 快照统一全量/增量逻辑 - 零后端 Chrome 扩展 xueqiu_sync（独立项目）
This commit is contained in:
gao
@@ -0,0 +1,5 @@
|
||||
"""数据源实现。
|
||||
|
||||
每个模块导出一个 `DataSource` 子类，由
|
||||
`app.core.datasource.registry.build_default_registry()` 在启动时统一注册。
|
||||
"""
|
||||
@@ -0,0 +1,159 @@
|
||||
"""Baostock 数据源。
|
||||
|
||||
提供：
|
||||
- 全市场股票基础信息（query_stock_basic）
|
||||
- 日 K 线（query_history_k_data_plus，免费，但 QPS 限制需要信号量）
|
||||
- 股票-行业映射（query_stock_industry）
|
||||
|
||||
免费、无 token，但需 bs.login() / bs.logout() 维护会话。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
from typing import Any, Optional
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from app.core.datasource.base import DataSource
|
||||
from app.core.datasource.utils import (
|
||||
code6_to_baostock,
|
||||
normalize_kline,
|
||||
)
|
||||
|
||||
|
||||
class BaostockSource(DataSource):
|
||||
key = "datasource_baostock"
|
||||
name = "Baostock"
|
||||
provides = ["kline_daily", "stock_basic", "industry"]
|
||||
requires_credential = False
|
||||
|
||||
# 进程内只 login 一次
|
||||
_login_lock = threading.Lock()
|
||||
_logged_in = False
|
||||
_semaphore = threading.BoundedSemaphore(3) # QPS 限流
|
||||
|
||||
def _ensure_login(self) -> None:
|
||||
if self._logged_in:
|
||||
return
|
||||
with self._login_lock:
|
||||
if self._logged_in:
|
||||
return
|
||||
import baostock as bs
|
||||
bs.login()
|
||||
self._logged_in = True
|
||||
|
||||
def is_available(self) -> tuple[bool, str]:
|
||||
try:
|
||||
self._ensure_login()
|
||||
return True, "ok"
|
||||
except Exception as e:
|
||||
return False, f"Baostock 登录失败: {e}"
|
||||
|
||||
def health_check(self) -> dict[str, Any]:
|
||||
"""连通性测试：复用 _ensure_login 避免重复 login 阻塞。"""
|
||||
try:
|
||||
self._ensure_login()
|
||||
import baostock as bs
|
||||
rs = bs.query_stock_basic(code="sh.600036")
|
||||
if rs.error_code != "0":
|
||||
return {"success": False, "message": f"Baostock 查询失败: {rs.error_msg}"}
|
||||
rows = []
|
||||
while rs.next():
|
||||
rows.append(rs.get_row_data())
|
||||
return {"success": True, "message": f"连接成功，查询到 {len(rows)} 条股票信息"}
|
||||
except Exception as e:
|
||||
return {"success": False, "message": f"Baostock 连接失败: {e}"}
|
||||
|
||||
def fetch_stock_basic(self) -> list[dict[str, Any]]:
|
||||
"""全市场 A 股基础信息。
|
||||
|
||||
baostock query_stock_basic 返回字段（按当前接口）：
|
||||
code, code_name, ipoDate, outDate, type, status
|
||||
按字段名索引，防止 Baostock 改顺序。
|
||||
"""
|
||||
import baostock as bs
|
||||
self._ensure_login()
|
||||
try:
|
||||
rs = bs.query_stock_basic()
|
||||
if rs.error_code != "0":
|
||||
raise RuntimeError(f"Baostock query_stock_basic 失败: {rs.error_msg}")
|
||||
fields = list(rs.fields)
|
||||
idx = {name: i for i, name in enumerate(fields)}
|
||||
rows = []
|
||||
while rs.next():
|
||||
row = rs.get_row_data()
|
||||
bs_code = row[idx["code"]]
|
||||
name = row[idx["code_name"]]
|
||||
stype = row[idx["type"]]
|
||||
status = row[idx["status"]]
|
||||
ipo = row[idx.get("ipoDate", -1)] if "ipoDate" in idx else ""
|
||||
if stype != "1" or status != "1":
|
||||
continue
|
||||
exchange = bs_code.split(".")[0].upper()
|
||||
if exchange not in ("SH", "SZ", "BJ"):
|
||||
continue
|
||||
code6 = bs_code.split(".")[-1]
|
||||
rows.append({
|
||||
"code": f"{exchange}{code6}",
|
||||
"name": name,
|
||||
"exchange": exchange,
|
||||
"list_date": ipo,
|
||||
"listing_status": "st" if (name.startswith("ST") or name.startswith("*ST")) else "normal",
|
||||
})
|
||||
return rows
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
def fetch_kline_daily(self, code6: str, start: str, end: str) -> pd.DataFrame:
|
||||
import baostock as bs
|
||||
self._ensure_login()
|
||||
bs_code = code6_to_baostock(code6)
|
||||
with self._semaphore:
|
||||
try:
|
||||
rs = bs.query_history_k_data_plus(
|
||||
bs_code,
|
||||
"date,open,high,low,close,volume",
|
||||
start_date=start, end_date=end,
|
||||
frequency="d", adjustflag="2", # 前复权
|
||||
)
|
||||
if rs.error_code != "0":
|
||||
return pd.DataFrame()
|
||||
rows = []
|
||||
while rs.next():
|
||||
rows.append(rs.get_row_data())
|
||||
if not rows:
|
||||
return pd.DataFrame()
|
||||
df = pd.DataFrame(rows, columns=["trade_date", "open", "high", "low", "close", "volume"])
|
||||
return normalize_kline(df)
|
||||
except Exception:
|
||||
return pd.DataFrame()
|
||||
|
||||
def fetch_industry_map(self) -> list[dict[str, Any]]:
|
||||
"""全市场股票-行业映射。"""
|
||||
import baostock as bs
|
||||
self._ensure_login()
|
||||
try:
|
||||
rs = bs.query_stock_industry(code="", date="")
|
||||
if rs.error_code != "0":
|
||||
raise RuntimeError(f"query_stock_industry 失败: {rs.error_msg}")
|
||||
fields = list(rs.fields) # 实际字段顺序：['updateDate', 'code', 'code_name', 'industry', 'industryClassification']
|
||||
idx = {name: i for i, name in enumerate(fields)}
|
||||
rows = []
|
||||
while rs.next():
|
||||
row = rs.get_row_data()
|
||||
industry = row[idx.get("industry", 3)] if "industry" in idx else ""
|
||||
if not industry:
|
||||
continue
|
||||
bs_code = row[idx.get("code", 1)] if "code" in idx else ""
|
||||
code6 = bs_code.split(".")[-1]
|
||||
classification = row[idx.get("industryClassification", 4)] if "industryClassification" in idx else ""
|
||||
update = row[idx.get("updateDate", 0)] if "updateDate" in idx else ""
|
||||
rows.append({
|
||||
"code": code6,
|
||||
"industry_name": industry.strip(),
|
||||
"industry_classification": (classification or "").strip(),
|
||||
"update_date": update or "",
|
||||
})
|
||||
return rows
|
||||
except Exception:
|
||||
return []
|
||||
@@ -0,0 +1,379 @@
|
||||
"""麦蕊智数（mairui）数据源。
|
||||
|
||||
提供：
|
||||
- 日 K 线（`hsstock/history/{code}.{ex}/d/n/{licence}`）
|
||||
- 5 分钟 K 线（`hsstock/history/{code}.{ex}/5/n/{licence}`）
|
||||
- 指数日 K 线（`hsindex/history/{code}.{ex}/d/{licence}`）
|
||||
- 资金流向（`hsstock/history/transaction/{code}.{ex}/{licence}`）
|
||||
|
||||
限速：1分钟300次（默认保守到 5 RPS = 1分钟300次）
|
||||
凭证：licence（无需登录态，从环境变量 MAIRUI_LICENCE 读取）
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
import urllib.request
|
||||
from typing import Any, Optional
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from app.core.datasource.base import DataSource
|
||||
from app.core.datasource.utils import (
|
||||
code6_to_mairui,
|
||||
filter_date_range,
|
||||
normalize_kline,
|
||||
)
|
||||
|
||||
|
||||
class MairuiSource(DataSource):
|
||||
key = "datasource_mairui"
|
||||
name = "麦蕊智数（mairui.club）"
|
||||
provides = ["kline_daily", "kline_5min", "index_daily", "stock_basic", "moneyflow"]
|
||||
requires_credential = True
|
||||
credential_key = "MAIRUI_LICENCE"
|
||||
|
||||
_HEADERS = {
|
||||
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)",
|
||||
"Accept": "application/json",
|
||||
}
|
||||
_BASE_URL = "https://api.mairuiapi.com"
|
||||
|
||||
# mairui 免费 licence 1 分钟 300 次 = 5 RPS（**单 licence**硬上限）。
|
||||
# 5 workers × 1 RPS/worker = 5 RPS 总 = 刚好踩满上限，避免风控。
|
||||
# 用 thread-local 计时：每个 worker 独立计自己的 last_ts，
|
||||
# 避免 N 个 worker 串行抢一把锁退化成 1 worker。
|
||||
_RPS_LIMIT = 1.0
|
||||
_rps_local = threading.local()
|
||||
|
||||
@classmethod
|
||||
def _wait_rps(cls):
|
||||
last = getattr(cls._rps_local, "last_ts", 0.0)
|
||||
now = time.time()
|
||||
elapsed = now - last
|
||||
min_interval = 1.0 / cls._RPS_LIMIT
|
||||
if elapsed < min_interval:
|
||||
time.sleep(min_interval - elapsed)
|
||||
now = time.time()
|
||||
cls._rps_local.last_ts = now
|
||||
|
||||
def _get_licence(self) -> Optional[str]:
|
||||
return os.environ.get(self.credential_key, "").strip() or None
|
||||
|
||||
def is_available(self) -> tuple[bool, str]:
|
||||
if not self._get_licence():
|
||||
return False, f"未配置 {self.credential_key}"
|
||||
return True, "ok"
|
||||
|
||||
def health_check(self) -> dict[str, Any]:
|
||||
lic = self._get_licence()
|
||||
if not lic:
|
||||
return {"success": False, "message": f"{self.credential_key} 未配置"}
|
||||
try:
|
||||
# 拿 1 条日线数据作为连通性 + 凭证测试
|
||||
url = f"{self._BASE_URL}/hsstock/history/600519.SH/d/n/{lic}?st=20260609&et=20260609"
|
||||
req = urllib.request.Request(url, headers=self._HEADERS)
|
||||
with urllib.request.urlopen(req, timeout=15) as resp:
|
||||
raw = resp.read().decode("utf-8", errors="replace")
|
||||
data = json.loads(raw)
|
||||
if isinstance(data, list) and data:
|
||||
return {"success": True, "message": f"连接成功，获取到 {len(data)} 条 K 线"}
|
||||
if isinstance(data, dict) and data.get("error"):
|
||||
return {"success": False, "message": f"麦蕊返回: {data['error']}"}
|
||||
return {"success": False, "message": f"麦蕊返回空/异常: {raw[:200]}"}
|
||||
except Exception as e:
|
||||
return {"success": False, "message": f"麦蕊连接失败: {e}"}
|
||||
|
||||
def _fetch(self, path: str, params: dict[str, Any], retry: int = 2) -> Any:
|
||||
"""通用 GET，含限速 + 重试。"""
|
||||
self._wait_rps()
|
||||
qs = "&".join(f"{k}={v}" for k, v in params.items() if v)
|
||||
url = f"{self._BASE_URL}{path}"
|
||||
if qs:
|
||||
url = f"{url}?{qs}"
|
||||
last_err = None
|
||||
for attempt in range(retry + 1):
|
||||
try:
|
||||
req = urllib.request.Request(url, headers=self._HEADERS)
|
||||
with urllib.request.urlopen(req, timeout=20) as resp:
|
||||
# 先按 gbk 试（中文乱码返回），再 fallback utf-8
|
||||
raw_bytes = resp.read()
|
||||
# 用 latin-1 永不失败地把 bytes 转成字符串（每字节 1 字符）
|
||||
raw = raw_bytes.decode("latin-1")
|
||||
# mairui 风控时返回 "接收数据异常，请稍后再试"（gbk 编码）
|
||||
if "请稍后再试" in raw or "codec can't decode" in raw or raw.startswith("'utf-8'") or "Error -3 while decompressing" in raw:
|
||||
raise RuntimeError(f"mairui 返回风控提示: {raw[:80]}")
|
||||
# 试 JSON parse；如果是 gbk 编码的中文提示，要先解码
|
||||
if raw.startswith("'") and raw.endswith("'"):
|
||||
# gbk 编码的 Python repr 字符串，如 "'接收数据异常，请稍后再试'"
|
||||
try:
|
||||
decoded = raw[1:-1].encode("latin-1").decode("gbk")
|
||||
if "请稍后再试" in decoded:
|
||||
raise RuntimeError(f"mairui 返回风控: {decoded}")
|
||||
except Exception:
|
||||
pass
|
||||
return json.loads(raw)
|
||||
except Exception as e:
|
||||
last_err = e
|
||||
if attempt < retry:
|
||||
time.sleep(1.0 * (attempt + 1)) # 风控重试退避长一点
|
||||
from app.core.utils.logging import get_logger
|
||||
get_logger("mairui").debug(f"mairui fetch {url} failed: {last_err}")
|
||||
return []
|
||||
|
||||
# ── 股票列表 ───────────────────────────────────────────
|
||||
|
||||
def fetch_stock_list(self) -> list[dict]:
|
||||
"""全市场沪深 A 股基础列表。
|
||||
|
||||
API: GET /hslt/list/{licence}
|
||||
返回字段：dm (代码.交易所，如 "000001.SZ"), mc (名称), jys (交易所 SZ/SH)
|
||||
"""
|
||||
lic = self._get_licence()
|
||||
if not lic:
|
||||
return []
|
||||
data = self._fetch(f"/hslt/list/{lic}", {})
|
||||
if not isinstance(data, list):
|
||||
return []
|
||||
rows = []
|
||||
for item in data:
|
||||
try:
|
||||
dm = str(item.get("dm", "")).strip()
|
||||
mc = str(item.get("mc", "")).strip()
|
||||
# Mairui 在 2 字简称中间填了空格（如 "万 科Ａ"），去掉多余空格
|
||||
mc = "".join(mc.split())
|
||||
jys = str(item.get("jys", "")).strip().upper()
|
||||
# dm 格式: "000001.SZ" → code6="000001", exchange="SZ"
|
||||
if "." in dm:
|
||||
code6, exch = dm.split(".", 1)
|
||||
else:
|
||||
code6, exch = dm, jys
|
||||
if not code6 or len(code6) != 6 or not code6.isdigit():
|
||||
continue
|
||||
exchange = exch.upper() or jys
|
||||
if exchange not in ("SH", "SZ"):
|
||||
continue
|
||||
rows.append({
|
||||
"code": f"{exchange}{code6}",
|
||||
"name": mc,
|
||||
"exchange": exchange,
|
||||
"list_date": "", # hslt/list 不返回 IPO 日期
|
||||
"listing_status": "st" if (mc.startswith("ST") or mc.startswith("*ST")) else "normal",
|
||||
})
|
||||
except Exception:
|
||||
continue
|
||||
return rows
|
||||
|
||||
# ── K 线 fetch ─────────────────────────────────────────
|
||||
|
||||
def fetch_kline_daily(self, code6: str, start: str, end: str) -> pd.DataFrame:
|
||||
lic = self._get_licence()
|
||||
if not lic:
|
||||
return pd.DataFrame()
|
||||
symbol = code6_to_mairui(code6)
|
||||
path = f"/hsstock/history/{symbol}/d/n/{lic}"
|
||||
params = {
|
||||
"st": (start or "").replace("-", ""),
|
||||
"et": (end or "").replace("-", ""),
|
||||
}
|
||||
data = self._fetch(path, params)
|
||||
if not isinstance(data, list):
|
||||
return pd.DataFrame()
|
||||
records = []
|
||||
for item in data:
|
||||
try:
|
||||
# t 字段：日线 "2026-06-09 00:00:00"，取日期部分
|
||||
t = item.get("t", "")
|
||||
d = t.split(" ")[0] if isinstance(t, str) else ""
|
||||
if not d:
|
||||
continue
|
||||
records.append({
|
||||
"trade_date": d,
|
||||
"open": float(item["o"]),
|
||||
"high": float(item["h"]),
|
||||
"low": float(item["l"]),
|
||||
"close": float(item["c"]),
|
||||
"volume": float(item.get("v") or 0),
|
||||
})
|
||||
except (KeyError, ValueError, TypeError):
|
||||
continue
|
||||
return filter_date_range(normalize_kline(pd.DataFrame(records)), start, end)
|
||||
|
||||
def fetch_kline_5min(self, code6: str, start: str, end: str) -> pd.DataFrame:
|
||||
"""5 分钟 K 线。返回标准列：bar_time, open, high, low, close, volume, amount。
|
||||
|
||||
mairui 用 level=5（数字，不是 "5m"）。
|
||||
"""
|
||||
lic = self._get_licence()
|
||||
if not lic:
|
||||
return pd.DataFrame()
|
||||
symbol = code6_to_mairui(code6)
|
||||
path = f"/hsstock/history/{symbol}/5/n/{lic}"
|
||||
params = {
|
||||
"st": (start or "").replace("-", ""),
|
||||
"et": (end or "").replace("-", ""),
|
||||
}
|
||||
data = self._fetch(path, params)
|
||||
if not isinstance(data, list):
|
||||
return pd.DataFrame()
|
||||
records = []
|
||||
for item in data:
|
||||
try:
|
||||
t = item.get("t", "")
|
||||
# 分钟级时间格式 "2026-06-09 14:50:00" → 替换空格为 T 让 pandas 识别
|
||||
bar_time = t.replace(" ", "T") if isinstance(t, str) else None
|
||||
if not bar_time:
|
||||
continue
|
||||
records.append({
|
||||
"bar_time": bar_time,
|
||||
"open": float(item["o"]),
|
||||
"high": float(item["h"]),
|
||||
"low": float(item["l"]),
|
||||
"close": float(item["c"]),
|
||||
"volume": float(item.get("v") or 0),
|
||||
"amount": float(item.get("a") or 0),
|
||||
})
|
||||
except (KeyError, ValueError, TypeError):
|
||||
continue
|
||||
if not records:
|
||||
return pd.DataFrame()
|
||||
df = pd.DataFrame(records)
|
||||
df["bar_time"] = pd.to_datetime(df["bar_time"], errors="coerce")
|
||||
df = df.dropna(subset=["bar_time"]).sort_values("bar_time").reset_index(drop=True)
|
||||
# 过滤日期范围（按日期部分，不含时分）
|
||||
if start:
|
||||
df = df[df["bar_time"] >= pd.Timestamp(start)]
|
||||
if end:
|
||||
# end 包含整天
|
||||
df = df[df["bar_time"] < pd.Timestamp(end) + pd.Timedelta(days=1)]
|
||||
return df
|
||||
|
||||
def fetch_index_daily(self, index_code: str, start: str, end: str) -> pd.DataFrame:
|
||||
"""指数日 K。index_code 用 mairui 格式（如 '000300.SH'）。"""
|
||||
lic = self._get_licence()
|
||||
if not lic:
|
||||
return pd.DataFrame()
|
||||
path = f"/hsindex/history/{index_code}/d/{lic}"
|
||||
params = {
|
||||
"st": (start or "").replace("-", ""),
|
||||
"et": (end or "").replace("-", ""),
|
||||
}
|
||||
data = self._fetch(path, params)
|
||||
if not isinstance(data, list):
|
||||
return pd.DataFrame()
|
||||
records = []
|
||||
for item in data:
|
||||
try:
|
||||
t = item.get("t", "")
|
||||
d = t.split(" ")[0] if isinstance(t, str) else ""
|
||||
if not d:
|
||||
continue
|
||||
records.append({
|
||||
"trade_date": d,
|
||||
"open": float(item["o"]),
|
||||
"high": float(item["h"]),
|
||||
"low": float(item["l"]),
|
||||
"close": float(item["c"]),
|
||||
"volume": float(item.get("v") or 0),
|
||||
})
|
||||
except (KeyError, ValueError, TypeError):
|
||||
continue
|
||||
return filter_date_range(normalize_kline(pd.DataFrame(records)), start, end)
|
||||
|
||||
# ── 资金流向 ───────────────────────────────────────────
|
||||
|
||||
def fetch_moneyflow(self, code6: str, start: str, end: str) -> pd.DataFrame:
|
||||
"""个股资金流向（主力/大/中/小单 净额）。
|
||||
|
||||
API: GET /hsstock/history/transaction/{code}.{ex}/{licence}?st=YYYYMMDD&et=YYYYMMDD
|
||||
返回字段（按买卖方向 × 单型 4×4 矩阵 + 主买/主卖/被动买/被动卖）：
|
||||
zmbtdcje 主买特大单成交额
|
||||
zmbddcje 主买大单成交额
|
||||
zmbzdcje 主买中单成交额
|
||||
zmbxdcje 主买小单成交额
|
||||
zmstdcje 主卖特大单成交额
|
||||
zmsddcje 主卖大单成交额
|
||||
zmszdcje 主卖中单成交额
|
||||
zmsxdcje 主卖小单成交额
|
||||
bdmbtdcje 被动买特大单成交额
|
||||
bdmbddcje 被动买大单成交额
|
||||
bdmbzdcje 被动买中单成交额
|
||||
bdmbxdcje 被动买小单成交额
|
||||
bdmstdcje 被动卖特大单成交额
|
||||
bdmsddcje 被动卖大单成交额
|
||||
bdmszdcje 被动卖中单成交额
|
||||
bdmsxdcje 被动卖小单成交额
|
||||
... 以及对应的成交量/笔数字段（zmbtdcjl 等），本接口暂只取成交额
|
||||
|
||||
单型口径（mairui 文档）：
|
||||
特大单：成交额 ≥ 100 万 或 成交量 ≥ 5000 手
|
||||
大单 ：成交额 ≥ 20 万 或 成交量 ≥ 1000 手
|
||||
中单 ：成交额 ≥ 4 万 或 成交量 ≥ 200 手
|
||||
小单 ：其他
|
||||
|
||||
输出字段：trade_date, main_net_inflow, large_net_inflow, medium_net_inflow, small_net_inflow
|
||||
净额口径（与 akshare stock_individual_fund_flow 保持一致）：
|
||||
主力净流入 = Σ主买四型 - Σ主卖四型 (zmb{t,d,z,x} - zms{t,d,z,x})
|
||||
大单净额 = 主买大 - 主卖大 (zmbddcje - zmsddcje)
|
||||
中单净额 = 主买中 - 主卖中 (zmbzdcje - zmszdcje)
|
||||
小单净额 = 主买小 - 主卖小 (zmbxdcje - zmsxdcje)
|
||||
注：不能加被动买/卖 —— 主动买 ≡ 被动卖、主动卖 ≡ 被动买
|
||||
（同一笔成交记在两边），加起来恒等 0。
|
||||
"""
|
||||
lic = self._get_licence()
|
||||
if not lic:
|
||||
return pd.DataFrame()
|
||||
symbol = code6_to_mairui(code6)
|
||||
path = f"/hsstock/history/transaction/{symbol}/{lic}"
|
||||
params = {
|
||||
"st": (start or "").replace("-", ""),
|
||||
"et": (end or "").replace("-", ""),
|
||||
}
|
||||
data = self._fetch(path, params)
|
||||
if not isinstance(data, list):
|
||||
return pd.DataFrame()
|
||||
|
||||
def f(item, k) -> float:
|
||||
try:
|
||||
v = item.get(k)
|
||||
return float(v) if v not in (None, "", "-") else 0.0
|
||||
except (TypeError, ValueError):
|
||||
return 0.0
|
||||
|
||||
records = []
|
||||
for item in data:
|
||||
t = item.get("t", "")
|
||||
d = t.split(" ")[0] if isinstance(t, str) else ""
|
||||
if not d:
|
||||
continue
|
||||
# 主买四型 / 主卖四型
|
||||
main_buy = (
|
||||
f(item, "zmbtdcje") + f(item, "zmbddcje")
|
||||
+ f(item, "zmbzdcje") + f(item, "zmbxdcje")
|
||||
)
|
||||
main_sell = (
|
||||
f(item, "zmstdcje") + f(item, "zmsddcje")
|
||||
+ f(item, "zmszdcje") + f(item, "zmsxdcje")
|
||||
)
|
||||
# 大/中/小 净额：只看主动方（不包含被动方，否则恒等 0）
|
||||
large_net = f(item, "zmbddcje") - f(item, "zmsddcje")
|
||||
medium_net = f(item, "zmbzdcje") - f(item, "zmszdcje")
|
||||
small_net = f(item, "zmbxdcje") - f(item, "zmsxdcje")
|
||||
records.append({
|
||||
"trade_date": d,
|
||||
"main_net_inflow": round(main_buy - main_sell, 2),
|
||||
"large_net_inflow": round(large_net, 2),
|
||||
"medium_net_inflow": round(medium_net, 2),
|
||||
"small_net_inflow": round(small_net, 2),
|
||||
})
|
||||
if not records:
|
||||
return pd.DataFrame()
|
||||
df = pd.DataFrame(records)
|
||||
# 日期过滤
|
||||
if start:
|
||||
df = df[df["trade_date"] >= start]
|
||||
if end:
|
||||
df = df[df["trade_date"] <= end]
|
||||
return df.sort_values("trade_date").reset_index(drop=True)
|
||||
@@ -0,0 +1,137 @@
|
||||
"""新浪财经数据源。
|
||||
|
||||
提供：
|
||||
- 日 K 线（主源，免费，无 token）
|
||||
- 指数日 K 线
|
||||
|
||||
接口：https://money.finance.sina.com.cn/quotes_service/api/json_v2.php/CN_MarketData.getKLineData
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import threading
|
||||
import time
|
||||
import urllib.request
|
||||
from typing import Any
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from app.core.datasource.base import DataSource
|
||||
from app.core.datasource.utils import (
|
||||
code6_to_sina,
|
||||
filter_date_range,
|
||||
normalize_kline,
|
||||
)
|
||||
|
||||
|
||||
class SinaSource(DataSource):
|
||||
key = "datasource_xinlang"
|
||||
name = "新浪财经"
|
||||
provides = ["kline_daily", "index_daily"]
|
||||
requires_credential = False
|
||||
|
||||
_HEADERS = {
|
||||
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)",
|
||||
"Referer": "https://finance.sina.com.cn",
|
||||
}
|
||||
_BASE_URL = (
|
||||
"https://money.finance.sina.com.cn/quotes_service/api/json_v2.php"
|
||||
"/CN_MarketData.getKLineData"
|
||||
)
|
||||
|
||||
# RPS 限流（线程安全）
|
||||
# 设为 3 防止触发新浪反爬（单 IP 频率过高会被临时封禁）
|
||||
_RPS_LIMIT = 3.0
|
||||
_rate_lock = threading.Lock()
|
||||
_last_request_ts = 0.0
|
||||
|
||||
@classmethod
|
||||
def _wait_rps(cls):
|
||||
"""全局 RPS 限流：所有线程共享同一个速率限制。"""
|
||||
with cls._rate_lock:
|
||||
now = time.time()
|
||||
elapsed = now - cls._last_request_ts
|
||||
min_interval = 1.0 / cls._RPS_LIMIT
|
||||
if elapsed < min_interval:
|
||||
time.sleep(min_interval - elapsed)
|
||||
cls._last_request_ts = time.time()
|
||||
|
||||
def is_available(self) -> tuple[bool, str]:
|
||||
return True, "ok"
|
||||
|
||||
def health_check(self) -> dict[str, Any]:
|
||||
try:
|
||||
url = (
|
||||
f"{self._BASE_URL}?symbol=sh600036&scale=240&ma=no&datalen=5"
|
||||
)
|
||||
req = urllib.request.Request(url, headers=self._HEADERS)
|
||||
with urllib.request.urlopen(req, timeout=15) as resp:
|
||||
data = json.loads(resp.read().decode("utf-8"))
|
||||
if not data or not isinstance(data, list):
|
||||
return {"success": False, "message": "新浪财经返回数据格式异常"}
|
||||
return {"success": True, "message": f"连接成功，获取到 {len(data)} 条 K 线数据"}
|
||||
except Exception as e:
|
||||
return {"success": False, "message": f"新浪财经连接失败: {e}"}
|
||||
|
||||
def _fetch(self, symbol: str, datalen: int = 5000, retry: int = 2) -> list:
|
||||
self._wait_rps()
|
||||
url = f"{self._BASE_URL}?symbol={symbol}&scale=240&ma=no&datalen={int(max(datalen, 260))}"
|
||||
last_err = None
|
||||
for attempt in range(retry + 1):
|
||||
try:
|
||||
req = urllib.request.Request(url, headers=self._HEADERS)
|
||||
with urllib.request.urlopen(req, timeout=20) as resp:
|
||||
raw = resp.read().decode("gbk", errors="replace")
|
||||
return json.loads(raw)
|
||||
except Exception as e:
|
||||
last_err = e
|
||||
if attempt < retry:
|
||||
time.sleep(0.5 * (attempt + 1)) # 短暂退避后重试
|
||||
continue
|
||||
if last_err:
|
||||
from app.core.utils.logging import get_logger
|
||||
get_logger("sina").debug(f"sina fetch {symbol} failed after {retry+1} attempts: {last_err}")
|
||||
return []
|
||||
|
||||
def fetch_kline_daily(self, code6: str, start: str, end: str) -> pd.DataFrame:
|
||||
symbol = code6_to_sina(code6)
|
||||
data = self._fetch(symbol)
|
||||
if not data or not isinstance(data, list):
|
||||
return pd.DataFrame()
|
||||
records = []
|
||||
for item in data:
|
||||
try:
|
||||
records.append({
|
||||
"trade_date": item["day"],
|
||||
"open": item["open"],
|
||||
"high": item["high"],
|
||||
"low": item["low"],
|
||||
"close": item["close"],
|
||||
"volume": item["volume"],
|
||||
})
|
||||
except KeyError:
|
||||
continue
|
||||
return filter_date_range(normalize_kline(pd.DataFrame(records)), start, end)
|
||||
|
||||
def fetch_index_daily(self, index_code: str, start: str, end: str) -> pd.DataFrame:
|
||||
if index_code.startswith(("5", "6", "9")):
|
||||
symbol = f"sh{index_code}"
|
||||
else:
|
||||
symbol = f"sz{index_code}"
|
||||
data = self._fetch(symbol)
|
||||
if not data or not isinstance(data, list):
|
||||
return pd.DataFrame()
|
||||
records = []
|
||||
for item in data:
|
||||
try:
|
||||
records.append({
|
||||
"trade_date": item["day"],
|
||||
"open": float(item["open"]),
|
||||
"high": float(item["high"]),
|
||||
"low": float(item["low"]),
|
||||
"close": float(item["close"]),
|
||||
"volume": float(item["volume"]),
|
||||
})
|
||||
except (KeyError, ValueError, TypeError):
|
||||
continue
|
||||
return filter_date_range(normalize_kline(pd.DataFrame(records)), start, end)
|
||||
@@ -0,0 +1,203 @@
|
||||
"""雪球数据源（pysnowball，非官方 SDK）。
|
||||
|
||||
提供：
|
||||
- 日 K 线复核（需要 token，免费但有限速）
|
||||
- 最新股本快照（quote_detail，单位原始股，需 ÷1e8 转为亿股）
|
||||
|
||||
没有 token 时降级：is_available() 返回 False。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from datetime import datetime
|
||||
from typing import Any, Optional
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from app.core.datasource.base import DataSource
|
||||
from app.core.datasource.utils import code6_to_xueqiu
|
||||
|
||||
logger = logging.getLogger("sync.xueqiu")
|
||||
|
||||
|
||||
class XueqiuSource(DataSource):
|
||||
key = "datasource_xueqiu"
|
||||
name = "雪球"
|
||||
provides = ["kline_daily", "share"]
|
||||
requires_credential = True
|
||||
credential_key = "XUEQIU_TOKEN"
|
||||
|
||||
# 进程内只 set_token 一次
|
||||
_token_lock = __import__("threading").Lock()
|
||||
_token_set = False
|
||||
|
||||
# RPS 限流（线程安全）
|
||||
# 雪球 token 限速 10 RPS（用户实测），用满 10 才能在合理时间内跑完全市场
|
||||
_RPS_LIMIT = 10.0
|
||||
_rate_lock = __import__("threading").Lock()
|
||||
_last_request = 0.0
|
||||
|
||||
def _read_token(self) -> str:
|
||||
return os.environ.get(self.credential_key, "").strip()
|
||||
|
||||
def _import_ball(self):
|
||||
"""延迟 + 显式 import。失败时打 warning（之前全被 except 吞了）。"""
|
||||
try:
|
||||
import pysnowball as ball
|
||||
return ball
|
||||
except ImportError as e:
|
||||
logger.error(
|
||||
"pysnowball 未安装，雪球数据源不可用。pip install pysnowball。错误: %s", e,
|
||||
)
|
||||
raise
|
||||
|
||||
def _set_token_once(self) -> None:
|
||||
if self._token_set:
|
||||
return
|
||||
with self._token_lock:
|
||||
if self._token_set:
|
||||
return
|
||||
token = self._read_token()
|
||||
if not token:
|
||||
return
|
||||
try:
|
||||
ball = self._import_ball()
|
||||
ball.set_token(token)
|
||||
self._token_set = True
|
||||
except ImportError:
|
||||
pass # _import_ball 已经打日志了
|
||||
|
||||
def _wait_rps(self) -> None:
|
||||
with self._rate_lock:
|
||||
now = time.time()
|
||||
elapsed = now - self._last_request
|
||||
min_interval = 1.0 / self._RPS_LIMIT
|
||||
if elapsed < min_interval:
|
||||
time.sleep(min_interval - elapsed)
|
||||
self._last_request = time.time()
|
||||
|
||||
def is_available(self) -> tuple[bool, str]:
|
||||
token = self._read_token()
|
||||
if not token:
|
||||
return False, f"未配置 {self.credential_key}"
|
||||
# 顺便检查 pysnowball 是否装了
|
||||
try:
|
||||
import pysnowball # noqa: F401
|
||||
except ImportError:
|
||||
return False, "pysnowball 未安装，pip install pysnowball"
|
||||
return True, "ok"
|
||||
|
||||
def health_check(self) -> dict[str, Any]:
|
||||
if not self._read_token():
|
||||
return {"success": False, "message": f"未配置 {self.credential_key}"}
|
||||
try:
|
||||
ball = self._import_ball()
|
||||
self._set_token_once()
|
||||
result = ball.quote_detail("SH600036")
|
||||
if result is None or result.get("error_code") != 0:
|
||||
return {
|
||||
"success": False,
|
||||
"message": f"雪球返回异常: error_code={result.get('error_code') if result else 'None'} "
|
||||
f"desc={result.get('error_description') if result else 'None'}",
|
||||
}
|
||||
quote = (result or {}).get("data", {}).get("quote", {})
|
||||
if not quote.get("total_shares") or not quote.get("float_shares"):
|
||||
return {"success": False, "message": "雪球返回成功，但缺少 total_shares/float_shares 字段"}
|
||||
return {"success": True, "message": "雪球连接成功，可读取股本快照"}
|
||||
except Exception as e:
|
||||
logger.exception("雪球 health_check 异常: %s", e)
|
||||
return {"success": False, "message": f"雪球连接失败: {e}"}
|
||||
|
||||
def fetch_kline_daily(self, code6: str, start: str, end: str) -> pd.DataFrame:
|
||||
if not self._read_token():
|
||||
return pd.DataFrame()
|
||||
try:
|
||||
ball = self._import_ball()
|
||||
self._set_token_once()
|
||||
self._wait_rps()
|
||||
|
||||
symbol = code6_to_xueqiu(code6)
|
||||
start_dt = datetime.strptime(start, "%Y-%m-%d")
|
||||
end_dt = datetime.strptime(end, "%Y-%m-%d")
|
||||
count = min(max((end_dt - start_dt).days + 60, 10), 5000)
|
||||
|
||||
# 加重试：雪球风控偶尔返回空/错误
|
||||
result = None
|
||||
for attempt in range(3):
|
||||
result = ball.kline(symbol, period="day", count=count)
|
||||
if result and result.get("error_code") == 0:
|
||||
break
|
||||
time.sleep(0.3 * (attempt + 1))
|
||||
if not result or result.get("error_code") != 0:
|
||||
logger.warning(
|
||||
"[kline %s] 雪球 kline 失败: error_code=%s desc=%s",
|
||||
code6,
|
||||
result.get("error_code") if result else "None",
|
||||
result.get("error_description") if result else "None",
|
||||
)
|
||||
return pd.DataFrame()
|
||||
data = result.get("data", {})
|
||||
columns = data.get("column", [])
|
||||
items = data.get("item", [])
|
||||
if not columns or not items:
|
||||
return pd.DataFrame()
|
||||
needed = {"timestamp": 0, "volume": 1, "open": 2, "high": 3, "low": 4, "close": 5}
|
||||
records = []
|
||||
for row in items:
|
||||
try:
|
||||
ts = row[needed["timestamp"]] / 1000
|
||||
records.append({
|
||||
"trade_date": datetime.fromtimestamp(ts).strftime("%Y-%m-%d"),
|
||||
"open": float(row[needed["open"]]),
|
||||
"high": float(row[needed["high"]]),
|
||||
"low": float(row[needed["low"]]),
|
||||
"close": float(row[needed["close"]]),
|
||||
"volume": float(row[needed["volume"]]),
|
||||
})
|
||||
except (IndexError, ValueError, TypeError, OSError):
|
||||
continue
|
||||
from app.core.datasource.utils import filter_date_range, normalize_kline
|
||||
return filter_date_range(normalize_kline(pd.DataFrame(records)), start, end)
|
||||
except Exception as e:
|
||||
logger.exception("[kline %s] 雪球 kline 异常: %s", code6, e)
|
||||
return pd.DataFrame()
|
||||
|
||||
def fetch_share_snapshot(self, code6: str) -> Optional[dict[str, Any]]:
|
||||
"""单只股票的最新股本快照。
|
||||
|
||||
返回：
|
||||
{total_share, float_share, trade_date} —— 单位：亿股
|
||||
"""
|
||||
if not self._read_token():
|
||||
return None
|
||||
try:
|
||||
ball = self._import_ball()
|
||||
self._set_token_once()
|
||||
self._wait_rps()
|
||||
|
||||
symbol = code6_to_xueqiu(code6)
|
||||
result = ball.quote_detail(symbol)
|
||||
if not result or result.get("error_code") != 0:
|
||||
logger.warning(
|
||||
"[share %s] 雪球 quote_detail 失败: error_code=%s desc=%s",
|
||||
code6,
|
||||
result.get("error_code") if result else "None",
|
||||
result.get("error_description") if result else "None",
|
||||
)
|
||||
return None
|
||||
quote = result.get("data", {}).get("quote", {})
|
||||
today_str = datetime.now().strftime("%Y-%m-%d")
|
||||
total = round(float(quote.get("total_shares") or 0) / 1e8, 4)
|
||||
flt = round(float(quote.get("float_shares") or 0) / 1e8, 4)
|
||||
if total <= 0 or flt <= 0:
|
||||
logger.warning("[share %s] 雪球返回成功但 total/float 为 0", code6)
|
||||
return None
|
||||
return {
|
||||
"trade_date": today_str,
|
||||
"total_share": total,
|
||||
"float_share": flt,
|
||||
}
|
||||
except Exception as e:
|
||||
logger.exception("[share %s] 雪球 quote_detail 异常: %s", code6, e)
|
||||
return None
|
||||
Reference in New Issue
Block a user