gao
8f016f25df
- ORM migration: models/ops 重构 - deploy: 标准化 systemd timer/service 部署体系 - MCP server: 5 个只读工具(任务/状态/数据集) - daily check: 数据一致性巡检 - watch: task_watch 后台监控 - kline_daily: 麦蕊优先,时间门禁15:00 - 删除: task_industry_sector, task_sector_features
228 lines
9.3 KiB
Python
228 lines
9.3 KiB
Python
"""雪球数据源（pysnowball，非官方 SDK）。
|
|
|
|
提供：
|
|
- 日 K 线复核（需要 token，免费但有限速）
|
|
- 最新股本快照（quote_detail，单位原始股，需 ÷1e8 转为亿股）
|
|
|
|
没有 token 时降级：is_available() 返回 False。"""
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import os
|
|
import time
|
|
from datetime import datetime
|
|
from typing import Any, Optional
|
|
|
|
import pandas as pd
|
|
|
|
from app.core.datasource.base import DataSource
|
|
from app.core.datasource.utils import call_with_timeout, code6_to_xueqiu, effective_market_date
|
|
|
|
logger = logging.getLogger("sync.xueqiu")
|
|
|
|
|
|
class XueqiuSource(DataSource):
|
|
key = "datasource_xueqiu"
|
|
name = "雪球"
|
|
provides = ["kline_daily", "share"]
|
|
requires_credential = True
|
|
credential_key = "XUEQIU_TOKEN"
|
|
|
|
# 进程内只 set_token 一次
|
|
_token_lock = __import__("threading").Lock()
|
|
_token_set = False
|
|
|
|
# RPS 限流（线程安全）
|
|
# 雪球 token 限速 10 RPS（用户实测），用满 10 才能在合理时间内跑完全市场
|
|
_RPS_LIMIT = 10.0
|
|
_rate_lock = __import__("threading").Lock()
|
|
_last_request = 0.0
|
|
|
|
def _read_token(self) -> str:
|
|
return os.environ.get(self.credential_key, "").strip()
|
|
|
|
def _import_ball(self):
|
|
"""延迟 + 显式 import。失败时打 warning（之前全被 except 吞了）。"""
|
|
try:
|
|
import pysnowball as ball
|
|
return ball
|
|
except ImportError as e:
|
|
logger.error(
|
|
"pysnowball 未安装，雪球数据源不可用。pip install pysnowball。错误: %s", e,
|
|
)
|
|
raise
|
|
|
|
def _set_token_once(self) -> None:
|
|
if self._token_set:
|
|
return
|
|
with self._token_lock:
|
|
if self._token_set:
|
|
return
|
|
token = self._read_token()
|
|
if not token:
|
|
return
|
|
try:
|
|
ball = self._import_ball()
|
|
ball.set_token(token)
|
|
self._token_set = True
|
|
except ImportError:
|
|
pass # _import_ball 已经打日志了
|
|
|
|
def _wait_rps(self) -> None:
|
|
with self._rate_lock:
|
|
now = time.time()
|
|
elapsed = now - self._last_request
|
|
min_interval = 1.0 / self._RPS_LIMIT
|
|
if elapsed < min_interval:
|
|
time.sleep(min_interval - elapsed)
|
|
self._last_request = time.time()
|
|
|
|
def is_available(self) -> tuple[bool, str]:
|
|
token = self._read_token()
|
|
if not token:
|
|
return False, f"未配置 {self.credential_key}"
|
|
# 顺便检查 pysnowball 是否装了
|
|
try:
|
|
import pysnowball # noqa: F401
|
|
except ImportError:
|
|
return False, "pysnowball 未安装，pip install pysnowball"
|
|
return True, "ok"
|
|
|
|
def health_check(self) -> dict[str, Any]:
|
|
if not self._read_token():
|
|
return {"success": False, "message": f"未配置 {self.credential_key}"}
|
|
try:
|
|
ball = self._import_ball()
|
|
self._set_token_once()
|
|
result = call_with_timeout(
|
|
ball.quote_detail, "SH600036",
|
|
timeout=10.0, on_timeout=None,
|
|
description="xueqiu.health_check",
|
|
)
|
|
if result is None or result.get("error_code") != 0:
|
|
return {
|
|
"success": False,
|
|
"message": f"雪球返回异常: error_code={result.get('error_code') if result else 'None'} "
|
|
f"desc={result.get('error_description') if result else 'None'}",
|
|
}
|
|
quote = (result or {}).get("data", {}).get("quote", {})
|
|
if not quote.get("total_shares") or not quote.get("float_shares"):
|
|
return {"success": False, "message": "雪球返回成功，但缺少 total_shares/float_shares 字段"}
|
|
return {"success": True, "message": "雪球连接成功，可读取股本快照"}
|
|
except Exception as e:
|
|
logger.exception("雪球 health_check 异常: %s", e)
|
|
return {"success": False, "message": f"雪球连接失败: {e}"}
|
|
|
|
def fetch_kline_daily(self, code6: str, start: str, end: str) -> pd.DataFrame:
|
|
if not self._read_token():
|
|
return pd.DataFrame()
|
|
try:
|
|
ball = self._import_ball()
|
|
self._set_token_once()
|
|
self._wait_rps()
|
|
|
|
symbol = code6_to_xueqiu(code6)
|
|
start_dt = datetime.strptime(start, "%Y-%m-%d")
|
|
end_dt = datetime.strptime(end, "%Y-%m-%d")
|
|
count = min(max((end_dt - start_dt).days + 60, 10), 5000)
|
|
|
|
# 加重试：雪球风控偶尔返回空/错误
|
|
# 用 call_with_timeout 包一层 — 2026-07-08 kline_daily 卡死 4h 根因
|
|
# 就是 pysnowball ball.kline() 底层 requests 无 read timeout,雪球
|
|
# 服务端卡住时这里永久阻塞。timeout=20s 与 mairui/sina 对齐。
|
|
result = None
|
|
for attempt in range(3):
|
|
result = call_with_timeout(
|
|
ball.kline, symbol, period="day", count=count,
|
|
timeout=20.0, on_timeout=None,
|
|
description=f"xueqiu.kline {symbol}",
|
|
)
|
|
if result and result.get("error_code") == 0:
|
|
break
|
|
if result is None and attempt < 2:
|
|
time.sleep(0.3 * (attempt + 1))
|
|
continue
|
|
if result and result.get("error_code") != 0:
|
|
time.sleep(0.3 * (attempt + 1))
|
|
if not result or result.get("error_code") != 0:
|
|
logger.warning(
|
|
"[kline %s] 雪球 kline 失败: error_code=%s desc=%s",
|
|
code6,
|
|
result.get("error_code") if result else "None",
|
|
result.get("error_description") if result else "None",
|
|
)
|
|
return pd.DataFrame()
|
|
data = result.get("data", {})
|
|
columns = data.get("column", [])
|
|
items = data.get("item", [])
|
|
if not columns or not items:
|
|
return pd.DataFrame()
|
|
needed = {"timestamp": 0, "volume": 1, "open": 2, "high": 3, "low": 4, "close": 5}
|
|
records = []
|
|
for row in items:
|
|
try:
|
|
ts = row[needed["timestamp"]] / 1000
|
|
records.append({
|
|
"trade_date": datetime.fromtimestamp(ts).strftime("%Y-%m-%d"),
|
|
"open": float(row[needed["open"]]),
|
|
"high": float(row[needed["high"]]),
|
|
"low": float(row[needed["low"]]),
|
|
"close": float(row[needed["close"]]),
|
|
"volume": float(row[needed["volume"]]),
|
|
})
|
|
except (IndexError, ValueError, TypeError, OSError):
|
|
continue
|
|
from app.core.datasource.utils import filter_date_range, normalize_kline
|
|
return filter_date_range(normalize_kline(pd.DataFrame(records)), start, end)
|
|
except Exception as e:
|
|
logger.exception("[kline %s] 雪球 kline 异常: %s", code6, e)
|
|
return pd.DataFrame()
|
|
|
|
def fetch_share_snapshot(self, code6: str) -> Optional[dict[str, Any]]:
|
|
"""单只股票的最新股本快照。
|
|
|
|
返回：
|
|
{total_share, float_share, trade_date} —— 单位：亿股
|
|
"""
|
|
if not self._read_token():
|
|
return None
|
|
try:
|
|
ball = self._import_ball()
|
|
self._set_token_once()
|
|
self._wait_rps()
|
|
|
|
symbol = code6_to_xueqiu(code6)
|
|
# ball.quote_detail 同样有 pysnowball 无 read timeout 风险,包一层
|
|
result = call_with_timeout(
|
|
ball.quote_detail, symbol,
|
|
timeout=15.0, on_timeout=None,
|
|
description=f"xueqiu.quote_detail {symbol}",
|
|
)
|
|
if not result or result.get("error_code") != 0:
|
|
logger.warning(
|
|
"[share %s] 雪球 quote_detail 失败: error_code=%s desc=%s",
|
|
code6,
|
|
result.get("error_code") if result else "None",
|
|
result.get("error_description") if result else "None",
|
|
)
|
|
return None
|
|
quote = result.get("data", {}).get("quote", {})
|
|
# 雪球 quote_detail 不返回明确"股本变动日",time 字段是 quote 当前
|
|
# 时间,不是交易日。股本数据本质是慢变快照,这里标"最近已完成交易日"
|
|
# (15:30 前=昨天,15:30 后=今天) — 比 datetime.now() 精确,避免盘前
|
|
# 拉到的快照被错标为"今天"。
|
|
as_of = effective_market_date()
|
|
total = round(float(quote.get("total_shares") or 0) / 1e8, 4)
|
|
flt = round(float(quote.get("float_shares") or 0) / 1e8, 4)
|
|
if total <= 0 or flt <= 0:
|
|
logger.warning("[share %s] 雪球返回成功但 total/float 为 0", code6)
|
|
return None
|
|
return {
|
|
"trade_date": as_of,
|
|
"total_share": total,
|
|
"float_share": flt,
|
|
}
|
|
except Exception as e:
|
|
logger.exception("[share %s] 雪球 quote_detail 异常: %s", code6, e)
|
|
return None
|