"""雪球数据源(pysnowball,非官方 SDK)。 提供: - 日 K 线复核(需要 token,免费但有限速) - 最新股本快照(quote_detail,单位原始股,需 ÷1e8 转为亿股) 没有 token 时降级:is_available() 返回 False。""" from __future__ import annotations import logging import os import time from datetime import datetime from typing import Any, Optional import pandas as pd from app.core.datasource.base import DataSource from app.core.datasource.utils import call_with_timeout, code6_to_xueqiu, effective_market_date logger = logging.getLogger("sync.xueqiu") class XueqiuSource(DataSource): key = "datasource_xueqiu" name = "雪球" provides = ["kline_daily", "share"] requires_credential = True credential_key = "XUEQIU_TOKEN" # 进程内只 set_token 一次 _token_lock = __import__("threading").Lock() _token_set = False # RPS 限流(线程安全) # 雪球 token 限速 10 RPS(用户实测),用满 10 才能在合理时间内跑完全市场 _RPS_LIMIT = 10.0 _rate_lock = __import__("threading").Lock() _last_request = 0.0 def _read_token(self) -> str: return os.environ.get(self.credential_key, "").strip() def _import_ball(self): """延迟 + 显式 import。失败时打 warning(之前全被 except 吞了)。""" try: import pysnowball as ball return ball except ImportError as e: logger.error( "pysnowball 未安装,雪球数据源不可用。pip install pysnowball。错误: %s", e, ) raise def _set_token_once(self) -> None: if self._token_set: return with self._token_lock: if self._token_set: return token = self._read_token() if not token: return try: ball = self._import_ball() ball.set_token(token) self._token_set = True except ImportError: pass # _import_ball 已经打日志了 def _wait_rps(self) -> None: with self._rate_lock: now = time.time() elapsed = now - self._last_request min_interval = 1.0 / self._RPS_LIMIT if elapsed < min_interval: time.sleep(min_interval - elapsed) self._last_request = time.time() def is_available(self) -> tuple[bool, str]: token = self._read_token() if not token: return False, f"未配置 {self.credential_key}" # 顺便检查 pysnowball 是否装了 try: import pysnowball # noqa: F401 except ImportError: return False, "pysnowball 未安装,pip install pysnowball" return True, "ok" def health_check(self) -> dict[str, Any]: if not self._read_token(): return {"success": False, "message": f"未配置 {self.credential_key}"} try: ball = self._import_ball() self._set_token_once() result = call_with_timeout( ball.quote_detail, "SH600036", timeout=10.0, on_timeout=None, description="xueqiu.health_check", ) if result is None or result.get("error_code") != 0: return { "success": False, "message": f"雪球返回异常: error_code={result.get('error_code') if result else 'None'} " f"desc={result.get('error_description') if result else 'None'}", } quote = (result or {}).get("data", {}).get("quote", {}) if not quote.get("total_shares") or not quote.get("float_shares"): return {"success": False, "message": "雪球返回成功,但缺少 total_shares/float_shares 字段"} return {"success": True, "message": "雪球连接成功,可读取股本快照"} except Exception as e: logger.exception("雪球 health_check 异常: %s", e) return {"success": False, "message": f"雪球连接失败: {e}"} def fetch_kline_daily(self, code6: str, start: str, end: str) -> pd.DataFrame: if not self._read_token(): return pd.DataFrame() try: ball = self._import_ball() self._set_token_once() self._wait_rps() symbol = code6_to_xueqiu(code6) start_dt = datetime.strptime(start, "%Y-%m-%d") end_dt = datetime.strptime(end, "%Y-%m-%d") count = min(max((end_dt - start_dt).days + 60, 10), 5000) # 加重试:雪球风控偶尔返回空/错误 # 用 call_with_timeout 包一层 — 2026-07-08 kline_daily 卡死 4h 根因 # 就是 pysnowball ball.kline() 底层 requests 无 read timeout,雪球 # 服务端卡住时这里永久阻塞。timeout=20s 与 mairui/sina 对齐。 result = None for attempt in range(3): result = call_with_timeout( ball.kline, symbol, period="day", count=count, timeout=20.0, on_timeout=None, description=f"xueqiu.kline {symbol}", ) if result and result.get("error_code") == 0: break if result is None and attempt < 2: time.sleep(0.3 * (attempt + 1)) continue if result and result.get("error_code") != 0: time.sleep(0.3 * (attempt + 1)) if not result or result.get("error_code") != 0: logger.warning( "[kline %s] 雪球 kline 失败: error_code=%s desc=%s", code6, result.get("error_code") if result else "None", result.get("error_description") if result else "None", ) return pd.DataFrame() data = result.get("data", {}) columns = data.get("column", []) items = data.get("item", []) if not columns or not items: return pd.DataFrame() needed = {"timestamp": 0, "volume": 1, "open": 2, "high": 3, "low": 4, "close": 5} records = [] for row in items: try: ts = row[needed["timestamp"]] / 1000 records.append({ "trade_date": datetime.fromtimestamp(ts).strftime("%Y-%m-%d"), "open": float(row[needed["open"]]), "high": float(row[needed["high"]]), "low": float(row[needed["low"]]), "close": float(row[needed["close"]]), "volume": float(row[needed["volume"]]), }) except (IndexError, ValueError, TypeError, OSError): continue from app.core.datasource.utils import filter_date_range, normalize_kline return filter_date_range(normalize_kline(pd.DataFrame(records)), start, end) except Exception as e: logger.exception("[kline %s] 雪球 kline 异常: %s", code6, e) return pd.DataFrame() def fetch_kline_5min(self, code6: str, start: str, end: str) -> pd.DataFrame: """5 分钟 K 线(雪球 period=5m)。返回标准列:bar_time, open, high, low, close, volume, amount。""" if not self._read_token(): return pd.DataFrame() try: ball = self._import_ball() self._set_token_once() self._wait_rps() symbol = code6_to_xueqiu(code6) start_dt = datetime.strptime(start, "%Y-%m-%d") end_dt = datetime.strptime(end, "%Y-%m-%d") days_needed = max((end_dt - start_dt).days + 2, 1) count = min(days_needed * 48, 5000) result = None for attempt in range(3): result = call_with_timeout( ball.kline, symbol, period="5m", count=count, timeout=20.0, on_timeout=None, description=f"xueqiu.kline.5m {symbol}", ) if result and result.get("error_code") == 0: break if result is None and attempt < 2: time.sleep(0.3 * (attempt + 1)) continue if result and result.get("error_code") != 0: time.sleep(0.3 * (attempt + 1)) if not result or result.get("error_code") != 0: logger.warning( "[kline5m %s] 雪球 5m 失败: error_code=%s desc=%s", code6, result.get("error_code") if result else "None", result.get("error_description") if result else "None", ) return pd.DataFrame() data = result.get("data", {}) columns = data.get("column", []) items = data.get("item", []) if not columns or not items: return pd.DataFrame() idx = {c: i for i, c in enumerate(columns)} needed = {"timestamp": idx.get("timestamp"), "volume": idx.get("volume"), "open": idx.get("open"), "high": idx.get("high"), "low": idx.get("low"), "close": idx.get("close")} if any(v is None for v in needed.values()): return pd.DataFrame() records = [] for row in items: try: ts = row[needed["timestamp"]] / 1000 records.append({ "bar_time": datetime.fromtimestamp(ts).strftime("%Y-%m-%d %H:%M:%S"), "open": float(row[needed["open"]]), "high": float(row[needed["high"]]), "low": float(row[needed["low"]]), "close": float(row[needed["close"]]), "volume": float(row[needed["volume"]]), "amount": 0.0, }) except (IndexError, ValueError, TypeError, OSError): continue if not records: return pd.DataFrame() df = pd.DataFrame(records) df["bar_time"] = pd.to_datetime(df["bar_time"], errors="coerce") df = df.dropna(subset=["bar_time"]).sort_values("bar_time").reset_index(drop=True) if start: df = df[df["bar_time"] >= pd.Timestamp(start)] if end: df = df[df["bar_time"] < pd.Timestamp(end) + pd.Timedelta(days=1)] return df except Exception as e: logger.exception("[kline5m %s] 雪球 5m 异常: %s", code6, e) return pd.DataFrame() def fetch_share_snapshot(self, code6: str) -> Optional[dict[str, Any]]: """单只股票的最新股本快照。 返回: {total_share, float_share, trade_date} —— 单位:亿股 """ if not self._read_token(): return None try: ball = self._import_ball() self._set_token_once() self._wait_rps() symbol = code6_to_xueqiu(code6) # ball.quote_detail 同样有 pysnowball 无 read timeout 风险,包一层 result = call_with_timeout( ball.quote_detail, symbol, timeout=15.0, on_timeout=None, description=f"xueqiu.quote_detail {symbol}", ) if not result or result.get("error_code") != 0: logger.warning( "[share %s] 雪球 quote_detail 失败: error_code=%s desc=%s", code6, result.get("error_code") if result else "None", result.get("error_description") if result else "None", ) return None quote = result.get("data", {}).get("quote", {}) # 雪球 quote_detail 不返回明确"股本变动日",time 字段是 quote 当前 # 时间,不是交易日。股本数据本质是慢变快照,这里标"最近已完成交易日" # (15:30 前=昨天,15:30 后=今天) — 比 datetime.now() 精确,避免盘前 # 拉到的快照被错标为"今天"。 as_of = effective_market_date() total = round(float(quote.get("total_shares") or 0) / 1e8, 4) flt = round(float(quote.get("float_shares") or 0) / 1e8, 4) if total <= 0 or flt <= 0: logger.warning("[share %s] 雪球返回成功但 total/float 为 0", code6) return None return { "trade_date": as_of, "total_share": total, "float_share": flt, } except Exception as e: logger.exception("[share %s] 雪球 quote_detail 异常: %s", code6, e) return None