chore: baseline — ORM migration + systemd deploy + MCP server + daily check
- ORM migration: models/ops 重构 - deploy: 标准化 systemd timer/service 部署体系 - MCP server: 5 个只读工具(任务/状态/数据集) - daily check: 数据一致性巡检 - watch: task_watch 后台监控 - kline_daily: 麦蕊优先,时间门禁15:00 - 删除: task_industry_sector, task_sector_features
This commit is contained in:
gao
+63
-1
@@ -32,7 +32,7 @@ from app.core.datasource.utils import (
|
||||
class MairuiSource(DataSource):
|
||||
key = "datasource_mairui"
|
||||
name = "麦蕊智数（mairui.club）"
|
||||
provides = ["kline_daily", "kline_5min", "index_daily", "stock_basic", "moneyflow", "tick_trade", "stock_node"]
|
||||
provides = ["kline_daily", "kline_5min", "index_daily", "stock_basic", "moneyflow", "tick_trade", "stock_node", "indicator_daily"]
|
||||
requires_credential = True
|
||||
credential_key = "MAIRUI_LICENCE"
|
||||
|
||||
@@ -285,6 +285,68 @@ class MairuiSource(DataSource):
|
||||
df = df[df["bar_time"] < pd.Timestamp(end) + pd.Timedelta(days=1)]
|
||||
return df
|
||||
|
||||
# ── 技术指标 fetch（MACD / KDJ / BOLL）─────────────────────
|
||||
# mairui 端点：/hsstock/history/{indicator}/{symbol}/d/n/{licence}
|
||||
# indicator ∈ {macd, kdj, boll}；日 K 级别 "/d/"；不复权 "/n/"
|
||||
# 各指标字段（除公共 "t" 外）：
|
||||
# macd → diff, dea, macd, ema12, ema26
|
||||
# kdj → k, d, j
|
||||
# boll → u(上轨), d(下轨), m(中轨) ← 注意 mairui 用 u/d/m
|
||||
_INDICATOR_FIELDS = {
|
||||
"macd": ["diff", "dea", "macd", "ema12", "ema26"],
|
||||
"kdj": ["k", "d", "j"],
|
||||
"boll": ["u", "d", "m"],
|
||||
}
|
||||
|
||||
def fetch_indicator_daily(
|
||||
self, indicator: str, code6: str, start: str = "", end: str = ""
|
||||
) -> pd.DataFrame:
|
||||
"""拉单只股票某个技术指标的日 K 序列。
|
||||
|
||||
返回列：trade_date + 该指标字段（macd/kdj/boll 各自的原始字段名）。
|
||||
start/end 为 "YYYY-MM-DD"（可空 → 全历史）。
|
||||
"""
|
||||
indicator = indicator.lower()
|
||||
fields = self._INDICATOR_FIELDS.get(indicator)
|
||||
if fields is None:
|
||||
raise ValueError(f"未知指标: {indicator!r}，可选 {list(self._INDICATOR_FIELDS)}")
|
||||
lic = self._get_licence()
|
||||
if not lic:
|
||||
return pd.DataFrame()
|
||||
symbol = code6_to_mairui(code6)
|
||||
path = f"/hsstock/history/{indicator}/{symbol}/d/n/{lic}"
|
||||
params = {
|
||||
"st": (start or "").replace("-", ""),
|
||||
"et": (end or "").replace("-", ""),
|
||||
}
|
||||
data = self._fetch(path, params)
|
||||
if not isinstance(data, list):
|
||||
return pd.DataFrame()
|
||||
records = []
|
||||
for item in data:
|
||||
try:
|
||||
t = item.get("t", "")
|
||||
d = t.split(" ")[0] if isinstance(t, str) else ""
|
||||
if not d:
|
||||
continue
|
||||
row = {"trade_date": d}
|
||||
for f in fields:
|
||||
v = item.get(f)
|
||||
row[f] = None if v is None else float(v)
|
||||
records.append(row)
|
||||
except (KeyError, ValueError, TypeError):
|
||||
continue
|
||||
if not records:
|
||||
return pd.DataFrame()
|
||||
df = pd.DataFrame(records)
|
||||
df["trade_date"] = pd.to_datetime(df["trade_date"], errors="coerce")
|
||||
df = df.dropna(subset=["trade_date"]).sort_values("trade_date").reset_index(drop=True)
|
||||
if start:
|
||||
df = df[df["trade_date"] >= pd.Timestamp(start)]
|
||||
if end:
|
||||
df = df[df["trade_date"] <= pd.Timestamp(end)]
|
||||
return df
|
||||
|
||||
def fetch_index_daily(self, index_code: str, start: str, end: str) -> pd.DataFrame:
|
||||
"""指数日 K。index_code 用 mairui 格式（如 '000300.SH'）。"""
|
||||
lic = self._get_licence()
|
||||
|
||||
+31
-7
@@ -16,7 +16,7 @@ from typing import Any, Optional
|
||||
import pandas as pd
|
||||
|
||||
from app.core.datasource.base import DataSource
|
||||
from app.core.datasource.utils import code6_to_xueqiu
|
||||
from app.core.datasource.utils import call_with_timeout, code6_to_xueqiu, effective_market_date
|
||||
|
||||
logger = logging.getLogger("sync.xueqiu")
|
||||
|
||||
@@ -94,7 +94,11 @@ class XueqiuSource(DataSource):
|
||||
try:
|
||||
ball = self._import_ball()
|
||||
self._set_token_once()
|
||||
result = ball.quote_detail("SH600036")
|
||||
result = call_with_timeout(
|
||||
ball.quote_detail, "SH600036",
|
||||
timeout=10.0, on_timeout=None,
|
||||
description="xueqiu.health_check",
|
||||
)
|
||||
if result is None or result.get("error_code") != 0:
|
||||
return {
|
||||
"success": False,
|
||||
@@ -123,12 +127,23 @@ class XueqiuSource(DataSource):
|
||||
count = min(max((end_dt - start_dt).days + 60, 10), 5000)
|
||||
|
||||
# 加重试：雪球风控偶尔返回空/错误
|
||||
# 用 call_with_timeout 包一层 — 2026-07-08 kline_daily 卡死 4h 根因
|
||||
# 就是 pysnowball ball.kline() 底层 requests 无 read timeout,雪球
|
||||
# 服务端卡住时这里永久阻塞。timeout=20s 与 mairui/sina 对齐。
|
||||
result = None
|
||||
for attempt in range(3):
|
||||
result = ball.kline(symbol, period="day", count=count)
|
||||
result = call_with_timeout(
|
||||
ball.kline, symbol, period="day", count=count,
|
||||
timeout=20.0, on_timeout=None,
|
||||
description=f"xueqiu.kline {symbol}",
|
||||
)
|
||||
if result and result.get("error_code") == 0:
|
||||
break
|
||||
time.sleep(0.3 * (attempt + 1))
|
||||
if result is None and attempt < 2:
|
||||
time.sleep(0.3 * (attempt + 1))
|
||||
continue
|
||||
if result and result.get("error_code") != 0:
|
||||
time.sleep(0.3 * (attempt + 1))
|
||||
if not result or result.get("error_code") != 0:
|
||||
logger.warning(
|
||||
"[kline %s] 雪球 kline 失败: error_code=%s desc=%s",
|
||||
@@ -177,7 +192,12 @@ class XueqiuSource(DataSource):
|
||||
self._wait_rps()
|
||||
|
||||
symbol = code6_to_xueqiu(code6)
|
||||
result = ball.quote_detail(symbol)
|
||||
# ball.quote_detail 同样有 pysnowball 无 read timeout 风险,包一层
|
||||
result = call_with_timeout(
|
||||
ball.quote_detail, symbol,
|
||||
timeout=15.0, on_timeout=None,
|
||||
description=f"xueqiu.quote_detail {symbol}",
|
||||
)
|
||||
if not result or result.get("error_code") != 0:
|
||||
logger.warning(
|
||||
"[share %s] 雪球 quote_detail 失败: error_code=%s desc=%s",
|
||||
@@ -187,14 +207,18 @@ class XueqiuSource(DataSource):
|
||||
)
|
||||
return None
|
||||
quote = result.get("data", {}).get("quote", {})
|
||||
today_str = datetime.now().strftime("%Y-%m-%d")
|
||||
# 雪球 quote_detail 不返回明确"股本变动日",time 字段是 quote 当前
|
||||
# 时间,不是交易日。股本数据本质是慢变快照,这里标"最近已完成交易日"
|
||||
# (15:30 前=昨天,15:30 后=今天) — 比 datetime.now() 精确,避免盘前
|
||||
# 拉到的快照被错标为"今天"。
|
||||
as_of = effective_market_date()
|
||||
total = round(float(quote.get("total_shares") or 0) / 1e8, 4)
|
||||
flt = round(float(quote.get("float_shares") or 0) / 1e8, 4)
|
||||
if total <= 0 or flt <= 0:
|
||||
logger.warning("[share %s] 雪球返回成功但 total/float 为 0", code6)
|
||||
return None
|
||||
return {
|
||||
"trade_date": today_str,
|
||||
"trade_date": as_of,
|
||||
"total_share": total,
|
||||
"float_share": flt,
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user