kyugao/market_sync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: baseline — ORM migration + systemd deploy + MCP server + daily check

Browse Source 
- ORM migration: models/ops 重构
- deploy: 标准化 systemd timer/service 部署体系
- MCP server: 5 个只读工具(任务/状态/数据集)
- daily check: 数据一致性巡检
- watch: task_watch 后台监控
- kline_daily: 麦蕊优先,时间门禁15:00
- 删除: task_industry_sector, task_sector_features
This commit is contained in:
gao
2026-07-21 16:37:00 +08:00
parent 91e83a3b0b
commit 8f016f25df
94 changed files with 4117 additions and 1176 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/core/db/models.py
+80 -94
View File
@@ -141,7 +141,6 @@ class Stock(ORMBase):
exchange: Mapped[str] = mapped_column(String(8), nullable=False, default="")
list_date: Mapped[Optional[date]] = mapped_column(Date)
listing_status: Mapped[str] = mapped_column(String(16), nullable=False, default="normal")
industry: Mapped[Optional[str]] = mapped_column(String(64), default="")
total_share: Mapped[float] = mapped_column(Float, default=0)
float_share: Mapped[float] = mapped_column(Float, default=0)
share_updated_at: Mapped[Optional[date]] = mapped_column(Date)
@@ -258,92 +257,7 @@ class Share(ORMBase):
float_share: Mapped[float] = mapped_column(Float, nullable=False)
# ──────────────────────── 10. sectors ────────────────────────
class Sectors(ORMBase):
"""行业字典（baostock 行业分类 / 申万 / 中证 等多种 taxonomy）。"""
__tablename__ = "sectors"
__table_args__ = (
Index("idx_sectors_sector_name", "sector_name"),
{"schema": "market_data"},
)
sector_key: Mapped[str] = mapped_column(String(64), primary_key=True)
sector_name: Mapped[str] = mapped_column(String(64), nullable=False, default="")
taxonomy: Mapped[Optional[str]] = mapped_column(String(64), default="")
level: Mapped[Optional[str]] = mapped_column(String(16), default="")
source: Mapped[Optional[str]] = mapped_column(String(32), default="")
enabled: Mapped[int] = mapped_column(Integer, default=1)
updated_at: Mapped[Optional[datetime]] = _updated_at()
# ──────────────────────── 11. stock_sector_map ────────────────────────
class StockSectorMap(ORMBase):
"""股票-行业多对多映射（一张股票可对应多个行业 / 概念板块）。"""
__tablename__ = "stock_sector_map"
__table_args__ = (
Index("idx_stock_sector_map_sector_key", "sector_key"),
{"schema": "market_data"},
)
stock_code: Mapped[str] = mapped_column(String(10), primary_key=True)
sector_key: Mapped[str] = mapped_column(String(64), nullable=False, default="")
updated_at: Mapped[Optional[datetime]] = _updated_at()
# ──────────────────────── 12. industry ────────────────────────
class Industry(ORMBase):
"""股票-行业映射（baostock 源，证监会分类标准）。"""
__tablename__ = "industry"
__table_args__ = (
Index("idx_industry_industry_name", "industry_name"),
{"schema": "market_data"},
)
code: Mapped[str] = mapped_column(String(10), primary_key=True)
industry_name: Mapped[Optional[str]] = mapped_column(String(64), default="")
industry_classification: Mapped[Optional[str]] = mapped_column(String(32), default="")
update_date: Mapped[Optional[date]] = mapped_column(Date)
updated_at: Mapped[Optional[datetime]] = _updated_at()
# ──────────────────────── 13. sector_indices ────────────────────────
class SectorIndices(ORMBase):
"""行业日线（基点 100，复合收益）。"""
__tablename__ = "sector_indices"
__table_args__ = (
PrimaryKeyConstraint("trade_date", "sector_name"),
Index("idx_sector_indices_sector_name", "sector_name"),
{"schema": "market_data"},
)
trade_date: Mapped[date] = mapped_column(Date, nullable=False)
sector_name: Mapped[str] = mapped_column(String(64), nullable=False)
close: Mapped[float] = mapped_column(Float, default=0)
sector_amplitude: Mapped[float] = mapped_column(Float, default=0)
# ──────────────────────── 14. sector_features_daily ────────────────────────
class SectorFeaturesDaily(ORMBase):
"""行业日特征：sector_ret / sector_amplitude / close / EMA / score。"""
__tablename__ = "sector_features_daily"
__table_args__ = (
PrimaryKeyConstraint("trade_date", "sector_name"),
Index("idx_sector_features_sector_name", "sector_name"),
{"schema": "market_data"},
)
trade_date: Mapped[date] = mapped_column(Date, nullable=False)
sector_name: Mapped[str] = mapped_column(String(64), nullable=False)
sector_ret: Mapped[float] = mapped_column(Float, default=0)
sector_amplitude: Mapped[float] = mapped_column(Float, default=0)
close: Mapped[float] = mapped_column(Float, default=0)
ema10: Mapped[float] = mapped_column(Float, default=0)
ema20: Mapped[float] = mapped_column(Float, default=0)
ema200: Mapped[float] = mapped_column(Float, default=0)
score: Mapped[int] = mapped_column(Integer, default=0)
# ──────────────────────── 15. market_regime_daily ────────────────────────
# ──────────────────────── 12. market_regime_daily ────────────────────────
class MarketRegimeDaily(ORMBase):
"""市场情绪衍生指标（基于本地 kline 聚合）。"""
__tablename__ = "market_regime_daily"
@@ -599,6 +513,80 @@ class StockNodeMap(ORMBase):
updated_at: Mapped[Optional[datetime]] = _updated_at()
# ──────────────────────── 22. kline_stock_macd_daily ────────────────────────
class KlineStockMACDDaily(ORMBase):
"""个股日 K 级别 MACD 指标（mairui /hsstock/history/macd 直拉）。
字段（mairui 原始字段名，与 K 线口径一致）：
diff = DIF（快慢 EMA 差）
dea = DEA（DIF 的 9 日 EMA，即 signal line）
macd = MACD 柱（2 ×（DIF − DEA））
ema12 / ema26 = 计算 DIF 用的两条 EMA
stock_code 用 hermes 格式（SH600519），与项目其他表一致。
"""
__tablename__ = "kline_stock_macd_daily"
__table_args__ = (
PrimaryKeyConstraint("stock_code", "trade_date"),
Index("idx_kline_stock_macd_date", "trade_date"),
{"schema": "market_data"},
)
stock_code: Mapped[str] = mapped_column(String(10), nullable=False)
trade_date: Mapped[date] = mapped_column(Date, nullable=False)
diff: Mapped[Optional[float]] = mapped_column(Float)
dea: Mapped[Optional[float]] = mapped_column(Float)
macd: Mapped[Optional[float]] = mapped_column(Float)
ema12: Mapped[Optional[float]] = mapped_column(Float)
ema26: Mapped[Optional[float]] = mapped_column(Float)
source: Mapped[Optional[str]] = mapped_column(String(32), default="mairui")
updated_at: Mapped[Optional[datetime]] = _updated_at()
# ──────────────────────── 23. kline_stock_kdj_daily ────────────────────────
class KlineStockKDJDaily(ORMBase):
"""个股日 K 级别 KDJ 指标（mairui /hsstock/history/kdj 直拉）。
k / d / j 三条随机指标线。stock_code 用 hermes 格式（SH600519）。
"""
__tablename__ = "kline_stock_kdj_daily"
__table_args__ = (
PrimaryKeyConstraint("stock_code", "trade_date"),
Index("idx_kline_stock_kdj_date", "trade_date"),
{"schema": "market_data"},
)
stock_code: Mapped[str] = mapped_column(String(10), nullable=False)
trade_date: Mapped[date] = mapped_column(Date, nullable=False)
k: Mapped[Optional[float]] = mapped_column(Float)
d: Mapped[Optional[float]] = mapped_column(Float)
j: Mapped[Optional[float]] = mapped_column(Float)
source: Mapped[Optional[str]] = mapped_column(String(32), default="mairui")
updated_at: Mapped[Optional[datetime]] = _updated_at()
# ──────────────────────── 24. kline_stock_boll_daily ────────────────────────
class KlineStockBOLLDaily(ORMBase):
"""个股日 K 级别 BOLL 布林带指标（mairui /hsstock/history/boll 直拉）。
mairui 原始字段：u=上轨, m=中轨, d=下轨。落库列名统一为 upper/mid/lower。
stock_code 用 hermes 格式（SH600519）。
"""
__tablename__ = "kline_stock_boll_daily"
__table_args__ = (
PrimaryKeyConstraint("stock_code", "trade_date"),
Index("idx_kline_stock_boll_date", "trade_date"),
{"schema": "market_data"},
)
stock_code: Mapped[str] = mapped_column(String(10), nullable=False)
trade_date: Mapped[date] = mapped_column(Date, nullable=False)
upper: Mapped[Optional[float]] = mapped_column(Float)
mid: Mapped[Optional[float]] = mapped_column(Float)
lower: Mapped[Optional[float]] = mapped_column(Float)
source: Mapped[Optional[str]] = mapped_column(String(32), default="mairui")
updated_at: Mapped[Optional[datetime]] = _updated_at()
__all__ = [
# 1-2
"Config",
@@ -612,13 +600,7 @@ __all__ = [
"Kline5Min",
"Moneyflow",
"Share",
# 10-12
"Sectors",
"StockSectorMap",
"Industry",
# 13-15
"SectorIndices",
"SectorFeaturesDaily",
# 10-11
"MarketRegimeDaily",
# 16
"TickTrade",
@@ -631,4 +613,8 @@ __all__ = [
"NodeCategory",
"Node",
"StockNodeMap",
# 22-24 (2026-07-08 mairui 技术指标 MACD/KDJ/BOLL - 日 K 级别)
"KlineStockMACDDaily",
"KlineStockKDJDaily",
"KlineStockBOLLDaily",
]
app/core/db/ops.py
+94 -157
View File
@@ -22,10 +22,12 @@ from sqlalchemy.dialects.postgresql import insert as pg_insert
from app.core.db.models import (
Config,
DatasetRegistry,
Industry,
Kline5Min,
KlineIndex,
KlineStock,
KlineStockBOLLDaily,
KlineStockKDJDaily,
KlineStockMACDDaily,
KlineStockMADaily,
LonghubangDaily,
LonghubangSeat,
@@ -34,13 +36,9 @@ from app.core.db.models import (
Moneyflow,
Node,
NodeCategory,
SectorFeaturesDaily,
SectorIndices,
Sectors,
Share,
Stock,
StockNodeMap,
StockSectorMap,
SyncHistory,
TickTrade,
)
@@ -451,19 +449,16 @@ def upsert_stock(
exchange: str,
list_date: str = "",
listing_status: str = "normal",
industry: str = "",
) -> None:
"""upsert 一只股票基础信息（单条）。
注意：MySQL 9.7.0 在 ON DUPLICATE KEY UPDATE 阶段对 DATE 字段的 VALUES()/new.col
求值存在 bug（会强制把空串塞进去，触发 1292 严格模式错误）。所以这里：
- list_date 字段在首次 INSERT 时写入；UPDATE 分支不更新（保持原值或 NULL）
- industry 字段也避开这个 bug
求值存在 bug（会强制把空串塞进去，触发 1292 严格模式错误）。所以这里
list_date 字段在首次 INSERT 时写入；UPDATE 分支不更新（保持原值或 NULL）。
批量写入请用 upsert_stocks_bulk()，性能高 10-20 倍。
"""
del list_date # 显式不接受 list_date 更新（旧值保留）
del industry # 同上
with get_session() as s:
stmt = _pg_upsert(
Stock,
@@ -477,7 +472,7 @@ def upsert_stock(
def upsert_stocks_bulk(rows: list[dict[str, Any]], chunk_size: int = 500) -> int:
"""批量 upsert 股票基础信息。
每条 row 需有: code, name, exchange, listing_status（可选 list_date / industry）
每条 row 需有: code, name, exchange, listing_status（可选 list_date）
性能：~5000 只股票从 ~50s 降到 ~3s。
"""
if not rows:
@@ -499,7 +494,7 @@ def upsert_stocks_bulk(rows: list[dict[str, Any]], chunk_size: int = 500) -> int
stmt = _pg_upsert(
Stock, values,
conflict_keys=["code"],
update_cols=["name", "exchange", "list_date", "listing_status", "industry"],
update_cols=["name", "exchange", "list_date", "listing_status"],
)
s.execute(stmt)
total += len(chunk)
@@ -507,15 +502,25 @@ def upsert_stocks_bulk(rows: list[dict[str, Any]], chunk_size: int = 500) -> int
def update_stock_share_snapshot(code: str, total_share: float, float_share: float, trade_date: str = "") -> None:
"""更新 stocks 表的股本字段。
Args:
trade_date: 数据源返回的股本快照"as-of"交易日(用于 share 表写入)。
注意:这个参数**不会**写到 stocks.share_updated_at 字段 —
那个字段的语义是"我们什么时候拉到的"(运行日,date.today()),
不是数据交易日。把 trade_date 写到 share_updated_at 会让
task_share_snapshot / task_stocks_basic 的 dedup 检查
(share_updated_at == today) 失效,见 task 层 fix。
"""
del trade_date # 显式不接受 — 见 docstring
with get_session() as s:
snap_date = _to_date_str(trade_date) if trade_date else None
s.execute(
update(Stock)
.where(Stock.code == code)
.values(
total_share=float(total_share),
float_share=float(float_share),
share_updated_at=snap_date,
share_updated_at=date.today(), # "拉取日",不是"数据日"
)
)
@@ -543,7 +548,6 @@ def _stock_row_to_dict(r: Stock) -> dict[str, Any]:
"exchange": r.exchange or "",
"list_date": _to_date_str(r.list_date),
"listing_status": r.listing_status or "normal",
"industry": r.industry or "",
"total_share": r.total_share if r.total_share is not None else 0,
"float_share": r.float_share if r.float_share is not None else 0,
"share_updated_at": _to_date_str(r.share_updated_at),
@@ -674,9 +678,13 @@ def get_kline_5min_snapshots() -> dict[str, Optional[datetime]]:
返回：{stock_code (6位): max(bar_time) 或 None}
用于 kline_5min 任务的"全量/增量"统一规划。
DB 里 stock_code 形如 "000001.SZ"（mairui 写入格式），函数剥掉
交易所后缀统一为 6 位。bar_time 是 DATETIME 字段，SQLAlchemy 2.x
native 返回 datetime；如果是 str（方言边界情况）手动解析。
DB 里 stock_code 是 hermes 格式 "SH600000"（带 SH/SZ/BJ 前缀，无点），
2026-07-09 之前误以为 mairui 原始格式 "000001.SZ"，用 split(".")[0] 剥
不到 → 全部股票被 skip → task 把整市场当"全量"重跑(实际是增量)。
修:用正则剥 SH/SZ/BJ 前缀。
bar_time 是 DATETIME 字段，SQLAlchemy 2.x native 返回 datetime；如果是
str（方言边界情况）手动解析。
"""
_ensure_schema()
out: dict[str, Optional[datetime]] = {}
@@ -686,7 +694,13 @@ def get_kline_5min_snapshots() -> dict[str, Optional[datetime]]:
.group_by(Kline5Min.stock_code)
).all()
for raw_code, latest in rows:
code6 = str(raw_code or "").strip().split(".")[0]
# 剥 SH/SZ/BJ 前缀(hermes 格式),容错 6 位裸码
code6 = str(raw_code or "").strip()
for prefix in ("SH", "SZ", "BJ"):
if code6.startswith(prefix):
code6 = code6[len(prefix):]
break
code6 = code6.split(".")[0] # 兜底剥 "000001.SZ" 老格式
if len(code6) != 6 or not code6.isdigit():
continue
if isinstance(latest, datetime):
@@ -834,143 +848,6 @@ def replace_all_stock_node_map(rows: list[dict[str, Any]]) -> None:
s.execute(stmt)
# ── 行业 / 概念板块 ─────────────────────────────────────────────────────
def replace_all_industries(rows: list[dict[str, Any]]) -> None:
"""全量替换 industry 表。rows: code, industry_name, industry_classification, update_date"""
if not rows:
return
with get_session() as s:
s.execute(delete(Industry))
values = [
{
"code": str(r.get("code", "")).zfill(6),
"industry_name": r.get("industry_name") or None,
"industry_classification": r.get("industry_classification") or None,
"update_date": _to_date_str(r.get("update_date")),
}
for r in rows
]
if values:
stmt = _pg_upsert(Industry, values,
conflict_keys=["code"],
update_cols=["industry_name", "industry_classification", "update_date"])
s.execute(stmt)
def fetch_all_industries() -> list[dict[str, Any]]:
_ensure_schema()
with get_session() as s:
rows = s.execute(
select(Industry.code, Industry.industry_name, Industry.industry_classification, Industry.update_date)
).all()
return [
{
"code": r.code,
"industry_name": r.industry_name,
"industry_classification": r.industry_classification,
"update_date": _to_date_str(r.update_date),
}
for r in rows
]
def replace_all_sectors(rows: list[dict[str, Any]]) -> None:
"""rows: sector_key, sector_name, taxonomy, level, source, enabled"""
if not rows:
return
with get_session() as s:
s.execute(delete(Sectors))
values = [
{
"sector_key": r.get("sector_key", ""),
"sector_name": r.get("sector_name", ""),
"taxonomy": r.get("taxonomy", ""),
"level": r.get("level", ""),
"source": r.get("source", ""),
"enabled": int(r.get("enabled", 1)),
}
for r in rows
]
if values:
stmt = _pg_upsert(Sectors, values,
conflict_keys=["sector_key"],
update_cols=["sector_name", "taxonomy", "level", "source", "enabled"])
s.execute(stmt)
def replace_all_stock_sector_map(rows: list[dict[str, Any]]) -> None:
"""rows: stock_code, sector_key"""
if not rows:
return
with get_session() as s:
s.execute(delete(StockSectorMap))
values = [
{
"stock_code": str(r.get("stock_code", "")).zfill(6),
"sector_key": r.get("sector_key", ""),
}
for r in rows
]
if values:
stmt = _pg_upsert(StockSectorMap, values,
conflict_keys=["stock_code"],
update_cols=["sector_key"])
s.execute(stmt)
# ── 行业聚合（衍生）─────────────────────────────────────────────────────
def replace_all_sector_indices(rows: list[dict[str, Any]]) -> None:
"""rows: trade_date, sector_name, close, sector_amplitude"""
if not rows:
return
with get_session() as s:
values = [
{
"trade_date": _to_date_str(r.get("trade_date")),
"sector_name": r.get("sector_name", ""),
"close": float(r.get("close") or 0),
"sector_amplitude": float(r.get("sector_amplitude") or 0),
}
for r in rows
]
if values:
stmt = _pg_upsert(SectorIndices, values,
conflict_keys=["trade_date", "sector_name"],
update_cols=["close", "sector_amplitude"])
s.execute(stmt)
def replace_all_sector_features(rows: list[dict[str, Any]]) -> None:
"""rows: trade_date, sector_name, sector_ret, sector_amplitude, close, ema10, ema20, ema200, score"""
if not rows:
return
with get_session() as s:
values = [
{
"trade_date": _to_date_str(r.get("trade_date")),
"sector_name": r.get("sector_name", ""),
"sector_ret": float(r.get("sector_ret") or 0),
"sector_amplitude": float(r.get("sector_amplitude") or 0),
"close": float(r.get("close") or 0),
"ema10": float(r.get("ema10") or 0),
"ema20": float(r.get("ema20") or 0),
"ema200": float(r.get("ema200") or 0),
"score": int(r.get("score") or 0),
}
for r in rows
]
if values:
stmt = _pg_upsert(SectorFeaturesDaily, values,
conflict_keys=["trade_date", "sector_name"],
update_cols=["sector_ret", "sector_amplitude", "close",
"ema10", "ema20", "ema200", "score"])
s.execute(stmt)
# ── 市场情绪（衍生）─────────────────────────────────────────────────────
@@ -1031,6 +908,51 @@ def upsert_kline_stock_ma_daily_rows(rows: list[dict[str, Any]]) -> None:
s.execute(stmt)
def upsert_kline_stock_macd_daily_rows(rows: list[dict[str, Any]]) -> int:
"""rows: stock_code, trade_date, diff, dea, macd, ema12, ema26, source"""
if not rows:
return 0
with get_session() as s:
return _bulk_upsert_orm(s, KlineStockMACDDaily, rows)
def upsert_kline_stock_kdj_daily_rows(rows: list[dict[str, Any]]) -> int:
"""rows: stock_code, trade_date, k, d, j, source"""
if not rows:
return 0
with get_session() as s:
return _bulk_upsert_orm(s, KlineStockKDJDaily, rows)
def upsert_kline_stock_boll_daily_rows(rows: list[dict[str, Any]]) -> int:
"""rows: stock_code, trade_date, upper, mid, lower, source"""
if not rows:
return 0
with get_session() as s:
return _bulk_upsert_orm(s, KlineStockBOLLDaily, rows)
_INDICATOR_MODEL = {
"macd": KlineStockMACDDaily,
"kdj": KlineStockKDJDaily,
"boll": KlineStockBOLLDaily,
}
def get_indicator_max_date(indicator: str, code: str) -> Optional[str]:
"""某只票某指标已入库的最大 trade_date（YYYY-MM-DD），无则 None。"""
model = _INDICATOR_MODEL.get(indicator.lower())
if model is None:
raise ValueError(f"未知指标: {indicator!r}")
_ensure_schema()
with get_session() as s:
d = s.execute(
select(func.max(model.trade_date))
.where(model.stock_code == code)
).scalar()
return _to_date_str(d) if d else None
def pd_isna(v: Any) -> bool:
"""避免直接 import pandas（开销大），手写 nan/None 检查。"""
if v is None:
@@ -1042,11 +964,26 @@ def pd_isna(v: Any) -> bool:
def get_stock_kline_max_date(code: str) -> Optional[str]:
"""某只票 kline_stock 已入库的最大 trade_date (YYYY-MM-DD),无则 None。
code 接受 6 位 ("600000") 或 hermes 格式 ("SH600000") — DB 里实际存的是
hermes 格式,2026-07-09 之前用 `where KlineStock.stock_code == '600000'`
永远查不到(hermes 有 SH/SZ/BJ 前缀),导致 incremental 把全市场当"无数据"
重跑。修:同时查 6 位和 hermes 两种格式,取较新结果。
"""
_ensure_schema()
c6 = str(code or "").strip()
hermes = f"SH{c6}" if c6.startswith(("5", "6", "9")) else (
f"SZ{c6}" if c6.startswith(("0", "2", "3")) else (
f"BJ{c6}" if c6.startswith(("4", "8")) else c6
))
candidates = [c6]
if hermes != c6:
candidates.append(hermes)
with get_session() as s:
d = s.execute(
select(func.max(KlineStock.trade_date))
.where(KlineStock.stock_code == code)
.where(KlineStock.stock_code.in_(candidates))
).scalar()
return _to_date_str(d) if d else None