diff --git a/app/core/config.py b/app/core/config.py
index 5b8a8b6..6558f60 100644
--- a/app/core/config.py
+++ b/app/core/config.py
@@ -1,4 +1,7 @@
-"""统一配置：从 .env / 环境变量读 MySQL、调度器、数据源开关等参数。"""
+"""统一配置：从 .env / 环境变量读 PG、调度器、数据源开关等参数。
+
+2026-06-16 重构：项目只支持 PostgreSQL，MySQL 字段全部移除。
+"""
from __future__ import annotations
import os
@@ -27,18 +30,6 @@ if _HAS_DOTENV:
class Settings(BaseSettings):
"""应用配置。所有字段都可以通过环境变量或 .env 文件覆盖。"""
- # MySQL
- mysql_host: str = "127.0.0.1"
- mysql_port: int = 3306
- mysql_user: str = "root"
- mysql_password: str = ""
- # 同步任务写入的目标库（原始市场数据）
- mysql_database: str = "market_data_sync_db"
- # 简版库（对照测试，可选）
- mysql_database_lite: str = "grid_seeker"
- # 业务主库（保留— 同步任务不再写入）
- mysql_database_business: str = "grid_seeker_model_base"
-
# Scheduler
scheduler_tick_seconds: int = 5
scheduler_timezone: str = "Asia/Shanghai"
@@ -57,6 +48,14 @@ class Settings(BaseSettings):
ds_sina_enabled: bool = True
xueqiu_token: str = ""
+ # PostgreSQL — 完整 URL（覆盖下面 PG_HOST 等）
+ pg_url: str = ""
+ pg_host: str = "127.0.0.1"
+ pg_port: int = 5432
+ pg_user: str = "market_sync"
+ pg_password: str = "market_sync"
+ pg_db_name: str = "market_data"
+
# Trading calendar
trading_holidays: str = ""
@@ -85,32 +84,15 @@ class Settings(BaseSettings):
return []
return [d.strip() for d in self.trading_holidays.split(",") if d.strip()]
- def mysql_url(self, database: Optional[str] = None) -> str:
- """构造 SQLAlchemy URL。"""
- db = database or self.mysql_database
+ def pg_sqlalchemy_url(self) -> str:
+ """构造 PG 的 SQLAlchemy URL（自动剥掉 +psycopg2 让 SQLAlchemy 决定 driver）。"""
+ if self.pg_url:
+ return self.pg_url
return (
- f"mysql+pymysql://{self.mysql_user}:{self.mysql_password}"
- f"@{self.mysql_host}:{self.mysql_port}/{db}?charset=utf8mb4"
+ f"postgresql+psycopg2://{self.pg_user}:{self.pg_password}"
+ f"@{self.pg_host}:{self.pg_port}/{self.pg_db_name}"
)
- def pymysql_connect_kwargs(self, database: Optional[str] = None) -> dict:
- """构造 pymysql.connect 的关键字参数。
-
- 内网自签名证书场景自动禁用 SSL。
- """
- db = database or self.mysql_database
- return {
- "host": self.mysql_host,
- "port": self.mysql_port,
- "user": self.mysql_user,
- "password": self.mysql_password,
- "database": db,
- "charset": "utf8mb4",
- "autocommit": False,
- # 兼容自签名证书的内网 MySQL — 禁用 SSL
- "ssl": None,
- }
-
_settings: Optional[Settings] = None
diff --git a/app/core/datasource/registry.py b/app/core/datasource/registry.py
index b620aa7..a49442e 100644
--- a/app/core/datasource/registry.py
+++ b/app/core/datasource/registry.py
@@ -65,6 +65,16 @@ _DATASOURCE_SEED: list[tuple[str, dict, str]] = [
},
"K 线主源（日线 + 5min + 指数）",
),
+ (
+ "datasource_akshare_lhb",
+ {
+ "name": "akshare 龙虎榜（东方财富封装）",
+ "provides": ["longhubang_daily", "longhubang_seat"],
+ "requiresCredential": False,
+ "note": "龙虎榜聚合层 + 席位层；无合理替代源，按 [[no-akshare]] 决策放行",
+ },
+ "龙虎榜聚合层 + 席位层（akshare → 东方财富）",
+ ),
]
@@ -247,4 +257,7 @@ def build_default_registry() -> None:
else:
# 即使没 token 也注册，is_available() 会拦截
registry.register(XueqiuSource())
- # akshare 已移除（项目级决策：永远不用）
\ No newline at end of file
+ # akshare：2026-07-01 决策更新——龙虎榜无替代源，烟测通过，可注册。
+ # 见 [[no-akshare]] 记忆：仅"无替代"+"akshare 烟测 OK"才允许用。
+ from app.sources.akshare_lhb import AkshareLhbSource
+ registry.register(AkshareLhbSource())
\ No newline at end of file
diff --git a/app/core/datasource/utils.py b/app/core/datasource/utils.py
index eecfcd0..5fe0dc3 100644
--- a/app/core/datasource/utils.py
+++ b/app/core/datasource/utils.py
@@ -48,6 +48,25 @@ def code6_to_mairui(code6: str) -> str:
return f"{code6}.{code6_to_exchange(code6)}"
+def to_hermes(code: str) -> str:
+ """6 位代码 → 带交易所前缀（项目标准：与 stocks.code 一致）。
+
+ '000001' → 'SZ000001'
+ '600519' → 'SH600519'
+ '830xxx' → 'BJ830xxx'（北证）
+ 'SH600519' → 'SH600519'（已是 hermes 格式则原样返回）
+ """
+ c = str(code).strip().upper()
+ # 已是 hermes 格式
+ if c.startswith(("SH", "SZ", "BJ")) and len(c) >= 8:
+ return c
+ # 6 位纯数字 → 加前缀
+ if len(c) == 6 and c.isdigit():
+ return f"{code6_to_exchange(c)}{c}"
+ # 兜底：原样返回
+ return c
+
+
def normalize_kline(df: pd.DataFrame) -> pd.DataFrame:
"""标准化日 K 线 DataFrame。
diff --git a/app/core/db/__init__.py b/app/core/db/__init__.py
index 6d7e5de..e1d1364 100644
--- a/app/core/db/__init__.py
+++ b/app/core/db/__init__.py
@@ -1,4 +1,4 @@
-"""DB 子包：连接管理 + schema + 业务 CRUD。"""
-from app.core.db.connection import get_mysql, init_mysql_schema, close_mysql
+"""DB 子包：ORM + 业务 CRUD。"""
+from app.core.db.orm import get_session, engine, SessionLocal, ORMBase
-__all__ = ["get_mysql", "init_mysql_schema", "close_mysql"]
+__all__ = ["get_session", "engine", "SessionLocal", "ORMBase"]
diff --git a/app/core/db/connection.py b/app/core/db/connection.py
deleted file mode 100644
index 560e30f..0000000
--- a/app/core/db/connection.py
+++ /dev/null
@@ -1,104 +0,0 @@
-"""MySQL 连接管理（thread-local，模式照搬 dashboard，但适配 market_data_sync 项目）。
-
-要点：
-- pymysql 连接不是线程安全的，每个线程各持一份
-- 内网自签名证书场景：传 ssl_disabled=True 跳过 SSL 校验
-- DATE / DATETIME / TIMESTAMP 统一转字符串，避免下游类型处理
-- DictCursor：返回字典风格行，方便业务拼装
-"""
-from __future__ import annotations
-
-import threading
-from typing import Optional
-
-import pymysql
-import pymysql.constants.FIELD_TYPE
-from pymysql.converters import conversions as _pymysql_conv, convert_date, convert_datetime
-from pymysql.cursors import DictCursor
-
-from app.core.config import settings
-
-_local = threading.local()
-
-
-def _strdate(obj):
- d = convert_date(obj)
- return d.strftime("%Y-%m-%d") if d else None
-
-
-def _strdatetime(obj):
- d = convert_datetime(obj)
- return d.strftime("%Y-%m-%d %H:%M:%S") if d else None
-
-
-# 包装默认 converter，让 DATE / DATETIME / TIMESTAMP 一律出字符串
-_conv = _pymysql_conv.copy()
-_conv[pymysql.constants.FIELD_TYPE.DATE] = _strdate
-_conv[pymysql.constants.FIELD_TYPE.DATETIME] = _strdatetime
-_conv[pymysql.constants.FIELD_TYPE.TIMESTAMP] = _strdatetime
-
-
-def get_mysql(database: Optional[str] = None) -> pymysql.connections.Connection:
- """取当前线程的 MySQL 连接。无则新建。
-
- 关键修复：高并发场景下，连接可能被服务端超时断开 (wait_timeout)。
- 用 ping(reconnect=True) 让 pymysql 自动重连，避免 stale conn 导致
- 2013 Lost connection 错误。
- """
- attr = f"conn_{database or 'default'}"
- conn = getattr(_local, attr, None)
- if conn is not None:
- try:
- conn.ping(reconnect=True)
- except Exception:
- try:
- conn.close()
- except Exception:
- pass
- conn = None
- if conn is None:
- kwargs = dict(
- host=settings.mysql_host,
- port=settings.mysql_port,
- user=settings.mysql_user,
- password=settings.mysql_password,
- database=database or settings.mysql_database,
- charset="utf8mb4",
- cursorclass=DictCursor,
- autocommit=False,
- conv=_conv,
- )
- # 内网自签名证书：禁用 SSL（用户场景）
- kwargs["ssl"] = None
- # 高并发时设较短 read_timeout / write_timeout
- kwargs["read_timeout"] = 30
- kwargs["write_timeout"] = 30
- conn = pymysql.connect(**kwargs)
- setattr(_local, attr, conn)
- return conn
-
-
-def close_mysql() -> None:
- """关闭当前线程的所有连接。"""
- for attr in list(vars(_local).keys()):
- conn = getattr(_local, attr, None)
- if conn is not None:
- try:
- conn.close()
- except Exception:
- pass
- try:
- delattr(_local, attr)
- except AttributeError:
- pass
-
-
-# ── Schema DDL 入口 ────────────────────────────────────────────────────────
-
-
-def init_mysql_schema() -> None:
- """集中创建/补齐所有表。被 db/ops.py 在第一次访问时调用。"""
- # 延后导入避免循环依赖
- from app.core.db.schema import ensure_all_tables
-
- ensure_all_tables(get_mysql())
diff --git a/app/core/db/models.py b/app/core/db/models.py
new file mode 100644
index 0000000..c911200
--- /dev/null
+++ b/app/core/db/models.py
@@ -0,0 +1,479 @@
+"""PostgreSQL ORM 模型（SQLAlchemy 2.x declarative）。
+
+设计原则：
+ - 全部 15 张表用 Python ORM 表达；`Base.metadata.create_all()` 一键建表
+ - 所有表放在 `market_data` schema 下（PG-native）
+ - 类型选择 PG-native：TIMESTAMPTZ / NUMERIC / JSONB / TEXT
+ - 字段名与 MySQL 版尽量保持一致，便于将来数据迁移直接 INSERT ... SELECT
+
+与原 MySQL 版的差异（按列类型一一对照）：
+ - DATETIME → TIMESTAMPTZ
+ - DOUBLE → Float（保持 double precision）
+ - VARCHAR JSON 字段 → JSONB（dataset_registry.dependency_ids）
+ - ON UPDATE CURRENT_TIMESTAMP → onupdate=func.now()（应用层处理，PG 没内置）
+ - PrimaryKeyConstraint → 全部用主键约束表达；复合主键用 __table_args__
+"""
+from __future__ import annotations
+
+from datetime import date, datetime
+from typing import Optional
+
+from sqlalchemy import (
+ Date,
+ DateTime,
+ Float,
+ Index,
+ Integer,
+ PrimaryKeyConstraint,
+ SmallInteger,
+ String,
+ Text,
+ func,
+)
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.orm import Mapped, mapped_column
+
+from app.core.db.orm import ORMBase
+
+
+# ──── 公共 updated_at 工厂 ────
+def _updated_at():
+ """`DEFAULT NOW() ON UPDATE NOW()` —— PG 应用层维护。"""
+ return mapped_column(
+ DateTime(timezone=True),
+ server_default=func.now(),
+ onupdate=func.now(),
+ )
+
+
+# ──────────────────────── 1. config ────────────────────────
+class Config(ORMBase):
+ """键值配置表（.env / 数据源开关 / 调度参数等）。"""
+ __tablename__ = "config"
+ __table_args__ = {"schema": "market_data"}
+
+ key: Mapped[str] = mapped_column(String(128), primary_key=True)
+ value: Mapped[str] = mapped_column(Text, nullable=False)
+ category: Mapped[str] = mapped_column(String(32), nullable=False, default="general")
+ description: Mapped[Optional[str]] = mapped_column(String(255), default="")
+ updated_at: Mapped[Optional[datetime]] = _updated_at()
+
+
+# ──────────────────────── 2. dataset_registry ────────────────────────
+class DatasetRegistry(ORMBase):
+ """同步任务状态机：idle → running → ok / warning / error / blocked。"""
+ __tablename__ = "dataset_registry"
+ __table_args__ = (
+ Index("idx_dataset_registry_sort_order", "sort_order"),
+ {"schema": "market_data"},
+ )
+
+ dataset_id: Mapped[str] = mapped_column(String(64), primary_key=True)
+ name: Mapped[str] = mapped_column(String(128), nullable=False, default="")
+ description: Mapped[Optional[str]] = mapped_column(Text)
+ storage_uri: Mapped[Optional[str]] = mapped_column(String(255), default="")
+ storage_layer: Mapped[Optional[str]] = mapped_column(String(32), default="")
+ management_role: Mapped[Optional[str]] = mapped_column(String(32), default="")
+ source: Mapped[Optional[str]] = mapped_column(String(255), default="")
+ sync_script: Mapped[Optional[str]] = mapped_column(String(255), default="")
+ # 原 schema 是 VARCHAR(4096) DEFAULT '[]'（序列化的 JSON 字符串），PG 改为原生 JSONB
+ dependency_ids: Mapped[Optional[list]] = mapped_column(JSONB, default=list)
+ enabled: Mapped[int] = mapped_column(Integer, default=1)
+ sort_order: Mapped[int] = mapped_column(Integer, default=0)
+ status: Mapped[Optional[str]] = mapped_column(String(16), default="idle")
+ trigger_source: Mapped[Optional[str]] = mapped_column(String(32), default="")
+ started_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True))
+ finished_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True))
+ last_success_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True))
+ last_failure_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True))
+ message: Mapped[Optional[str]] = mapped_column(Text)
+ last_error: Mapped[Optional[str]] = mapped_column(Text)
+ needs_resync: Mapped[int] = mapped_column(Integer, default=0)
+ progress_current: Mapped[int] = mapped_column(Integer, default=0)
+ progress_total: Mapped[int] = mapped_column(Integer, default=0)
+ current_step: Mapped[Optional[str]] = mapped_column(String(255), default="")
+ updated_at: Mapped[Optional[datetime]] = _updated_at()
+
+
+# ──────────────────────── 3. stocks ────────────────────────
+class Stock(ORMBase):
+ """股票基础信息 + 最新股本快照（冗余缓存）。"""
+ __tablename__ = "stocks"
+ __table_args__ = {"schema": "market_data"}
+
+ code: Mapped[str] = mapped_column(String(10), primary_key=True)
+ name: Mapped[str] = mapped_column(String(32), nullable=False, default="")
+ exchange: Mapped[str] = mapped_column(String(8), nullable=False, default="")
+ list_date: Mapped[Optional[date]] = mapped_column(Date)
+ listing_status: Mapped[str] = mapped_column(String(16), nullable=False, default="normal")
+ industry: Mapped[Optional[str]] = mapped_column(String(64), default="")
+ total_share: Mapped[float] = mapped_column(Float, default=0)
+ float_share: Mapped[float] = mapped_column(Float, default=0)
+ share_updated_at: Mapped[Optional[date]] = mapped_column(Date)
+ kline_synced_at: Mapped[Optional[date]] = mapped_column(Date)
+ updated_at: Mapped[Optional[datetime]] = _updated_at()
+
+
+# ──────────────────────── 4. indices ────────────────────────
+# 类的 Python 名 `MarketIndex`（避免和 `from sqlalchemy import Index` 冲突），
+# 但 __tablename__ 仍是 "indices" 跟原 MySQL 一致。
+class MarketIndex(ORMBase):
+ """六大指数基础信息。"""
+ __tablename__ = "indices"
+ __table_args__ = {"schema": "market_data"}
+
+ index_code: Mapped[str] = mapped_column(String(10), primary_key=True)
+ index_name: Mapped[str] = mapped_column(String(64), nullable=False, default="")
+ market: Mapped[Optional[str]] = mapped_column(String(16), default="")
+ category: Mapped[Optional[str]] = mapped_column(String(16), default="")
+ source: Mapped[Optional[str]] = mapped_column(String(32), default="")
+ enabled: Mapped[int] = mapped_column(Integer, default=1)
+ updated_at: Mapped[Optional[datetime]] = _updated_at()
+
+
+# ──────────────────────── 5. kline_stock ────────────────────────
+class KlineStock(ORMBase):
+ """全市场 A 股日频 OHLCV。"""
+ __tablename__ = "kline_stock"
+ __table_args__ = (
+ PrimaryKeyConstraint("stock_code", "trade_date"),
+ Index("idx_kline_stock_date", "trade_date"),
+ {"schema": "market_data"},
+ )
+
+ stock_code: Mapped[str] = mapped_column(String(10), nullable=False)
+ trade_date: Mapped[date] = mapped_column(Date, nullable=False)
+ open: Mapped[float] = mapped_column(Float, nullable=False)
+ high: Mapped[float] = mapped_column(Float, nullable=False)
+ low: Mapped[float] = mapped_column(Float, nullable=False)
+ close: Mapped[float] = mapped_column(Float, nullable=False)
+ volume: Mapped[float] = mapped_column(Float, nullable=False)
+
+
+# ──────────────────────── 6. kline_index ────────────────────────
+class KlineIndex(ORMBase):
+ """六大指数日 K。"""
+ __tablename__ = "kline_index"
+ __table_args__ = (
+ PrimaryKeyConstraint("index_code", "trade_date"),
+ Index("idx_kline_index_date", "trade_date"),
+ {"schema": "market_data"},
+ )
+
+ index_code: Mapped[str] = mapped_column(String(10), nullable=False)
+ trade_date: Mapped[date] = mapped_column(Date, nullable=False)
+ open: Mapped[float] = mapped_column(Float, nullable=False)
+ high: Mapped[float] = mapped_column(Float, nullable=False)
+ low: Mapped[float] = mapped_column(Float, nullable=False)
+ close: Mapped[float] = mapped_column(Float, nullable=False)
+ volume: Mapped[float] = mapped_column(Float, nullable=False)
+
+
+# ──────────────────────── 7. kline_5min ────────────────────────
+class Kline5Min(ORMBase):
+ """全市场 A 股 5 分钟 K 线。"""
+ __tablename__ = "kline_5min"
+ __table_args__ = (
+ PrimaryKeyConstraint("stock_code", "bar_time"),
+ Index("idx_kline_5min_bar_time", "bar_time"),
+ {"schema": "market_data"},
+ )
+
+ stock_code: Mapped[str] = mapped_column(String(10), nullable=False)
+ bar_time: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+ open: Mapped[Optional[float]] = mapped_column(Float)
+ high: Mapped[Optional[float]] = mapped_column(Float)
+ low: Mapped[Optional[float]] = mapped_column(Float)
+ close: Mapped[Optional[float]] = mapped_column(Float)
+ volume: Mapped[Optional[float]] = mapped_column(Float)
+ amount: Mapped[Optional[float]] = mapped_column(Float)
+ turnover_rate: Mapped[Optional[float]] = mapped_column(Float)
+
+
+# ──────────────────────── 8. moneyflow ────────────────────────
+class Moneyflow(ORMBase):
+ """个股资金流（mairui 源：主力/大/中/小单 净额，单位元）。"""
+ __tablename__ = "moneyflow"
+ __table_args__ = (
+ PrimaryKeyConstraint("stock_code", "trade_date"),
+ Index("idx_moneyflow_date", "trade_date"),
+ {"schema": "market_data"},
+ )
+
+ stock_code: Mapped[str] = mapped_column(String(10), nullable=False)
+ trade_date: Mapped[date] = mapped_column(Date, nullable=False)
+ main_net_inflow: Mapped[float] = mapped_column(Float, default=0)
+ large_net_inflow: Mapped[float] = mapped_column(Float, default=0)
+ medium_net_inflow: Mapped[float] = mapped_column(Float, default=0)
+ small_net_inflow: Mapped[float] = mapped_column(Float, default=0)
+
+
+# ──────────────────────── 9. share ────────────────────────
+class Share(ORMBase):
+ """个股股本快照（雪球 quote_detail，单位亿股）。"""
+ __tablename__ = "share"
+ __table_args__ = (
+ PrimaryKeyConstraint("stock_code", "trade_date"),
+ {"schema": "market_data"},
+ )
+
+ stock_code: Mapped[str] = mapped_column(String(10), nullable=False)
+ trade_date: Mapped[date] = mapped_column(Date, nullable=False)
+ total_share: Mapped[float] = mapped_column(Float, nullable=False)
+ float_share: Mapped[float] = mapped_column(Float, nullable=False)
+
+
+# ──────────────────────── 10. sectors ────────────────────────
+class Sectors(ORMBase):
+ """行业字典（baostock 行业分类 / 申万 / 中证 等多种 taxonomy）。"""
+ __tablename__ = "sectors"
+ __table_args__ = (
+ Index("idx_sectors_sector_name", "sector_name"),
+ {"schema": "market_data"},
+ )
+
+ sector_key: Mapped[str] = mapped_column(String(64), primary_key=True)
+ sector_name: Mapped[str] = mapped_column(String(64), nullable=False, default="")
+ taxonomy: Mapped[Optional[str]] = mapped_column(String(64), default="")
+ level: Mapped[Optional[str]] = mapped_column(String(16), default="")
+ source: Mapped[Optional[str]] = mapped_column(String(32), default="")
+ enabled: Mapped[int] = mapped_column(Integer, default=1)
+ updated_at: Mapped[Optional[datetime]] = _updated_at()
+
+
+# ──────────────────────── 11. stock_sector_map ────────────────────────
+class StockSectorMap(ORMBase):
+ """股票-行业多对多映射（一张股票可对应多个行业 / 概念板块）。"""
+ __tablename__ = "stock_sector_map"
+ __table_args__ = (
+ Index("idx_stock_sector_map_sector_key", "sector_key"),
+ {"schema": "market_data"},
+ )
+
+ stock_code: Mapped[str] = mapped_column(String(10), primary_key=True)
+ sector_key: Mapped[str] = mapped_column(String(64), nullable=False, default="")
+ updated_at: Mapped[Optional[datetime]] = _updated_at()
+
+
+# ──────────────────────── 12. industry ────────────────────────
+class Industry(ORMBase):
+ """股票-行业映射（baostock 源，证监会分类标准）。"""
+ __tablename__ = "industry"
+ __table_args__ = (
+ Index("idx_industry_industry_name", "industry_name"),
+ {"schema": "market_data"},
+ )
+
+ code: Mapped[str] = mapped_column(String(10), primary_key=True)
+ industry_name: Mapped[Optional[str]] = mapped_column(String(64), default="")
+ industry_classification: Mapped[Optional[str]] = mapped_column(String(32), default="")
+ update_date: Mapped[Optional[date]] = mapped_column(Date)
+ updated_at: Mapped[Optional[datetime]] = _updated_at()
+
+
+# ──────────────────────── 13. sector_indices ────────────────────────
+class SectorIndices(ORMBase):
+ """行业日线（基点 100，复合收益）。"""
+ __tablename__ = "sector_indices"
+ __table_args__ = (
+ PrimaryKeyConstraint("trade_date", "sector_name"),
+ Index("idx_sector_indices_sector_name", "sector_name"),
+ {"schema": "market_data"},
+ )
+
+ trade_date: Mapped[date] = mapped_column(Date, nullable=False)
+ sector_name: Mapped[str] = mapped_column(String(64), nullable=False)
+ close: Mapped[float] = mapped_column(Float, default=0)
+ sector_amplitude: Mapped[float] = mapped_column(Float, default=0)
+
+
+# ──────────────────────── 14. sector_features_daily ────────────────────────
+class SectorFeaturesDaily(ORMBase):
+ """行业日特征：sector_ret / sector_amplitude / close / EMA / score。"""
+ __tablename__ = "sector_features_daily"
+ __table_args__ = (
+ PrimaryKeyConstraint("trade_date", "sector_name"),
+ Index("idx_sector_features_sector_name", "sector_name"),
+ {"schema": "market_data"},
+ )
+
+ trade_date: Mapped[date] = mapped_column(Date, nullable=False)
+ sector_name: Mapped[str] = mapped_column(String(64), nullable=False)
+ sector_ret: Mapped[float] = mapped_column(Float, default=0)
+ sector_amplitude: Mapped[float] = mapped_column(Float, default=0)
+ close: Mapped[float] = mapped_column(Float, default=0)
+ ema10: Mapped[float] = mapped_column(Float, default=0)
+ ema20: Mapped[float] = mapped_column(Float, default=0)
+ ema200: Mapped[float] = mapped_column(Float, default=0)
+ score: Mapped[int] = mapped_column(Integer, default=0)
+
+
+# ──────────────────────── 15. market_regime_daily ────────────────────────
+class MarketRegimeDaily(ORMBase):
+ """市场情绪衍生指标（基于本地 kline 聚合）。"""
+ __tablename__ = "market_regime_daily"
+ __table_args__ = (
+ Index("idx_market_regime_daily_date", "trade_date"),
+ {"schema": "market_data"},
+ )
+
+ trade_date: Mapped[date] = mapped_column(Date, primary_key=True)
+ advancers: Mapped[float] = mapped_column(Float, default=0)
+ decliners: Mapped[float] = mapped_column(Float, default=0)
+ advance_ratio: Mapped[float] = mapped_column(Float, default=0)
+ turnover: Mapped[float] = mapped_column(Float, default=0)
+ turnover_avg_5d: Mapped[float] = mapped_column(Float, default=0)
+ turnover_ratio_5d: Mapped[float] = mapped_column(Float, default=0)
+ source: Mapped[Optional[str]] = mapped_column(String(32), default="")
+ is_extreme_panic: Mapped[int] = mapped_column(Integer, default=0)
+ updated_at: Mapped[Optional[datetime]] = _updated_at()
+
+
+# ──────────────────────── 16. tick_trade ────────────────────────
+class TickTrade(ORMBase):
+ """个股当天逐笔成交（mairui 源）。
+
+ mairui API（`hsrl/zbjy/{code6}/{licence}`）不返唯一 trade_id；
+ 6 字段组合 `(stock_code, trade_date, trade_time, price, volume, direction_code)`
+ 作为复合主键保证幂等 upsert。
+
+ direction_code: 0=中性盘 / 1=买入 / 2=卖出（mairui 原始值）。
+ direction: 派生 'neutral' / 'buy' / 'sell'（冗余存便于按文本查）。
+ amount: 派生 price * volume（元；mairui 不返）。
+ """
+ __tablename__ = "tick_trade"
+ __table_args__ = (
+ PrimaryKeyConstraint(
+ "stock_code", "trade_date", "trade_time",
+ "price", "volume", "direction_code",
+ ),
+ Index("idx_tick_trade_date", "trade_date"),
+ Index("idx_tick_trade_code_date", "stock_code", "trade_date"),
+ {"schema": "market_data"},
+ )
+
+ stock_code: Mapped[str] = mapped_column(String(10), nullable=False)
+ trade_date: Mapped[date] = mapped_column(Date, nullable=False)
+ trade_time: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+ price: Mapped[float] = mapped_column(Float, nullable=False)
+ volume: Mapped[float] = mapped_column(Float, nullable=False)
+ direction_code: Mapped[int] = mapped_column(Integer, nullable=False)
+ direction: Mapped[str] = mapped_column(String(8), nullable=False, default="")
+ amount: Mapped[float] = mapped_column(Float, nullable=False, default=0)
+
+
+# ──────────────────────── 17. longhubang_daily ────────────────────────
+class LonghubangDaily(ORMBase):
+ """龙虎榜每日上榜股票汇总（聚合层，akshare 源）。
+
+ 数据源：`ak.stock_lhb_detail_em(start_date, end_date)` —— 东方财富私有 API 封装。
+ 每个上榜日 × 每只上榜股票 = 1 行。
+
+ 字段（21 列）：
+ 基础：rank_idx, stock_code, stock_name, trade_date, comment
+ 当日：close, pct_chg
+ 资金：lhb_net_buy, lhb_buy_amt, lhb_sell_amt, lhb_total_amt, market_total_amt
+ 比率：net_buy_ratio, total_amt_ratio, turnover_rate, float_mv
+ 原因：reason
+ 后效：post_1d_pct, post_2d_pct, post_5d_pct, post_10d_pct
+
+ 注意：`stock_code` 含交易所前缀（SH600000 / SZ000001），与项目其他表一致。
+ """
+ __tablename__ = "longhubang_daily"
+ __table_args__ = (
+ PrimaryKeyConstraint("stock_code", "trade_date"),
+ Index("idx_lhb_daily_date", "trade_date"),
+ {"schema": "market_data"},
+ )
+
+ stock_code: Mapped[str] = mapped_column(String(10), nullable=False)
+ stock_name: Mapped[str] = mapped_column(String(64), nullable=False, default="")
+ trade_date: Mapped[date] = mapped_column(Date, nullable=False)
+ rank_idx: Mapped[Optional[int]] = mapped_column(Integer, nullable=True)
+ comment: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
+ close: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
+ pct_chg: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
+ lhb_net_buy: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
+ lhb_buy_amt: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
+ lhb_sell_amt: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
+ lhb_total_amt: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
+ market_total_amt: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
+ net_buy_ratio: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
+ total_amt_ratio: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
+ turnover_rate: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
+ float_mv: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
+ reason: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
+ post_1d_pct: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
+ post_2d_pct: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
+ post_5d_pct: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
+ post_10d_pct: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
+
+
+# ──────────────────────── 18. longhubang_seat ────────────────────────
+class LonghubangSeat(ORMBase):
+ """龙虎榜单只票买卖前五营业部（席位层，akshare 源）。
+
+ 数据源：`ak.stock_lhb_stock_detail_em(symbol, date, flag)` —— 东方财富私有 API 封装。
+ 每只上榜票 × 每方向（buy/sell）= 5 行（共 10 行/票/日）。
+
+ 字段（11 列 + 上榜原因）：
+ 基础：stock_code, trade_date, direction（buy/sell）, rank_idx
+ 营业部：branch_name（akshare 不返唯一 branch_code，用 name 作 PK 兜底）
+ 金额：buy_amount, sell_amount, amount, net_amount, amount_ratio
+ 当日：pct_chg, close, stock_total_amt
+ 原因：explanation
+
+ 注意：akshare 的 `stock_lhb_stock_detail_em` 不返 `branch_code`，只用 `branch_name`
+ 作 PK 兜底（不同营业部名冲突概率极低；如未来需要可补 eastmoney 私有 API 抓 code）。
+ """
+ __tablename__ = "longhubang_seat"
+ __table_args__ = (
+ PrimaryKeyConstraint("stock_code", "trade_date", "direction", "rank_idx"),
+ Index("idx_lhb_seat_date", "trade_date"),
+ Index("idx_lhb_seat_branch_name", "branch_name"),
+ {"schema": "market_data"},
+ )
+
+ stock_code: Mapped[str] = mapped_column(String(10), nullable=False)
+ trade_date: Mapped[date] = mapped_column(Date, nullable=False)
+ direction: Mapped[str] = mapped_column(String(8), nullable=False) # "buy" / "sell"
+ rank_idx: Mapped[int] = mapped_column(Integer, nullable=False)
+ branch_name: Mapped[Optional[str]] = mapped_column(String(128), nullable=True)
+ buy_amount: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
+ sell_amount: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
+ amount: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
+ net_amount: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
+ amount_ratio: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
+ pct_chg: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
+ close: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
+ stock_total_amt: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
+ explanation: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
+
+
+__all__ = [
+ # 1-2
+ "Config",
+ "DatasetRegistry",
+ # 3-4
+ "Stock",
+ "MarketIndex",
+ # 5-9
+ "KlineStock",
+ "KlineIndex",
+ "Kline5Min",
+ "Moneyflow",
+ "Share",
+ # 10-12
+ "Sectors",
+ "StockSectorMap",
+ "Industry",
+ # 13-15
+ "SectorIndices",
+ "SectorFeaturesDaily",
+ "MarketRegimeDaily",
+ # 16
+ "TickTrade",
+ # 17-18 (2026-07-01 龙虎榜)
+ "LonghubangDaily",
+ "LonghubangSeat",
+]
diff --git a/app/core/db/ops.py b/app/core/db/ops.py
index 18bb7ac..1e6c7f5 100644
--- a/app/core/db/ops.py
+++ b/app/core/db/ops.py
@@ -1,110 +1,173 @@
-"""MySQL 业务 CRUD 函数集合（参考 dashboard/api/services/mysql_ops.py）。
+"""PG 业务 CRUD 函数集合（SQLAlchemy 2.x ORM，PostgreSQL only）。
-本文件只包含 market_data_sync 同步任务需要的最小函数集，不涉及 dashboard 的
-accounts / positions / trade_record / scoring 等业务表。
+2026-06-16 重构：项目只支持 PG。MySQL 路径已删除。
设计要点：
-- 每个 ensure_xxx_table() 调用会触发 init_mysql_schema()（幂等）
-- 批量 upsert 走 executemany，效率高
-- 统一使用 DictCursor
+ - 用 SQLAlchemy `select()` / `pg_insert().on_conflict_do_update()` / `update()` / `delete()` 拼 ORM 操作
+ - datetime/date 类型经 `_to_dt_str()` / `_to_date_str()` 转字符串返回（与原 pymysql 行为一致）
+ - dict 形状保留：消费者按 `row["key"]` 索引，所以每个 SELECT 用 `.scalars()` + 列表推导 或 `_row_to_dict`
+ - 增量查询 (`get_*_max_date`) 返回 `Optional[str]` (`YYYY-MM-DD`) 而不是 `date`，行为不变
+ - 所有表在 PG `market_data` schema 下；建表由 ``app.core.db.pg_bootstrap`` 负责
"""
from __future__ import annotations
import json
-from contextlib import contextmanager
-from datetime import datetime
+from datetime import date, datetime
from typing import Any, Iterable, Optional
-from app.core.db.connection import get_mysql, init_mysql_schema
+from sqlalchemy import and_, case, delete, exists, func, or_, select, update
+from sqlalchemy.dialects.postgresql import insert as pg_insert
+
+from app.core.db.models import (
+ Config,
+ DatasetRegistry,
+ Industry,
+ Kline5Min,
+ KlineIndex,
+ KlineStock,
+ LonghubangDaily,
+ LonghubangSeat,
+ MarketIndex,
+ MarketRegimeDaily,
+ Moneyflow,
+ SectorFeaturesDaily,
+ SectorIndices,
+ Sectors,
+ Share,
+ Stock,
+ StockSectorMap,
+ TickTrade,
+)
+from app.core.db.orm import get_session
+
+
+# ──────── PG upsert helper ────────
+
+
+def _pg_upsert(model, rows, conflict_keys, update_cols=None):
+ """PG upsert：ON CONFLICT (cols) DO UPDATE SET ... = EXCLUDED.。
+
+ conflict_keys: 主键列列表（用于 ON CONFLICT）
+ update_cols: 要在 upsert 时更新的列（None=除 PK 外的所有列）
+ """
+ if update_cols is None:
+ update_cols = [c.name for c in model.__table__.columns if c.name not in conflict_keys]
+ stmt = pg_insert(model).values(rows)
+ return stmt.on_conflict_do_update(
+ index_elements=conflict_keys,
+ set_={c: getattr(stmt.excluded, c) for c in update_cols},
+ )
# ── 内部 helpers ────────────────────────────────────────────────────────
-def _now_local_ts() -> str:
- return datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-
-
-def _today_str() -> str:
- return datetime.now().strftime("%Y-%m-%d")
-
-
-def _date_or_none(val) -> str | None:
+def _to_dt_str(val) -> Optional[str]:
+ """datetime / date / str → 'YYYY-MM-DD HH:MM:SS' 或 None。"""
if val is None:
return None
+ if isinstance(val, datetime):
+ return val.strftime("%Y-%m-%d %H:%M:%S")
+ if isinstance(val, date):
+ return val.strftime("%Y-%m-%d %H:%M:%S")
s = str(val).strip()
return s if s else None
-@contextmanager
-def _cursor():
- """带 init_mysql_schema 的 cursor 上下文。"""
- init_mysql_schema()
- conn = get_mysql()
- cur = conn.cursor()
- try:
- yield cur
- conn.commit()
- except Exception:
- conn.rollback()
- raise
- finally:
- cur.close()
+def _to_date_str(val) -> Optional[str]:
+ """date / datetime / str → 'YYYY-MM-DD' 或 None。"""
+ if val is None:
+ return None
+ if isinstance(val, datetime):
+ return val.date().strftime("%Y-%m-%d")
+ if isinstance(val, date):
+ return val.strftime("%Y-%m-%d")
+ s = str(val).strip()
+ return s if s else None
+
+
+def _row_to_dict(row) -> dict[str, Any]:
+ """SQLAlchemy Row → dict（保留原 pymysql DictCursor 行为）。"""
+ return row._mapping if hasattr(row, "_mapping") else dict(row._asdict())
+
+
+def _ensure_schema() -> None:
+ """所有读函数开头调用一次，幂等。
+
+ PG 的表由 ``app.core.db.pg_bootstrap`` 一次性建好（人工触发），这里 no-op 即可。
+ 保留这个函数是为了不破坏 9 个 sync task 的调用约定（行 14 个调用点）。
+ """
+ pass
# ── config 表 ───────────────────────────────────────────────────────────
def fetch_all_config() -> list[dict[str, Any]]:
- init_mysql_schema()
- conn = get_mysql()
- with conn.cursor() as cur:
- cur.execute("SELECT `key`, `value`, category, description, updated_at FROM config")
- return cur.fetchall()
+ _ensure_schema()
+ with get_session() as s:
+ rows = s.execute(select(Config)).scalars().all()
+ return [
+ {
+ "key": r.key,
+ "value": r.value,
+ "category": r.category,
+ "description": r.description or "",
+ "updated_at": _to_dt_str(r.updated_at),
+ }
+ for r in rows
+ ]
def fetch_config_by_key(key: str) -> dict[str, Any] | None:
"""按 key 查单条 config（比 fetch_all_config 高效）。"""
- init_mysql_schema()
- conn = get_mysql()
- with conn.cursor() as cur:
- cur.execute(
- "SELECT `key`, `value`, category, description, updated_at FROM config WHERE `key` = %s",
- (key,),
- )
- return cur.fetchone()
+ _ensure_schema()
+ with get_session() as s:
+ r = s.get(Config, key)
+ if r is None:
+ return None
+ return {
+ "key": r.key,
+ "value": r.value,
+ "category": r.category,
+ "description": r.description or "",
+ "updated_at": _to_dt_str(r.updated_at),
+ }
def fetch_configs_by_category(category: str) -> list[dict[str, Any]]:
"""按 category 查 config（比 fetch_all_config + 内存过滤高效）。"""
- init_mysql_schema()
- conn = get_mysql()
- with conn.cursor() as cur:
- cur.execute(
- "SELECT `key`, `value`, category, description, updated_at FROM config WHERE category = %s",
- (category,),
- )
- return cur.fetchall()
+ _ensure_schema()
+ with get_session() as s:
+ rows = s.execute(
+ select(Config).where(Config.category == category)
+ ).scalars().all()
+ return [
+ {
+ "key": r.key,
+ "value": r.value,
+ "category": r.category,
+ "description": r.description or "",
+ "updated_at": _to_dt_str(r.updated_at),
+ }
+ for r in rows
+ ]
def upsert_config(key: str, value: str, category: str = "general", description: str = "") -> None:
- with _cursor() as cur:
- cur.execute(
- """
- INSERT INTO config (`key`, `value`, category, description)
- VALUES (%s, %s, %s, %s)
- ON DUPLICATE KEY UPDATE
- `value` = VALUES(`value`),
- category = VALUES(category),
- description = VALUES(description)
- """,
- (key, value, category, description),
+ with get_session() as s:
+ stmt = _pg_upsert(
+ Config,
+ [{"key": key, "value": value, "category": category, "description": description}],
+ conflict_keys=["key"],
+ update_cols=["value", "category", "description"],
)
+ s.execute(stmt)
def delete_config(key: str) -> None:
- with _cursor() as cur:
- cur.execute("DELETE FROM config WHERE `key` = %s", (key,))
+ with get_session() as s:
+ s.execute(delete(Config).where(Config.key == key))
# ── dataset_registry ────────────────────────────────────────────────────
@@ -114,52 +177,75 @@ def upsert_dataset_registry_rows(rows: list[dict[str, Any]]) -> None:
"""批量 upsert 同步任务定义到 dataset_registry。"""
if not rows:
return
- with _cursor() as cur:
- for r in rows:
- dep_json = r.get("dependency_ids")
- if isinstance(dep_json, (list, dict)):
- dep_json = json.dumps(dep_json, ensure_ascii=False)
- cur.execute(
- """
- INSERT INTO dataset_registry
- (dataset_id, name, description, storage_uri, storage_layer,
- management_role, source, sync_script, dependency_ids,
- enabled, sort_order)
- VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
- ON DUPLICATE KEY UPDATE
- name = VALUES(name),
- description = VALUES(description),
- storage_uri = VALUES(storage_uri),
- storage_layer = VALUES(storage_layer),
- management_role = VALUES(management_role),
- source = VALUES(source),
- sync_script = VALUES(sync_script),
- dependency_ids = VALUES(dependency_ids),
- enabled = VALUES(enabled),
- sort_order = VALUES(sort_order)
- """,
- (
- r.get("dataset_id", ""),
- r.get("name", ""),
- r.get("description", ""),
- r.get("storage_uri", ""),
- r.get("storage_layer", ""),
- r.get("management_role", ""),
- r.get("source", ""),
- r.get("sync_script", ""),
- dep_json or "[]",
- r.get("enabled", 1),
- r.get("sort_order", 0),
- ),
- )
+ # 把 dict / list 的 dependency_ids 序列化成 JSON 字符串（PG 的 JSONB 字段会再反序列化）
+ cleaned = []
+ for r in rows:
+ dep = r.get("dependency_ids")
+ if isinstance(dep, (list, dict)):
+ dep = json.dumps(dep, ensure_ascii=False)
+ elif dep is None:
+ dep = "[]"
+ cleaned.append({
+ "dataset_id": r.get("dataset_id", ""),
+ "name": r.get("name", ""),
+ "description": r.get("description", ""),
+ "storage_uri": r.get("storage_uri", ""),
+ "storage_layer": r.get("storage_layer", ""),
+ "management_role": r.get("management_role", ""),
+ "source": r.get("source", ""),
+ "sync_script": r.get("sync_script", ""),
+ "dependency_ids": dep,
+ "enabled": r.get("enabled", 1),
+ "sort_order": r.get("sort_order", 0),
+ })
+ with get_session() as s:
+ stmt = _pg_upsert(
+ DatasetRegistry,
+ cleaned,
+ conflict_keys=["dataset_id"],
+ update_cols=["name", "description", "storage_uri", "storage_layer",
+ "management_role", "source", "sync_script",
+ "dependency_ids", "enabled", "sort_order"],
+ )
+ s.execute(stmt)
def fetch_dataset_registry_rows() -> list[dict[str, Any]]:
- init_mysql_schema()
- conn = get_mysql()
- with conn.cursor() as cur:
- cur.execute("SELECT * FROM dataset_registry ORDER BY sort_order, dataset_id")
- return cur.fetchall()
+ _ensure_schema()
+ with get_session() as s:
+ rows = s.execute(
+ select(DatasetRegistry).order_by(DatasetRegistry.sort_order, DatasetRegistry.dataset_id)
+ ).scalars().all()
+ return [_dataset_registry_row_to_dict(r) for r in rows]
+
+
+def _dataset_registry_row_to_dict(r: DatasetRegistry) -> dict[str, Any]:
+ return {
+ "dataset_id": r.dataset_id,
+ "name": r.name or "",
+ "description": r.description,
+ "storage_uri": r.storage_uri or "",
+ "storage_layer": r.storage_layer or "",
+ "management_role": r.management_role or "",
+ "source": r.source or "",
+ "sync_script": r.sync_script or "",
+ "dependency_ids": r.dependency_ids or "[]",
+ "enabled": r.enabled if r.enabled is not None else 1,
+ "sort_order": r.sort_order if r.sort_order is not None else 0,
+ "status": r.status or "idle",
+ "trigger_source": r.trigger_source or "",
+ "started_at": _to_dt_str(r.started_at),
+ "finished_at": _to_dt_str(r.finished_at),
+ "last_success_at": _to_dt_str(r.last_success_at),
+ "last_failure_at": _to_dt_str(r.last_failure_at),
+ "message": r.message,
+ "last_error": r.last_error,
+ "needs_resync": r.needs_resync if r.needs_resync is not None else 0,
+ "progress_current": r.progress_current if r.progress_current is not None else 0,
+ "progress_total": r.progress_total if r.progress_total is not None else 0,
+ "current_step": r.current_step or "",
+ "updated_at": _to_dt_str(r.updated_at),
+ }
def fetch_dataset_registry_map() -> dict[str, dict[str, Any]]:
@@ -181,33 +267,35 @@ def update_dataset_registry_state(dataset_id: str, **kwargs: Any) -> None:
if not payload:
return
if "updated_at" not in payload:
- payload["updated_at"] = _now_local_ts()
- set_clause = ", ".join(f"`{k}` = %s" for k in payload)
- values = list(payload.values()) + [dataset_id]
- with _cursor() as cur:
- cur.execute(
- f"UPDATE dataset_registry SET {set_clause} WHERE dataset_id = %s",
- values,
+ # ORM 的 onupdate 会在 commit 时自动刷新；但显式赋值更可靠
+ payload["updated_at"] = datetime.now()
+ with get_session() as s:
+ s.execute(
+ update(DatasetRegistry)
+ .where(DatasetRegistry.dataset_id == dataset_id)
+ .values(**payload)
)
def recover_interrupted_dataset_registry() -> int:
"""把状态卡在 running 的同步任务标记为 failed（启动时调用）。"""
- with _cursor() as cur:
- cur.execute(
- """
- UPDATE dataset_registry
- SET status = 'failed',
- finished_at = %s,
- last_failure_at = %s,
- last_error = CONCAT('进程在状态为 running 时被中断，已自动恢复。', IFNULL(last_error, '')),
- message = CONCAT('[recovered] ', IFNULL(message, '同步被中断')),
- needs_resync = 1
- WHERE status = 'running'
- """,
- (_now_local_ts(), _now_local_ts()),
- )
- return cur.rowcount
+ with get_session() as s:
+ now = datetime.now()
+ # CONCAT 在 ORM 跨方言时较麻烦 —— 直接读 + 写
+ running = s.execute(
+ select(DatasetRegistry).where(DatasetRegistry.status == "running")
+ ).scalars().all()
+ for r in running:
+ r.status = "failed"
+ r.finished_at = now
+ r.last_failure_at = now
+ r.last_error = (
+ f"进程在状态为 running 时被中断，已自动恢复。{r.last_error or ''}"
+ )
+ r.message = f"[recovered] {r.message or '同步被中断'}"
+ r.needs_resync = 1
+ s.add_all(running)
+ return len(running)
# ── stocks ──────────────────────────────────────────────────────────────
@@ -232,18 +320,14 @@ def upsert_stock(
"""
del list_date # 显式不接受 list_date 更新（旧值保留）
del industry # 同上
- with _cursor() as cur:
- cur.execute(
- """
- INSERT INTO stocks (code, name, exchange, listing_status)
- VALUES (%s, %s, %s, %s)
- ON DUPLICATE KEY UPDATE
- name = VALUES(name),
- exchange = VALUES(exchange),
- listing_status = VALUES(listing_status)
- """,
- (code, name, exchange, listing_status),
+ with get_session() as s:
+ stmt = _pg_upsert(
+ Stock,
+ [{"code": code, "name": name, "exchange": exchange, "listing_status": listing_status}],
+ conflict_keys=["code"],
+ update_cols=["name", "exchange", "listing_status"],
)
+ s.execute(stmt)
def upsert_stocks_bulk(rows: list[dict[str, Any]], chunk_size: int = 500) -> int:
@@ -254,72 +338,89 @@ def upsert_stocks_bulk(rows: list[dict[str, Any]], chunk_size: int = 500) -> int
"""
if not rows:
return 0
- sql = (
- "INSERT INTO stocks (code, name, exchange, listing_status) "
- "VALUES (%s, %s, %s, %s) "
- "ON DUPLICATE KEY UPDATE "
- " name = VALUES(name), "
- " exchange = VALUES(exchange), "
- " listing_status = VALUES(listing_status)"
- )
- with _cursor() as cur:
- total = 0
+ total = 0
+ with get_session() as s:
for i in range(0, len(rows), chunk_size):
chunk = rows[i: i + chunk_size]
values = [
- (r.get("code", ""), r.get("name", ""), r.get("exchange", ""), r.get("listing_status", "normal"))
+ {
+ "code": r.get("code", ""),
+ "name": r.get("name", ""),
+ "exchange": r.get("exchange", ""),
+ "listing_status": r.get("listing_status", "normal"),
+ }
for r in chunk
]
- cur.executemany(sql, values)
+ # executemany-style via dialect-aware upsert
+ stmt = _pg_upsert(
+ Stock, values,
+ conflict_keys=["code"],
+ update_cols=["name", "exchange", "list_date", "listing_status", "industry"],
+ )
+ s.execute(stmt)
total += len(chunk)
- return total
+ return total
def update_stock_share_snapshot(code: str, total_share: float, float_share: float, trade_date: str = "") -> None:
- with _cursor() as cur:
- cur.execute(
- """
- UPDATE stocks
- SET total_share = %s, float_share = %s,
- share_updated_at = COALESCE(NULLIF(%s, ''), CURDATE())
- WHERE code = %s
- """,
- (float(total_share), float(float_share), trade_date or "", code),
+ with get_session() as s:
+ snap_date = _to_date_str(trade_date) if trade_date else None
+ s.execute(
+ update(Stock)
+ .where(Stock.code == code)
+ .values(
+ total_share=float(total_share),
+ float_share=float(float_share),
+ share_updated_at=snap_date,
+ )
)
def update_stock_kline_synced_at(code: str, synced_at: str) -> None:
- with _cursor() as cur:
- cur.execute(
- "UPDATE stocks SET kline_synced_at = %s WHERE code = %s",
- (_date_or_none(synced_at), code),
+ with get_session() as s:
+ s.execute(
+ update(Stock)
+ .where(Stock.code == code)
+ .values(kline_synced_at=_to_date_str(synced_at))
)
def fetch_all_stocks() -> list[dict[str, Any]]:
- init_mysql_schema()
- conn = get_mysql()
- with conn.cursor() as cur:
- cur.execute("SELECT * FROM stocks ORDER BY code")
- return cur.fetchall()
+ _ensure_schema()
+ with get_session() as s:
+ rows = s.execute(select(Stock).order_by(Stock.code)).scalars().all()
+ return [_stock_row_to_dict(r) for r in rows]
+
+
+def _stock_row_to_dict(r: Stock) -> dict[str, Any]:
+ return {
+ "code": r.code,
+ "name": r.name or "",
+ "exchange": r.exchange or "",
+ "list_date": _to_date_str(r.list_date),
+ "listing_status": r.listing_status or "normal",
+ "industry": r.industry or "",
+ "total_share": r.total_share if r.total_share is not None else 0,
+ "float_share": r.float_share if r.float_share is not None else 0,
+ "share_updated_at": _to_date_str(r.share_updated_at),
+ "kline_synced_at": _to_date_str(r.kline_synced_at),
+ "updated_at": _to_dt_str(r.updated_at),
+ }
def fetch_stock_by_code(code: str) -> Optional[dict[str, Any]]:
- init_mysql_schema()
- conn = get_mysql()
- with conn.cursor() as cur:
- cur.execute("SELECT * FROM stocks WHERE code = %s", (code,))
- row = cur.fetchone()
- return dict(row) if row else None
+ _ensure_schema()
+ with get_session() as s:
+ r = s.get(Stock, code)
+ if r is None:
+ return None
+ return _stock_row_to_dict(r)
def stock_count() -> int:
- init_mysql_schema()
- conn = get_mysql()
- with conn.cursor() as cur:
- cur.execute("SELECT COUNT(*) AS cnt FROM stocks")
- row = cur.fetchone()
- return int(row["cnt"]) if row else 0
+ _ensure_schema()
+ with get_session() as s:
+ return s.execute(select(func.count()).select_from(Stock)).scalar() or 0
def iter_stock_codes(active_only: bool = True) -> Iterable[str]:
@@ -349,112 +450,101 @@ def upsert_index(
source: str = "",
enabled: bool = True,
) -> None:
- with _cursor() as cur:
- cur.execute(
- """
- INSERT INTO indices (index_code, index_name, market, category, source, enabled)
- VALUES (%s, %s, %s, %s, %s, %s)
- ON DUPLICATE KEY UPDATE
- index_name = VALUES(index_name),
- market = VALUES(market),
- category = VALUES(category),
- source = VALUES(source),
- enabled = VALUES(enabled)
- """,
- (index_code, index_name, market, category, source, int(enabled)),
+ with get_session() as s:
+ stmt = _pg_upsert(
+ MarketIndex,
+ [{
+ "index_code": index_code, "index_name": index_name,
+ "market": market, "category": category,
+ "source": source, "enabled": int(enabled),
+ }],
+ conflict_keys=["index_code"],
+ update_cols=["index_name", "market", "category", "source", "enabled"],
)
+ s.execute(stmt)
# ── kline 通用批量 upsert ──────────────────────────────────────────────
-def _bulk_upsert(
- cur,
- table: str,
- columns: list[str],
+def _bulk_upsert_orm(
+ s,
+ model_cls,
rows: list[dict[str, Any]],
- conflict_keys: list[str],
chunk_size: int = 1000,
) -> int:
- """通用批量 INSERT ... ON DUPLICATE KEY UPDATE 工具。"""
+ """通用批量 upsert（PG only）：INSERT ... ON CONFLICT DO UPDATE SET。
+
+ model_cls 的所有 mapped_column 都参与 upsert（PK 列也参与 INSERT，作为 ON CONFLICT 目标）。
+ """
if not rows:
return 0
- placeholders = ", ".join(["%s"] * len(columns))
- col_list = ", ".join(f"`{c}`" for c in columns)
- update_clause = ", ".join(
- f"`{c}` = VALUES(`{c}`)"
- for c in columns
- if c not in conflict_keys
- )
- sql = (
- f"INSERT INTO {table} ({col_list}) VALUES ({placeholders}) "
- f"ON DUPLICATE KEY UPDATE {update_clause}"
- )
+ # PK 列名（用于 ON CONFLICT）
+ pk_cols = [c.name for c in model_cls.__table__.primary_key.columns]
+ # 全部 ORM 列名（含 PK）— INSERT 需要全部列
+ all_cols = [c.name for c in model_cls.__table__.columns]
+ # 非 PK 列名（用于 DO UPDATE SET）
+ update_cols = [c for c in all_cols if c not in pk_cols]
total = 0
for i in range(0, len(rows), chunk_size):
chunk = rows[i: i + chunk_size]
- values = [
- tuple(_date_or_none(r.get(c)) if c.endswith("date") else r.get(c) for c in columns)
- for r in chunk
- ]
- cur.executemany(sql, values)
+ values = []
+ for r in chunk:
+ row_dict = {}
+ for c in all_cols:
+ v = r.get(c)
+ # DATE/DATETIME 字段：空串 → None，让 DB default 生效
+ if isinstance(v, str) and c.endswith(("date", "time")) and v.strip() == "":
+ v = None
+ row_dict[c] = v
+ values.append(row_dict)
+ stmt = _pg_upsert(model_cls, values,
+ conflict_keys=pk_cols,
+ update_cols=update_cols)
+ s.execute(stmt)
total += len(chunk)
return total
def upsert_kline_stock(rows: list[dict[str, Any]]) -> int:
"""rows: stock_code, trade_date, open, high, low, close, volume"""
- with _cursor() as cur:
- return _bulk_upsert(
- cur, "kline_stock",
- ["stock_code", "trade_date", "open", "high", "low", "close", "volume"],
- rows, conflict_keys=["stock_code", "trade_date"],
- )
+ with get_session() as s:
+ return _bulk_upsert_orm(s, KlineStock, rows)
def upsert_kline_index(rows: list[dict[str, Any]]) -> int:
"""rows: index_code, trade_date, open, high, low, close, volume"""
- with _cursor() as cur:
- return _bulk_upsert(
- cur, "kline_index",
- ["index_code", "trade_date", "open", "high", "low", "close", "volume"],
- rows, conflict_keys=["index_code", "trade_date"],
- )
+ with get_session() as s:
+ return _bulk_upsert_orm(s, KlineIndex, rows)
def upsert_kline_5min(rows: list[dict[str, Any]]) -> int:
"""rows: stock_code, bar_time, open, high, low, close, volume, amount, turnover_rate"""
- with _cursor() as cur:
- return _bulk_upsert(
- cur, "kline_5min",
- ["stock_code", "bar_time", "open", "high", "low", "close", "volume", "amount", "turnover_rate"],
- rows, conflict_keys=["stock_code", "bar_time"],
- )
+ with get_session() as s:
+ return _bulk_upsert_orm(s, Kline5Min, rows)
def get_kline_5min_snapshots() -> dict[str, Optional[datetime]]:
"""一次 SQL 拉 kline_5min 全表每只股票的最新 bar_time。
返回：{stock_code (6位): max(bar_time) 或 None}
- 用于 kline_5min 任务的"全量/增量"统一规划：
- - DB 里有 → 增量（从 max - 2 天 到今天）
- - DB 里没 → 全量（从 mairui 历史深度起点 到今天）
+ 用于 kline_5min 任务的"全量/增量"统一规划。
DB 里 stock_code 形如 "000001.SZ"（mairui 写入格式），函数剥掉
- 交易所后缀统一为 6 位。bar_time 是 DATETIME 字段，pymysql DictCursor
- 在某些连接参数下会返回 str，这里手动解析为 datetime（解析失败置 None）。
+ 交易所后缀统一为 6 位。bar_time 是 DATETIME 字段，SQLAlchemy 2.x
+ native 返回 datetime；如果是 str（方言边界情况）手动解析。
"""
- init_mysql_schema()
- conn = get_mysql()
+ _ensure_schema()
out: dict[str, Optional[datetime]] = {}
- with conn.cursor() as cur:
- cur.execute("SELECT stock_code, MAX(bar_time) AS latest FROM kline_5min GROUP BY stock_code")
- for r in cur.fetchall():
- raw_code = str(r["stock_code"] or "").strip()
- code6 = raw_code.split(".")[0] # "000001.SZ" → "000001"
+ with get_session() as s:
+ rows = s.execute(
+ select(Kline5Min.stock_code, func.max(Kline5Min.bar_time).label("latest"))
+ .group_by(Kline5Min.stock_code)
+ ).all()
+ for raw_code, latest in rows:
+ code6 = str(raw_code or "").strip().split(".")[0]
if len(code6) != 6 or not code6.isdigit():
continue
- latest = r["latest"]
if isinstance(latest, datetime):
pass
elif isinstance(latest, str) and latest:
@@ -470,22 +560,45 @@ def get_kline_5min_snapshots() -> dict[str, Optional[datetime]]:
def upsert_moneyflow(rows: list[dict[str, Any]]) -> int:
"""rows: stock_code, trade_date, main_net_inflow, large_net_inflow, medium_net_inflow, small_net_inflow"""
- with _cursor() as cur:
- return _bulk_upsert(
- cur, "moneyflow",
- ["stock_code", "trade_date", "main_net_inflow", "large_net_inflow", "medium_net_inflow", "small_net_inflow"],
- rows, conflict_keys=["stock_code", "trade_date"],
- )
+ with get_session() as s:
+ return _bulk_upsert_orm(s, Moneyflow, rows)
def upsert_share(rows: list[dict[str, Any]]) -> int:
"""rows: stock_code, trade_date, total_share, float_share"""
- with _cursor() as cur:
- return _bulk_upsert(
- cur, "share",
- ["stock_code", "trade_date", "total_share", "float_share"],
- rows, conflict_keys=["stock_code", "trade_date"],
- )
+ with get_session() as s:
+ return _bulk_upsert_orm(s, Share, rows)
+
+
+def upsert_tick_trade(rows: list[dict[str, Any]]) -> int:
+ """rows: stock_code, trade_date, trade_time, price, volume,
+ direction_code, direction, amount。
+
+ chunk_size 调到 2000（比 5min K 的 1000 还小）—— 单只股票单日 tick
+ 量大（10万+），避免一次 pg_insert 撑爆 driver。
+ """
+ with get_session() as s:
+ return _bulk_upsert_orm(s, TickTrade, rows, chunk_size=2000)
+
+
+def upsert_longhubang_daily(rows: list[dict[str, Any]]) -> int:
+ """rows: 21 列聚合层字段（见 LonghubangDaily ORM 注释）。
+ 每日全市场最多 ~100 行上榜票，chunk=500 足够。
+ """
+ if not rows:
+ return 0
+ with get_session() as s:
+ return _bulk_upsert_orm(s, LonghubangDaily, rows, chunk_size=500)
+
+
+def upsert_longhubang_seat(rows: list[dict[str, Any]]) -> int:
+ """rows: 席位层字段（见 LonghubangSeat ORM 注释）。
+ 每日全市场最多 ~1000 行（100 只票 × 2 方向 × 5 行），chunk=500 足够。
+ """
+ if not rows:
+ return 0
+ with get_session() as s:
+ return _bulk_upsert_orm(s, LonghubangSeat, rows, chunk_size=500)
# ── 行业 / 概念板块 ─────────────────────────────────────────────────────
@@ -495,79 +608,83 @@ def replace_all_industries(rows: list[dict[str, Any]]) -> None:
"""全量替换 industry 表。rows: code, industry_name, industry_classification, update_date"""
if not rows:
return
- with _cursor() as cur:
- cur.execute("DELETE FROM industry")
- for r in rows:
- cur.execute(
- """
- INSERT INTO industry (code, industry_name, industry_classification, update_date)
- VALUES (%s, %s, %s, %s)
- ON DUPLICATE KEY UPDATE
- industry_name = VALUES(industry_name),
- industry_classification = VALUES(industry_classification),
- update_date = VALUES(update_date)
- """,
- (
- str(r.get("code", "")).zfill(6),
- r.get("industry_name") or None,
- r.get("industry_classification") or None,
- _date_or_none(r.get("update_date")),
- ),
- )
+ with get_session() as s:
+ s.execute(delete(Industry))
+ values = [
+ {
+ "code": str(r.get("code", "")).zfill(6),
+ "industry_name": r.get("industry_name") or None,
+ "industry_classification": r.get("industry_classification") or None,
+ "update_date": _to_date_str(r.get("update_date")),
+ }
+ for r in rows
+ ]
+ if values:
+ stmt = _pg_upsert(Industry, values,
+ conflict_keys=["code"],
+ update_cols=["industry_name", "industry_classification", "update_date"])
+ s.execute(stmt)
def fetch_all_industries() -> list[dict[str, Any]]:
- init_mysql_schema()
- conn = get_mysql()
- with conn.cursor() as cur:
- cur.execute("SELECT code, industry_name, industry_classification, update_date FROM industry")
- return cur.fetchall()
+ _ensure_schema()
+ with get_session() as s:
+ rows = s.execute(
+ select(Industry.code, Industry.industry_name, Industry.industry_classification, Industry.update_date)
+ ).all()
+ return [
+ {
+ "code": r.code,
+ "industry_name": r.industry_name,
+ "industry_classification": r.industry_classification,
+ "update_date": _to_date_str(r.update_date),
+ }
+ for r in rows
+ ]
def replace_all_sectors(rows: list[dict[str, Any]]) -> None:
"""rows: sector_key, sector_name, taxonomy, level, source, enabled"""
if not rows:
return
- with _cursor() as cur:
- cur.execute("DELETE FROM sectors")
- for r in rows:
- cur.execute(
- """
- INSERT INTO sectors (sector_key, sector_name, taxonomy, level, source, enabled)
- VALUES (%s, %s, %s, %s, %s, %s)
- ON DUPLICATE KEY UPDATE
- sector_name = VALUES(sector_name),
- taxonomy = VALUES(taxonomy),
- level = VALUES(level),
- source = VALUES(source),
- enabled = VALUES(enabled)
- """,
- (
- r.get("sector_key", ""),
- r.get("sector_name", ""),
- r.get("taxonomy", ""),
- r.get("level", ""),
- r.get("source", ""),
- int(r.get("enabled", 1)),
- ),
- )
+ with get_session() as s:
+ s.execute(delete(Sectors))
+ values = [
+ {
+ "sector_key": r.get("sector_key", ""),
+ "sector_name": r.get("sector_name", ""),
+ "taxonomy": r.get("taxonomy", ""),
+ "level": r.get("level", ""),
+ "source": r.get("source", ""),
+ "enabled": int(r.get("enabled", 1)),
+ }
+ for r in rows
+ ]
+ if values:
+ stmt = _pg_upsert(Sectors, values,
+ conflict_keys=["sector_key"],
+ update_cols=["sector_name", "taxonomy", "level", "source", "enabled"])
+ s.execute(stmt)
def replace_all_stock_sector_map(rows: list[dict[str, Any]]) -> None:
"""rows: stock_code, sector_key"""
if not rows:
return
- with _cursor() as cur:
- cur.execute("DELETE FROM stock_sector_map")
- for r in rows:
- cur.execute(
- """
- INSERT INTO stock_sector_map (stock_code, sector_key)
- VALUES (%s, %s)
- ON DUPLICATE KEY UPDATE sector_key = VALUES(sector_key)
- """,
- (str(r.get("stock_code", "")).zfill(6), r.get("sector_key", "")),
- )
+ with get_session() as s:
+ s.execute(delete(StockSectorMap))
+ values = [
+ {
+ "stock_code": str(r.get("stock_code", "")).zfill(6),
+ "sector_key": r.get("sector_key", ""),
+ }
+ for r in rows
+ ]
+ if values:
+ stmt = _pg_upsert(StockSectorMap, values,
+ conflict_keys=["stock_code"],
+ update_cols=["sector_key"])
+ s.execute(stmt)
# ── 行业聚合（衍生）─────────────────────────────────────────────────────
@@ -577,57 +694,48 @@ def replace_all_sector_indices(rows: list[dict[str, Any]]) -> None:
"""rows: trade_date, sector_name, close, sector_amplitude"""
if not rows:
return
- with _cursor() as cur:
- for r in rows:
- cur.execute(
- """
- INSERT INTO sector_indices (trade_date, sector_name, `close`, sector_amplitude)
- VALUES (%s, %s, %s, %s)
- ON DUPLICATE KEY UPDATE
- `close` = VALUES(`close`),
- sector_amplitude = VALUES(sector_amplitude)
- """,
- (
- _date_or_none(r.get("trade_date")),
- r.get("sector_name", ""),
- float(r.get("close") or 0),
- float(r.get("sector_amplitude") or 0),
- ),
- )
+ with get_session() as s:
+ values = [
+ {
+ "trade_date": _to_date_str(r.get("trade_date")),
+ "sector_name": r.get("sector_name", ""),
+ "close": float(r.get("close") or 0),
+ "sector_amplitude": float(r.get("sector_amplitude") or 0),
+ }
+ for r in rows
+ ]
+ if values:
+ stmt = _pg_upsert(SectorIndices, values,
+ conflict_keys=["trade_date", "sector_name"],
+ update_cols=["close", "sector_amplitude"])
+ s.execute(stmt)
def replace_all_sector_features(rows: list[dict[str, Any]]) -> None:
"""rows: trade_date, sector_name, sector_ret, sector_amplitude, close, ema10, ema20, ema200, score"""
if not rows:
return
- with _cursor() as cur:
- for r in rows:
- cur.execute(
- """
- INSERT INTO sector_features_daily
- (trade_date, sector_name, sector_ret, sector_amplitude, `close`, ema10, ema20, ema200, score)
- VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
- ON DUPLICATE KEY UPDATE
- sector_ret = VALUES(sector_ret),
- sector_amplitude = VALUES(sector_amplitude),
- `close` = VALUES(`close`),
- ema10 = VALUES(ema10),
- ema20 = VALUES(ema20),
- ema200 = VALUES(ema200),
- score = VALUES(score)
- """,
- (
- _date_or_none(r.get("trade_date")),
- r.get("sector_name", ""),
- float(r.get("sector_ret") or 0),
- float(r.get("sector_amplitude") or 0),
- float(r.get("close") or 0),
- float(r.get("ema10") or 0),
- float(r.get("ema20") or 0),
- float(r.get("ema200") or 0),
- int(r.get("score") or 0),
- ),
- )
+ with get_session() as s:
+ values = [
+ {
+ "trade_date": _to_date_str(r.get("trade_date")),
+ "sector_name": r.get("sector_name", ""),
+ "sector_ret": float(r.get("sector_ret") or 0),
+ "sector_amplitude": float(r.get("sector_amplitude") or 0),
+ "close": float(r.get("close") or 0),
+ "ema10": float(r.get("ema10") or 0),
+ "ema20": float(r.get("ema20") or 0),
+ "ema200": float(r.get("ema200") or 0),
+ "score": int(r.get("score") or 0),
+ }
+ for r in rows
+ ]
+ if values:
+ stmt = _pg_upsert(SectorFeaturesDaily, values,
+ conflict_keys=["trade_date", "sector_name"],
+ update_cols=["sector_ret", "sector_amplitude", "close",
+ "ema10", "ema20", "ema200", "score"])
+ s.execute(stmt)
# ── 市场情绪（衍生）─────────────────────────────────────────────────────
@@ -637,92 +745,68 @@ def upsert_market_regime_rows(rows: list[dict[str, Any]]) -> None:
"""rows: trade_date, advancers, decliners, advance_ratio, turnover, turnover_avg_5d, turnover_ratio_5d, source, is_extreme_panic"""
if not rows:
return
- with _cursor() as cur:
- for r in rows:
- cur.execute(
- """
- INSERT INTO market_regime_daily
- (trade_date, advancers, decliners, advance_ratio, turnover,
- turnover_avg_5d, turnover_ratio_5d, source, is_extreme_panic)
- VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
- ON DUPLICATE KEY UPDATE
- advancers = VALUES(advancers),
- decliners = VALUES(decliners),
- advance_ratio = VALUES(advance_ratio),
- turnover = VALUES(turnover),
- turnover_avg_5d = VALUES(turnover_avg_5d),
- turnover_ratio_5d = VALUES(turnover_ratio_5d),
- source = VALUES(source),
- is_extreme_panic = VALUES(is_extreme_panic)
- """,
- (
- _date_or_none(r.get("trade_date")),
- float(r.get("advancers") or 0),
- float(r.get("decliners") or 0),
- float(r.get("advance_ratio") or 0),
- float(r.get("turnover") or 0),
- float(r.get("turnover_avg_5d") or 0),
- float(r.get("turnover_ratio_5d") or 0),
- r.get("source", ""),
- int(r.get("is_extreme_panic") or 0),
- ),
- )
+ with get_session() as s:
+ values = [
+ {
+ "trade_date": _to_date_str(r.get("trade_date")),
+ "advancers": float(r.get("advancers") or 0),
+ "decliners": float(r.get("decliners") or 0),
+ "advance_ratio": float(r.get("advance_ratio") or 0),
+ "turnover": float(r.get("turnover") or 0),
+ "turnover_avg_5d": float(r.get("turnover_avg_5d") or 0),
+ "turnover_ratio_5d": float(r.get("turnover_ratio_5d") or 0),
+ "source": r.get("source", ""),
+ "is_extreme_panic": int(r.get("is_extreme_panic") or 0),
+ }
+ for r in rows
+ ]
+ if values:
+ stmt = _pg_upsert(MarketRegimeDaily, values,
+ conflict_keys=["trade_date"],
+ update_cols=["advancers", "decliners", "advance_ratio",
+ "turnover", "turnover_avg_5d", "turnover_ratio_5d",
+ "source", "is_extreme_panic", "updated_at"])
+ s.execute(stmt)
# ── kline 查询（同步时用于判断增量起点）─────────────────────────────────
def get_stock_kline_max_date(code: str) -> Optional[str]:
- init_mysql_schema()
- conn = get_mysql()
- with conn.cursor() as cur:
- cur.execute(
- "SELECT MAX(trade_date) AS d FROM kline_stock WHERE stock_code = %s",
- (code,),
- )
- row = cur.fetchone()
- if row and row.get("d"):
- return str(row["d"])
- return None
+ _ensure_schema()
+ with get_session() as s:
+ d = s.execute(
+ select(func.max(KlineStock.trade_date))
+ .where(KlineStock.stock_code == code)
+ ).scalar()
+ return _to_date_str(d) if d else None
def get_index_kline_max_date(index_code: str) -> Optional[str]:
- init_mysql_schema()
- conn = get_mysql()
- with conn.cursor() as cur:
- cur.execute(
- "SELECT MAX(trade_date) AS d FROM kline_index WHERE index_code = %s",
- (index_code,),
- )
- row = cur.fetchone()
- if row and row.get("d"):
- return str(row["d"])
- return None
+ _ensure_schema()
+ with get_session() as s:
+ d = s.execute(
+ select(func.max(KlineIndex.trade_date))
+ .where(KlineIndex.index_code == index_code)
+ ).scalar()
+ return _to_date_str(d) if d else None
def get_5min_max_bar_time(code: str) -> Optional[str]:
- init_mysql_schema()
- conn = get_mysql()
- with conn.cursor() as cur:
- cur.execute(
- "SELECT MAX(bar_time) AS d FROM kline_5min WHERE stock_code = %s",
- (code,),
- )
- row = cur.fetchone()
- if row and row.get("d"):
- return str(row["d"])
- return None
+ _ensure_schema()
+ with get_session() as s:
+ d = s.execute(
+ select(func.max(Kline5Min.bar_time))
+ .where(Kline5Min.stock_code == code)
+ ).scalar()
+ return _to_dt_str(d) if d else None
def get_moneyflow_max_date(code: str) -> Optional[str]:
- init_mysql_schema()
- conn = get_mysql()
- with conn.cursor() as cur:
- cur.execute(
- "SELECT MAX(trade_date) AS d FROM moneyflow WHERE stock_code = %s",
- (code,),
- )
- row = cur.fetchone()
- if row and row.get("d"):
- return str(row["d"])
- return None
+ _ensure_schema()
+ with get_session() as s:
+ d = s.execute(
+ select(func.max(Moneyflow.trade_date))
+ .where(Moneyflow.stock_code == code)
+ ).scalar()
+ return _to_date_str(d) if d else None
diff --git a/app/core/db/orm.py b/app/core/db/orm.py
new file mode 100644
index 0000000..3ad7664
--- /dev/null
+++ b/app/core/db/orm.py
@@ -0,0 +1,59 @@
+"""SQLAlchemy 2.x ORM 基础设施（PostgreSQL only）。
+
+2026-06-16 重构：项目放弃 MySQL，只保留 PG。
+所有模型 class 仍带 ``__table_args__ = {"schema": "market_data"}``，指向 PG schema。
+PG 数据库 / schema / role / 表 由 ``app.core.db.pg_bootstrap`` 一次性建好。
+"""
+from __future__ import annotations
+
+from contextlib import contextmanager
+from typing import Iterator
+
+from sqlalchemy import create_engine
+from sqlalchemy.engine import Engine
+from sqlalchemy.orm import DeclarativeBase, Session, sessionmaker
+
+from app.core.config import settings
+
+
+class ORMBase(DeclarativeBase):
+ """所有 ORM 模型的基类。"""
+
+
+def _build_engine() -> Engine:
+ """构造 PG 的 SQLAlchemy Engine。"""
+ return create_engine(
+ settings.pg_sqlalchemy_url(),
+ pool_pre_ping=True,
+ pool_recycle=3600,
+ future=True,
+ echo=False,
+ )
+
+
+engine: Engine = _build_engine()
+
+# sessionmaker 工厂
+SessionLocal = sessionmaker(
+ bind=engine,
+ expire_on_commit=False,
+ autoflush=False,
+ future=True,
+)
+
+
+@contextmanager
+def get_session() -> Iterator[Session]:
+ """事务上下文：异常自动 rollback，正常自动 commit。"""
+ s = SessionLocal()
+ try:
+ yield s
+ s.commit()
+ except Exception:
+ s.rollback()
+ raise
+ finally:
+ s.close()
+
+
+__all__ = ["ORMBase", "engine", "SessionLocal", "get_session"]
diff --git a/app/core/db/pg_bootstrap.py b/app/core/db/pg_bootstrap.py
new file mode 100644
index 0000000..0244b5b
--- /dev/null
+++ b/app/core/db/pg_bootstrap.py
@@ -0,0 +1,235 @@
+"""PostgreSQL bootstrap — 一键建库 / 建 schema / 建 role / 建表。
+
+用法：
+ .venv/bin/python -m app.core.db.pg_bootstrap
+
+按顺序做：
+ 1) 以 superuser 身份连默认 'postgres' 库
+ 2) CREATE DATABASE market_data (IF NOT EXISTS)
+ 3) \\connect 到 market_data
+ 4) CREATE SCHEMA market_data, grid_seeker
+ 5) CREATE ROLE market_sync LOGIN PASSWORD ... (从 .env 读 PG_MARKET_SYNC_PASSWORD)
+ 6) GRANT 权限 + ALTER DEFAULT PRIVILEGES
+ 7) SQLAlchemy ORM Base.metadata.create_all() 一键建所有表
+
+幂等：所有 DDL 都用 IF NOT EXISTS / OR REPLACE；可以重复跑。
+"""
+from __future__ import annotations
+
+import os
+
+import psycopg2
+from psycopg2 import sql
+from sqlalchemy import create_engine, text
+from sqlalchemy.engine import URL
+
+from app.core.db.orm import ORMBase as Base
+# 重要：必须显式 import models 模块，所有 @mapped_column 才会注册到 ORMBase.metadata
+# 漏了这一步 Base.metadata.tables 是空的，create_all() 不会建任何表
+# 改用 getattr 触发副作用，绕开 IDE "unused import" 警告
+_ = __import__("app.core.db.models", fromlist=["*"])
+
+
+# ──── 配置（从 .env 读） ────
+
+def _env(key: str, default: str) -> str:
+ return os.environ.get(key, default).strip() or default
+
+
+SUPERUSER = _env("PG_SUPERUSER", "postgres")
+SUPERPASS = _env("PG_SUPERUSER_PASSWORD", "postgres")
+SUPERHOST = _env("PG_SUPERHOST", "127.0.0.1")
+SUPERPORT = int(_env("PG_SUPERPORT", "5432"))
+
+DB_NAME = _env("PG_DB_NAME", "market_data")
+SCHEMAS = _env("PG_SCHEMAS", "market_data,grid_seeker").split(",")
+
+MARKET_SYNC_ROLE = _env("PG_MARKET_SYNC_ROLE", "market_sync")
+MARKET_SYNC_PASS = _env("PG_MARKET_SYNC_PASSWORD", "market_sync")
+
+LOG_PREFIX = "[pg_bootstrap]"
+
+
+def log(msg: str) -> None:
+ print(f"{LOG_PREFIX} {msg}", flush=True)
+
+
+def _connect(dbname: str):
+ return psycopg2.connect(
+ host=SUPERHOST, port=SUPERPORT,
+ user=SUPERUSER, password=SUPERPASS,
+ dbname=dbname, connect_timeout=10,
+ )
+
+
+# ──── Step 1: 创建数据库 ────
+
+def step1_create_database() -> None:
+ log(f"Step 1: CREATE DATABASE {DB_NAME} (if not exists)")
+ conn = _connect("postgres")
+ conn.autocommit = True
+ try:
+ with conn.cursor() as cur:
+ cur.execute("SELECT 1 FROM pg_database WHERE datname = %s", (DB_NAME,))
+ if cur.fetchone():
+ log(f" ✓ {DB_NAME} 已存在，跳过")
+ else:
+ cur.execute(sql.SQL("CREATE DATABASE {}").format(sql.Identifier(DB_NAME)))
+ log(f" ✓ {DB_NAME} 已创建")
+ finally:
+ conn.close()
+
+
+# ──── Step 2: 创建 schema ────
+
+def step2_create_schemas() -> None:
+ log(f"Step 2: CREATE SCHEMA for {SCHEMAS}")
+ conn = _connect(DB_NAME)
+ conn.autocommit = True
+ try:
+ with conn.cursor() as cur:
+ for schema in SCHEMAS:
+ schema = schema.strip()
+ if not schema:
+ continue
+ cur.execute(
+ sql.SQL("CREATE SCHEMA IF NOT EXISTS {}").format(sql.Identifier(schema))
+ )
+ log(f" ✓ schema '{schema}' 就绪")
+ finally:
+ conn.close()
+
+
+# ──── Step 3: 创建角色 + 授权 ────
+
+def step3_create_role() -> None:
+ log(f"Step 3: CREATE ROLE {MARKET_SYNC_ROLE} + GRANT")
+ conn = _connect(DB_NAME)
+ conn.autocommit = True
+ try:
+ with conn.cursor() as cur:
+ # 角色（IF NOT EXISTS 9.0+ 支持，旧版会报错 — 我们用 try/except 兜底）
+ cur.execute(
+ "SELECT 1 FROM pg_roles WHERE rolname = %s",
+ (MARKET_SYNC_ROLE,),
+ )
+ if cur.fetchone():
+ log(f" ✓ role '{MARKET_SYNC_ROLE}' 已存在，跳过")
+ else:
+ cur.execute(
+ sql.SQL("CREATE ROLE {} LOGIN PASSWORD %s").format(
+ sql.Identifier(MARKET_SYNC_ROLE)
+ ),
+ (MARKET_SYNC_PASS,),
+ )
+ log(f" ✓ role '{MARKET_SYNC_ROLE}' 已创建")
+
+ # 连接 DB 权限
+ cur.execute(
+ sql.SQL("GRANT CONNECT ON DATABASE {} TO {}").format(
+ sql.Identifier(DB_NAME), sql.Identifier(MARKET_SYNC_ROLE)
+ )
+ )
+
+ # 4 个 schema 都有 USAGE
+ for schema in SCHEMAS:
+ schema = schema.strip()
+ if not schema:
+ continue
+ cur.execute(
+ sql.SQL("GRANT USAGE ON SCHEMA {} TO {}").format(
+ sql.Identifier(schema), sql.Identifier(MARKET_SYNC_ROLE)
+ )
+ )
+
+ # market_data: RW（当前项目用的）
+ cur.execute(
+ sql.SQL("GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA market_data TO {}").format(
+ sql.Identifier(MARKET_SYNC_ROLE)
+ )
+ )
+ cur.execute(
+ sql.SQL("GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA market_data TO {}").format(
+ sql.Identifier(MARKET_SYNC_ROLE)
+ )
+ )
+ # grid_seeker: 暂时也 RW（未来 grid_seeker 自己做权限时再收紧）
+ cur.execute(
+ sql.SQL("GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA grid_seeker TO {}").format(
+ sql.Identifier(MARKET_SYNC_ROLE)
+ )
+ )
+ cur.execute(
+ sql.SQL("GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA grid_seeker TO {}").format(
+ sql.Identifier(MARKET_SYNC_ROLE)
+ )
+ )
+
+ # 关键：未来新表自动继承（不然新加的表 market_sync 没权限）
+ for schema in SCHEMAS:
+ schema = schema.strip()
+ if not schema:
+ continue
+ cur.execute(
+ sql.SQL(
+ "ALTER DEFAULT PRIVILEGES IN SCHEMA {s} "
+ "GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO {r}"
+ ).format(s=sql.Identifier(schema), r=sql.Identifier(MARKET_SYNC_ROLE))
+ )
+ cur.execute(
+ sql.SQL(
+ "ALTER DEFAULT PRIVILEGES IN SCHEMA {s} "
+ "GRANT USAGE, SELECT ON SEQUENCES TO {r}"
+ ).format(s=sql.Identifier(schema), r=sql.Identifier(MARKET_SYNC_ROLE))
+ )
+ log(f" ✓ {MARKET_SYNC_ROLE} 在所有 schema 上有 RW + 未来新表自动继承")
+ finally:
+ conn.close()
+
+
+# ──── Step 4: SQLAlchemy ORM 建所有表 ────
+
+def step4_create_tables() -> None:
+ log("Step 4: SQLAlchemy Base.metadata.create_all() 建所有表")
+ # 临时以 superuser 连接 market_data（建表后权限自动由 ALTER DEFAULT 接管）
+ url = URL.create(
+ drivername="postgresql+psycopg2",
+ username=SUPERUSER, password=SUPERPASS,
+ host=SUPERHOST, port=SUPERPORT,
+ database=DB_NAME,
+ )
+ engine = create_engine(url, future=True, echo=False)
+ try:
+ Base.metadata.create_all(engine)
+ with engine.connect() as c:
+ rows = c.execute(
+ text(
+ "SELECT table_schema, table_name FROM information_schema.tables "
+ "WHERE table_schema IN ('market_data', 'grid_seeker') "
+ "ORDER BY table_schema, table_name"
+ )
+ ).fetchall()
+ log(f" ✓ 当前 schema 下的表（{len(rows)} 张）:")
+ for schema, tbl in rows:
+ log(f" - {schema}.{tbl}")
+ finally:
+ engine.dispose()
+
+
+# ──── main ────
+
+def main() -> None:
+ log(f"连接 superuser {SUPERUSER}@{SUPERHOST}:{SUPERPORT}")
+ step1_create_database()
+ step2_create_schemas()
+ step3_create_role()
+ step4_create_tables()
+ log("✅ PG bootstrap 完成")
+ log(f" DB: {DB_NAME}")
+ log(f" 角色: {MARKET_SYNC_ROLE} (password 见 .env PG_MARKET_SYNC_PASSWORD)")
+ log(f" schemas: {', '.join(SCHEMAS)}")
+ log(f" 下一步：在 .env 设 PG_URL=postgresql+psycopg2://{MARKET_SYNC_ROLE}:{MARKET_SYNC_PASS}@{SUPERHOST}:{SUPERPORT}/{DB_NAME}")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/app/core/db/schema.py b/app/core/db/schema.py
deleted file mode 100644
index b4fb520..0000000
--- a/app/core/db/schema.py
+++ /dev/null
@@ -1,264 +0,0 @@
-"""Schema DDL — 集中所有 CREATE TABLE IF NOT EXISTS。
-
-参考 dashboard/api/services/mysql_conn.py::init_mysql_schema()，但补齐 dashboard 缺的几张表
-（kline_stock / kline_5min / kline_index / moneyflow / share / sector_features_daily / sector_indices
-/ stock_scores / registry），这些是 grid_seeker_model_base 生产库已存在但 mysql_conn.py 没建的表。
-
-所有 DDL 写在同一个文件，方便审阅、diff 和补字段。"""
-from __future__ import annotations
-
-from typing import TYPE_CHECKING
-
-if TYPE_CHECKING:
- import pymysql.connections
-
-
-# DDL 列表（顺序无关，全部 IF NOT EXISTS）
-_DDL: list[str] = [
- # ─── config ─────────────────────────────────────────────────────────
- """
- CREATE TABLE IF NOT EXISTS config (
- `key` VARCHAR(128) PRIMARY KEY,
- `value` TEXT NOT NULL,
- category VARCHAR(32) NOT NULL DEFAULT 'general',
- description VARCHAR(255) DEFAULT '',
- updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
- ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
- """,
- # ─── dataset_registry (同步状态机) ────────────────────────────────────
- """
- CREATE TABLE IF NOT EXISTS dataset_registry (
- dataset_id VARCHAR(64) PRIMARY KEY,
- name VARCHAR(128) NOT NULL DEFAULT '',
- description TEXT,
- storage_uri VARCHAR(255) DEFAULT '',
- storage_layer VARCHAR(32) DEFAULT '',
- management_role VARCHAR(32) DEFAULT '',
- source VARCHAR(255) DEFAULT '',
- sync_script VARCHAR(255) DEFAULT '',
- dependency_ids VARCHAR(4096) DEFAULT '[]',
- enabled INT DEFAULT 1,
- sort_order INT DEFAULT 0,
- status VARCHAR(16) DEFAULT 'idle',
- trigger_source VARCHAR(32) DEFAULT '',
- started_at DATETIME DEFAULT NULL,
- finished_at DATETIME DEFAULT NULL,
- last_success_at DATETIME DEFAULT NULL,
- last_failure_at DATETIME DEFAULT NULL,
- message TEXT,
- last_error TEXT,
- needs_resync INT DEFAULT 0,
- progress_current INT DEFAULT 0,
- progress_total INT DEFAULT 0,
- current_step VARCHAR(255) DEFAULT '',
- updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
- INDEX idx_dataset_registry_sort_order (sort_order)
- ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
- """,
- # ─── stocks (基础信息 + 股本字段) ─────────────────────────────────────
- """
- CREATE TABLE IF NOT EXISTS stocks (
- code VARCHAR(10) PRIMARY KEY,
- name VARCHAR(32) NOT NULL DEFAULT '',
- exchange VARCHAR(8) NOT NULL DEFAULT '',
- list_date DATE DEFAULT NULL,
- listing_status VARCHAR(16) NOT NULL DEFAULT 'normal',
- industry VARCHAR(64) DEFAULT '',
- total_share DOUBLE DEFAULT 0,
- float_share DOUBLE DEFAULT 0,
- share_updated_at DATE DEFAULT NULL,
- kline_synced_at DATE DEFAULT NULL,
- updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
- ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
- """,
- # ─── indices (指数字典) ───────────────────────────────────────────────
- """
- CREATE TABLE IF NOT EXISTS indices (
- index_code VARCHAR(10) PRIMARY KEY,
- index_name VARCHAR(64) NOT NULL DEFAULT '',
- market VARCHAR(16) DEFAULT '',
- category VARCHAR(16) DEFAULT '',
- source VARCHAR(32) DEFAULT '',
- enabled INT DEFAULT 1,
- updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
- ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
- """,
- # ─── kline_stock (日 K 线，按股票) ────────────────────────────────────
- """
- CREATE TABLE IF NOT EXISTS kline_stock (
- stock_code VARCHAR(10) NOT NULL,
- trade_date DATE NOT NULL,
- `open` DOUBLE NOT NULL,
- `high` DOUBLE NOT NULL,
- `low` DOUBLE NOT NULL,
- `close` DOUBLE NOT NULL,
- volume DOUBLE NOT NULL,
- PRIMARY KEY (stock_code, trade_date),
- INDEX idx_kline_stock_date (trade_date)
- ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
- """,
- # ─── kline_index (指数日 K 线) ────────────────────────────────────────
- """
- CREATE TABLE IF NOT EXISTS kline_index (
- index_code VARCHAR(10) NOT NULL,
- trade_date DATE NOT NULL,
- `open` DOUBLE NOT NULL,
- `high` DOUBLE NOT NULL,
- `low` DOUBLE NOT NULL,
- `close` DOUBLE NOT NULL,
- volume DOUBLE NOT NULL,
- PRIMARY KEY (index_code, trade_date),
- INDEX idx_kline_index_date (trade_date)
- ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
- """,
- # ─── kline_5min (5 分钟 K 线) ────────────────────────────────────────
- """
- CREATE TABLE IF NOT EXISTS kline_5min (
- stock_code VARCHAR(10) NOT NULL,
- bar_time DATETIME NOT NULL,
- `open` DOUBLE DEFAULT NULL,
- `high` DOUBLE DEFAULT NULL,
- `low` DOUBLE DEFAULT NULL,
- `close` DOUBLE DEFAULT NULL,
- volume DOUBLE DEFAULT NULL,
- amount DOUBLE DEFAULT NULL,
- turnover_rate DOUBLE DEFAULT NULL,
- PRIMARY KEY (stock_code, bar_time),
- INDEX idx_kline_5min_bar_time (bar_time)
- ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
- """,
- # ─── moneyflow (资金流，主力/大/中/小单净额) ──────────────────────────
- """
- CREATE TABLE IF NOT EXISTS moneyflow (
- stock_code VARCHAR(10) NOT NULL,
- trade_date DATE NOT NULL,
- main_net_inflow DOUBLE DEFAULT 0,
- large_net_inflow DOUBLE DEFAULT 0,
- medium_net_inflow DOUBLE DEFAULT 0,
- small_net_inflow DOUBLE DEFAULT 0,
- PRIMARY KEY (stock_code, trade_date),
- INDEX idx_moneyflow_date (trade_date)
- ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
- """,
- # ─── share (股本快照) ────────────────────────────────────────────────
- """
- CREATE TABLE IF NOT EXISTS share (
- stock_code VARCHAR(10) NOT NULL,
- trade_date DATE NOT NULL,
- total_share DOUBLE NOT NULL,
- float_share DOUBLE NOT NULL,
- PRIMARY KEY (stock_code, trade_date)
- ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
- """,
- # ─── sectors (行业字典) ──────────────────────────────────────────────
- """
- CREATE TABLE IF NOT EXISTS sectors (
- sector_key VARCHAR(64) PRIMARY KEY,
- sector_name VARCHAR(64) NOT NULL DEFAULT '',
- taxonomy VARCHAR(64) DEFAULT '',
- level VARCHAR(16) DEFAULT '',
- source VARCHAR(32) DEFAULT '',
- enabled INT DEFAULT 1,
- updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
- INDEX idx_sectors_sector_name (sector_name)
- ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
- """,
- # ─── stock_sector_map (股票→行业映射) ─────────────────────────────────
- """
- CREATE TABLE IF NOT EXISTS stock_sector_map (
- stock_code VARCHAR(6) PRIMARY KEY,
- sector_key VARCHAR(64) NOT NULL DEFAULT '',
- updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
- INDEX idx_stock_sector_map_sector_key (sector_key)
- ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
- """,
- # ─── industry (股票-行业映射，datasource 实际写入用这张) ──────────────
- """
- CREATE TABLE IF NOT EXISTS industry (
- code VARCHAR(6) PRIMARY KEY,
- industry_name VARCHAR(64) DEFAULT '',
- industry_classification VARCHAR(32) DEFAULT '',
- update_date DATE DEFAULT NULL,
- updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
- INDEX idx_industry_industry_name (industry_name)
- ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
- """,
- # ─── sector_indices (行业聚合指数时序) ─────────────────────────────────
- """
- CREATE TABLE IF NOT EXISTS sector_indices (
- trade_date DATE NOT NULL,
- sector_name VARCHAR(64) NOT NULL,
- `close` DOUBLE DEFAULT 0,
- sector_amplitude DOUBLE DEFAULT 0,
- PRIMARY KEY (trade_date, sector_name),
- INDEX idx_sector_indices_sector_name (sector_name)
- ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
- """,
- # ─── sector_features_daily (行业特征衍生) ─────────────────────────────
- """
- CREATE TABLE IF NOT EXISTS sector_features_daily (
- trade_date DATE NOT NULL,
- sector_name VARCHAR(64) NOT NULL,
- sector_ret DOUBLE DEFAULT 0,
- sector_amplitude DOUBLE DEFAULT 0,
- `close` DOUBLE DEFAULT 0,
- ema10 DOUBLE DEFAULT 0,
- ema20 DOUBLE DEFAULT 0,
- ema200 DOUBLE DEFAULT 0,
- score INT DEFAULT 0,
- PRIMARY KEY (trade_date, sector_name),
- INDEX idx_sector_features_sector_name (sector_name)
- ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
- """,
- # ─── market_regime_daily (市场情绪) ───────────────────────────────────
- """
- CREATE TABLE IF NOT EXISTS market_regime_daily (
- trade_date DATE PRIMARY KEY,
- advancers DOUBLE DEFAULT 0,
- decliners DOUBLE DEFAULT 0,
- advance_ratio DOUBLE DEFAULT 0,
- turnover DOUBLE DEFAULT 0,
- turnover_avg_5d DOUBLE DEFAULT 0,
- turnover_ratio_5d DOUBLE DEFAULT 0,
- source VARCHAR(32) DEFAULT '',
- is_extreme_panic INT DEFAULT 0,
- updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
- INDEX idx_market_regime_daily_date (trade_date)
- ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
- """,
- # ─── registry (模型注册，dashboard 业务无关但库里有) ─────────────────
- """
- CREATE TABLE IF NOT EXISTS registry (
- model_name VARCHAR(64) NOT NULL,
- role VARCHAR(32) NOT NULL,
- version VARCHAR(16) DEFAULT NULL,
- artifact_path VARCHAR(256) NOT NULL,
- architecture VARCHAR(32) DEFAULT NULL,
- enabled TINYINT(1) DEFAULT 1,
- updated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
- PRIMARY KEY (model_name, role)
- ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
- """,
- # ─── stock_scores (评分快照，dashboard 业务无关但库里有) ─────────────
- """
- CREATE TABLE IF NOT EXISTS stock_scores (
- `date` DATE NOT NULL,
- stock_code VARCHAR(10) NOT NULL,
- predicted_rounds DOUBLE DEFAULT 0,
- stacking_probability DOUBLE DEFAULT 0,
- meta_ranker_score DOUBLE DEFAULT NULL,
- latest_close DOUBLE DEFAULT 0,
- `rank` INT DEFAULT 0,
- PRIMARY KEY (`date`, stock_code),
- INDEX idx_stock_scores_date (`date`)
- ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
- """,
-]
-
-
-def ensure_all_tables(conn) -> None:
- """遍历 _DDL 列表，逐条执行 CREATE TABLE IF NOT EXISTS。"""
- with conn.cursor() as cur:
- for ddl in _DDL:
- cur.execute(ddl)
- conn.commit()
diff --git a/app/core/scheduler/scheduler.py b/app/core/scheduler/scheduler.py
index c11bb41..592ad97 100644
--- a/app/core/scheduler/scheduler.py
+++ b/app/core/scheduler/scheduler.py
@@ -157,6 +157,22 @@ def update_job_status(key: str, status: str, message: str) -> None:
_config_cache_ts = 0.0
+def update_job_status_for_dataset(dataset_id: str, status: str, message: str) -> None:
+ """按 dataset_id 找到对应 schedule config 并镜像 lastRun/lastStatus/lastMessage。
+
+ 用法：SyncTask.run() 完成后对非 schedule 触发的运行（cli / runall_once / manual）
+ 调用，让 schedule config 视图与实际跑过保持同步（fix Bug 2,2026-07-01）。
+ """
+ for r in db_ops.fetch_configs_by_category("schedule"):
+ try:
+ obj = json.loads(r.get("value", "") or "{}")
+ except (json.JSONDecodeError, TypeError):
+ continue
+ if obj.get("job") == dataset_id:
+ update_job_status(r["key"], status, message)
+ return
+
+
# ── 调度循环 ──────────────────────────────────────────────────────────
def _loop() -> None:
@@ -300,12 +316,12 @@ DEFAULT_SCHEDULES: list[tuple[str, dict, str]] = [
"schedule_moneyflow",
{
"name": "盘后资金流",
- "time": "16:30",
+ "time": "21:35",
"condition": "trading_day",
"job": "moneyflow",
"enabled": True,
},
- "每个交易日 16:30 拉取个股资金流",
+ "每个交易日 21:35 拉取个股资金流（mairui 21:30 发布）",
),
(
"schedule_industry_sector",
@@ -340,6 +356,17 @@ DEFAULT_SCHEDULES: list[tuple[str, dict, str]] = [
},
"每个交易日 16:20 刷新个股总股本/流通股本快照",
),
+ (
+ "schedule_longhubang",
+ {
+ "name": "盘后龙虎榜",
+ "time": "22:00",
+ "condition": "trading_day",
+ "job": "longhubang",
+ "enabled": True,
+ },
+ "每个交易日 22:00 拉龙虎榜聚合层 + 席位层(akshare/东方财富 19:00~21:00 出齐)",
+ ),
]
@@ -380,6 +407,7 @@ def register_sync_jobs() -> None:
for dataset_id in [
"stock_basic", "kline_daily", "kline_index", "kline_5min",
"moneyflow", "industry_sector", "share_snapshot", "market_regime",
+ "longhubang",
]:
def _make_job(did=dataset_id):
diff --git a/app/core/sync/base.py b/app/core/sync/base.py
index 8330bf6..87c535d 100644
--- a/app/core/sync/base.py
+++ b/app/core/sync/base.py
@@ -19,6 +19,7 @@ from app.core.sync.registry import (
mark_sync_progress,
mark_sync_running,
mark_sync_success,
+ mark_sync_warning,
)
from app.core.utils.logging import get_logger
@@ -79,15 +80,20 @@ class SyncTask:
error=result.get("error", message),
status="blocked",
)
- else: # warning / partial
+ elif status == "warning":
+ # 部分失败:走专用 warning 路径,不写 last_failure_at / last_error
+ mark_sync_warning(self.dataset_id, message=message)
+ else: # 其他未识别状态默认走 failed
mark_sync_failed(
self.dataset_id,
message=message,
error=result.get("error", message),
- status="warning",
+ status="failed",
)
result["elapsed_sec"] = elapsed
logger.info(f"[{self.dataset_id}] 完成 {status}: {message} ({elapsed}s)")
+ # 同步 schedule config 的 lastRun（fix Bug 2）
+ self._maybe_mirror_schedule_status(trigger_source, status, message)
return result
except Exception as e:
elapsed = round(time.time() - t0, 1)
@@ -139,3 +145,28 @@ class SyncTask:
progress_total=total,
current_step=current_step,
)
+
+ def _maybe_mirror_schedule_status(
+ self, trigger_source: str, status: str, message: str
+ ) -> None:
+ """把非 schedule 触发的运行结果同步到 schedule config 表（fix Bug 2）。
+
+ 背景：bin/runall_once.py（15:30 systemd 触发）和 python -m app.entrypoints.cli sync
+ （手动触发）走的都是 task.run(trigger_source="cli" or "manual")，不会经过
+ app.core.scheduler 的 update_job_status。结果：schedule config 表的 lastRun /
+ lastStatus 一直停在 scheduler 真正跑过的那天（实际是 6/15 之后就没动过）。
+ UI 看到的就是"schedule 视图"和"实际跑过"脱节。
+
+ 解法：每次 task.run() 完成后，lazy import scheduler，找到这个 dataset_id 对应的
+ schedule config 并镜像 lastRun/lastStatus/lastMessage。
+
+ scheduler 触发的（trigger_source="schedule"）会自己 update_job_status，跳过即可。
+ """
+ if trigger_source == "schedule":
+ return
+ try:
+ from app.core.scheduler.scheduler import update_job_status_for_dataset
+ update_job_status_for_dataset(self.dataset_id, status, message)
+ except Exception as e:
+ # 镜像失败不影响主流程；只记 debug log
+ logger.debug(f"[_maybe_mirror_schedule_status] {self.dataset_id} 镜像失败: {e}")
diff --git a/app/core/sync/registry.py b/app/core/sync/registry.py
index f252ac2..fffe616 100644
--- a/app/core/sync/registry.py
+++ b/app/core/sync/registry.py
@@ -26,8 +26,8 @@ SYNC_DEFINITIONS: list[dict[str, Any]] = [
"dataset_id": "stock_basic",
"name": "股票基础信息（含最新股本）",
"description": "全市场 A 股代码 / 名称 / 交易所 / 上市状态 + 股本快照（雪球）",
- "storage_uri": "MySQL market_data_sync_db.stocks",
- "storage_layer": "mysql",
+ "storage_uri": "PG market_data.stocks",
+ "storage_layer": "pg",
"management_role": "基础字典",
"source": "Baostock query_stock_basic + 雪球 quote_detail",
"sync_script": "app.tasks.task_stocks_basic:run",
@@ -38,8 +38,8 @@ SYNC_DEFINITIONS: list[dict[str, Any]] = [
"dataset_id": "kline_daily",
"name": "原始日K线数据",
"description": "全市场 A 股日频 OHLCV，多源交叉校验（雪球/新浪/Baostock）",
- "storage_uri": "MySQL market_data_sync_db.kline_stock",
- "storage_layer": "mysql",
+ "storage_uri": "PG market_data.kline_stock",
+ "storage_layer": "pg",
"management_role": "原始源",
"source": "雪球主源 + 新浪/Baostock 复核",
"sync_script": "app.tasks.task_kline_daily:run",
@@ -50,8 +50,8 @@ SYNC_DEFINITIONS: list[dict[str, Any]] = [
"dataset_id": "kline_index",
"name": "六大指数日线",
"description": "上证 / 深证 / 创业板 / 沪深300 / 中证500 / 中证1000",
- "storage_uri": "MySQL market_data_sync_db.kline_index + indices",
- "storage_layer": "mysql",
+ "storage_uri": "PG market_data.kline_index + indices",
+ "storage_layer": "pg",
"management_role": "原始源",
"source": "新浪指数日K",
"sync_script": "app.tasks.task_kline_index:run",
@@ -62,22 +62,34 @@ SYNC_DEFINITIONS: list[dict[str, Any]] = [
"dataset_id": "kline_5min",
"name": "5 分钟 K 线",
"description": "全市场 A 股 5 分钟 K 线（mairui 源）",
- "storage_uri": "MySQL market_data_sync_db.kline_5min",
- "storage_layer": "mysql",
+ "storage_uri": "PG market_data.kline_5min",
+ "storage_layer": "pg",
"management_role": "原始源",
"source": "mairui stockMin",
"sync_script": "app.tasks.task_kline_5min:run",
"dependency_ids": ["stock_basic"],
"sort_order": 40,
},
+ {
+ "dataset_id": "tick_trade",
+ "name": "当天逐笔交易",
+ "description": "全市场 A 股当天逐笔成交（mairui hsrl/zbjy 源；每日 21:00 发布，仅当天数据）",
+ "storage_uri": "PG market_data.tick_trade",
+ "storage_layer": "pg",
+ "management_role": "原始源",
+ "source": "mairui hsrl/zbjy",
+ "sync_script": "app.tasks.task_tick_trade:run",
+ "dependency_ids": ["stock_basic"],
+ "sort_order": 45,
+ },
{
"dataset_id": "moneyflow",
"name": "资金流",
- "description": "个股资金流（主力/大/中/小单净额），mairui 源",
- "storage_uri": "MySQL market_data_sync_db.moneyflow",
- "storage_layer": "mysql",
+ "description": "个股资金流（主力/大/中/小单净额），mairui 源（每日 21:30 发布，由 market-sync-moneyflow.timer 21:35 触发）",
+ "storage_uri": "PG market_data.moneyflow",
+ "storage_layer": "pg",
"management_role": "原始源",
- "source": "mairui hsstock/history/transaction",
+ "source": "mairui hsstock/history/transaction (21:30 publish)",
"sync_script": "app.tasks.task_moneyflow:run",
"dependency_ids": ["stock_basic"],
"sort_order": 50,
@@ -86,8 +98,8 @@ SYNC_DEFINITIONS: list[dict[str, Any]] = [
"dataset_id": "industry_sector",
"name": "股票-行业映射",
"description": "股票-行业映射 + 行业字典（Baostock）",
- "storage_uri": "MySQL market_data_sync_db.industry + sectors + stock_sector_map",
- "storage_layer": "mysql",
+ "storage_uri": "PG market_data.industry + sectors + stock_sector_map",
+ "storage_layer": "pg",
"management_role": "基础字典",
"source": "Baostock query_stock_industry",
"sync_script": "app.tasks.task_industry_sector:run",
@@ -98,8 +110,8 @@ SYNC_DEFINITIONS: list[dict[str, Any]] = [
"dataset_id": "sector_features",
"name": "行业聚合特征",
"description": "由 kline_stock + industry 衍生：行业日收益 / 行业指数 close / EMA10-20-200 / score。纯本地计算，无外部 API。",
- "storage_uri": "MySQL market_data_sync_db.sector_indices + sector_features_daily",
- "storage_layer": "mysql",
+ "storage_uri": "PG market_data.sector_indices + sector_features_daily",
+ "storage_layer": "pg",
"management_role": "衍生源",
"source": "本地计算（kline_stock + industry）",
"sync_script": "app.tasks.task_sector_features:run",
@@ -110,8 +122,8 @@ SYNC_DEFINITIONS: list[dict[str, Any]] = [
"dataset_id": "share_snapshot",
"name": "股本快照",
"description": "全市场 A 股最新股本（雪球 quote_detail，单位亿股）",
- "storage_uri": "MySQL market_data_sync_db.stocks.total_share/float_share + share 表",
- "storage_layer": "mysql",
+ "storage_uri": "PG market_data.stocks.total_share/float_share + share 表",
+ "storage_layer": "pg",
"management_role": "基础字典",
"source": "雪球 quote_detail",
"sync_script": "app.tasks.task_share_snapshot:run",
@@ -122,14 +134,26 @@ SYNC_DEFINITIONS: list[dict[str, Any]] = [
"dataset_id": "market_regime",
"name": "市场情绪 (Market Regime)",
"description": "基于本地日K线聚合的 advance_ratio / 恐慌标记（衍生源）",
- "storage_uri": "MySQL market_data_sync_db.market_regime_daily",
- "storage_layer": "mysql",
+ "storage_uri": "PG market_data.market_regime_daily",
+ "storage_layer": "pg",
"management_role": "衍生源",
"source": "本地日K线聚合（无外部接口）",
"sync_script": "app.tasks.task_market_regime:run",
"dependency_ids": ["kline_daily"],
"sort_order": 80,
},
+ {
+ "dataset_id": "longhubang",
+ "name": "龙虎榜(聚合 + 席位)",
+ "description": "akshare 源（东方财富封装）;每日 22:00 timer 触发,新增聚合层 + 席位层两张表",
+ "storage_uri": "PG market_data.longhubang_daily + longhubang_seat",
+ "storage_layer": "pg",
+ "management_role": "原始源",
+ "source": "akshare stock_lhb_detail_em + stock_lhb_stock_detail_em (东方财富)",
+ "sync_script": "app.tasks.task_longhubang:run",
+ "dependency_ids": ["stock_basic"],
+ "sort_order": 85,
+ },
]
@@ -236,6 +260,26 @@ def mark_sync_failed(
)
+def mark_sync_warning(dataset_id: str, *, message: str = "") -> None:
+ """部分失败的标记:成功 + 失败混合 → status=warning。
+
+ 与 mark_sync_failed 的区别：warning 不写 last_failure_at / last_error，
+ 也不触发 needs_resync=1（warning 状态下数据可能部分可用，不是硬错误）。
+ 历史 bug（2026-07-01 修复）：SyncTask.run() 把 warning 走 mark_sync_failed 路径
+ 导致 UI 误显示"失败"（last_failure_at 有值、last_error 非空）。
+ """
+ now = _now_ts()
+ db_ops.update_dataset_registry_state(
+ dataset_id,
+ status="warning",
+ finished_at=now,
+ last_success_at=now, # 部分成功也算成功一刻，记到 last_success
+ message=message or "部分失败",
+ current_step="",
+ updated_at=now,
+ )
+
+
def mark_sync_blocked(dataset_id: str, *, message: str) -> None:
mark_sync_failed(dataset_id, message=message, error=message, status="blocked")
diff --git a/app/entrypoints/cli.py b/app/entrypoints/cli.py
index 3ce1a6e..2298cbf 100644
--- a/app/entrypoints/cli.py
+++ b/app/entrypoints/cli.py
@@ -24,6 +24,7 @@ def main():
p_sync.add_argument("--start", help="起始日期 YYYY-MM-DD", default=None)
p_sync.add_argument("--end", help="结束日期 YYYY-MM-DD", default=None)
p_sync.add_argument("--workers", type=int, default=10, help="并发数")
+ p_sync.add_argument("--force", action="store_true", help="跳过时间门控（tick_trade 用）")
sub.add_parser("list", help="列出所有同步任务")
sub.add_parser("status", help="查看整体状态（调度器 / 数据源 / 同步任务）")
@@ -55,6 +56,8 @@ def main():
kwargs["end"] = args.end
if args.workers:
kwargs["max_workers"] = args.workers
+ if args.force:
+ kwargs["force"] = True
result = task.run(trigger_source="cli", **kwargs)
print(json.dumps(result, ensure_ascii=False, indent=2))
sys.exit(0 if result.get("status") in ("ok", "warning") else 2)
diff --git a/app/sources/akshare_lhb.py b/app/sources/akshare_lhb.py
new file mode 100644
index 0000000..991a116
--- /dev/null
+++ b/app/sources/akshare_lhb.py
@@ -0,0 +1,216 @@
+"""akshare 龙虎榜数据源（2026-07-01 引入）。
+
+提供：
+ - 龙虎榜聚合层（`ak.stock_lhb_detail_em`）：每日上榜股票汇总
+ - 龙虎榜席位层（`ak.stock_lhb_stock_detail_em`）：单只上榜票的买卖前五营业部
+
+设计原则：
+ - 严格遵循 [[no-akshare]] 当前规则：仅"无替代源"场景可用。
+ 龙虎榜无合理替代（baostock / mairui / 雪球均无龙虎榜接口）→ 用 akshare。
+ - 速率限制：~0.5 RPS 主动 sleep（akshare 经验值；东财未明确限速）
+ - 失败处理：席位层对"未上榜"的股票会触发 akshare 内部 `'NoneType' object is not subscriptable`
+ → 在外层 try/except 吞掉该异常，按"空数据"处理。
+ - 每次调用前显式 sleep 0.4s，避免被东财风控。
+"""
+from __future__ import annotations
+
+import threading
+import time
+from typing import Any, Optional
+
+import pandas as pd
+
+from app.core.datasource.base import DataSource
+from app.core.utils.logging import get_logger
+
+logger = get_logger("akshare_lhb")
+
+
+class AkshareLhbSource(DataSource):
+ """akshare 龙虎榜数据源（基于东方财富私有 API 的 akshare 封装）。"""
+
+ key = "datasource_akshare_lhb"
+ name = "akshare 龙虎榜（东方财富封装）"
+ provides = ["longhubang_daily", "longhubang_seat"]
+ requires_credential = False
+
+ # 全局 RPS 限流（akshare → 东财私有 API）
+ # 设 2.5 RPS（0.4s 间隔）—— 经验值比 mairui 钻石档保守，避免被东财风控
+ _RPS_LIMIT = 2.5
+ _rate_lock = threading.Lock()
+ _last_request_ts = 0.0
+
+ @classmethod
+ def _wait_rps(cls) -> None:
+ """全局 RPS 限流：每次 akshare 调用前 sleep。"""
+ with cls._rate_lock:
+ now = time.time()
+ elapsed = now - cls._last_request_ts
+ min_interval = 1.0 / cls._RPS_LIMIT
+ if elapsed < min_interval:
+ time.sleep(min_interval - elapsed)
+ cls._last_request_ts = time.time()
+
+ def is_available(self) -> tuple[bool, str]:
+ try:
+ import akshare # noqa: F401
+ return True, "akshare 已安装"
+ except ImportError as e:
+ return False, f"akshare 未安装: {e}"
+
+ def health_check(self) -> dict[str, Any]:
+ try:
+ import akshare as ak
+ # 用一次历史数据作烟测（取一个肯定有数据的工作日；2024-06-14 是周五）
+ self._wait_rps()
+ try:
+ df = ak.stock_lhb_detail_em(start_date="20240614", end_date="20240614")
+ except Exception as inner_e:
+ # akshare 在节假日可能返 None 然后内部崩，catch 后报失败
+ return {"success": False, "message": f"akshare 龙虎榜失败: {inner_e}"}
+ if df is None or df.empty:
+ return {"success": False, "message": "akshare 龙虎榜返回空（可能是节假日）"}
+ return {"success": True, "message": f"akshare 龙虎榜 OK,样例 {len(df)} 行"}
+ except Exception as e:
+ return {"success": False, "message": f"akshare 龙虎榜失败: {e}"}
+
+ # ── 聚合层 ───────────────────────────────────────────
+
+ def fetch_longhubang_daily(
+ self,
+ start_date: str,
+ end_date: str,
+ ) -> pd.DataFrame:
+ """龙虎榜每日上榜股票汇总。
+
+ Args:
+ start_date / end_date: 'YYYY-MM-DD' 或 'YYYYMMDD'（akshare 接受两种）
+ Returns:
+ DataFrame 字段：
+ 序号 / 代码 / 名称 / 上榜日 / 解读 / 收盘价 / 涨跌幅 /
+ 龙虎榜净买额 / 龙虎榜买入额 / 龙虎榜卖出额 / 龙虎榜成交额 /
+ 市场总成交额 / 净买额占总成交比 / 成交额占总成交比 / 换手率 /
+ 流通市值 / 上榜原因 / 上榜后1日 / 上榜后2日 / 上榜后5日 / 上榜后10日
+ """
+ try:
+ import akshare as ak
+ except ImportError:
+ return pd.DataFrame()
+ # 兼容两种日期格式：YYYYMMDD 直接传，YYYY-MM-DD 剥横线
+ sd = start_date.replace("-", "")
+ ed = end_date.replace("-", "")
+ self._wait_rps()
+ try:
+ df = ak.stock_lhb_detail_em(start_date=sd, end_date=ed)
+ except Exception as e:
+ logger.warning(f"[akshare] stock_lhb_detail_em({sd},{ed}) failed: {e}")
+ return pd.DataFrame()
+ if df is None or df.empty:
+ return pd.DataFrame()
+ # akshare 6 位代码不带交易所前缀；project 标准是 SH600000 / SZ000001 → 加前缀
+ df = df.copy()
+ df["stock_code"] = df["代码"].astype(str).str.zfill(6).apply(self._prefix_code)
+ df = df.rename(columns={
+ "名称": "stock_name",
+ "上榜日": "trade_date",
+ "解读": "comment",
+ "收盘价": "close",
+ "涨跌幅": "pct_chg",
+ "龙虎榜净买额": "lhb_net_buy",
+ "龙虎榜买入额": "lhb_buy_amt",
+ "龙虎榜卖出额": "lhb_sell_amt",
+ "龙虎榜成交额": "lhb_total_amt",
+ "市场总成交额": "market_total_amt",
+ "净买额占总成交比": "net_buy_ratio",
+ "成交额占总成交比": "total_amt_ratio",
+ "换手率": "turnover_rate",
+ "流通市值": "float_mv",
+ "上榜原因": "reason",
+ "上榜后1日": "post_1d_pct",
+ "上榜后2日": "post_2d_pct",
+ "上榜后5日": "post_5d_pct",
+ "上榜后10日": "post_10d_pct",
+ "序号": "rank_idx",
+ })
+ # 去重：akshare 同一天同一只票可能因多上榜原因（涨幅+换手率）返回多行，
+ # 数值相同只 reason 不同 → 按 (stock_code, trade_date) group by，
+ # reason 用 " | " 拼接保留所有原因，数值列 first()
+ agg_cols = {
+ "stock_name": "first",
+ "rank_idx": "min",
+ "comment": "first",
+ "close": "first",
+ "pct_chg": "first",
+ "lhb_net_buy": "first",
+ "lhb_buy_amt": "first",
+ "lhb_sell_amt": "first",
+ "lhb_total_amt": "first",
+ "market_total_amt": "first",
+ "net_buy_ratio": "first",
+ "total_amt_ratio": "first",
+ "turnover_rate": "first",
+ "float_mv": "first",
+ "reason": lambda s: " | ".join(sorted(set(v for v in s if v))),
+ "post_1d_pct": "first",
+ "post_2d_pct": "first",
+ "post_5d_pct": "first",
+ "post_10d_pct": "first",
+ }
+ df = (
+ df.groupby(["stock_code", "trade_date"], as_index=False)
+ .agg(agg_cols)
+ )
+ return df
+
+ @staticmethod
+ def _prefix_code(code6: str) -> str:
+ """6 位代码 → SH/SZ 前缀。"""
+ if code6.startswith(("5", "6", "9")):
+ return f"SH{code6}"
+ return f"SZ{code6}"
+
+ # ── 席位层 ───────────────────────────────────────────
+
+ def fetch_longhubang_seat(
+ self,
+ code6: str,
+ trade_date: str,
+ direction: str = "buy",
+ ) -> pd.DataFrame:
+ """单只上榜股票单日的买/卖前 5 营业部。
+
+ Args:
+ code6: 6 位股票代码（无交易所前缀）
+ trade_date: 'YYYY-MM-DD'（akshare 要求此格式）
+ direction: 'buy' / 'sell'（中文：'买入' / '卖出'）
+ Returns:
+ DataFrame 字段：
+ 序号 / 交易营业部名称 / 买入金额 / 买入金额-占总成交比例 /
+ 卖出金额 / 卖出金额-占总成交比例 / 净额 / 类型
+
+ 注意：若该股当日未上榜，akshare 内部抛 `'NoneType' object is not subscriptable`，
+ 我们 try/except 吞掉返回空 DataFrame（正常情况，不是错）。
+ """
+ try:
+ import akshare as ak
+ except ImportError:
+ return pd.DataFrame()
+ flag = "买入" if direction == "buy" else "卖出"
+ # akshare 的 stock_lhb_stock_detail_em 要 YYYYMMDD 格式（不带横线），
+ # YYYY-MM-DD 会返回 None 然后内部崩在 'NoneType' object is not subscriptable
+ date_yyyymmdd = str(trade_date).replace("-", "")
+ self._wait_rps()
+ try:
+ df = ak.stock_lhb_stock_detail_em(
+ symbol=str(code6).zfill(6),
+ date=date_yyyymmdd,
+ flag=flag,
+ )
+ except Exception as e:
+ # 'NoneType' object is not subscriptable = 股票未上榜（akshare bug）
+ # 其他真错也兜住，返回空
+ logger.debug(f"[akshare] stock_lhb_stock_detail_em({code6},{date_yyyymmdd},{flag}) → {type(e).__name__}: {e}")
+ return pd.DataFrame()
+ if df is None or df.empty:
+ return pd.DataFrame()
+ return df
diff --git a/app/sources/mairui.py b/app/sources/mairui.py
index 2c506da..f9c86c8 100644
--- a/app/sources/mairui.py
+++ b/app/sources/mairui.py
@@ -5,6 +5,7 @@
- 5 分钟 K 线（`hsstock/history/{code}.{ex}/5/n/{licence}`）
- 指数日 K 线（`hsindex/history/{code}.{ex}/d/{licence}`）
- 资金流向（`hsstock/history/transaction/{code}.{ex}/{licence}`）
+ - 当天逐笔交易（`hsrl/zbjy/{code6}/{licence}`）
限速：1分钟300次（默认保守到 5 RPS = 1分钟300次）
凭证：licence（无需登录态，从环境变量 MAIRUI_LICENCE 读取）
@@ -31,7 +32,7 @@ from app.core.datasource.utils import (
class MairuiSource(DataSource):
key = "datasource_mairui"
name = "麦蕊智数（mairui.club）"
- provides = ["kline_daily", "kline_5min", "index_daily", "stock_basic", "moneyflow"]
+ provides = ["kline_daily", "kline_5min", "index_daily", "stock_basic", "moneyflow", "tick_trade"]
requires_credential = True
credential_key = "MAIRUI_LICENCE"
@@ -41,11 +42,20 @@ class MairuiSource(DataSource):
}
_BASE_URL = "https://api.mairuiapi.com"
- # mairui 免费 licence 1 分钟 300 次 = 5 RPS（**单 licence**硬上限）。
- # 5 workers × 1 RPS/worker = 5 RPS 总 = 刚好踩满上限，避免风控。
+ # mairui licence 限速（按 tier）：
+ # 免费版: 1 min/300 次 = 5 RPS
+ # 体验版: 1 min/1千次 = 16.67 RPS
+ # 包年版: 1 min/3千次 = 50 RPS
+ # 钻石版: 1 min/6千次 = 100 RPS ← 当前项目用
+ #
+ # 用法：5 workers × _RPS_LIMIT RPS/worker = 总 RPS。
+ # 默认 10.0 → 5 workers = 50 RPS（钻石 100 RPS 的一半，留 50% buffer 防风控）。
+ # 如需调：env `MAIRUI_RPS_LIMIT=20` → 100 RPS (踩满钻石);
+ # `MAIRUI_RPS_LIMIT=1` → 5 RPS (回退到免费档兼容)。
+ #
# 用 thread-local 计时：每个 worker 独立计自己的 last_ts，
# 避免 N 个 worker 串行抢一把锁退化成 1 worker。
- _RPS_LIMIT = 1.0
+ _RPS_LIMIT = float(os.environ.get("MAIRUI_RPS_LIMIT", "10"))
_rps_local = threading.local()
@classmethod
@@ -87,7 +97,18 @@ class MairuiSource(DataSource):
return {"success": False, "message": f"麦蕊连接失败: {e}"}
def _fetch(self, path: str, params: dict[str, Any], retry: int = 2) -> Any:
- """通用 GET，含限速 + 重试。"""
+ """通用 GET，含限速 + 重试。
+
+ 编码探测顺序 (2026-07-01 修复):
+ 1) UTF-8 — 正常 JSON 响应
+ 2) GBK — mairui 风控时的中文错误消息 ("接收数据异常，请稍后再试")
+ 3) latin-1 兜底 — 防御性 fallback (历史 Python repr'd 字符串)
+
+ 历史 bug (2026-06-16 之前): 无条件 `raw_bytes.decode("latin-1")`，
+ 把 UTF-8 中文名全部 double-encoded (Latin-1 → UTF-8) → 写入 DB
+ 后 stock_name 看起来正常但 hex 是错的双倍长度字节。
+ 修复后正常 JSON 走 UTF-8，错误消息走 GBK，Latin-1 仅作兜底。
+ """
self._wait_rps()
qs = "&".join(f"{k}={v}" for k, v in params.items() if v)
url = f"{self._BASE_URL}{path}"
@@ -98,16 +119,14 @@ class MairuiSource(DataSource):
try:
req = urllib.request.Request(url, headers=self._HEADERS)
with urllib.request.urlopen(req, timeout=20) as resp:
- # 先按 gbk 试（中文乱码返回），再 fallback utf-8
raw_bytes = resp.read()
- # 用 latin-1 永不失败地把 bytes 转成字符串（每字节 1 字符）
- raw = raw_bytes.decode("latin-1")
- # mairui 风控时返回 "接收数据异常，请稍后再试"（gbk 编码）
+ # 编码探测: 优先 UTF-8, fallback GBK, 兜底 latin-1
+ raw = self._decode_response(raw_bytes)
+ # 风控提示 (GBK 编码的中文消息)
if "请稍后再试" in raw or "codec can't decode" in raw or raw.startswith("'utf-8'") or "Error -3 while decompressing" in raw:
raise RuntimeError(f"mairui 返回风控提示: {raw[:80]}")
- # 试 JSON parse；如果是 gbk 编码的中文提示，要先解码
+ # Python repr'd 字符串: "'接收数据异常，请稍后再试'" (GBK bytes 用 latin-1 解码后又被 Python repr)
if raw.startswith("'") and raw.endswith("'"):
- # gbk 编码的 Python repr 字符串，如 "'接收数据异常，请稍后再试'"
try:
decoded = raw[1:-1].encode("latin-1").decode("gbk")
if "请稍后再试" in decoded:
@@ -123,6 +142,22 @@ class MairuiSource(DataSource):
get_logger("mairui").debug(f"mairui fetch {url} failed: {last_err}")
return []
+ @staticmethod
+ def _decode_response(raw_bytes: bytes) -> str:
+ """探测响应字节流的编码,优先 UTF-8,fallback GBK,latin-1 兜底。"""
+ # 1) UTF-8: 99% 的情况 (正常 JSON 响应)
+ try:
+ return raw_bytes.decode("utf-8")
+ except UnicodeDecodeError:
+ pass
+ # 2) GBK: mairui 风控时返 GBK 编码的中文错误消息
+ try:
+ return raw_bytes.decode("gbk")
+ except UnicodeDecodeError:
+ pass
+ # 3) latin-1 兜底: 永不失败,但可能产生错乱字符
+ return raw_bytes.decode("latin-1")
+
# ── 股票列表 ───────────────────────────────────────────
def fetch_stock_list(self) -> list[dict]:
@@ -376,4 +411,72 @@ class MairuiSource(DataSource):
df = df[df["trade_date"] >= start]
if end:
df = df[df["trade_date"] <= end]
- return df.sort_values("trade_date").reset_index(drop=True)
\ No newline at end of file
+ return df.sort_values("trade_date").reset_index(drop=True)
+
+ # ── 当天逐笔交易 ───────────────────────────────────────
+
+ def fetch_tick_trade(self, code6: str) -> pd.DataFrame:
+ """当天逐笔交易（mairui 不支持历史回溯，仅当天数据）。
+
+ API: GET /hsrl/zbjy/{code6}/{licence} （无 st/et 参数）
+ 文档：https://mairui.club/hsdata → "当天逐笔交易"
+ 更新：每日 21:00。
+ 排序：按时间倒序。
+ 限速：1 min/300 次（与其它接口共享 licence 配额）。
+
+ 返回字段：
+ d string 数据归属日期（yyyy-MM-dd）
+ t string 时间（HH:mm:dd，文档笔误，应为 HH:mm:ss）
+ v number 成交量（股）
+ p number 成交价（元）
+ ts number 交易方向 0=中性盘 / 1=买入 / 2=卖出
+
+ 派生字段：
+ trade_time = d + 'T' + t（datetime64[ns, UTC]，无 tz 直接当本地时区）
+ direction = 0/1/2 → 'neutral'/'buy'/'sell'
+ amount = price * volume（元）
+ stock_code = 外部传入的 6 位 code（保持与其它表一致）
+ """
+ lic = self._get_licence()
+ if not lic:
+ return pd.DataFrame()
+ path = f"/hsrl/zbjy/{code6}/{lic}"
+ data = self._fetch(path, {}) # 200/200 —— 不需要 start/end
+ if not isinstance(data, list) or not data:
+ return pd.DataFrame()
+
+ direction_map = {0: "neutral", 1: "buy", 2: "sell"}
+ records = []
+ for item in data:
+ try:
+ d = str(item.get("d", "") or "").strip()
+ t = str(item.get("t", "") or "").strip()
+ if not d or not t:
+ continue
+ # mairui `t` 形如 "14:53:21"，拼成 ISO 字符串让 pandas 解析
+ iso = f"{d}T{t}"
+ ts_val = pd.to_datetime(iso, errors="coerce")
+ if pd.isna(ts_val):
+ continue
+ price = float(item.get("p") or 0)
+ volume = float(item.get("v") or 0)
+ if price <= 0 or volume <= 0:
+ continue
+ ts_code = int(item.get("ts") or 0)
+ records.append({
+ "stock_code": code6,
+ "trade_date": d,
+ "trade_time": ts_val.to_pydatetime(),
+ "price": price,
+ "volume": volume,
+ "direction_code": ts_code,
+ "direction": direction_map.get(ts_code, ""),
+ "amount": round(price * volume, 2),
+ })
+ except (KeyError, ValueError, TypeError):
+ continue
+ if not records:
+ return pd.DataFrame()
+ df = pd.DataFrame(records)
+ # mairui 按时间倒序，统一升序便于入库
+ return df.sort_values("trade_time").reset_index(drop=True)
\ No newline at end of file
diff --git a/app/tasks/__init__.py b/app/tasks/__init__.py
index ad5310a..9c04c14 100644
--- a/app/tasks/__init__.py
+++ b/app/tasks/__init__.py
@@ -7,11 +7,13 @@ from app.tasks.task_industry_sector import SyncIndustrySector
from app.tasks.task_kline_5min import SyncKline5Min
from app.tasks.task_kline_daily import SyncKlineDaily
from app.tasks.task_kline_index import SyncKlineIndex
+from app.tasks.task_longhubang import SyncLonghubang
from app.tasks.task_market_regime import SyncMarketRegime
from app.tasks.task_moneyflow import SyncMoneyflow
from app.tasks.task_sector_features import SyncSectorFeatures
from app.tasks.task_share_snapshot import SyncShareSnapshot
from app.tasks.task_stocks_basic import SyncStocksBasic
+from app.tasks.task_tick_trade import SyncTickTrade
TASKS: dict[str, type] = {
cls.dataset_id: cls
@@ -20,11 +22,13 @@ TASKS: dict[str, type] = {
SyncKlineDaily,
SyncKlineIndex,
SyncKline5Min,
+ SyncTickTrade,
SyncMoneyflow,
SyncIndustrySector,
SyncSectorFeatures,
SyncShareSnapshot,
SyncMarketRegime,
+ SyncLonghubang,
)
}
diff --git a/app/tasks/task_industry_sector.py b/app/tasks/task_industry_sector.py
index 02d7c2f..51dc54e 100644
--- a/app/tasks/task_industry_sector.py
+++ b/app/tasks/task_industry_sector.py
@@ -6,9 +6,13 @@ from __future__ import annotations
import time
from typing import Any
+from sqlalchemy import update
+
from app.core.db import ops as db_ops
+from app.core.db.models import Stock
from app.core.datasource.base import registry as ds_registry
from app.core.datasource.registry import is_source_ready
+from app.core.datasource.utils import to_hermes
from app.core.sync.base import SyncTask
from app.core.sync.registry import mark_sync_blocked
from app.core.utils.logging import get_logger
@@ -65,9 +69,9 @@ class SyncIndustrySector(SyncTask):
"source": "baostock_query_stock_industry",
"enabled": 1,
}
- stock_sector_rows.append({"stock_code": code6, "sector_key": sector_key})
+ stock_sector_rows.append({"stock_code": to_hermes(code6), "sector_key": sector_key})
industry_rows.append({
- "code": code6,
+ "code": to_hermes(code6),
"industry_name": name,
"industry_classification": classification,
"update_date": r.get("update_date", ""),
@@ -82,12 +86,9 @@ class SyncIndustrySector(SyncTask):
return {"status": "error", "message": f"写库失败: {e}"}
# 回填 stocks.industry 字段（让 stocks 表也能直接看到）
- # 使用批量 UPDATE ... CASE WHEN 方式一次性完成
+ # 走 ORM update(Stock) + IN (SHxxx, SZxxx, BJxxx, ...)
try:
- from app.core.db.connection import get_mysql
- from app.core.db.schema import ensure_all_tables
- ensure_all_tables(get_mysql())
- # 按 industry_name 分组，对每组用一个 IN 查询批量更新
+ # 按 industry_name 分组，对每组用一个 IN 批量更新
by_industry: dict[str, list[str]] = {}
for r in industry_rows:
name = r["industry_name"]
@@ -95,19 +96,17 @@ class SyncIndustrySector(SyncTask):
if name not in by_industry:
by_industry[name] = []
by_industry[name].append(code)
- with get_mysql().cursor() as cur:
- for ind_name, codes in by_industry.items():
- # 构造 IN 子句：code IN (SHxxx, SZxxx, BJxxx)
- placeholders = []
- params = [ind_name]
- for c in codes:
- for prefix in ("SH", "SZ", "BJ"):
- placeholders.append("%s")
- params.append(f"{prefix}{c}")
- in_clause = ", ".join(placeholders)
- sql = f"UPDATE stocks SET industry = %s WHERE code IN ({in_clause})"
- cur.execute(sql, params)
- get_mysql().commit()
+ for ind_name, codes in by_industry.items():
+ # 6 位 code × 3 个交易所前缀（SH/SZ/BJ）
+ code_variants = [
+ f"{prefix}{c}" for c in codes for prefix in ("SH", "SZ", "BJ")
+ ]
+ with db_ops.get_session() as s:
+ s.execute(
+ update(Stock)
+ .where(Stock.code.in_(code_variants))
+ .values(industry=ind_name)
+ )
except Exception as e:
logger.warning(f"回填 stocks.industry 失败（不影响主任务）: {e}")
diff --git a/app/tasks/task_kline_5min.py b/app/tasks/task_kline_5min.py
index 47e1780..f5799f8 100644
--- a/app/tasks/task_kline_5min.py
+++ b/app/tasks/task_kline_5min.py
@@ -18,13 +18,13 @@ from __future__ import annotations
import os
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
from typing import Any, Optional
from app.core.db import ops as db_ops
from app.core.datasource.base import registry as ds_registry
from app.core.datasource.registry import is_source_ready
-from app.core.datasource.utils import is_a_share_code, to_code6
+from app.core.datasource.utils import is_a_share_code, to_code6, to_hermes
from app.core.sync.base import SyncTask
from app.core.sync.registry import mark_sync_blocked
from app.core.utils.logging import get_logger
@@ -32,7 +32,7 @@ from app.core.utils.logging import get_logger
logger = get_logger("sync.kline_5min")
# mairui 5min K 实测历史深度起点（实测 2023-06-14 才有数据，更早就 0）
-MAIRUI_5MIN_FLOOR = datetime(2023, 6, 14)
+MAIRUI_5MIN_FLOOR = datetime(2023, 6, 14, tzinfo=timezone.utc)
# 单次请求窗口大小：1 年（mairui 单次最多 ~11640 条 5min）
WINDOW_DAYS = 365
# 增量时往前多取的天数（防交易日历边界漏当天）
@@ -98,7 +98,8 @@ class SyncKline5Min(SyncTask):
logger.info(f"[5min] DB 里已有 {len(snapshots)} 只的 kline_5min 数据")
# ── 2) 内存算计划 ──
- end = datetime.now()
+ # PG TIMESTAMPTZ 是 timezone-aware，所以 end 也必须 tz-aware 否则比较会炸
+ end = datetime.now(timezone.utc)
plans = self._plan(stock_codes, end, snapshots)
if not plans:
msg = f"5min K线 — 全部 {len(stock_codes)} 只都已最新（无需同步）"
@@ -174,10 +175,11 @@ class SyncKline5Min(SyncTask):
w_end.strftime("%Y-%m-%d"),
)
if df is not None and not df.empty:
+ hermes = to_hermes(code6)
for _, r in df.iterrows():
bar_time = r["bar_time"]
all_rows.append({
- "stock_code": code6,
+ "stock_code": hermes,
"bar_time": bar_time.strftime("%Y-%m-%d %H:%M:%S")
if hasattr(bar_time, "strftime") else str(bar_time),
"open": float(r.get("open") or 0),
diff --git a/app/tasks/task_kline_daily.py b/app/tasks/task_kline_daily.py
index 4154a2c..6d38ca7 100644
--- a/app/tasks/task_kline_daily.py
+++ b/app/tasks/task_kline_daily.py
@@ -26,7 +26,7 @@ import pandas as pd
from app.core.db import ops as db_ops
from app.core.datasource.base import registry as ds_registry
-from app.core.datasource.utils import is_a_share_code, to_code6
+from app.core.datasource.utils import is_a_share_code, to_code6, to_hermes
from app.core.sync.base import SyncTask, _effective_sync_end
from app.core.utils.logging import get_logger
@@ -145,11 +145,14 @@ class SyncKlineDaily(SyncTask):
return (code6, None, str(e)[:120])
if df is None or df.empty:
return (code6, None, "no data")
+ # 写库用 hermes（带 SH/SZ 前缀）—— 2026-07-01 重构：历史上 kline_stock 用
+ # 6位 code 与 stocks.code(SH600519) 格式不一致，跨表 JOIN 失败。现统一。
+ hermes = to_hermes(code6)
rows = []
for _, r in df.iterrows():
td = r["trade_date"]
rows.append({
- "stock_code": code6,
+ "stock_code": hermes,
"trade_date": td.strftime("%Y-%m-%d") if hasattr(td, "strftime") else str(td)[:10],
"open": float(r["open"]),
"high": float(r["high"]),
@@ -226,11 +229,12 @@ class SyncKlineDaily(SyncTask):
return {"status": "fail", "error": str(e)}
if df is None or df.empty:
return {"status": "fail", "error": "no data"}
+ hermes = to_hermes(code6)
rows = []
for _, r in df.iterrows():
td = r["trade_date"]
rows.append({
- "stock_code": code6,
+ "stock_code": hermes,
"trade_date": td.strftime("%Y-%m-%d") if hasattr(td, "strftime") else str(td)[:10],
"open": float(r["open"]),
"high": float(r["high"]),
diff --git a/app/tasks/task_longhubang.py b/app/tasks/task_longhubang.py
new file mode 100644
index 0000000..029a3f2
--- /dev/null
+++ b/app/tasks/task_longhubang.py
@@ -0,0 +1,245 @@
+"""同步任务：龙虎榜聚合层 + 席位层（akshare 源）。
+
+数据源：
+ - 聚合层：`ak.stock_lhb_detail_em(start_date, end_date)` → `longhubang_daily` 表
+ - 席位层：`ak.stock_lhb_stock_detail_em(symbol, date, flag)` → `longhubang_seat` 表
+
+设计要点：
+ 1) 22:00 触发（mairui 21:30 跑 moneyflow；龙虎榜 19:00-21:00 陆续出齐，22:00 兜底）
+ 2) 默认"今天"增量；CLI 支持 --backfill YYYYMMDD 单日；--backfill-range YYYYMMDD YYYYMMDD 日期范围
+ 3) 聚合层一次性拉日期范围；席位层先抓聚合层取出 stock_code 集合再逐只抓（节省调用）
+ 4) 串行执行（每日调用 ~3 次聚合层 + ~200 次席位层；3-5 分钟内完成）
+ 5) akshare 对"未上榜"的股票会抛 `'NoneType' object is not subscriptable`
+ → source 层 try/except 兜住，按"空数据"处理
+ 6) 数据是历史回填友好（全量 4 年 ≈ 25 分钟，2026-07-01 烟测通过）
+"""
+from __future__ import annotations
+
+import time
+from datetime import datetime, timedelta
+from typing import Any
+
+from app.core.db import ops as db_ops
+from app.core.datasource.base import registry as ds_registry
+from app.core.datasource.registry import is_source_ready
+from app.core.sync.base import SyncTask
+from app.core.sync.registry import mark_sync_blocked
+from app.core.utils.logging import get_logger
+
+logger = get_logger("sync.longhubang")
+
+
+def _parse_yyyymmdd(s: str) -> str:
+ """'YYYYMMDD' 或 'YYYY-MM-DD' → 'YYYY-MM-DD'。"""
+ s = str(s).strip()
+ if len(s) == 8 and s.isdigit():
+ return f"{s[:4]}-{s[4:6]}-{s[6:8]}"
+ return s
+
+
+def _date_range(start: str, end: str) -> list[str]:
+ """返回 start..end（含）期间所有日期字符串列表（YYYY-MM-DD）。"""
+ s = datetime.strptime(start, "%Y-%m-%d")
+ e = datetime.strptime(end, "%Y-%m-%d")
+ if e < s:
+ s, e = e, s
+ out = []
+ cur = s
+ while cur <= e:
+ out.append(cur.strftime("%Y-%m-%d"))
+ cur += timedelta(days=1)
+ return out
+
+
+class SyncLonghubang(SyncTask):
+ """龙虎榜聚合层 + 席位层同步。
+
+ CLI 用法：
+ python -m app.entrypoints.cli sync longhubang # 今天增量
+ ... sync longhubang --backfill 20240614 # 单日
+ ... sync longhubang --backfill-range 20240101 20240614 # 日期范围
+ ... sync longhubang --start 20240101 --end 20240614 # 等价于 backfill-range
+ """
+ dataset_id = "longhubang"
+
+ def _run(
+ self,
+ *,
+ trigger_source: str = "manual",
+ backfill: str | None = None,
+ backfill_range: tuple[str, str] | None = None,
+ start: str | None = None,
+ end: str | None = None,
+ **kwargs,
+ ) -> dict[str, Any]:
+ src = ds_registry.get("datasource_akshare_lhb")
+ if src is None:
+ return {"status": "error", "message": "akshare_lhb 数据源未注册"}
+ ok, reason = is_source_ready(src.key)
+ if not ok:
+ mark_sync_blocked(self.dataset_id, message=f"akshare_lhb 未就绪: {reason}")
+ return {"status": "blocked", "message": f"akshare_lhb 未就绪: {reason}"}
+
+ # ── 决定要拉哪几天 ──
+ if backfill_range:
+ s, e = _parse_yyyymmdd(backfill_range[0]), _parse_yyyymmdd(backfill_range[1])
+ elif start and end:
+ s, e = _parse_yyyymmdd(start), _parse_yyyymmdd(end)
+ elif backfill:
+ s = e = _parse_yyyymmdd(backfill)
+ else:
+ # 默认"今天"（盘后未收盘时 fallback 到昨天）
+ from app.core.sync.base import _effective_sync_end
+ s = e = _effective_sync_end()
+
+ dates = _date_range(s, e)
+ if not dates:
+ return {"status": "error", "message": f"日期范围无效: {s}..{e}"}
+ logger.info(f"[longhubang] 计划: {len(dates)} 天 ({s} ~ {e})")
+
+ # ── 逐日同步 ──
+ t0 = time.time()
+ daily_ok = daily_fail = 0
+ daily_total = seat_total = 0
+ for i, d in enumerate(dates, 1):
+ self._progress(
+ message=f"longhubang 进度 {i}/{len(dates)} 当前 {d}",
+ current=i, total=len(dates), current_step=d,
+ )
+ try:
+ d_rows, s_rows = self._sync_one_day(src, d)
+ daily_ok += 1
+ daily_total += d_rows
+ seat_total += s_rows
+ except Exception as e:
+ daily_fail += 1
+ logger.warning(f"[longhubang {d}] 失败: {e}")
+
+ elapsed = round(time.time() - t0, 1)
+ msg = (
+ f"龙虎榜 {len(dates)}天 daily_ok={daily_ok} daily_fail={daily_fail} "
+ f"聚合层={daily_total}行 席位层={seat_total}行, {elapsed}s"
+ )
+ return {
+ "status": "ok" if daily_fail == 0 else "warning",
+ "message": msg,
+ "ok": daily_ok, "fail": daily_fail,
+ "daily_rows": daily_total, "seat_rows": seat_total,
+ "elapsed_sec": elapsed,
+ }
+
+ # ── 单日同步（聚合 + 席位）──
+
+ def _sync_one_day(self, src, trade_date: str) -> tuple[int, int]:
+ """返回 (聚合层行数, 席位层行数)。"""
+ # 1) 聚合层（1 调用/天）
+ df_daily = src.fetch_longhubang_daily(trade_date, trade_date)
+ daily_rows = 0
+ stock_codes: list[str] = []
+ if df_daily is not None and not df_daily.empty:
+ rows = []
+ for _, r in df_daily.iterrows():
+ rows.append({
+ "stock_code": str(r.get("stock_code") or "").strip(),
+ "stock_name": str(r.get("stock_name") or "").strip(),
+ "trade_date": str(r.get("trade_date") or trade_date)[:10],
+ "rank_idx": self._to_int(r.get("rank_idx")),
+ "comment": self._to_str(r.get("comment")),
+ "close": self._to_float(r.get("close")),
+ "pct_chg": self._to_float(r.get("pct_chg")),
+ "lhb_net_buy": self._to_float(r.get("lhb_net_buy")),
+ "lhb_buy_amt": self._to_float(r.get("lhb_buy_amt")),
+ "lhb_sell_amt": self._to_float(r.get("lhb_sell_amt")),
+ "lhb_total_amt": self._to_float(r.get("lhb_total_amt")),
+ "market_total_amt": self._to_float(r.get("market_total_amt")),
+ "net_buy_ratio": self._to_float(r.get("net_buy_ratio")),
+ "total_amt_ratio": self._to_float(r.get("total_amt_ratio")),
+ "turnover_rate": self._to_float(r.get("turnover_rate")),
+ "float_mv": self._to_float(r.get("float_mv")),
+ "reason": self._to_str(r.get("reason")),
+ "post_1d_pct": self._to_float(r.get("post_1d_pct")),
+ "post_2d_pct": self._to_float(r.get("post_2d_pct")),
+ "post_5d_pct": self._to_float(r.get("post_5d_pct")),
+ "post_10d_pct": self._to_float(r.get("post_10d_pct")),
+ })
+ if rows:
+ db_ops.upsert_longhubang_daily(rows)
+ daily_rows = len(rows)
+ # 取出 stock_code（去交易所前缀得 6 位，供席位层调用）
+ for r in rows:
+ code6 = str(r["stock_code"]).replace("SH", "").replace("SZ", "").replace("BJ", "")
+ if len(code6) == 6 and code6.isdigit():
+ stock_codes.append(code6)
+ logger.info(f"[longhubang {trade_date}] 聚合层 {daily_rows} 行, {len(stock_codes)} 只上榜")
+
+ # 2) 席位层（每只票 2 调用：buy + sell）
+ seat_rows_total = 0
+ if stock_codes:
+ seat_rows = []
+ for code6 in stock_codes:
+ for direction in ("buy", "sell"):
+ df_seat = src.fetch_longhubang_seat(code6, trade_date, direction)
+ if df_seat is None or df_seat.empty:
+ continue
+ for _, r in df_seat.iterrows():
+ # 字段映射（akshare 6 中文列名）
+ if direction == "buy":
+ buy_amt = self._to_float(r.get("买入金额"))
+ sell_amt = self._to_float(r.get("卖出金额"))
+ else:
+ buy_amt = self._to_float(r.get("买入金额"))
+ sell_amt = self._to_float(r.get("卖出金额"))
+ # 买方向的占比列叫"买入金额-占总成交比例"，卖方向叫"卖出金额-..."
+ amt_ratio = (
+ self._to_float(r.get("买入金额-占总成交比例"))
+ if direction == "buy"
+ else self._to_float(r.get("卖出金额-占总成交比例"))
+ )
+ # amount=主金额：买方向=买入金额，卖方向=卖出金额
+ amt = buy_amt if direction == "buy" else sell_amt
+ seat_rows.append({
+ "stock_code": f"{'SH' if code6.startswith(('5','6','9')) else 'SZ'}{code6}",
+ "trade_date": trade_date,
+ "direction": direction,
+ "rank_idx": self._to_int(r.get("序号")) or 0,
+ "branch_name": self._to_str(r.get("交易营业部名称")),
+ "buy_amount": buy_amt,
+ "sell_amount": sell_amt,
+ "amount": amt,
+ "net_amount": self._to_float(r.get("净额")),
+ "amount_ratio": amt_ratio,
+ "explanation": self._to_str(r.get("类型")),
+ })
+ if seat_rows:
+ db_ops.upsert_longhubang_seat(seat_rows)
+ seat_rows_total = len(seat_rows)
+ logger.info(f"[longhubang {trade_date}] 席位层 {seat_rows_total} 行")
+ return daily_rows, seat_rows_total
+
+ # ── 类型转换 helper ──
+
+ @staticmethod
+ def _to_float(v) -> float | None:
+ if v is None:
+ return None
+ try:
+ f = float(v)
+ return f if f == f else None # NaN → None
+ except (TypeError, ValueError):
+ return None
+
+ @staticmethod
+ def _to_int(v) -> int | None:
+ if v is None:
+ return None
+ try:
+ return int(v)
+ except (TypeError, ValueError):
+ return None
+
+ @staticmethod
+ def _to_str(v) -> str | None:
+ if v is None:
+ return None
+ s = str(v).strip()
+ return s if s else None
diff --git a/app/tasks/task_market_regime.py b/app/tasks/task_market_regime.py
index ef1b281..4f3646b 100644
--- a/app/tasks/task_market_regime.py
+++ b/app/tasks/task_market_regime.py
@@ -8,8 +8,10 @@ from typing import Any
import numpy as np
import pandas as pd
+from sqlalchemy import text
from app.core.db import ops as db_ops
+from app.core.db.orm import engine as pg_engine
from app.core.sync.base import SyncTask
from app.core.utils.logging import get_logger
@@ -40,22 +42,19 @@ class SyncMarketRegime(SyncTask):
if not codes:
return {"status": "error", "message": "stocks 表为空"}
- # 2. 拉所有 K 线（按股票）- 这里用 SQL GROUP BY 一次性算 daily turnover
- from app.core.db.connection import get_mysql
-
- sql = """
+ # 2. 拉所有 K 线（按股票）— 走 SA text() 走 PG
+ sql = text("""
SELECT
stock_code,
trade_date,
- `close`,
+ "close",
volume
- FROM kline_stock
- WHERE trade_date BETWEEN %s AND %s
+ FROM market_data.kline_stock
+ WHERE trade_date BETWEEN :start AND :end
ORDER BY stock_code, trade_date
- """
- with get_mysql().cursor() as cur:
- cur.execute(sql, (start, end))
- rows = cur.fetchall()
+ """)
+ with pg_engine.connect() as conn:
+ rows = conn.execute(sql, {"start": start, "end": end}).mappings().all()
if not rows:
return {"status": "warning", "message": f"kline_stock 在 {start} ~ {end} 区间无数据"}
diff --git a/app/tasks/task_moneyflow.py b/app/tasks/task_moneyflow.py
index 66d4bfd..bfba306 100644
--- a/app/tasks/task_moneyflow.py
+++ b/app/tasks/task_moneyflow.py
@@ -15,7 +15,7 @@ from typing import Any
from app.core.db import ops as db_ops
from app.core.datasource.base import registry as ds_registry
from app.core.datasource.registry import is_source_ready
-from app.core.datasource.utils import is_a_share_code, to_code6
+from app.core.datasource.utils import is_a_share_code, to_code6, to_hermes
from app.core.sync.base import SyncTask
from app.core.sync.registry import mark_sync_blocked
from app.core.utils.logging import get_logger
@@ -102,7 +102,7 @@ class SyncMoneyflow(SyncTask):
return {"status": "fail", "error": "no data"}
rows = [
{
- "stock_code": code6,
+ "stock_code": to_hermes(code6),
"trade_date": str(r["trade_date"]),
"main_net_inflow": float(r.get("main_net_inflow") or 0),
"large_net_inflow": float(r.get("large_net_inflow") or 0),
diff --git a/app/tasks/task_sector_features.py b/app/tasks/task_sector_features.py
index a352b0f..4e681a6 100644
--- a/app/tasks/task_sector_features.py
+++ b/app/tasks/task_sector_features.py
@@ -5,7 +5,7 @@
- sector_indices ：行业日线（trade_date × sector_name × close × sector_amplitude）
- sector_features_daily ：行业特征（+ sector_ret + ema10/20/200 + score）
-算法（参考 dashboard/api/services/sync/sector.py 重写，持久层改为 MySQL）：
+算法（参考 dashboard/api/services/sync/sector.py 重写，持久层改为 PG）：
1) 每只股票日涨跌幅 pct_chg = (close/prev_close - 1) * 100
2) 按行业聚合 sector_ret = 该行业所有股票当日 pct_chg 的均值
3) 行业指数 close = 100 × ∏(1 + sector_ret/100) （基点 100，复合收益）
@@ -26,7 +26,6 @@ from sqlalchemy import create_engine
from app.core.config import settings
from app.core.db import ops as db_ops
-from app.core.db.connection import get_mysql
from app.core.sync.base import SyncTask
from app.core.utils.logging import get_logger
@@ -56,11 +55,11 @@ class SyncSectorFeatures(SyncTask):
logger.info("[sector] 启动行业聚合特征计算")
# ── 1) 拉 kline_stock（日线）只取需要的列 ──
- # 用 SQLAlchemy engine（pymysql 裸连接 + pd.read_sql 偶发只读 1 行的 bug）
- engine = create_engine(settings.mysql_url())
+ # 走 SQLAlchemy engine + PG URL（pd.read_sql 一次性读 1.1kw 行 → pandas DataFrame）
+ engine = create_engine(settings.pg_sqlalchemy_url())
df_kline = pd.read_sql(
- "SELECT stock_code, trade_date, open, high, low, `close`, volume "
- "FROM kline_stock ORDER BY stock_code, trade_date",
+ 'SELECT stock_code, trade_date, open, high, low, "close", volume '
+ 'FROM market_data.kline_stock ORDER BY stock_code, trade_date',
engine,
)
if df_kline.empty:
@@ -72,7 +71,7 @@ class SyncSectorFeatures(SyncTask):
# ── 2) 拉 industry（股票→行业映射）──
df_ind = pd.read_sql(
- "SELECT code, industry_name FROM industry WHERE industry_name IS NOT NULL",
+ "SELECT code, industry_name FROM market_data.industry WHERE industry_name IS NOT NULL",
engine,
)
if df_ind.empty:
diff --git a/app/tasks/task_share_snapshot.py b/app/tasks/task_share_snapshot.py
index 5250d4f..a2e1e13 100644
--- a/app/tasks/task_share_snapshot.py
+++ b/app/tasks/task_share_snapshot.py
@@ -87,7 +87,10 @@ class SyncShareSnapshot(SyncTask):
if r is None:
fail_cnt += 1
continue
- # 写 stocks
+ # 写 stocks + share 表 —— 两张表 stock_code 都用 hermes（带 SH/SZ 前缀）
+ # 与 stocks.code 保持一致，方便 JOIN。
+ # 历史 bug：之前 share 表写的是裸 6 位 c6，导致 share JOIN stocks 失败。
+ # 一次性 UPDATE（2026-07-01）已把存量 6.3 万行加上前缀。
hermes = self._to_hermes(c6)
db_ops.update_stock_share_snapshot(
code=hermes,
@@ -95,9 +98,8 @@ class SyncShareSnapshot(SyncTask):
float_share=r["float_share"],
trade_date=r.get("trade_date", today),
)
- # 也写 share 表
rows_to_share_table.append({
- "stock_code": c6,
+ "stock_code": hermes,
"trade_date": r.get("trade_date", today),
"total_share": r["total_share"],
"float_share": r["float_share"],
diff --git a/app/tasks/task_tick_trade.py b/app/tasks/task_tick_trade.py
new file mode 100644
index 0000000..2701c9c
--- /dev/null
+++ b/app/tasks/task_tick_trade.py
@@ -0,0 +1,168 @@
+"""同步任务：当天逐笔交易（mairui 源）。
+
+数据源：mairui `hsrl/zbjy/{code6}/{licence}`
+口径：单笔成交 — 时间 / 价 / 量 / 买卖方向
+更新：每日 21:00（mairui 当天数据此时才发布；21:00 之前 API 返空或昨日数据）
+
+设计要点：
+ 1) 起步门控：21:00 之前（未传 force=True）→ 早返 warning
+ 避免 0 行空跑（5000 只股票 × 1s 限速 ≈ 17 分钟浪费）
+ 2) 5 worker × 1 RPS/worker = 5 RPS（守住 mairui 上限）
+ 3) 不做"今日已同步"dedup —— mairui "当天"实际是"最近已发布交易日"，
+ trade_date 可能比 wall-clock 早一天；重跑靠 PK + ON CONFLICT 幂等
+ 4) 单只股票日均 5w-10w tick，CHUNK=2000 upsert 防 driver 撑爆
+ 5) 数据是"当天 only"，无 start/end 窗口；不存增量概念
+"""
+from __future__ import annotations
+
+import os
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from datetime import datetime
+from typing import Any
+
+from app.core.db import ops as db_ops
+from app.core.datasource.base import registry as ds_registry
+from app.core.datasource.registry import is_source_ready
+from app.core.datasource.utils import is_a_share_code, to_code6, to_hermes
+from app.core.sync.base import SyncTask
+from app.core.sync.registry import mark_sync_blocked
+from app.core.utils.logging import get_logger
+
+logger = get_logger("sync.tick_trade")
+
+# mairui 文档：「更新：每日 21:00」—— 早于此时 mairui 仍返昨日/空数据
+MAIRUI_TICK_TRADE_PUBLISH_HOUR = 21
+MAIRUI_TICK_TRADE_PUBLISH_MIN = 0
+# 21:00 之后再跑（留 5 分钟缓冲，等 mairui 完整入库）
+TICK_TRADE_RUN_GATE = (MAIRUI_TICK_TRADE_PUBLISH_HOUR, MAIRUI_TICK_TRADE_PUBLISH_MIN + 5)
+
+
+def _passes_publish_gate(now: Optional[datetime] = None, *, force: bool = False) -> tuple[bool, str]:
+ """判断当前时间是否已过 mairui 21:00 数据发布门控。
+
+ force=True 跳过门控（手动 / 测试用）。
+ """
+ if force:
+ return True, "force=True 跳过门控"
+ n = now or datetime.now()
+ gate = n.replace(hour=TICK_TRADE_RUN_GATE[0], minute=TICK_TRADE_RUN_GATE[1], second=0, microsecond=0)
+ if n < gate:
+ return False, f"mairui 逐笔数据 {MAIRUI_TICK_TRADE_PUBLISH_HOUR:02d}:{MAIRUI_TICK_TRADE_PUBLISH_MIN:02d} 发布；当前 {n.strftime('%H:%M')} 早于门控，skip"
+ return True, "ok"
+
+
+class SyncTickTrade(SyncTask):
+ dataset_id = "tick_trade"
+
+ def _run(
+ self,
+ *,
+ trigger_source: str = "manual",
+ codes: list[str] | None = None,
+ max_workers: int = 5,
+ force: bool = False,
+ **kwargs,
+ ) -> dict[str, Any]:
+ mr = ds_registry.get("datasource_mairui")
+ if mr is None:
+ return {"status": "error", "message": "mairui 数据源未注册"}
+ ok, reason = is_source_ready(mr.key)
+ if not ok:
+ mark_sync_blocked(self.dataset_id, message=f"mairui 未就绪: {reason}")
+ return {"status": "blocked", "message": f"mairui 未就绪: {reason}"}
+
+ # ── 21:00 门控（force=True 跳过）──
+ gate_ok, gate_msg = _passes_publish_gate(force=force)
+ if not gate_ok:
+ logger.info(f"[tick_trade] 门控未过: {gate_msg}")
+ return {"status": "warning", "message": gate_msg, "ok": 0, "fail": 0, "rows": 0, "elapsed_sec": 0.0}
+
+ # ── 取股票列表 ──
+ if codes:
+ stock_codes = [to_code6(c) for c in codes]
+ else:
+ stock_codes = [c for c in db_ops.iter_stock_codes(active_only=True) if is_a_share_code(c)]
+ limit_raw = os.environ.get("MARKET_DATA_STOCK_LIMIT", "").strip()
+ if limit_raw.isdigit() and int(limit_raw) > 0:
+ stock_codes = stock_codes[: int(limit_raw)]
+ if not stock_codes:
+ return {"status": "error", "message": "无股票代码"}
+
+ # 注：不做"今日已同步"dedup。原因：
+ # - mairui 的"当天"实际是"最近一个已发布的交易日"（21:00 之前是昨日），
+ # 用 today_str 做去重会失效（即使刚跑完，trade_date 还是昨日）。
+ # - PK 含 trade_date，重跑是 ON CONFLICT DO UPDATE，安全且幂等。
+ # - 17 分钟全市场开销可接受；非交易日用 `MARKET_DATA_STOCK_LIMIT=0` 跳。
+
+ total = len(stock_codes)
+ if total == 0:
+ return {"status": "ok", "message": "无股票需要同步", "ok": 0, "fail": 0, "rows": 0, "elapsed_sec": 0.0}
+
+ # ── 预估耗时：5 RPS 上限 ──
+ est_sec = total / 5.0
+ logger.info(f"[tick_trade] 计划: {total} 只, 预估 {est_sec:.0f}s ({est_sec/60:.1f}min)")
+ self._progress(
+ message=f"计划: {total} 只, 预估 {est_sec/60:.1f}min @5RPS",
+ current=0, total=total, current_step="planning",
+ )
+
+ # ── 执行 ──
+ t0 = time.time()
+ ok_cnt = fail_cnt = 0
+ rows_total = 0
+ with ThreadPoolExecutor(max_workers=max_workers) as pool:
+ futures = {pool.submit(self._sync_one, mr, c6): c6 for c6 in stock_codes}
+ for i, future in enumerate(as_completed(futures), 1):
+ c6 = futures[future]
+ try:
+ res = future.result()
+ if res["status"] == "ok":
+ ok_cnt += 1
+ rows_total += res["rows"]
+ else:
+ fail_cnt += 1
+ if res.get("error"):
+ logger.warning(f"[tick_trade {c6}] {res['error']}")
+ except Exception as e:
+ fail_cnt += 1
+ logger.warning(f"[tick_trade {c6}] {e}")
+ if i % 50 == 0 or i == total:
+ self._progress(
+ message=f"tick_trade 进度 {i}/{total} OK:{ok_cnt} FAIL:{fail_cnt} 行:{rows_total}",
+ current=i, total=total, current_step=c6,
+ )
+ elapsed = round(time.time() - t0, 1)
+ msg = f"逐笔 {ok_cnt}成 {fail_cnt}败 共{rows_total}行, {elapsed}s"
+ return {
+ "status": "ok" if fail_cnt == 0 else "warning",
+ "message": msg,
+ "ok": ok_cnt, "fail": fail_cnt, "rows": rows_total, "elapsed_sec": elapsed,
+ }
+
+ def _sync_one(self, mr, code6: str) -> dict:
+ try:
+ df = mr.fetch_tick_trade(code6)
+ except Exception as e:
+ return {"status": "fail", "error": str(e)}
+ if df is None or df.empty:
+ return {"status": "fail", "error": "no data (mairui 21:00 前可能为空)"}
+ try:
+ rows = [
+ {
+ "stock_code": to_hermes(code6),
+ "trade_date": str(r["trade_date"]),
+ "trade_time": r["trade_time"].strftime("%Y-%m-%d %H:%M:%S")
+ if hasattr(r["trade_time"], "strftime") else str(r["trade_time"]),
+ "price": float(r.get("price") or 0),
+ "volume": float(r.get("volume") or 0),
+ "direction_code": int(r.get("direction_code") or 0),
+ "direction": str(r.get("direction") or ""),
+ "amount": float(r.get("amount") or 0),
+ }
+ for _, r in df.iterrows()
+ ]
+ db_ops.upsert_tick_trade(rows)
+ return {"status": "ok", "rows": len(rows)}
+ except Exception as e:
+ return {"status": "fail", "error": f"db write: {e}"}
diff --git a/bin/archive/migrate_mysql_to_pg.py b/bin/archive/migrate_mysql_to_pg.py
new file mode 100644
index 0000000..17301e3
--- /dev/null
+++ b/bin/archive/migrate_mysql_to_pg.py
@@ -0,0 +1,288 @@
+"""[DEPRECATED 2026-06-16] MySQL → PostgreSQL 数据迁移工具。
+
+⚠️ 项目从 2026-06-16 起放弃 MySQL，只支持 PostgreSQL。
+ 本脚本已归档到 ``bin/archive/``，仅作历史参考 / 应急回滚。
+ 如果你想从 MySQL 备份恢复数据到 PG，可以临时把它移回 ``bin/`` 用一次。
+
+同步目标：把 MySQL `market_data_sync_db` 库 15 张表的数据搬到 PG `market_data.schema` 下。
+
+策略（不重新建表，建表由 pg_bootstrap 负责）：
+ - 用 pymysql 读 MySQL 流式分页（避免 1 亿行一次占满内存）
+ - 用 psycopg2.copy_from 或 executemany 写 PG（按表选择最高效方式）
+ - 顺序：先小表（配置/字典）→ 后大表（kline_5min 等）
+
+数据量预估（按当时 MySQL 实测，2026-06-16）：
+ config: 20 行
+ dataset_registry: 15 行
+ industry: 5,207 行
+ sectors: 83 行
+ stock_sector_map: 5,207 行
+ indices: 6 行
+ market_regime_daily: 2,046 行
+ share: 6,141 行
+ stocks: 5,210 行
+ sector_indices: 103,084 行
+ sector_features_daily: 416,521 行
+ kline_index: 37,480 行
+ moneyflow: 3,188,301 行
+ kline_stock: 11,605,541 行
+ kline_5min: 121,945,849 行 ← 最大，约 1.2 亿
+
+用法（应急时）：
+ mv bin/archive/migrate_mysql_to_pg.py bin/migrate_mysql_to_pg.py
+ .venv/bin/python -m bin.migrate_mysql_to_pg
+"""
+from __future__ import annotations
+
+import argparse
+import sys
+import time
+from io import StringIO
+from typing import Iterable
+
+import pymysql
+import psycopg2
+from dotenv import load_dotenv
+
+load_dotenv(".env", override=False)
+
+# 15 张表的列定义（MySQL → PG 列名映射）
+# 大部分列名一致；少数 PG 用 JSONB（dataset_registry.dependency_ids）需要 JSON 解析
+TABLES = [
+ # (table_name, mysql_order_by, list_of_column_names)
+ ("config", "key", ["key", "value", "category", "description", "updated_at"]),
+ ("dataset_registry", "dataset_id", ["dataset_id", "name", "description", "storage_uri", "storage_layer",
+ "management_role", "source", "sync_script", "dependency_ids",
+ "enabled", "sort_order", "status", "trigger_source",
+ "started_at", "finished_at", "last_success_at", "last_failure_at",
+ "message", "last_error", "needs_resync", "progress_current",
+ "progress_total", "current_step", "updated_at"]),
+ ("indices", "index_code", ["index_code", "index_name", "market", "category", "source", "enabled", "updated_at"]),
+ ("industry", "code", ["code", "industry_name", "industry_classification", "update_date", "updated_at"]),
+ ("sectors", "sector_key", ["sector_key", "sector_name", "taxonomy", "level", "source", "enabled", "updated_at"]),
+ ("stock_sector_map", "stock_code", ["stock_code", "sector_key", "updated_at"]),
+ ("stocks", "code", ["code", "name", "exchange", "list_date", "listing_status", "industry",
+ "total_share", "float_share", "share_updated_at", "kline_synced_at",
+ "updated_at"]),
+ ("market_regime_daily", "trade_date", ["trade_date", "advancers", "decliners", "advance_ratio", "turnover",
+ "turnover_avg_5d", "turnover_ratio_5d", "source", "is_extreme_panic",
+ "updated_at"]),
+ ("share", "stock_code,trade_date", ["stock_code", "trade_date", "total_share", "float_share"]),
+ ("sector_indices", "trade_date,sector_name", ["trade_date", "sector_name", "close", "sector_amplitude"]),
+ ("sector_features_daily", "trade_date,sector_name", ["trade_date", "sector_name", "sector_ret", "sector_amplitude",
+ "close", "ema10", "ema20", "ema200", "score"]),
+ ("kline_index", "index_code,trade_date", ["index_code", "trade_date", "open", "high", "low", "close", "volume"]),
+ ("moneyflow", "stock_code,trade_date", ["stock_code", "trade_date", "main_net_inflow",
+ "large_net_inflow", "medium_net_inflow", "small_net_inflow"]),
+ ("kline_stock", "stock_code,trade_date", ["stock_code", "trade_date", "open", "high", "low", "close", "volume"]),
+ ("kline_5min", "stock_code,bar_time", ["stock_code", "bar_time", "open", "high", "low", "close",
+ "volume", "amount", "turnover_rate"]),
+]
+
+
+def mysql_conn():
+ return pymysql.connect(
+ host=__import__("os").environ.get("MYSQL_HOST", "127.0.0.1"),
+ port=int(__import__("os").environ.get("MYSQL_PORT", "3306")),
+ user=__import__("os").environ.get("MYSQL_USER", "root"),
+ password=__import__("os").environ.get("MYSQL_PASSWORD", ""),
+ database=__import__("os").environ.get("MYSQL_DATABASE", "market_data_sync_db"),
+ charset="utf8mb4",
+ cursorclass=pymysql.cursors.SSDictCursor, # 服务端 cursor，流式不撑内存
+ )
+
+
+def pg_conn():
+ import os as _os
+ pg_url = _os.environ.get("PG_URL", "").strip()
+ # SQLAlchemy 格式 → psycopg2 格式（剥掉 +psycopg2）
+ if pg_url.startswith("postgresql+psycopg2://"):
+ pg_url = pg_url.replace("postgresql+psycopg2://", "postgresql://", 1)
+ if not pg_url:
+ host = _os.environ.get("PG_HOST", "127.0.0.1")
+ port = _os.environ.get("PG_PORT", "5432")
+ user = _os.environ.get("PG_USER", "market_sync")
+ pw = _os.environ.get("PG_PASSWORD", "market_sync")
+ db = _os.environ.get("PG_DB_NAME", "market_data")
+ pg_url = f"postgresql://{user}:{pw}@{host}:{port}/{db}"
+ return psycopg2.connect(pg_url)
+
+
+def _convert_value(col: str, v):
+ """MySQL → PG 类型兼容转换。"""
+ if v is None:
+ return None
+ if col == "dependency_ids" and isinstance(v, str):
+ # MySQL 存的是 Python list repr（如 ['stock_info']），不是合法 JSON。
+ # 先后备方案：先 json.loads（合法 JSON），失败再 ast.literal_eval（Python repr）。
+ import ast
+ import json
+ if not v.strip():
+ return []
+ # 试 1: 当 JSON 解析
+ try:
+ return json.loads(v)
+ except json.JSONDecodeError:
+ pass
+ # 试 2: 当 Python list/tuple repr 解析
+ try:
+ parsed = ast.literal_eval(v)
+ if isinstance(parsed, (list, tuple)):
+ return list(parsed)
+ except (ValueError, SyntaxError):
+ pass
+ # 试 3: 逗号分隔的纯字符串列表（兜底）
+ return [s.strip().strip("'\"") for s in v.strip("[]").split(",") if s.strip()]
+ if isinstance(v, str) and v == "":
+ return None
+ return v
+
+
+def _copy_from_path(conn_pg, table_name: str, columns: list[str], tmp_path: str) -> int:
+ """从已写好的 temp file COPY 到 PG。
+
+ temp file 由调用方在游标活跃时填充好。
+ """
+ cur = conn_pg.cursor()
+ with open(tmp_path, "rb") as fb:
+ full_table = f"market_data.{table_name}"
+ cur.copy_expert(
+ f"COPY {full_table} ({','.join(columns)}) FROM STDIN WITH (FORMAT text, NULL '')",
+ fb,
+ )
+ conn_pg.commit()
+ return 0 # 实际行数已经在外层统计了
+
+
+def _truncate_pg_table(conn_pg, table_name: str) -> None:
+ """清空 PG 表。用 DELETE 而不是 TRUNCATE（TRUNCATE 需要额外权限）。
+
+ DELETE 在大表上慢，但仅迁移时用一次可以接受。
+ """
+ cur = conn_pg.cursor()
+ cur.execute(f"DELETE FROM market_data.{table_name}")
+ conn_pg.commit()
+
+
+def migrate_table(mysql, pg, table_name: str, order_by: str, columns: list[str],
+ chunk_size: int = 100_000) -> int:
+ """单表迁移：先 TRUNCATE，再流式读 MySQL + COPY 写 PG。"""
+ t0 = time.time()
+ col_list = ", ".join(f"`{c}`" for c in columns)
+
+ with mysql.cursor() as cur:
+ cur.execute(f"SELECT COUNT(*) FROM {table_name}")
+ n_total = cur.fetchone()["COUNT(*)"]
+
+ if n_total == 0:
+ print(f" {table_name:<25} 0 行（跳过）")
+ return 0
+
+ # 先清空 PG 表（确保是干净迁移，不重复）
+ _truncate_pg_table(pg, table_name)
+
+ # 读 MySQL（流式：SSCursor 不缓存全表）→ 写 PG 临时文件 → COPY
+ # 关键：迭代 + 写 temp file 必须在 with 块内完成（cur 关闭后再迭代会失败）
+ print(f" {table_name:<25} {n_total:>12,} 行 ...", end=" ", flush=True)
+ import json as _json
+ import tempfile
+ JSONB_COLS = {"dependency_ids"}
+ tmp = tempfile.NamedTemporaryFile(
+ mode="w", encoding="utf-8", prefix=f"migrate_{table_name}_", suffix=".tsv",
+ delete=False,
+ )
+ tmp_path = tmp.name
+ written = 0
+ try:
+ with mysql.cursor() as cur:
+ order_by_quoted = ", ".join(f"`{c.strip()}`" for c in order_by.split(","))
+ cur.execute(f"SELECT {col_list} FROM `{table_name}` ORDER BY {order_by_quoted}")
+ for r in cur: # SSCursor：一次一行，不缓存全表
+ cells = []
+ for col, v in zip(columns, (r.get(col) for col in columns)):
+ if v is None:
+ cells.append("")
+ elif col in JSONB_COLS and not isinstance(v, str):
+ cells.append(_json.dumps(v, ensure_ascii=False))
+ else:
+ s = str(v)
+ cells.append(s.replace("\\", "\\\\").replace("\t", "\\t").replace("\n", "\\n"))
+ tmp.write("\t".join(cells))
+ tmp.write("\n")
+ written += 1
+ # with 块结束，cur 已关闭。temp file 写完。关 file 准备 COPY。
+ tmp.flush()
+ tmp.close()
+ # COPY 从磁盘 temp file 读
+ with open(tmp_path, "rb") as fb:
+ cur_pg = pg.cursor()
+ full_table = f"market_data.{table_name}"
+ cur_pg.copy_expert(
+ f"COPY {full_table} ({','.join(columns)}) FROM STDIN WITH (FORMAT text, NULL '')",
+ fb,
+ )
+ pg.commit()
+ finally:
+ try:
+ import os as _os
+ _os.unlink(tmp_path)
+ except OSError:
+ pass
+ elapsed = time.time() - t0
+ print(f"→ {written:>12,} 行 ({elapsed:.1f}s)")
+ return written
+
+
+def main():
+ p = argparse.ArgumentParser()
+ p.add_argument("--only", help="逗号分隔的表名列表，只迁指定的")
+ p.add_argument("--chunk-size", type=int, default=100_000)
+ p.add_argument("--skip", help="逗号分隔要跳过的表")
+ args = p.parse_args()
+
+ only = set((args.only or "").split(",")) if args.only else None
+ skip = set((args.skip or "").split(",")) if args.skip else set()
+
+ targets = [
+ t for t in TABLES
+ if (only is None or t[0] in only) and t[0] not in skip
+ ]
+
+ print(f"=== MySQL → PG 数据迁移 ===")
+ print(f"目标表: {len(targets)} 张")
+ print(f"源: MySQL market_data_sync_db (127.0.0.1:3306)")
+ print(f"目标: PG market_data 库 market_data schema")
+ print()
+
+ mysql = mysql_conn()
+ pg = pg_conn()
+
+ grand_total = 0
+ grand_t0 = time.time()
+ for table_name, order_by, columns in targets:
+ try:
+ n = migrate_table(mysql, pg, table_name, order_by, columns, args.chunk_size)
+ grand_total += n
+ except Exception as e:
+ print(f" ✗ {table_name} 失败: {e}")
+ grand_elapsed = time.time() - grand_t0
+ print()
+ print(f"=== 全部完成：{grand_total:,} 行 / 耗时 {grand_elapsed:.1f}s ===")
+
+ # 简单校验：每张表行数对比
+ print("\n=== 校验：MySQL vs PG 行数对比 ===")
+ with mysql.cursor() as cur, pg.cursor() as cur_pg:
+ for table_name, _, _ in targets:
+ cur.execute(f"SELECT COUNT(*) AS n FROM {table_name}")
+ n_mysql = cur.fetchone()["n"]
+ cur_pg.execute(f"SELECT COUNT(*) FROM market_data.{table_name}")
+ n_pg = cur_pg.fetchone()[0]
+ ok = "✓" if n_mysql == n_pg else "✗"
+ print(f" {ok} {table_name:<25} MySQL={n_mysql:>12,} PG={n_pg:>12,}")
+
+ mysql.close()
+ pg.close()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/bin/market_sync_lhb_run.sh b/bin/market_sync_lhb_run.sh
new file mode 100755
index 0000000..e282e18
--- /dev/null
+++ b/bin/market_sync_lhb_run.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+# systemd 调度的入口（专跑龙虎榜）
+#
+# 设计：
+# - 与 market_sync_moneyflow_run.sh 同构,只跑 longhubang 一个 task
+# - 22:00 触发（龙虎榜 19:00~21:00 陆续出齐；与 moneyflow 21:35 错开避免并发限速）
+# - 单日调用 ~3 次聚合层 + ~200 次席位层，3-5 分钟内完成
+# - 单独日志到 logs/lhb_.log
+# - exit code 透传
+set -u
+
+PROJECT_ROOT="/home/gao/Development/quant_home/market_sync"
+LOG_DIR="$PROJECT_ROOT/logs"
+mkdir -p "$LOG_DIR"
+
+TS=$(date +%Y%m%d_%H%M%S)
+LOG="$LOG_DIR/lhb_${TS}.log"
+
+echo "[market-sync-lhb] start ts=$TS log=$LOG" | tee -a "$LOG"
+
+# 注：不做法定节假日过滤 —— 与其他 3 个 timer 一致的 fail-open 原则。
+# 非交易日 akshare 仍可能返当周数据 → task 报 warning，不丢历史。
+
+# ── 跑 longhubang ──
+cd "$PROJECT_ROOT"
+.venv/bin/python -m app.entrypoints.cli sync longhubang 2>&1 | tee -a "$LOG"
+EXIT_CODE=${PIPESTATUS[0]}
+echo "[market-sync-lhb] done exit_code=$EXIT_CODE" | tee -a "$LOG"
+exit $EXIT_CODE
diff --git a/bin/market_sync_moneyflow_run.sh b/bin/market_sync_moneyflow_run.sh
new file mode 100755
index 0000000..3e7de1d
--- /dev/null
+++ b/bin/market_sync_moneyflow_run.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+# systemd 调度的入口（专跑 moneyflow）
+#
+# 设计：
+# - 与 market_sync_tick_run.sh 同构,只跑 moneyflow 一个 task
+# - mairui 文档：「更新：每日 21:30」—— systemd 21:35 触发,留 5min buffer
+# - CLI 不传 --force：让 task 走 21:30 时间门控（moneyflow 暂无门控；为一致仍走通路径）
+# - 单独日志到 logs/moneyflow_.log
+# - exit code 透传
+set -u
+
+PROJECT_ROOT="/home/gao/Development/quant_home/market_sync"
+LOG_DIR="$PROJECT_ROOT/logs"
+mkdir -p "$LOG_DIR"
+
+TS=$(date +%Y%m%d_%H%M%S)
+LOG="$LOG_DIR/moneyflow_${TS}.log"
+
+echo "[market-sync-moneyflow] start ts=$TS log=$LOG" | tee -a "$LOG"
+
+# 注：不做法定节假日过滤 —— 与 market_sync_run.sh 同样的 fail-open 原则。
+# 大不了非交易日 mairui 返回空 → task 报 warning，不丢历史。
+
+# ── 跑 moneyflow ──
+cd "$PROJECT_ROOT"
+.venv/bin/python -m app.entrypoints.cli sync moneyflow 2>&1 | tee -a "$LOG"
+EXIT_CODE=${PIPESTATUS[0]}
+echo "[market-sync-moneyflow] done exit_code=$EXIT_CODE" | tee -a "$LOG"
+exit $EXIT_CODE
diff --git a/bin/market_sync_run.sh b/bin/market_sync_run.sh
new file mode 100755
index 0000000..250a34f
--- /dev/null
+++ b/bin/market_sync_run.sh
@@ -0,0 +1,92 @@
+#!/bin/bash
+# systemd 调度的入口：跑一次全量同步 + 配套结构化 watch
+#
+# 设计：
+# - 后台启动 runall_once.py（输出重定向到时间戳 log）
+# - 同时后台启动 structured_watch.sh 盯进程（13 checkpoint × 递增间隔）
+# - wrapper 自身 fast-poll runall PID（每 5s 一次）
+# - runall 一退出就 SIGTERM watch（不靠 watch 自己的 checkpoint 轮询，避开 30min 长 sleep）
+# - watch 自然结束后整个脚本退出
+# - 退出码透传 runall_once.py 的 exit code，systemd 据此判断成功 / 失败
+#
+# 为什么不只用 structured_watch：
+# - 看 [[background-watcher-pitfalls]] 记忆：先承诺后验证是三大坑之一
+# - structured_watch.sh 的 schedule 在后期是 30min / 60min 间隔
+# - runall 死后，watch 最坏要等 60min 才发现进程没了
+# - wrapper fast-poll 让响应时间 < 5s
+#
+# 为什么不让 systemd TimeoutStartSec= 自动收尾：
+# - 超时是 SIGTERM → SIGKILL，但 SIGKILL 不给 runall / watch 写日志的机会
+# - 自己 fast-poll 能优雅收尾（先 SIGTERM watch 留出写日志时间，再 SIGKILL 兜底）
+set -u
+
+PROJECT_ROOT="/home/gao/Development/quant_home/market_sync"
+LOG_DIR="$PROJECT_ROOT/logs"
+mkdir -p "$LOG_DIR"
+
+TS=$(date +%Y%m%d_%H%M%S)
+RUNALL_LOG="$LOG_DIR/runall_${TS}.log"
+WATCH_LOG="$LOG_DIR/watch_${TS}.log"
+
+echo "[market-sync-run] start ts=$TS"
+echo "[market-sync-run] runall_log=$RUNALL_LOG"
+echo "[market-sync-run] watch_log=$WATCH_LOG"
+
+# 注：不做法定节假日过滤。理由（fail-open 原则）：
+# - 节假日配置（TRADING_HOLIDAYS / config 表）一旦出错（误把今天加进去、
+# 漏更新本年度、笔误等）= 直接漏一天数据，且没声音
+# - 大不了非交易日跑出来 mairui/baostock 返回空数据，task 报 warning 但不丢历史
+# - 周末过滤由 systemd timer `OnCalendar=Mon..Fri` 完成，足够
+# ── 1) 后台启动 runall_once.py ──
+cd "$PROJECT_ROOT"
+.venv/bin/python bin/runall_once.py > "$RUNALL_LOG" 2>&1 &
+RUNALL_PID=$!
+echo "[market-sync-run] runall pid=$RUNALL_PID"
+
+# ── 2) 后台启动 structured_watch.sh（独立进程，不阻塞 wrapper）
+# PATTERN 不含 "python" → 触发 watch 内部 case 第二分支，前缀 [p]ython.* 拼出
+"$PROJECT_ROOT/bin/structured_watch.sh" \
+ "runall_once" \
+ "$RUNALL_LOG" \
+ "$WATCH_LOG" \
+ "market-sync-full" > /dev/null 2>&1 &
+WATCH_PID=$!
+echo "[market-sync-run] watch pid=$WATCH_PID"
+
+# ── 3) wrapper 自身 fast-poll runall PID（每 5s）
+# 这是关键修复：runall 死后 < 5s 内就触发 watch 退出
+RUNALL_EXIT_CODE=1
+while kill -0 "$RUNALL_PID" 2>/dev/null; do
+ sleep 5
+done
+
+# runall 已退出，取 exit code
+wait "$RUNALL_PID" 2>/dev/null
+RUNALL_EXIT_CODE=$?
+echo "[market-sync-run] runall 已退出 exit_code=$RUNALL_EXIT_CODE"
+
+# ── 4) 通知 watch 退出（如果还在跑）──
+if kill -0 "$WATCH_PID" 2>/dev/null; then
+ echo "[market-sync-run] 通知 watch 退出（pid=$WATCH_PID）"
+ kill -TERM "$WATCH_PID" 2>/dev/null || true
+ # 给 watch 30s 自然退出（不要立即 KILL，留它写最后日志）
+ for _ in 1 2 3 4 5 6; do
+ sleep 5
+ kill -0 "$WATCH_PID" 2>/dev/null || break
+ done
+ if kill -0 "$WATCH_PID" 2>/dev/null; then
+ echo "[market-sync-run] ⚠️ watch 30s 内未退出，强制 kill -9"
+ kill -KILL "$WATCH_PID" 2>/dev/null || true
+ fi
+fi
+
+# ── 5) 输出 summary ──
+SUMMARY="$LOG_DIR/runall_summary.json"
+if [ -f "$SUMMARY" ]; then
+ echo "[market-sync-run] === summary ==="
+ cat "$SUMMARY"
+ echo "[market-sync-run] === /summary ==="
+fi
+
+echo "[market-sync-run] done exit_code=$RUNALL_EXIT_CODE"
+exit "$RUNALL_EXIT_CODE"
\ No newline at end of file
diff --git a/bin/market_sync_tick_run.sh b/bin/market_sync_tick_run.sh
new file mode 100755
index 0000000..b1f06d8
--- /dev/null
+++ b/bin/market_sync_tick_run.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+# systemd 调度的入口（专跑 tick_trade）
+#
+# 设计：
+# - 比 market_sync_run.sh 简单：只跑一个 task (tick_trade)
+# - 21:00 门控由 task 内部处理；systemd 触发时间 21:05 留 5min buffer
+# - CLI 加 --force 跳过 21:00 门控（systemd 触发时已是 21:05+，force 不会真正"跳过"
+# 任何东西，但保持与 runall_once 同步任务的 force 语义一致）
+# - 单独日志到 logs/tick_.log
+# - exit code 透传（systemd 据此判成功 / 失败）
+set -u
+
+PROJECT_ROOT="/home/gao/Development/quant_home/market_sync"
+LOG_DIR="$PROJECT_ROOT/logs"
+mkdir -p "$LOG_DIR"
+
+TS=$(date +%Y%m%d_%H%M%S)
+LOG="$LOG_DIR/tick_${TS}.log"
+
+echo "[market-sync-tick] start ts=$TS log=$LOG" | tee -a "$LOG"
+
+# 注：不做法定节假日过滤 —— 与 market_sync_run.sh 同样的 fail-open 原则。
+# 大不了非交易日 mairui 返回空 → task 报 warning，不丢历史。
+
+# ── 跑 tick_trade（force=True 跳过 21:00 门控，因为 systemd 触发时间 21:05 已过门控）──
+cd "$PROJECT_ROOT"
+.venv/bin/python -m app.entrypoints.cli sync tick_trade --force 2>&1 | tee -a "$LOG"
+EXIT_CODE=${PIPESTATUS[0]}
+echo "[market-sync-tick] done exit_code=$EXIT_CODE" | tee -a "$LOG"
+exit $EXIT_CODE
diff --git a/bin/run_remaining.sh b/bin/run_remaining.sh
index 23508b3..d96597c 100755
--- a/bin/run_remaining.sh
+++ b/bin/run_remaining.sh
@@ -1,6 +1,7 @@
#!/bin/bash
-# 跑剩余三个任务：moneyflow → share_snapshot → market_regime
+# 跑剩余任务：tick_trade → moneyflow → share_snapshot → market_regime
# 跳过 kline_5min（全量回填 6 年太慢，下次有空再补）
+# tick_trade 用 --force 跳过 21:00 门控（人工补跑场景）
set -e
cd "$(dirname "$0")/.."
export PYTHONPATH="$(pwd)"
@@ -9,11 +10,12 @@ mkdir -p logs
# 先清理上次可能留下的卡死记录（recover_interrupted_syncs 启动时自动处理，但显式更稳）
echo "=== 启动时间: $(date) ===" | tee logs/runall_remaining.log
-for TASK in moneyflow share_snapshot market_regime; do
+for TASK in "tick_trade --force" moneyflow share_snapshot market_regime; do
echo "" | tee -a logs/runall_remaining.log
echo "=== [$TASK] 开始 $(date) ===" | tee -a logs/runall_remaining.log
T0=$(date +%s)
- if .venv/bin/python -m app.entrypoints.cli sync "$TASK" 2>&1 | tee -a logs/runall_remaining.log; then
+ # shellcheck disable=SC2086
+ if .venv/bin/python -m app.entrypoints.cli sync $TASK 2>&1 | tee -a logs/runall_remaining.log; then
T1=$(date +%s)
echo "=== [$TASK] 完成，耗时 $((T1 - T0))s ===" | tee -a logs/runall_remaining.log
else
diff --git a/bin/runall_once.py b/bin/runall_once.py
index 60f1901..7144389 100644
--- a/bin/runall_once.py
+++ b/bin/runall_once.py
@@ -1,10 +1,14 @@
"""一次性跑完所有 sync task（按依赖顺序）。
-跳过 industry_sector（已 ok）。
-按顺序触发：stock_basic → kline_daily → kline_index → kline_5min → moneyflow
- → share_snapshot → market_regime
+按 sort_order 触发：stock_basic → kline_daily → kline_index → kline_5min
+ → industry_sector → sector_features
+ → share_snapshot → market_regime
-每个 task 跑完写日志 + 把 result 落到 summary.json。
+每个 task 跑完写日志 + 把 result 落到 logs/runall_summary.json。
+
+注：以下 task **不**在 runall 链中 —— 各自有独立的 systemd timer：
+ - tick_trade : mairui 21:00 发布 → market-sync-tick.timer (21:05)
+ - moneyflow : mairui 21:30 发布 → market-sync-moneyflow.timer (21:35)
"""
from __future__ import annotations
@@ -28,20 +32,21 @@ seed_sync_registry()
from app.tasks import get_task
-# 顺序执行，便于排查
+# 顺序执行，按 sort_order 走（便于排查 + 避免外部 API 限流）
+# 注：tick_trade / moneyflow 由独立 timer 跑（21:05 / 21:35）
TASKS = [
"stock_basic",
- "industry_sector", # 已 ok，但再跑一次全量也 OK (用 SKIP 标志)
"kline_index",
"kline_daily",
"kline_5min",
- "moneyflow",
+ "industry_sector",
+ "sector_features",
"share_snapshot",
"market_regime",
]
-# industry_sector 跑过一次（291s 全量）已 ok，跳过
-SKIP = {"industry_sector"}
+# 不跳任何 task — 全量跑
+SKIP: set[str] = set()
results = {}
t_all = time.time()
diff --git a/bin/systemd/market-sync-lhb.service b/bin/systemd/market-sync-lhb.service
new file mode 100644
index 0000000..1f759ce
--- /dev/null
+++ b/bin/systemd/market-sync-lhb.service
@@ -0,0 +1,33 @@
+[Unit]
+Description=Market data 龙虎榜 sync (akshare 源, daily 22:00)
+Documentation=file:///home/gao/Development/quant_home/market_sync/bin/market_sync_lhb_run.sh
+After=network-online.target market-sync.service market-sync-tick.service market-sync-moneyflow.service
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+WorkingDirectory=/home/gao/Development/quant_home/market_sync
+User=gao
+Group=gao
+
+# 入口脚本：单独跑 longhubang 一个 task
+# 龙虎榜数据 19:00~21:00 陆续出齐 → 22:00 触发（与 moneyflow 21:35 错开避免并发）
+ExecStart=/home/gao/Development/quant_home/market_sync/bin/market_sync_lhb_run.sh
+
+# 硬上限 1h（每日 ~200 次 akshare 调用 + DB upsert，3-5 分钟内完成，留 buffer）
+TimeoutStartSec=3600
+
+# 不要 Restart=（oneshot 失败就让 OnFailure= 发通知，别自动重跑——
+# 重跑会撞外部 API 限流窗口，浪费资源）
+# Restart=no 是 oneshot 默认值
+
+# 日志走 journald（journalctl -u market-sync-lhb.service -f）
+StandardOutput=journal
+StandardError=journal
+SyslogIdentifier=market-sync-lhb
+
+# 环境（不读 /etc/environment，只带这几个；.env 由 cli 内部 load_dotenv）
+Environment=PYTHONUNBUFFERED=1
+
+[Install]
+WantedBy=multi-user.target
diff --git a/bin/systemd/market-sync-lhb.timer b/bin/systemd/market-sync-lhb.timer
new file mode 100644
index 0000000..98de850
--- /dev/null
+++ b/bin/systemd/market-sync-lhb.timer
@@ -0,0 +1,16 @@
+[Unit]
+Description=Schedule 龙虎榜 sync — Mon..Fri 22:00 Asia/Shanghai
+# market-sync-lhb.service 是这个 timer 的执行单元
+# 龙虎榜数据 19:00~21:00 陆续出齐 → 22:00 触发（与 moneyflow 21:35 错开）
+
+[Timer]
+# 龙虎榜出齐 → 22:00 触发
+OnCalendar=Mon..Fri 22:00:00 Asia/Shanghai
+# 系统关机 / 错过执行时，下次开机补跑一次（避免漏数据）
+Persistent=true
+# 单位（service）的精确名称
+Unit=market-sync-lhb.service
+# 不要 AccuracySec（默认 1min 漂移够用，避免 22:00:00 整点打堆）
+
+[Install]
+WantedBy=timers.target
diff --git a/bin/systemd/market-sync-moneyflow.service b/bin/systemd/market-sync-moneyflow.service
new file mode 100644
index 0000000..98f4aff
--- /dev/null
+++ b/bin/systemd/market-sync-moneyflow.service
@@ -0,0 +1,33 @@
+[Unit]
+Description=Market data moneyflow sync (mairui transaction, daily 21:35)
+Documentation=file:///home/gao/Development/quant_home/market_sync/bin/market_sync_moneyflow_run.sh
+After=network-online.target market-sync.service market-sync-tick.service
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+WorkingDirectory=/home/gao/Development/quant_home/market_sync
+User=gao
+Group=gao
+
+# 入口脚本：单独跑 moneyflow 一个 task
+# mairui 文档：「更新：每日 21:30」—— 21:35 触发留 5min buffer
+ExecStart=/home/gao/Development/quant_home/market_sync/bin/market_sync_moneyflow_run.sh
+
+# 硬上限 2h（5206 只 × 50 RPS = 104s 理论上限，留 buffer 给重试 / 慢响应）
+TimeoutStartSec=7200
+
+# 不要 Restart=（oneshot 失败就让 OnFailure= 发通知，别自动重跑——
+# 重跑会撞外部 API 限流窗口，浪费 1.5h）
+# Restart=no 是 oneshot 默认值
+
+# 日志走 journald（journalctl -u market-sync-moneyflow.service -f）
+StandardOutput=journal
+StandardError=journal
+SyslogIdentifier=market-sync-moneyflow
+
+# 环境（不读 /etc/environment，只带这几个；.env 由 cli 内部 load_dotenv）
+Environment=PYTHONUNBUFFERED=1
+
+[Install]
+WantedBy=multi-user.target
diff --git a/bin/systemd/market-sync-moneyflow.timer b/bin/systemd/market-sync-moneyflow.timer
new file mode 100644
index 0000000..5f20a5e
--- /dev/null
+++ b/bin/systemd/market-sync-moneyflow.timer
@@ -0,0 +1,16 @@
+[Unit]
+Description=Schedule moneyflow sync — Mon..Fri 21:35 Asia/Shanghai
+# market-sync-moneyflow.service 是这个 timer 的执行单元
+# mairui 文档：「更新：每日 21:30」—— 21:35 触发留 5 分钟 buffer
+
+[Timer]
+# mairui 21:30 发布数据 → 21:35 触发
+OnCalendar=Mon..Fri 21:35:00 Asia/Shanghai
+# 系统关机 / 错过执行时，下次开机补跑一次（避免漏数据）
+Persistent=true
+# 单位（service）的精确名称
+Unit=market-sync-moneyflow.service
+# 不要 AccuracySec（默认 1min 漂移够用，避免 21:35:00 整点打堆）
+
+[Install]
+WantedBy=timers.target
diff --git a/bin/systemd/market-sync-tick.service b/bin/systemd/market-sync-tick.service
new file mode 100644
index 0000000..4a3742d
--- /dev/null
+++ b/bin/systemd/market-sync-tick.service
@@ -0,0 +1,33 @@
+[Unit]
+Description=Market data tick-by-tick trade sync (mairui hsrl/zbjy, daily 21:05)
+Documentation=file:///home/gao/Development/quant_home/market_sync/bin/market_sync_tick_run.sh
+After=network-online.target market-sync.service
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+WorkingDirectory=/home/gao/Development/quant_home/market_sync
+User=gao
+Group=gao
+
+# 入口脚本：单独跑 tick_trade 一个 task（mairui 21:00 发布数据，
+# 这个 service 在 21:05 触发，留 5min buffer；force=True 绕过 task 内部 21:00 门控）
+ExecStart=/home/gao/Development/quant_home/market_sync/bin/market_sync_tick_run.sh
+
+# 硬上限 2h（5000 只 × 1 RPS ≈ 17min，留 buffer 给 retry / 慢任务）
+TimeoutStartSec=7200
+
+# 不要 Restart=（oneshot 失败就让 OnFailure= 发通知，别自动重跑——
+# 重跑会撞外部 API 限流窗口，浪费 1.5h）
+# Restart=no 是 oneshot 默认值
+
+# 日志走 journald（journalctl -u market-sync-tick.service -f）
+StandardOutput=journal
+StandardError=journal
+SyslogIdentifier=market-sync-tick
+
+# 环境（不读 /etc/environment，只带这几个；.env 由 cli 内部 load_dotenv）
+Environment=PYTHONUNBUFFERED=1
+
+[Install]
+WantedBy=multi-user.target
diff --git a/bin/systemd/market-sync-tick.timer b/bin/systemd/market-sync-tick.timer
new file mode 100644
index 0000000..05ace68
--- /dev/null
+++ b/bin/systemd/market-sync-tick.timer
@@ -0,0 +1,17 @@
+[Unit]
+Description=Schedule tick_trade sync — Mon..Fri 21:05 Asia/Shanghai
+# market-sync-tick.service 是这个 timer 的执行单元
+# 触发顺序：market-sync.service (15:30) 之后 market-sync-tick.service (21:05)
+# 但 systemd timer 独立触发，After= 只是声明依赖关系（非强制）
+
+[Timer]
+# mairui 文档：「更新：每日 21:00」—— 21:05 触发留 5 分钟 buffer
+OnCalendar=Mon..Fri 21:05:00 Asia/Shanghai
+# 系统关机 / 错过执行时，下次开机补跑一次（避免漏数据）
+Persistent=true
+# 单位（service）的精确名称
+Unit=market-sync-tick.service
+# 不要 AccuracySec（默认 1min 漂移够用，避免 21:05:00 整点打堆）
+
+[Install]
+WantedBy=timers.target
diff --git a/bin/systemd/market-sync.service b/bin/systemd/market-sync.service
new file mode 100644
index 0000000..d75396e
--- /dev/null
+++ b/bin/systemd/market-sync.service
@@ -0,0 +1,32 @@
+[Unit]
+Description=Market Data Full Sync (PostgreSQL, 9 sync tasks end-to-end)
+Documentation=file:///home/gao/Development/quant_home/market_sync/bin/market_sync_run.sh
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+WorkingDirectory=/home/gao/Development/quant_home/market_sync
+User=gao
+Group=gao
+
+# 入口脚本（runall_once.py + structured_watch.sh 二合一）
+ExecStart=/home/gao/Development/quant_home/market_sync/bin/market_sync_run.sh
+
+# 硬上限 2h（实际 ~80min，留 buffer 给 retry / 慢任务）
+TimeoutStartSec=7200
+
+# 不要 Restart=（oneshot 失败就让 OnFailure= 发通知，别自动重跑——
+# 重跑会撞外部 API 限流窗口，浪费 1.5h）
+# Restart=no 是 oneshot 默认值
+
+# 日志走 journald（journalctl -u market-sync.service -f）
+StandardOutput=journal
+StandardError=journal
+SyslogIdentifier=market-sync
+
+# 环境（不读 /etc/environment，只带这几个；.env 由 runall_once.py 内部 load_dotenv）
+Environment=PYTHONUNBUFFERED=1
+
+[Install]
+WantedBy=multi-user.target
\ No newline at end of file
diff --git a/bin/systemd/market-sync.timer b/bin/systemd/market-sync.timer
new file mode 100644
index 0000000..013560e
--- /dev/null
+++ b/bin/systemd/market-sync.timer
@@ -0,0 +1,15 @@
+[Unit]
+Description=Schedule market data full sync — Mon..Fri 15:30 Asia/Shanghai
+# market-sync.service 是这个 timer 的执行单元
+
+[Timer]
+# A 股收盘时间 15:00，30 分钟 buffer 让最后一笔 tick 落地
+OnCalendar=Mon..Fri 15:30:00 Asia/Shanghai
+# 系统关机 / 错过执行时，下次开机补跑一次（避免漏数据）
+Persistent=true
+# 单位（service）的精确名称
+Unit=market-sync.service
+# 不要 AccuracySec（默认 1min 漂移够用，避免 15:30:00 整点打堆）
+
+[Install]
+WantedBy=timers.target
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index 0ef5b2c..147cd33 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -20,7 +20,10 @@ requests>=2.30.0
tenacity>=8.0.0
# Datasources — A 股免费源
-# akshare 永远不使用（项目级决策）
+# akshare 仅用于"无稳定替代源"的数据集（2026-07-01 决策）。
+# 当前仅龙虎榜使用（聚合层 stock_lhb_detail_em + 席位层 stock_lhb_stock_detail_em）。
+# 5min K / moneyflow / kline_daily 等已有 baostock / mairui 替代，禁 akshare。
+akshare>=1.18.0
baostock>=0.9.0
# 雪球非官方 SDK：需要 XUEQIU_TOKEN（环境变量）才能拿到股本 / 日K 复核
# 没装 → share / kline_daily 任务静默失败（ImportError 被 except 吞）
diff --git a/scripts/run_all_sync.sh b/scripts/run_all_sync.sh
index b7b1d4d..2d62e7f 100755
--- a/scripts/run_all_sync.sh
+++ b/scripts/run_all_sync.sh
@@ -57,6 +57,10 @@ case "${1:-all}" in
4|kline_daily) run_step "4_kline_daily" "kline_daily" "--workers 20" ;;
5|share_snapshot) run_step "5_share_snapshot" "share_snapshot" "--workers 10" ;;
6|market_regime) run_step "6_market_regime" "market_regime" ;;
+ 7|kline_5min) run_step "7_kline_5min" "kline_5min" "--workers 10" ;;
+ 8|moneyflow) run_step "8_moneyflow" "moneyflow" "--workers 10" ;;
+ 9|sector_features) run_step "9_sector_features" "sector_features" ;;
+ 10|tick_trade) run_step "10_tick_trade" "tick_trade" "--workers 5 --force" ;;
all)
run_step "1_stock_basic" "stock_basic" || exit 1
run_step "2_industry_sector" "industry_sector" || exit 1
@@ -64,10 +68,14 @@ case "${1:-all}" in
run_step "4_kline_daily" "kline_daily" "--workers 20" || exit 1
run_step "5_share_snapshot" "share_snapshot" "--workers 10" || exit 1
run_step "6_market_regime" "market_regime" || exit 1
- echo "$LOG_PREFIX ✓ 全部 6 步完成" | tee -a "$RUN_LOG"
+ run_step "7_kline_5min" "kline_5min" "--workers 10" || exit 1
+ run_step "8_moneyflow" "moneyflow" "--workers 10" || exit 1
+ run_step "9_sector_features" "sector_features" || exit 1
+ run_step "10_tick_trade" "tick_trade" "--workers 5 --force" || exit 1
+ echo "$LOG_PREFIX ✓ 全部 10 步完成" | tee -a "$RUN_LOG"
;;
*)
- echo "usage: $0 [1|2|3|4|5|6|all]"
+ echo "usage: $0 [1|2|3|4|5|6|7|8|9|10|all]"
exit 1
;;
esac
diff --git a/tests/test_schema_models.py b/tests/test_schema_models.py
new file mode 100644
index 0000000..50d18c1
--- /dev/null
+++ b/tests/test_schema_models.py
@@ -0,0 +1,38 @@
+"""ORM 模型 metadata 一致性测试（PostgreSQL only，2026-06-16 重构后）。
+
+历史背景：原 test_schema_models.py 还测了 schema.py vs models.py 的 DDL 漂移。
+schema.py 在这次重构中已删除（MySQL DDL 不再需要），所以只保留 ORM metadata 完整性测试。
+"""
+from __future__ import annotations
+
+
+def test_orm_metadata_registers_all_tables():
+ """ORMBase.metadata 应该注册 18 张 PG 表（项目主业务表）。"""
+ from app.core.db import models # 触发全部模型 import
+ from app.core.db.orm import ORMBase
+
+ # 注意：SA 2.x 的 MetaData.tables 既能按 bare name 也能按 schema-qualified name 索引
+ # （keyed 容器），所以用 values() 拿 Table 对象再用 .name 拿 bare 名
+ bare_names = {t.name for t in ORMBase.metadata.tables.values()}
+ assert len(bare_names) == 18, f"期望 18 张 ORM 表，实际 {len(bare_names)}: {bare_names}"
+ expected = {
+ "config", "dataset_registry", "stocks", "indices",
+ "kline_stock", "kline_index", "kline_5min", "moneyflow", "share",
+ "sectors", "stock_sector_map", "industry",
+ "sector_indices", "sector_features_daily", "market_regime_daily",
+ "tick_trade",
+ "longhubang_daily", "longhubang_seat", # 2026-07-01 龙虎榜
+ }
+ assert bare_names == expected, f"ORM 表名集合与期望不符: {bare_names ^ expected}"
+
+
+def test_orm_tables_in_market_data_schema():
+ """所有 ORM 表应该在 market_data PG schema 下（不是 public，不是 None）。"""
+ from app.core.db import models # 触发全部模型 import
+ from app.core.db.orm import ORMBase
+
+ bad = []
+ for table in ORMBase.metadata.tables.values():
+ if table.schema != "market_data":
+ bad.append((table.name, table.schema))
+ assert not bad, f"以下表 schema 不是 market_data: {bad}"
diff --git a/tests/test_smoke.py b/tests/test_smoke.py
index f1149b2..6fd16a4 100644
--- a/tests/test_smoke.py
+++ b/tests/test_smoke.py
@@ -8,7 +8,7 @@ import pytest
def test_import_config():
from app.core.config import settings
assert settings is not None
- assert hasattr(settings, "mysql_host")
+ assert hasattr(settings, "pg_host")
def test_import_data_source_base():
@@ -35,9 +35,10 @@ def test_tasks_registry():
from app.tasks import TASKS, get_task
assert isinstance(TASKS, dict)
# 注：每次新增 task 都要更新这里的数字
- # 当前 9 个：stock_basic, kline_daily, kline_index, kline_5min, moneyflow,
- # industry_sector, sector_features, share_snapshot, market_regime
- assert len(TASKS) == 9
+ # 当前 11 个：stock_basic, kline_daily, kline_index, kline_5min, tick_trade,
+ # moneyflow, industry_sector, sector_features, share_snapshot, market_regime,
+ # longhubang
+ assert len(TASKS) == 11
# get_task 应该返回实例
task = get_task("kline_daily")
assert task.dataset_id == "kline_daily"
@@ -45,10 +46,18 @@ def test_tasks_registry():
def test_sync_definitions():
from app.core.sync.registry import SYNC_DEFINITIONS
- assert len(SYNC_DEFINITIONS) == 9
+ assert len(SYNC_DEFINITIONS) == 11
# 所有 dataset_id 应唯一
ids = [d["dataset_id"] for d in SYNC_DEFINITIONS]
- assert len(set(ids)) == 9
+ assert len(set(ids)) == 11
+ # tick_trade 应在其中且 sort_order=45
+ tt = next(d for d in SYNC_DEFINITIONS if d["dataset_id"] == "tick_trade")
+ assert tt["sort_order"] == 45
+ assert "stock_basic" in tt["dependency_ids"]
+ # longhubang 应在其中且 sort_order=85
+ lh = next(d for d in SYNC_DEFINITIONS if d["dataset_id"] == "longhubang")
+ assert lh["sort_order"] == 85
+ assert "stock_basic" in lh["dependency_ids"]
def test_build_default_registry():
@@ -60,8 +69,9 @@ def test_build_default_registry():
assert "datasource_xinlang" in keys
assert "datasource_baostock" in keys
assert "datasource_mairui" in keys
- # akshare 已移除（项目级决策：永远不用）
- assert "datasource_akshare" not in keys
+ # akshare：2026-07-01 决策更新——龙虎榜无替代源，烟测通过，可注册。
+ # 见 [[no-akshare]] 记忆：仅"无替代"+"akshare 烟测 OK"才允许用。
+ assert "datasource_akshare_lhb" in keys
assert "datasource_xueqiu" in keys