refactor: PG-only 迁移 + 龙虎榜/tick/moneyflow 同步 + stock_code 统一 + mairui 编码修复
## 1. PG-only 重构
- 删 app/core/db/connection.py + schema.py (MySQL 路径)
- 新 app/core/db/{orm,models,pg_bootstrap}.py — SQLAlchemy 2.x ORM 一键建表
- 16 张业务表全在 market_data schema,原生 TIMESTAMPTZ / JSONB / Float / TEXT
- requirements.txt 删 PyMySQL 路径,加 psycopg2
## 2. 同步任务扩展(3 个新 task)
- **tick_trade** (mairui hsrl/zbjy):当天逐笔交易,21:00 发布
- **moneyflow** (mairui hsstock/history/transaction):个股资金流,21:30 发布
- **longhubang** (akshare):龙虎榜聚合层 + 席位层(2 张新表)
- 长虎榜放宽 akshare 政策:仅"无替代源 + 烟测通过"场景允许
- data_eastmoney 私有 API 不需要(akshare 烟测通过)
- 3-timer 设计:
- 15:30 market-sync.service (8 base tasks via runall_once)
- 21:05 market-sync-tick.service (tick_trade)
- 21:35 market-sync-moneyflow.service (moneyflow)
- 22:00 market-sync-lhb.service (longhubang,新加)
- bin/systemd/ 新增 tick / moneyflow / lhb 各 1 对 service+timer
- bin/market_sync_*_run.sh wrapper 脚本(不做法定节假日过滤,fail-open)
## 3. stock_code 统一为带 SH/SZ/BJ 前缀
- 历史 bug:stocks.code 用 SH600519,但 kline/moneyflow/tick_trade/kline_5min
/stock_sector_map/industry 6 张表用纯 6 位 600519,跨表 JOIN 全部 0 行
- 新增 to_hermes() 工具:6位 / 9位(mairui `000001.SZ` 格式)→ 统一 SH000001
- 5 个 task 改写:用 to_hermes(code6) 写入 stock_code
- 一次性迁移 6 张表存量 154M 行(CASE WHEN 探测 + 去重 + 加前缀)
- ORM: stock_sector_map.stock_code / industry.code String(6)→String(10)
## 4. Bug 修复
- **share table stock_code 格式**:之前写 6 位不带前缀,与 stocks 不一致
→ 修 task_share_snapshot + 一次性 UPDATE 63,417 行加前缀
- **share_snapshot warning 状态错填 last_error**:
→ 加 mark_sync_warning() 走专用路径,不写 last_failure_at / last_error
- **schedule config lastRun 不同步**:
→ 加 update_job_status_for_dataset(),SyncTask.run() 完成后自动镜像
→ cli/runall 触发的 task 也能更新 schedule config
## 5. mairui UTF-8 编码修复
- 历史 bug:mairui.py:_fetch 用 latin-1 兜底解码,把所有 UTF-8 中文名
double-encoded 写入 stocks.name(如 `歌华有线` 变成 `æ\xad\x8cå\x8d\x8e...`)
- 加 _decode_response():UTF-8 → GBK → latin-1 兜底
- 一次性修复 stocks.name 5,213 行:
- 4,370 行 (encode('latin-1').decode('utf-8') 反向解码)
- 616 行 (含 fullwidth A,宽松 printable 检查)
- 820 行 (mid-character 截断,重新从 mairui 拉)
## 6. 测试
- tests/test_smoke.py: TASKS 10→11, SYNC_DEFINITIONS 10→11
- tests/test_schema_models.py: 16→18 张表,新增 longhubang_daily/seat
- pytest 11/11 passed
## 验证
- 6 张表 0 残留无前缀行
- stocks JOIN kline_stock / kline_5min / moneyflow / tick_trade / stock_sector_map:88-100% 命中
- 5,213 stocks.name 全部正确 UTF-8 中文
- pytest 11/11 passed
This commit is contained in:
+18
-36
@@ -1,4 +1,7 @@
|
|||||||
"""统一配置:从 .env / 环境变量读 MySQL、调度器、数据源开关等参数。"""
|
"""统一配置:从 .env / 环境变量读 PG、调度器、数据源开关等参数。
|
||||||
|
|
||||||
|
2026-06-16 重构:项目只支持 PostgreSQL,MySQL 字段全部移除。
|
||||||
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import os
|
import os
|
||||||
@@ -27,18 +30,6 @@ if _HAS_DOTENV:
|
|||||||
class Settings(BaseSettings):
|
class Settings(BaseSettings):
|
||||||
"""应用配置。所有字段都可以通过环境变量或 .env 文件覆盖。"""
|
"""应用配置。所有字段都可以通过环境变量或 .env 文件覆盖。"""
|
||||||
|
|
||||||
# MySQL
|
|
||||||
mysql_host: str = "127.0.0.1"
|
|
||||||
mysql_port: int = 3306
|
|
||||||
mysql_user: str = "root"
|
|
||||||
mysql_password: str = ""
|
|
||||||
# 同步任务写入的目标库(原始市场数据)
|
|
||||||
mysql_database: str = "market_data_sync_db"
|
|
||||||
# 简版库(对照测试,可选)
|
|
||||||
mysql_database_lite: str = "grid_seeker"
|
|
||||||
# 业务主库(保留— 同步任务不再写入)
|
|
||||||
mysql_database_business: str = "grid_seeker_model_base"
|
|
||||||
|
|
||||||
# Scheduler
|
# Scheduler
|
||||||
scheduler_tick_seconds: int = 5
|
scheduler_tick_seconds: int = 5
|
||||||
scheduler_timezone: str = "Asia/Shanghai"
|
scheduler_timezone: str = "Asia/Shanghai"
|
||||||
@@ -57,6 +48,14 @@ class Settings(BaseSettings):
|
|||||||
ds_sina_enabled: bool = True
|
ds_sina_enabled: bool = True
|
||||||
xueqiu_token: str = ""
|
xueqiu_token: str = ""
|
||||||
|
|
||||||
|
# PostgreSQL — 完整 URL(覆盖下面 PG_HOST 等)
|
||||||
|
pg_url: str = ""
|
||||||
|
pg_host: str = "127.0.0.1"
|
||||||
|
pg_port: int = 5432
|
||||||
|
pg_user: str = "market_sync"
|
||||||
|
pg_password: str = "market_sync"
|
||||||
|
pg_db_name: str = "market_data"
|
||||||
|
|
||||||
# Trading calendar
|
# Trading calendar
|
||||||
trading_holidays: str = ""
|
trading_holidays: str = ""
|
||||||
|
|
||||||
@@ -85,32 +84,15 @@ class Settings(BaseSettings):
|
|||||||
return []
|
return []
|
||||||
return [d.strip() for d in self.trading_holidays.split(",") if d.strip()]
|
return [d.strip() for d in self.trading_holidays.split(",") if d.strip()]
|
||||||
|
|
||||||
def mysql_url(self, database: Optional[str] = None) -> str:
|
def pg_sqlalchemy_url(self) -> str:
|
||||||
"""构造 SQLAlchemy URL。"""
|
"""构造 PG 的 SQLAlchemy URL(自动剥掉 +psycopg2 让 SQLAlchemy 决定 driver)。"""
|
||||||
db = database or self.mysql_database
|
if self.pg_url:
|
||||||
|
return self.pg_url
|
||||||
return (
|
return (
|
||||||
f"mysql+pymysql://{self.mysql_user}:{self.mysql_password}"
|
f"postgresql+psycopg2://{self.pg_user}:{self.pg_password}"
|
||||||
f"@{self.mysql_host}:{self.mysql_port}/{db}?charset=utf8mb4"
|
f"@{self.pg_host}:{self.pg_port}/{self.pg_db_name}"
|
||||||
)
|
)
|
||||||
|
|
||||||
def pymysql_connect_kwargs(self, database: Optional[str] = None) -> dict:
|
|
||||||
"""构造 pymysql.connect 的关键字参数。
|
|
||||||
|
|
||||||
内网自签名证书场景自动禁用 SSL。
|
|
||||||
"""
|
|
||||||
db = database or self.mysql_database
|
|
||||||
return {
|
|
||||||
"host": self.mysql_host,
|
|
||||||
"port": self.mysql_port,
|
|
||||||
"user": self.mysql_user,
|
|
||||||
"password": self.mysql_password,
|
|
||||||
"database": db,
|
|
||||||
"charset": "utf8mb4",
|
|
||||||
"autocommit": False,
|
|
||||||
# 兼容自签名证书的内网 MySQL — 禁用 SSL
|
|
||||||
"ssl": None,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
_settings: Optional[Settings] = None
|
_settings: Optional[Settings] = None
|
||||||
|
|
||||||
|
|||||||
@@ -65,6 +65,16 @@ _DATASOURCE_SEED: list[tuple[str, dict, str]] = [
|
|||||||
},
|
},
|
||||||
"K 线主源(日线 + 5min + 指数)",
|
"K 线主源(日线 + 5min + 指数)",
|
||||||
),
|
),
|
||||||
|
(
|
||||||
|
"datasource_akshare_lhb",
|
||||||
|
{
|
||||||
|
"name": "akshare 龙虎榜(东方财富封装)",
|
||||||
|
"provides": ["longhubang_daily", "longhubang_seat"],
|
||||||
|
"requiresCredential": False,
|
||||||
|
"note": "龙虎榜聚合层 + 席位层;无合理替代源,按 [[no-akshare]] 决策放行",
|
||||||
|
},
|
||||||
|
"龙虎榜聚合层 + 席位层(akshare → 东方财富)",
|
||||||
|
),
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
@@ -247,4 +257,7 @@ def build_default_registry() -> None:
|
|||||||
else:
|
else:
|
||||||
# 即使没 token 也注册,is_available() 会拦截
|
# 即使没 token 也注册,is_available() 会拦截
|
||||||
registry.register(XueqiuSource())
|
registry.register(XueqiuSource())
|
||||||
# akshare 已移除(项目级决策:永远不用)
|
# akshare:2026-07-01 决策更新——龙虎榜无替代源,烟测通过,可注册。
|
||||||
|
# 见 [[no-akshare]] 记忆:仅"无替代"+"akshare 烟测 OK"才允许用。
|
||||||
|
from app.sources.akshare_lhb import AkshareLhbSource
|
||||||
|
registry.register(AkshareLhbSource())
|
||||||
@@ -48,6 +48,25 @@ def code6_to_mairui(code6: str) -> str:
|
|||||||
return f"{code6}.{code6_to_exchange(code6)}"
|
return f"{code6}.{code6_to_exchange(code6)}"
|
||||||
|
|
||||||
|
|
||||||
|
def to_hermes(code: str) -> str:
|
||||||
|
"""6 位代码 → 带交易所前缀(项目标准:与 stocks.code 一致)。
|
||||||
|
|
||||||
|
'000001' → 'SZ000001'
|
||||||
|
'600519' → 'SH600519'
|
||||||
|
'830xxx' → 'BJ830xxx'(北证)
|
||||||
|
'SH600519' → 'SH600519'(已是 hermes 格式则原样返回)
|
||||||
|
"""
|
||||||
|
c = str(code).strip().upper()
|
||||||
|
# 已是 hermes 格式
|
||||||
|
if c.startswith(("SH", "SZ", "BJ")) and len(c) >= 8:
|
||||||
|
return c
|
||||||
|
# 6 位纯数字 → 加前缀
|
||||||
|
if len(c) == 6 and c.isdigit():
|
||||||
|
return f"{code6_to_exchange(c)}{c}"
|
||||||
|
# 兜底:原样返回
|
||||||
|
return c
|
||||||
|
|
||||||
|
|
||||||
def normalize_kline(df: pd.DataFrame) -> pd.DataFrame:
|
def normalize_kline(df: pd.DataFrame) -> pd.DataFrame:
|
||||||
"""标准化日 K 线 DataFrame。
|
"""标准化日 K 线 DataFrame。
|
||||||
|
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
"""DB 子包:连接管理 + schema + 业务 CRUD。"""
|
"""DB 子包:ORM + 业务 CRUD。"""
|
||||||
from app.core.db.connection import get_mysql, init_mysql_schema, close_mysql
|
from app.core.db.orm import get_session, engine, SessionLocal, ORMBase
|
||||||
|
|
||||||
__all__ = ["get_mysql", "init_mysql_schema", "close_mysql"]
|
__all__ = ["get_session", "engine", "SessionLocal", "ORMBase"]
|
||||||
|
|||||||
@@ -1,104 +0,0 @@
|
|||||||
"""MySQL 连接管理(thread-local,模式照搬 dashboard,但适配 market_data_sync 项目)。
|
|
||||||
|
|
||||||
要点:
|
|
||||||
- pymysql 连接不是线程安全的,每个线程各持一份
|
|
||||||
- 内网自签名证书场景:传 ssl_disabled=True 跳过 SSL 校验
|
|
||||||
- DATE / DATETIME / TIMESTAMP 统一转字符串,避免下游类型处理
|
|
||||||
- DictCursor:返回字典风格行,方便业务拼装
|
|
||||||
"""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import threading
|
|
||||||
from typing import Optional
|
|
||||||
|
|
||||||
import pymysql
|
|
||||||
import pymysql.constants.FIELD_TYPE
|
|
||||||
from pymysql.converters import conversions as _pymysql_conv, convert_date, convert_datetime
|
|
||||||
from pymysql.cursors import DictCursor
|
|
||||||
|
|
||||||
from app.core.config import settings
|
|
||||||
|
|
||||||
_local = threading.local()
|
|
||||||
|
|
||||||
|
|
||||||
def _strdate(obj):
|
|
||||||
d = convert_date(obj)
|
|
||||||
return d.strftime("%Y-%m-%d") if d else None
|
|
||||||
|
|
||||||
|
|
||||||
def _strdatetime(obj):
|
|
||||||
d = convert_datetime(obj)
|
|
||||||
return d.strftime("%Y-%m-%d %H:%M:%S") if d else None
|
|
||||||
|
|
||||||
|
|
||||||
# 包装默认 converter,让 DATE / DATETIME / TIMESTAMP 一律出字符串
|
|
||||||
_conv = _pymysql_conv.copy()
|
|
||||||
_conv[pymysql.constants.FIELD_TYPE.DATE] = _strdate
|
|
||||||
_conv[pymysql.constants.FIELD_TYPE.DATETIME] = _strdatetime
|
|
||||||
_conv[pymysql.constants.FIELD_TYPE.TIMESTAMP] = _strdatetime
|
|
||||||
|
|
||||||
|
|
||||||
def get_mysql(database: Optional[str] = None) -> pymysql.connections.Connection:
|
|
||||||
"""取当前线程的 MySQL 连接。无则新建。
|
|
||||||
|
|
||||||
关键修复:高并发场景下,连接可能被服务端超时断开 (wait_timeout)。
|
|
||||||
用 ping(reconnect=True) 让 pymysql 自动重连,避免 stale conn 导致
|
|
||||||
2013 Lost connection 错误。
|
|
||||||
"""
|
|
||||||
attr = f"conn_{database or 'default'}"
|
|
||||||
conn = getattr(_local, attr, None)
|
|
||||||
if conn is not None:
|
|
||||||
try:
|
|
||||||
conn.ping(reconnect=True)
|
|
||||||
except Exception:
|
|
||||||
try:
|
|
||||||
conn.close()
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
conn = None
|
|
||||||
if conn is None:
|
|
||||||
kwargs = dict(
|
|
||||||
host=settings.mysql_host,
|
|
||||||
port=settings.mysql_port,
|
|
||||||
user=settings.mysql_user,
|
|
||||||
password=settings.mysql_password,
|
|
||||||
database=database or settings.mysql_database,
|
|
||||||
charset="utf8mb4",
|
|
||||||
cursorclass=DictCursor,
|
|
||||||
autocommit=False,
|
|
||||||
conv=_conv,
|
|
||||||
)
|
|
||||||
# 内网自签名证书:禁用 SSL(用户场景)
|
|
||||||
kwargs["ssl"] = None
|
|
||||||
# 高并发时设较短 read_timeout / write_timeout
|
|
||||||
kwargs["read_timeout"] = 30
|
|
||||||
kwargs["write_timeout"] = 30
|
|
||||||
conn = pymysql.connect(**kwargs)
|
|
||||||
setattr(_local, attr, conn)
|
|
||||||
return conn
|
|
||||||
|
|
||||||
|
|
||||||
def close_mysql() -> None:
|
|
||||||
"""关闭当前线程的所有连接。"""
|
|
||||||
for attr in list(vars(_local).keys()):
|
|
||||||
conn = getattr(_local, attr, None)
|
|
||||||
if conn is not None:
|
|
||||||
try:
|
|
||||||
conn.close()
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
try:
|
|
||||||
delattr(_local, attr)
|
|
||||||
except AttributeError:
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
# ── Schema DDL 入口 ────────────────────────────────────────────────────────
|
|
||||||
|
|
||||||
|
|
||||||
def init_mysql_schema() -> None:
|
|
||||||
"""集中创建/补齐所有表。被 db/ops.py 在第一次访问时调用。"""
|
|
||||||
# 延后导入避免循环依赖
|
|
||||||
from app.core.db.schema import ensure_all_tables
|
|
||||||
|
|
||||||
ensure_all_tables(get_mysql())
|
|
||||||
@@ -0,0 +1,479 @@
|
|||||||
|
"""PostgreSQL ORM 模型(SQLAlchemy 2.x declarative)。
|
||||||
|
|
||||||
|
设计原则:
|
||||||
|
- 全部 15 张表用 Python ORM 表达;`Base.metadata.create_all()` 一键建表
|
||||||
|
- 所有表放在 `market_data` schema 下(PG-native)
|
||||||
|
- 类型选择 PG-native:TIMESTAMPTZ / NUMERIC / JSONB / TEXT
|
||||||
|
- 字段名与 MySQL 版尽量保持一致,便于将来数据迁移直接 INSERT ... SELECT
|
||||||
|
|
||||||
|
与原 MySQL 版的差异(按列类型一一对照):
|
||||||
|
- DATETIME → TIMESTAMPTZ
|
||||||
|
- DOUBLE → Float(保持 double precision)
|
||||||
|
- VARCHAR JSON 字段 → JSONB(dataset_registry.dependency_ids)
|
||||||
|
- ON UPDATE CURRENT_TIMESTAMP → onupdate=func.now()(应用层处理,PG 没内置)
|
||||||
|
- PrimaryKeyConstraint → 全部用主键约束表达;复合主键用 __table_args__
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import date, datetime
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
from sqlalchemy import (
|
||||||
|
Date,
|
||||||
|
DateTime,
|
||||||
|
Float,
|
||||||
|
Index,
|
||||||
|
Integer,
|
||||||
|
PrimaryKeyConstraint,
|
||||||
|
SmallInteger,
|
||||||
|
String,
|
||||||
|
Text,
|
||||||
|
func,
|
||||||
|
)
|
||||||
|
from sqlalchemy.dialects.postgresql import JSONB
|
||||||
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
|
from app.core.db.orm import ORMBase
|
||||||
|
|
||||||
|
|
||||||
|
# ──── 公共 updated_at 工厂 ────
|
||||||
|
def _updated_at():
|
||||||
|
"""`DEFAULT NOW() ON UPDATE NOW()` —— PG 应用层维护。"""
|
||||||
|
return mapped_column(
|
||||||
|
DateTime(timezone=True),
|
||||||
|
server_default=func.now(),
|
||||||
|
onupdate=func.now(),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ──────────────────────── 1. config ────────────────────────
|
||||||
|
class Config(ORMBase):
|
||||||
|
"""键值配置表(.env / 数据源开关 / 调度参数等)。"""
|
||||||
|
__tablename__ = "config"
|
||||||
|
__table_args__ = {"schema": "market_data"}
|
||||||
|
|
||||||
|
key: Mapped[str] = mapped_column(String(128), primary_key=True)
|
||||||
|
value: Mapped[str] = mapped_column(Text, nullable=False)
|
||||||
|
category: Mapped[str] = mapped_column(String(32), nullable=False, default="general")
|
||||||
|
description: Mapped[Optional[str]] = mapped_column(String(255), default="")
|
||||||
|
updated_at: Mapped[Optional[datetime]] = _updated_at()
|
||||||
|
|
||||||
|
|
||||||
|
# ──────────────────────── 2. dataset_registry ────────────────────────
|
||||||
|
class DatasetRegistry(ORMBase):
|
||||||
|
"""同步任务状态机:idle → running → ok / warning / error / blocked。"""
|
||||||
|
__tablename__ = "dataset_registry"
|
||||||
|
__table_args__ = (
|
||||||
|
Index("idx_dataset_registry_sort_order", "sort_order"),
|
||||||
|
{"schema": "market_data"},
|
||||||
|
)
|
||||||
|
|
||||||
|
dataset_id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||||
|
name: Mapped[str] = mapped_column(String(128), nullable=False, default="")
|
||||||
|
description: Mapped[Optional[str]] = mapped_column(Text)
|
||||||
|
storage_uri: Mapped[Optional[str]] = mapped_column(String(255), default="")
|
||||||
|
storage_layer: Mapped[Optional[str]] = mapped_column(String(32), default="")
|
||||||
|
management_role: Mapped[Optional[str]] = mapped_column(String(32), default="")
|
||||||
|
source: Mapped[Optional[str]] = mapped_column(String(255), default="")
|
||||||
|
sync_script: Mapped[Optional[str]] = mapped_column(String(255), default="")
|
||||||
|
# 原 schema 是 VARCHAR(4096) DEFAULT '[]'(序列化的 JSON 字符串),PG 改为原生 JSONB
|
||||||
|
dependency_ids: Mapped[Optional[list]] = mapped_column(JSONB, default=list)
|
||||||
|
enabled: Mapped[int] = mapped_column(Integer, default=1)
|
||||||
|
sort_order: Mapped[int] = mapped_column(Integer, default=0)
|
||||||
|
status: Mapped[Optional[str]] = mapped_column(String(16), default="idle")
|
||||||
|
trigger_source: Mapped[Optional[str]] = mapped_column(String(32), default="")
|
||||||
|
started_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True))
|
||||||
|
finished_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True))
|
||||||
|
last_success_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True))
|
||||||
|
last_failure_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True))
|
||||||
|
message: Mapped[Optional[str]] = mapped_column(Text)
|
||||||
|
last_error: Mapped[Optional[str]] = mapped_column(Text)
|
||||||
|
needs_resync: Mapped[int] = mapped_column(Integer, default=0)
|
||||||
|
progress_current: Mapped[int] = mapped_column(Integer, default=0)
|
||||||
|
progress_total: Mapped[int] = mapped_column(Integer, default=0)
|
||||||
|
current_step: Mapped[Optional[str]] = mapped_column(String(255), default="")
|
||||||
|
updated_at: Mapped[Optional[datetime]] = _updated_at()
|
||||||
|
|
||||||
|
|
||||||
|
# ──────────────────────── 3. stocks ────────────────────────
|
||||||
|
class Stock(ORMBase):
|
||||||
|
"""股票基础信息 + 最新股本快照(冗余缓存)。"""
|
||||||
|
__tablename__ = "stocks"
|
||||||
|
__table_args__ = {"schema": "market_data"}
|
||||||
|
|
||||||
|
code: Mapped[str] = mapped_column(String(10), primary_key=True)
|
||||||
|
name: Mapped[str] = mapped_column(String(32), nullable=False, default="")
|
||||||
|
exchange: Mapped[str] = mapped_column(String(8), nullable=False, default="")
|
||||||
|
list_date: Mapped[Optional[date]] = mapped_column(Date)
|
||||||
|
listing_status: Mapped[str] = mapped_column(String(16), nullable=False, default="normal")
|
||||||
|
industry: Mapped[Optional[str]] = mapped_column(String(64), default="")
|
||||||
|
total_share: Mapped[float] = mapped_column(Float, default=0)
|
||||||
|
float_share: Mapped[float] = mapped_column(Float, default=0)
|
||||||
|
share_updated_at: Mapped[Optional[date]] = mapped_column(Date)
|
||||||
|
kline_synced_at: Mapped[Optional[date]] = mapped_column(Date)
|
||||||
|
updated_at: Mapped[Optional[datetime]] = _updated_at()
|
||||||
|
|
||||||
|
|
||||||
|
# ──────────────────────── 4. indices ────────────────────────
|
||||||
|
# 类的 Python 名 `MarketIndex`(避免和 `from sqlalchemy import Index` 冲突),
|
||||||
|
# 但 __tablename__ 仍是 "indices" 跟原 MySQL 一致。
|
||||||
|
class MarketIndex(ORMBase):
|
||||||
|
"""六大指数基础信息。"""
|
||||||
|
__tablename__ = "indices"
|
||||||
|
__table_args__ = {"schema": "market_data"}
|
||||||
|
|
||||||
|
index_code: Mapped[str] = mapped_column(String(10), primary_key=True)
|
||||||
|
index_name: Mapped[str] = mapped_column(String(64), nullable=False, default="")
|
||||||
|
market: Mapped[Optional[str]] = mapped_column(String(16), default="")
|
||||||
|
category: Mapped[Optional[str]] = mapped_column(String(16), default="")
|
||||||
|
source: Mapped[Optional[str]] = mapped_column(String(32), default="")
|
||||||
|
enabled: Mapped[int] = mapped_column(Integer, default=1)
|
||||||
|
updated_at: Mapped[Optional[datetime]] = _updated_at()
|
||||||
|
|
||||||
|
|
||||||
|
# ──────────────────────── 5. kline_stock ────────────────────────
|
||||||
|
class KlineStock(ORMBase):
|
||||||
|
"""全市场 A 股日频 OHLCV。"""
|
||||||
|
__tablename__ = "kline_stock"
|
||||||
|
__table_args__ = (
|
||||||
|
PrimaryKeyConstraint("stock_code", "trade_date"),
|
||||||
|
Index("idx_kline_stock_date", "trade_date"),
|
||||||
|
{"schema": "market_data"},
|
||||||
|
)
|
||||||
|
|
||||||
|
stock_code: Mapped[str] = mapped_column(String(10), nullable=False)
|
||||||
|
trade_date: Mapped[date] = mapped_column(Date, nullable=False)
|
||||||
|
open: Mapped[float] = mapped_column(Float, nullable=False)
|
||||||
|
high: Mapped[float] = mapped_column(Float, nullable=False)
|
||||||
|
low: Mapped[float] = mapped_column(Float, nullable=False)
|
||||||
|
close: Mapped[float] = mapped_column(Float, nullable=False)
|
||||||
|
volume: Mapped[float] = mapped_column(Float, nullable=False)
|
||||||
|
|
||||||
|
|
||||||
|
# ──────────────────────── 6. kline_index ────────────────────────
|
||||||
|
class KlineIndex(ORMBase):
|
||||||
|
"""六大指数日 K。"""
|
||||||
|
__tablename__ = "kline_index"
|
||||||
|
__table_args__ = (
|
||||||
|
PrimaryKeyConstraint("index_code", "trade_date"),
|
||||||
|
Index("idx_kline_index_date", "trade_date"),
|
||||||
|
{"schema": "market_data"},
|
||||||
|
)
|
||||||
|
|
||||||
|
index_code: Mapped[str] = mapped_column(String(10), nullable=False)
|
||||||
|
trade_date: Mapped[date] = mapped_column(Date, nullable=False)
|
||||||
|
open: Mapped[float] = mapped_column(Float, nullable=False)
|
||||||
|
high: Mapped[float] = mapped_column(Float, nullable=False)
|
||||||
|
low: Mapped[float] = mapped_column(Float, nullable=False)
|
||||||
|
close: Mapped[float] = mapped_column(Float, nullable=False)
|
||||||
|
volume: Mapped[float] = mapped_column(Float, nullable=False)
|
||||||
|
|
||||||
|
|
||||||
|
# ──────────────────────── 7. kline_5min ────────────────────────
|
||||||
|
class Kline5Min(ORMBase):
|
||||||
|
"""全市场 A 股 5 分钟 K 线。"""
|
||||||
|
__tablename__ = "kline_5min"
|
||||||
|
__table_args__ = (
|
||||||
|
PrimaryKeyConstraint("stock_code", "bar_time"),
|
||||||
|
Index("idx_kline_5min_bar_time", "bar_time"),
|
||||||
|
{"schema": "market_data"},
|
||||||
|
)
|
||||||
|
|
||||||
|
stock_code: Mapped[str] = mapped_column(String(10), nullable=False)
|
||||||
|
bar_time: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||||
|
open: Mapped[Optional[float]] = mapped_column(Float)
|
||||||
|
high: Mapped[Optional[float]] = mapped_column(Float)
|
||||||
|
low: Mapped[Optional[float]] = mapped_column(Float)
|
||||||
|
close: Mapped[Optional[float]] = mapped_column(Float)
|
||||||
|
volume: Mapped[Optional[float]] = mapped_column(Float)
|
||||||
|
amount: Mapped[Optional[float]] = mapped_column(Float)
|
||||||
|
turnover_rate: Mapped[Optional[float]] = mapped_column(Float)
|
||||||
|
|
||||||
|
|
||||||
|
# ──────────────────────── 8. moneyflow ────────────────────────
|
||||||
|
class Moneyflow(ORMBase):
|
||||||
|
"""个股资金流(mairui 源:主力/大/中/小单 净额,单位元)。"""
|
||||||
|
__tablename__ = "moneyflow"
|
||||||
|
__table_args__ = (
|
||||||
|
PrimaryKeyConstraint("stock_code", "trade_date"),
|
||||||
|
Index("idx_moneyflow_date", "trade_date"),
|
||||||
|
{"schema": "market_data"},
|
||||||
|
)
|
||||||
|
|
||||||
|
stock_code: Mapped[str] = mapped_column(String(10), nullable=False)
|
||||||
|
trade_date: Mapped[date] = mapped_column(Date, nullable=False)
|
||||||
|
main_net_inflow: Mapped[float] = mapped_column(Float, default=0)
|
||||||
|
large_net_inflow: Mapped[float] = mapped_column(Float, default=0)
|
||||||
|
medium_net_inflow: Mapped[float] = mapped_column(Float, default=0)
|
||||||
|
small_net_inflow: Mapped[float] = mapped_column(Float, default=0)
|
||||||
|
|
||||||
|
|
||||||
|
# ──────────────────────── 9. share ────────────────────────
|
||||||
|
class Share(ORMBase):
|
||||||
|
"""个股股本快照(雪球 quote_detail,单位亿股)。"""
|
||||||
|
__tablename__ = "share"
|
||||||
|
__table_args__ = (
|
||||||
|
PrimaryKeyConstraint("stock_code", "trade_date"),
|
||||||
|
{"schema": "market_data"},
|
||||||
|
)
|
||||||
|
|
||||||
|
stock_code: Mapped[str] = mapped_column(String(10), nullable=False)
|
||||||
|
trade_date: Mapped[date] = mapped_column(Date, nullable=False)
|
||||||
|
total_share: Mapped[float] = mapped_column(Float, nullable=False)
|
||||||
|
float_share: Mapped[float] = mapped_column(Float, nullable=False)
|
||||||
|
|
||||||
|
|
||||||
|
# ──────────────────────── 10. sectors ────────────────────────
|
||||||
|
class Sectors(ORMBase):
|
||||||
|
"""行业字典(baostock 行业分类 / 申万 / 中证 等多种 taxonomy)。"""
|
||||||
|
__tablename__ = "sectors"
|
||||||
|
__table_args__ = (
|
||||||
|
Index("idx_sectors_sector_name", "sector_name"),
|
||||||
|
{"schema": "market_data"},
|
||||||
|
)
|
||||||
|
|
||||||
|
sector_key: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||||
|
sector_name: Mapped[str] = mapped_column(String(64), nullable=False, default="")
|
||||||
|
taxonomy: Mapped[Optional[str]] = mapped_column(String(64), default="")
|
||||||
|
level: Mapped[Optional[str]] = mapped_column(String(16), default="")
|
||||||
|
source: Mapped[Optional[str]] = mapped_column(String(32), default="")
|
||||||
|
enabled: Mapped[int] = mapped_column(Integer, default=1)
|
||||||
|
updated_at: Mapped[Optional[datetime]] = _updated_at()
|
||||||
|
|
||||||
|
|
||||||
|
# ──────────────────────── 11. stock_sector_map ────────────────────────
|
||||||
|
class StockSectorMap(ORMBase):
|
||||||
|
"""股票-行业多对多映射(一张股票可对应多个行业 / 概念板块)。"""
|
||||||
|
__tablename__ = "stock_sector_map"
|
||||||
|
__table_args__ = (
|
||||||
|
Index("idx_stock_sector_map_sector_key", "sector_key"),
|
||||||
|
{"schema": "market_data"},
|
||||||
|
)
|
||||||
|
|
||||||
|
stock_code: Mapped[str] = mapped_column(String(10), primary_key=True)
|
||||||
|
sector_key: Mapped[str] = mapped_column(String(64), nullable=False, default="")
|
||||||
|
updated_at: Mapped[Optional[datetime]] = _updated_at()
|
||||||
|
|
||||||
|
|
||||||
|
# ──────────────────────── 12. industry ────────────────────────
|
||||||
|
class Industry(ORMBase):
|
||||||
|
"""股票-行业映射(baostock 源,证监会分类标准)。"""
|
||||||
|
__tablename__ = "industry"
|
||||||
|
__table_args__ = (
|
||||||
|
Index("idx_industry_industry_name", "industry_name"),
|
||||||
|
{"schema": "market_data"},
|
||||||
|
)
|
||||||
|
|
||||||
|
code: Mapped[str] = mapped_column(String(10), primary_key=True)
|
||||||
|
industry_name: Mapped[Optional[str]] = mapped_column(String(64), default="")
|
||||||
|
industry_classification: Mapped[Optional[str]] = mapped_column(String(32), default="")
|
||||||
|
update_date: Mapped[Optional[date]] = mapped_column(Date)
|
||||||
|
updated_at: Mapped[Optional[datetime]] = _updated_at()
|
||||||
|
|
||||||
|
|
||||||
|
# ──────────────────────── 13. sector_indices ────────────────────────
|
||||||
|
class SectorIndices(ORMBase):
|
||||||
|
"""行业日线(基点 100,复合收益)。"""
|
||||||
|
__tablename__ = "sector_indices"
|
||||||
|
__table_args__ = (
|
||||||
|
PrimaryKeyConstraint("trade_date", "sector_name"),
|
||||||
|
Index("idx_sector_indices_sector_name", "sector_name"),
|
||||||
|
{"schema": "market_data"},
|
||||||
|
)
|
||||||
|
|
||||||
|
trade_date: Mapped[date] = mapped_column(Date, nullable=False)
|
||||||
|
sector_name: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||||
|
close: Mapped[float] = mapped_column(Float, default=0)
|
||||||
|
sector_amplitude: Mapped[float] = mapped_column(Float, default=0)
|
||||||
|
|
||||||
|
|
||||||
|
# ──────────────────────── 14. sector_features_daily ────────────────────────
|
||||||
|
class SectorFeaturesDaily(ORMBase):
|
||||||
|
"""行业日特征:sector_ret / sector_amplitude / close / EMA / score。"""
|
||||||
|
__tablename__ = "sector_features_daily"
|
||||||
|
__table_args__ = (
|
||||||
|
PrimaryKeyConstraint("trade_date", "sector_name"),
|
||||||
|
Index("idx_sector_features_sector_name", "sector_name"),
|
||||||
|
{"schema": "market_data"},
|
||||||
|
)
|
||||||
|
|
||||||
|
trade_date: Mapped[date] = mapped_column(Date, nullable=False)
|
||||||
|
sector_name: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||||
|
sector_ret: Mapped[float] = mapped_column(Float, default=0)
|
||||||
|
sector_amplitude: Mapped[float] = mapped_column(Float, default=0)
|
||||||
|
close: Mapped[float] = mapped_column(Float, default=0)
|
||||||
|
ema10: Mapped[float] = mapped_column(Float, default=0)
|
||||||
|
ema20: Mapped[float] = mapped_column(Float, default=0)
|
||||||
|
ema200: Mapped[float] = mapped_column(Float, default=0)
|
||||||
|
score: Mapped[int] = mapped_column(Integer, default=0)
|
||||||
|
|
||||||
|
|
||||||
|
# ──────────────────────── 15. market_regime_daily ────────────────────────
|
||||||
|
class MarketRegimeDaily(ORMBase):
|
||||||
|
"""市场情绪衍生指标(基于本地 kline 聚合)。"""
|
||||||
|
__tablename__ = "market_regime_daily"
|
||||||
|
__table_args__ = (
|
||||||
|
Index("idx_market_regime_daily_date", "trade_date"),
|
||||||
|
{"schema": "market_data"},
|
||||||
|
)
|
||||||
|
|
||||||
|
trade_date: Mapped[date] = mapped_column(Date, primary_key=True)
|
||||||
|
advancers: Mapped[float] = mapped_column(Float, default=0)
|
||||||
|
decliners: Mapped[float] = mapped_column(Float, default=0)
|
||||||
|
advance_ratio: Mapped[float] = mapped_column(Float, default=0)
|
||||||
|
turnover: Mapped[float] = mapped_column(Float, default=0)
|
||||||
|
turnover_avg_5d: Mapped[float] = mapped_column(Float, default=0)
|
||||||
|
turnover_ratio_5d: Mapped[float] = mapped_column(Float, default=0)
|
||||||
|
source: Mapped[Optional[str]] = mapped_column(String(32), default="")
|
||||||
|
is_extreme_panic: Mapped[int] = mapped_column(Integer, default=0)
|
||||||
|
updated_at: Mapped[Optional[datetime]] = _updated_at()
|
||||||
|
|
||||||
|
|
||||||
|
# ──────────────────────── 16. tick_trade ────────────────────────
|
||||||
|
class TickTrade(ORMBase):
|
||||||
|
"""个股当天逐笔成交(mairui 源)。
|
||||||
|
|
||||||
|
mairui API(`hsrl/zbjy/{code6}/{licence}`)不返唯一 trade_id;
|
||||||
|
6 字段组合 `(stock_code, trade_date, trade_time, price, volume, direction_code)`
|
||||||
|
作为复合主键保证幂等 upsert。
|
||||||
|
|
||||||
|
direction_code: 0=中性盘 / 1=买入 / 2=卖出(mairui 原始值)。
|
||||||
|
direction: 派生 'neutral' / 'buy' / 'sell'(冗余存便于按文本查)。
|
||||||
|
amount: 派生 price * volume(元;mairui 不返)。
|
||||||
|
"""
|
||||||
|
__tablename__ = "tick_trade"
|
||||||
|
__table_args__ = (
|
||||||
|
PrimaryKeyConstraint(
|
||||||
|
"stock_code", "trade_date", "trade_time",
|
||||||
|
"price", "volume", "direction_code",
|
||||||
|
),
|
||||||
|
Index("idx_tick_trade_date", "trade_date"),
|
||||||
|
Index("idx_tick_trade_code_date", "stock_code", "trade_date"),
|
||||||
|
{"schema": "market_data"},
|
||||||
|
)
|
||||||
|
|
||||||
|
stock_code: Mapped[str] = mapped_column(String(10), nullable=False)
|
||||||
|
trade_date: Mapped[date] = mapped_column(Date, nullable=False)
|
||||||
|
trade_time: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||||
|
price: Mapped[float] = mapped_column(Float, nullable=False)
|
||||||
|
volume: Mapped[float] = mapped_column(Float, nullable=False)
|
||||||
|
direction_code: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||||
|
direction: Mapped[str] = mapped_column(String(8), nullable=False, default="")
|
||||||
|
amount: Mapped[float] = mapped_column(Float, nullable=False, default=0)
|
||||||
|
|
||||||
|
|
||||||
|
# ──────────────────────── 17. longhubang_daily ────────────────────────
|
||||||
|
class LonghubangDaily(ORMBase):
|
||||||
|
"""龙虎榜每日上榜股票汇总(聚合层,akshare 源)。
|
||||||
|
|
||||||
|
数据源:`ak.stock_lhb_detail_em(start_date, end_date)` —— 东方财富私有 API 封装。
|
||||||
|
每个上榜日 × 每只上榜股票 = 1 行。
|
||||||
|
|
||||||
|
字段(21 列):
|
||||||
|
基础:rank_idx, stock_code, stock_name, trade_date, comment
|
||||||
|
当日:close, pct_chg
|
||||||
|
资金:lhb_net_buy, lhb_buy_amt, lhb_sell_amt, lhb_total_amt, market_total_amt
|
||||||
|
比率:net_buy_ratio, total_amt_ratio, turnover_rate, float_mv
|
||||||
|
原因:reason
|
||||||
|
后效:post_1d_pct, post_2d_pct, post_5d_pct, post_10d_pct
|
||||||
|
|
||||||
|
注意:`stock_code` 含交易所前缀(SH600000 / SZ000001),与项目其他表一致。
|
||||||
|
"""
|
||||||
|
__tablename__ = "longhubang_daily"
|
||||||
|
__table_args__ = (
|
||||||
|
PrimaryKeyConstraint("stock_code", "trade_date"),
|
||||||
|
Index("idx_lhb_daily_date", "trade_date"),
|
||||||
|
{"schema": "market_data"},
|
||||||
|
)
|
||||||
|
|
||||||
|
stock_code: Mapped[str] = mapped_column(String(10), nullable=False)
|
||||||
|
stock_name: Mapped[str] = mapped_column(String(64), nullable=False, default="")
|
||||||
|
trade_date: Mapped[date] = mapped_column(Date, nullable=False)
|
||||||
|
rank_idx: Mapped[Optional[int]] = mapped_column(Integer, nullable=True)
|
||||||
|
comment: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
|
||||||
|
close: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
|
||||||
|
pct_chg: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
|
||||||
|
lhb_net_buy: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
|
||||||
|
lhb_buy_amt: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
|
||||||
|
lhb_sell_amt: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
|
||||||
|
lhb_total_amt: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
|
||||||
|
market_total_amt: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
|
||||||
|
net_buy_ratio: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
|
||||||
|
total_amt_ratio: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
|
||||||
|
turnover_rate: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
|
||||||
|
float_mv: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
|
||||||
|
reason: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
|
||||||
|
post_1d_pct: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
|
||||||
|
post_2d_pct: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
|
||||||
|
post_5d_pct: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
|
||||||
|
post_10d_pct: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
|
||||||
|
|
||||||
|
|
||||||
|
# ──────────────────────── 18. longhubang_seat ────────────────────────
|
||||||
|
class LonghubangSeat(ORMBase):
|
||||||
|
"""龙虎榜单只票买卖前五营业部(席位层,akshare 源)。
|
||||||
|
|
||||||
|
数据源:`ak.stock_lhb_stock_detail_em(symbol, date, flag)` —— 东方财富私有 API 封装。
|
||||||
|
每只上榜票 × 每方向(buy/sell)= 5 行(共 10 行/票/日)。
|
||||||
|
|
||||||
|
字段(11 列 + 上榜原因):
|
||||||
|
基础:stock_code, trade_date, direction(buy/sell), rank_idx
|
||||||
|
营业部:branch_name(akshare 不返唯一 branch_code,用 name 作 PK 兜底)
|
||||||
|
金额:buy_amount, sell_amount, amount, net_amount, amount_ratio
|
||||||
|
当日:pct_chg, close, stock_total_amt
|
||||||
|
原因:explanation
|
||||||
|
|
||||||
|
注意:akshare 的 `stock_lhb_stock_detail_em` 不返 `branch_code`,只用 `branch_name`
|
||||||
|
作 PK 兜底(不同营业部名冲突概率极低;如未来需要可补 eastmoney 私有 API 抓 code)。
|
||||||
|
"""
|
||||||
|
__tablename__ = "longhubang_seat"
|
||||||
|
__table_args__ = (
|
||||||
|
PrimaryKeyConstraint("stock_code", "trade_date", "direction", "rank_idx"),
|
||||||
|
Index("idx_lhb_seat_date", "trade_date"),
|
||||||
|
Index("idx_lhb_seat_branch_name", "branch_name"),
|
||||||
|
{"schema": "market_data"},
|
||||||
|
)
|
||||||
|
|
||||||
|
stock_code: Mapped[str] = mapped_column(String(10), nullable=False)
|
||||||
|
trade_date: Mapped[date] = mapped_column(Date, nullable=False)
|
||||||
|
direction: Mapped[str] = mapped_column(String(8), nullable=False) # "buy" / "sell"
|
||||||
|
rank_idx: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||||
|
branch_name: Mapped[Optional[str]] = mapped_column(String(128), nullable=True)
|
||||||
|
buy_amount: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
|
||||||
|
sell_amount: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
|
||||||
|
amount: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
|
||||||
|
net_amount: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
|
||||||
|
amount_ratio: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
|
||||||
|
pct_chg: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
|
||||||
|
close: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
|
||||||
|
stock_total_amt: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
|
||||||
|
explanation: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
# 1-2
|
||||||
|
"Config",
|
||||||
|
"DatasetRegistry",
|
||||||
|
# 3-4
|
||||||
|
"Stock",
|
||||||
|
"MarketIndex",
|
||||||
|
# 5-9
|
||||||
|
"KlineStock",
|
||||||
|
"KlineIndex",
|
||||||
|
"Kline5Min",
|
||||||
|
"Moneyflow",
|
||||||
|
"Share",
|
||||||
|
# 10-12
|
||||||
|
"Sectors",
|
||||||
|
"StockSectorMap",
|
||||||
|
"Industry",
|
||||||
|
# 13-15
|
||||||
|
"SectorIndices",
|
||||||
|
"SectorFeaturesDaily",
|
||||||
|
"MarketRegimeDaily",
|
||||||
|
# 16
|
||||||
|
"TickTrade",
|
||||||
|
# 17-18 (2026-07-01 龙虎榜)
|
||||||
|
"LonghubangDaily",
|
||||||
|
"LonghubangSeat",
|
||||||
|
]
|
||||||
+524
-440
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,59 @@
|
|||||||
|
"""SQLAlchemy 2.x ORM 基础设施(PostgreSQL only)。
|
||||||
|
|
||||||
|
2026-06-16 重构:项目放弃 MySQL,只保留 PG。
|
||||||
|
所有模型 class 仍带 ``__table_args__ = {"schema": "market_data"}``,指向 PG schema。
|
||||||
|
PG 数据库 / schema / role / 表 由 ``app.core.db.pg_bootstrap`` 一次性建好。
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from contextlib import contextmanager
|
||||||
|
from typing import Iterator
|
||||||
|
|
||||||
|
from sqlalchemy import create_engine
|
||||||
|
from sqlalchemy.engine import Engine
|
||||||
|
from sqlalchemy.orm import DeclarativeBase, Session, sessionmaker
|
||||||
|
|
||||||
|
from app.core.config import settings
|
||||||
|
|
||||||
|
|
||||||
|
class ORMBase(DeclarativeBase):
|
||||||
|
"""所有 ORM 模型的基类。"""
|
||||||
|
|
||||||
|
|
||||||
|
def _build_engine() -> Engine:
|
||||||
|
"""构造 PG 的 SQLAlchemy Engine。"""
|
||||||
|
return create_engine(
|
||||||
|
settings.pg_sqlalchemy_url(),
|
||||||
|
pool_pre_ping=True,
|
||||||
|
pool_recycle=3600,
|
||||||
|
future=True,
|
||||||
|
echo=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
engine: Engine = _build_engine()
|
||||||
|
|
||||||
|
# sessionmaker 工厂
|
||||||
|
SessionLocal = sessionmaker(
|
||||||
|
bind=engine,
|
||||||
|
expire_on_commit=False,
|
||||||
|
autoflush=False,
|
||||||
|
future=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def get_session() -> Iterator[Session]:
|
||||||
|
"""事务上下文:异常自动 rollback,正常自动 commit。"""
|
||||||
|
s = SessionLocal()
|
||||||
|
try:
|
||||||
|
yield s
|
||||||
|
s.commit()
|
||||||
|
except Exception:
|
||||||
|
s.rollback()
|
||||||
|
raise
|
||||||
|
finally:
|
||||||
|
s.close()
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["ORMBase", "engine", "SessionLocal", "get_session"]
|
||||||
@@ -0,0 +1,235 @@
|
|||||||
|
"""PostgreSQL bootstrap — 一键建库 / 建 schema / 建 role / 建表。
|
||||||
|
|
||||||
|
用法:
|
||||||
|
.venv/bin/python -m app.core.db.pg_bootstrap
|
||||||
|
|
||||||
|
按顺序做:
|
||||||
|
1) 以 superuser 身份连默认 'postgres' 库
|
||||||
|
2) CREATE DATABASE market_data (IF NOT EXISTS)
|
||||||
|
3) \\connect 到 market_data
|
||||||
|
4) CREATE SCHEMA market_data, grid_seeker
|
||||||
|
5) CREATE ROLE market_sync LOGIN PASSWORD ... (从 .env 读 PG_MARKET_SYNC_PASSWORD)
|
||||||
|
6) GRANT 权限 + ALTER DEFAULT PRIVILEGES
|
||||||
|
7) SQLAlchemy ORM Base.metadata.create_all() 一键建所有表
|
||||||
|
|
||||||
|
幂等:所有 DDL 都用 IF NOT EXISTS / OR REPLACE;可以重复跑。
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
|
||||||
|
import psycopg2
|
||||||
|
from psycopg2 import sql
|
||||||
|
from sqlalchemy import create_engine, text
|
||||||
|
from sqlalchemy.engine import URL
|
||||||
|
|
||||||
|
from app.core.db.orm import ORMBase as Base
|
||||||
|
# 重要:必须显式 import models 模块,所有 @mapped_column 才会注册到 ORMBase.metadata
|
||||||
|
# 漏了这一步 Base.metadata.tables 是空的,create_all() 不会建任何表
|
||||||
|
# 改用 getattr 触发副作用,绕开 IDE "unused import" 警告
|
||||||
|
_ = __import__("app.core.db.models", fromlist=["*"])
|
||||||
|
|
||||||
|
|
||||||
|
# ──── 配置(从 .env 读) ────
|
||||||
|
|
||||||
|
def _env(key: str, default: str) -> str:
|
||||||
|
return os.environ.get(key, default).strip() or default
|
||||||
|
|
||||||
|
|
||||||
|
SUPERUSER = _env("PG_SUPERUSER", "postgres")
|
||||||
|
SUPERPASS = _env("PG_SUPERUSER_PASSWORD", "postgres")
|
||||||
|
SUPERHOST = _env("PG_SUPERHOST", "127.0.0.1")
|
||||||
|
SUPERPORT = int(_env("PG_SUPERPORT", "5432"))
|
||||||
|
|
||||||
|
DB_NAME = _env("PG_DB_NAME", "market_data")
|
||||||
|
SCHEMAS = _env("PG_SCHEMAS", "market_data,grid_seeker").split(",")
|
||||||
|
|
||||||
|
MARKET_SYNC_ROLE = _env("PG_MARKET_SYNC_ROLE", "market_sync")
|
||||||
|
MARKET_SYNC_PASS = _env("PG_MARKET_SYNC_PASSWORD", "market_sync")
|
||||||
|
|
||||||
|
LOG_PREFIX = "[pg_bootstrap]"
|
||||||
|
|
||||||
|
|
||||||
|
def log(msg: str) -> None:
|
||||||
|
print(f"{LOG_PREFIX} {msg}", flush=True)
|
||||||
|
|
||||||
|
|
||||||
|
def _connect(dbname: str):
|
||||||
|
return psycopg2.connect(
|
||||||
|
host=SUPERHOST, port=SUPERPORT,
|
||||||
|
user=SUPERUSER, password=SUPERPASS,
|
||||||
|
dbname=dbname, connect_timeout=10,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ──── Step 1: 创建数据库 ────
|
||||||
|
|
||||||
|
def step1_create_database() -> None:
|
||||||
|
log(f"Step 1: CREATE DATABASE {DB_NAME} (if not exists)")
|
||||||
|
conn = _connect("postgres")
|
||||||
|
conn.autocommit = True
|
||||||
|
try:
|
||||||
|
with conn.cursor() as cur:
|
||||||
|
cur.execute("SELECT 1 FROM pg_database WHERE datname = %s", (DB_NAME,))
|
||||||
|
if cur.fetchone():
|
||||||
|
log(f" ✓ {DB_NAME} 已存在,跳过")
|
||||||
|
else:
|
||||||
|
cur.execute(sql.SQL("CREATE DATABASE {}").format(sql.Identifier(DB_NAME)))
|
||||||
|
log(f" ✓ {DB_NAME} 已创建")
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
# ──── Step 2: 创建 schema ────
|
||||||
|
|
||||||
|
def step2_create_schemas() -> None:
|
||||||
|
log(f"Step 2: CREATE SCHEMA for {SCHEMAS}")
|
||||||
|
conn = _connect(DB_NAME)
|
||||||
|
conn.autocommit = True
|
||||||
|
try:
|
||||||
|
with conn.cursor() as cur:
|
||||||
|
for schema in SCHEMAS:
|
||||||
|
schema = schema.strip()
|
||||||
|
if not schema:
|
||||||
|
continue
|
||||||
|
cur.execute(
|
||||||
|
sql.SQL("CREATE SCHEMA IF NOT EXISTS {}").format(sql.Identifier(schema))
|
||||||
|
)
|
||||||
|
log(f" ✓ schema '{schema}' 就绪")
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
# ──── Step 3: 创建角色 + 授权 ────
|
||||||
|
|
||||||
|
def step3_create_role() -> None:
|
||||||
|
log(f"Step 3: CREATE ROLE {MARKET_SYNC_ROLE} + GRANT")
|
||||||
|
conn = _connect(DB_NAME)
|
||||||
|
conn.autocommit = True
|
||||||
|
try:
|
||||||
|
with conn.cursor() as cur:
|
||||||
|
# 角色(IF NOT EXISTS 9.0+ 支持,旧版会报错 — 我们用 try/except 兜底)
|
||||||
|
cur.execute(
|
||||||
|
"SELECT 1 FROM pg_roles WHERE rolname = %s",
|
||||||
|
(MARKET_SYNC_ROLE,),
|
||||||
|
)
|
||||||
|
if cur.fetchone():
|
||||||
|
log(f" ✓ role '{MARKET_SYNC_ROLE}' 已存在,跳过")
|
||||||
|
else:
|
||||||
|
cur.execute(
|
||||||
|
sql.SQL("CREATE ROLE {} LOGIN PASSWORD %s").format(
|
||||||
|
sql.Identifier(MARKET_SYNC_ROLE)
|
||||||
|
),
|
||||||
|
(MARKET_SYNC_PASS,),
|
||||||
|
)
|
||||||
|
log(f" ✓ role '{MARKET_SYNC_ROLE}' 已创建")
|
||||||
|
|
||||||
|
# 连接 DB 权限
|
||||||
|
cur.execute(
|
||||||
|
sql.SQL("GRANT CONNECT ON DATABASE {} TO {}").format(
|
||||||
|
sql.Identifier(DB_NAME), sql.Identifier(MARKET_SYNC_ROLE)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
# 4 个 schema 都有 USAGE
|
||||||
|
for schema in SCHEMAS:
|
||||||
|
schema = schema.strip()
|
||||||
|
if not schema:
|
||||||
|
continue
|
||||||
|
cur.execute(
|
||||||
|
sql.SQL("GRANT USAGE ON SCHEMA {} TO {}").format(
|
||||||
|
sql.Identifier(schema), sql.Identifier(MARKET_SYNC_ROLE)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
# market_data: RW(当前项目用的)
|
||||||
|
cur.execute(
|
||||||
|
sql.SQL("GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA market_data TO {}").format(
|
||||||
|
sql.Identifier(MARKET_SYNC_ROLE)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
cur.execute(
|
||||||
|
sql.SQL("GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA market_data TO {}").format(
|
||||||
|
sql.Identifier(MARKET_SYNC_ROLE)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
# grid_seeker: 暂时也 RW(未来 grid_seeker 自己做权限时再收紧)
|
||||||
|
cur.execute(
|
||||||
|
sql.SQL("GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA grid_seeker TO {}").format(
|
||||||
|
sql.Identifier(MARKET_SYNC_ROLE)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
cur.execute(
|
||||||
|
sql.SQL("GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA grid_seeker TO {}").format(
|
||||||
|
sql.Identifier(MARKET_SYNC_ROLE)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
# 关键:未来新表自动继承(不然新加的表 market_sync 没权限)
|
||||||
|
for schema in SCHEMAS:
|
||||||
|
schema = schema.strip()
|
||||||
|
if not schema:
|
||||||
|
continue
|
||||||
|
cur.execute(
|
||||||
|
sql.SQL(
|
||||||
|
"ALTER DEFAULT PRIVILEGES IN SCHEMA {s} "
|
||||||
|
"GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO {r}"
|
||||||
|
).format(s=sql.Identifier(schema), r=sql.Identifier(MARKET_SYNC_ROLE))
|
||||||
|
)
|
||||||
|
cur.execute(
|
||||||
|
sql.SQL(
|
||||||
|
"ALTER DEFAULT PRIVILEGES IN SCHEMA {s} "
|
||||||
|
"GRANT USAGE, SELECT ON SEQUENCES TO {r}"
|
||||||
|
).format(s=sql.Identifier(schema), r=sql.Identifier(MARKET_SYNC_ROLE))
|
||||||
|
)
|
||||||
|
log(f" ✓ {MARKET_SYNC_ROLE} 在所有 schema 上有 RW + 未来新表自动继承")
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
# ──── Step 4: SQLAlchemy ORM 建所有表 ────
|
||||||
|
|
||||||
|
def step4_create_tables() -> None:
|
||||||
|
log("Step 4: SQLAlchemy Base.metadata.create_all() 建所有表")
|
||||||
|
# 临时以 superuser 连接 market_data(建表后权限自动由 ALTER DEFAULT 接管)
|
||||||
|
url = URL.create(
|
||||||
|
drivername="postgresql+psycopg2",
|
||||||
|
username=SUPERUSER, password=SUPERPASS,
|
||||||
|
host=SUPERHOST, port=SUPERPORT,
|
||||||
|
database=DB_NAME,
|
||||||
|
)
|
||||||
|
engine = create_engine(url, future=True, echo=False)
|
||||||
|
try:
|
||||||
|
Base.metadata.create_all(engine)
|
||||||
|
with engine.connect() as c:
|
||||||
|
rows = c.execute(
|
||||||
|
text(
|
||||||
|
"SELECT table_schema, table_name FROM information_schema.tables "
|
||||||
|
"WHERE table_schema IN ('market_data', 'grid_seeker') "
|
||||||
|
"ORDER BY table_schema, table_name"
|
||||||
|
)
|
||||||
|
).fetchall()
|
||||||
|
log(f" ✓ 当前 schema 下的表({len(rows)} 张):")
|
||||||
|
for schema, tbl in rows:
|
||||||
|
log(f" - {schema}.{tbl}")
|
||||||
|
finally:
|
||||||
|
engine.dispose()
|
||||||
|
|
||||||
|
|
||||||
|
# ──── main ────
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
log(f"连接 superuser {SUPERUSER}@{SUPERHOST}:{SUPERPORT}")
|
||||||
|
step1_create_database()
|
||||||
|
step2_create_schemas()
|
||||||
|
step3_create_role()
|
||||||
|
step4_create_tables()
|
||||||
|
log("✅ PG bootstrap 完成")
|
||||||
|
log(f" DB: {DB_NAME}")
|
||||||
|
log(f" 角色: {MARKET_SYNC_ROLE} (password 见 .env PG_MARKET_SYNC_PASSWORD)")
|
||||||
|
log(f" schemas: {', '.join(SCHEMAS)}")
|
||||||
|
log(f" 下一步:在 .env 设 PG_URL=postgresql+psycopg2://{MARKET_SYNC_ROLE}:{MARKET_SYNC_PASS}@{SUPERHOST}:{SUPERPORT}/{DB_NAME}")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -1,264 +0,0 @@
|
|||||||
"""Schema DDL — 集中所有 CREATE TABLE IF NOT EXISTS。
|
|
||||||
|
|
||||||
参考 dashboard/api/services/mysql_conn.py::init_mysql_schema(),但补齐 dashboard 缺的几张表
|
|
||||||
(kline_stock / kline_5min / kline_index / moneyflow / share / sector_features_daily / sector_indices
|
|
||||||
/ stock_scores / registry),这些是 grid_seeker_model_base 生产库已存在但 mysql_conn.py 没建的表。
|
|
||||||
|
|
||||||
所有 DDL 写在同一个文件,方便审阅、diff 和补字段。"""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from typing import TYPE_CHECKING
|
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
|
||||||
import pymysql.connections
|
|
||||||
|
|
||||||
|
|
||||||
# DDL 列表(顺序无关,全部 IF NOT EXISTS)
|
|
||||||
_DDL: list[str] = [
|
|
||||||
# ─── config ─────────────────────────────────────────────────────────
|
|
||||||
"""
|
|
||||||
CREATE TABLE IF NOT EXISTS config (
|
|
||||||
`key` VARCHAR(128) PRIMARY KEY,
|
|
||||||
`value` TEXT NOT NULL,
|
|
||||||
category VARCHAR(32) NOT NULL DEFAULT 'general',
|
|
||||||
description VARCHAR(255) DEFAULT '',
|
|
||||||
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
|
|
||||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
|
|
||||||
""",
|
|
||||||
# ─── dataset_registry (同步状态机) ────────────────────────────────────
|
|
||||||
"""
|
|
||||||
CREATE TABLE IF NOT EXISTS dataset_registry (
|
|
||||||
dataset_id VARCHAR(64) PRIMARY KEY,
|
|
||||||
name VARCHAR(128) NOT NULL DEFAULT '',
|
|
||||||
description TEXT,
|
|
||||||
storage_uri VARCHAR(255) DEFAULT '',
|
|
||||||
storage_layer VARCHAR(32) DEFAULT '',
|
|
||||||
management_role VARCHAR(32) DEFAULT '',
|
|
||||||
source VARCHAR(255) DEFAULT '',
|
|
||||||
sync_script VARCHAR(255) DEFAULT '',
|
|
||||||
dependency_ids VARCHAR(4096) DEFAULT '[]',
|
|
||||||
enabled INT DEFAULT 1,
|
|
||||||
sort_order INT DEFAULT 0,
|
|
||||||
status VARCHAR(16) DEFAULT 'idle',
|
|
||||||
trigger_source VARCHAR(32) DEFAULT '',
|
|
||||||
started_at DATETIME DEFAULT NULL,
|
|
||||||
finished_at DATETIME DEFAULT NULL,
|
|
||||||
last_success_at DATETIME DEFAULT NULL,
|
|
||||||
last_failure_at DATETIME DEFAULT NULL,
|
|
||||||
message TEXT,
|
|
||||||
last_error TEXT,
|
|
||||||
needs_resync INT DEFAULT 0,
|
|
||||||
progress_current INT DEFAULT 0,
|
|
||||||
progress_total INT DEFAULT 0,
|
|
||||||
current_step VARCHAR(255) DEFAULT '',
|
|
||||||
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
|
||||||
INDEX idx_dataset_registry_sort_order (sort_order)
|
|
||||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
|
|
||||||
""",
|
|
||||||
# ─── stocks (基础信息 + 股本字段) ─────────────────────────────────────
|
|
||||||
"""
|
|
||||||
CREATE TABLE IF NOT EXISTS stocks (
|
|
||||||
code VARCHAR(10) PRIMARY KEY,
|
|
||||||
name VARCHAR(32) NOT NULL DEFAULT '',
|
|
||||||
exchange VARCHAR(8) NOT NULL DEFAULT '',
|
|
||||||
list_date DATE DEFAULT NULL,
|
|
||||||
listing_status VARCHAR(16) NOT NULL DEFAULT 'normal',
|
|
||||||
industry VARCHAR(64) DEFAULT '',
|
|
||||||
total_share DOUBLE DEFAULT 0,
|
|
||||||
float_share DOUBLE DEFAULT 0,
|
|
||||||
share_updated_at DATE DEFAULT NULL,
|
|
||||||
kline_synced_at DATE DEFAULT NULL,
|
|
||||||
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
|
|
||||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
|
|
||||||
""",
|
|
||||||
# ─── indices (指数字典) ───────────────────────────────────────────────
|
|
||||||
"""
|
|
||||||
CREATE TABLE IF NOT EXISTS indices (
|
|
||||||
index_code VARCHAR(10) PRIMARY KEY,
|
|
||||||
index_name VARCHAR(64) NOT NULL DEFAULT '',
|
|
||||||
market VARCHAR(16) DEFAULT '',
|
|
||||||
category VARCHAR(16) DEFAULT '',
|
|
||||||
source VARCHAR(32) DEFAULT '',
|
|
||||||
enabled INT DEFAULT 1,
|
|
||||||
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
|
|
||||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
|
|
||||||
""",
|
|
||||||
# ─── kline_stock (日 K 线,按股票) ────────────────────────────────────
|
|
||||||
"""
|
|
||||||
CREATE TABLE IF NOT EXISTS kline_stock (
|
|
||||||
stock_code VARCHAR(10) NOT NULL,
|
|
||||||
trade_date DATE NOT NULL,
|
|
||||||
`open` DOUBLE NOT NULL,
|
|
||||||
`high` DOUBLE NOT NULL,
|
|
||||||
`low` DOUBLE NOT NULL,
|
|
||||||
`close` DOUBLE NOT NULL,
|
|
||||||
volume DOUBLE NOT NULL,
|
|
||||||
PRIMARY KEY (stock_code, trade_date),
|
|
||||||
INDEX idx_kline_stock_date (trade_date)
|
|
||||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
|
|
||||||
""",
|
|
||||||
# ─── kline_index (指数日 K 线) ────────────────────────────────────────
|
|
||||||
"""
|
|
||||||
CREATE TABLE IF NOT EXISTS kline_index (
|
|
||||||
index_code VARCHAR(10) NOT NULL,
|
|
||||||
trade_date DATE NOT NULL,
|
|
||||||
`open` DOUBLE NOT NULL,
|
|
||||||
`high` DOUBLE NOT NULL,
|
|
||||||
`low` DOUBLE NOT NULL,
|
|
||||||
`close` DOUBLE NOT NULL,
|
|
||||||
volume DOUBLE NOT NULL,
|
|
||||||
PRIMARY KEY (index_code, trade_date),
|
|
||||||
INDEX idx_kline_index_date (trade_date)
|
|
||||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
|
|
||||||
""",
|
|
||||||
# ─── kline_5min (5 分钟 K 线) ────────────────────────────────────────
|
|
||||||
"""
|
|
||||||
CREATE TABLE IF NOT EXISTS kline_5min (
|
|
||||||
stock_code VARCHAR(10) NOT NULL,
|
|
||||||
bar_time DATETIME NOT NULL,
|
|
||||||
`open` DOUBLE DEFAULT NULL,
|
|
||||||
`high` DOUBLE DEFAULT NULL,
|
|
||||||
`low` DOUBLE DEFAULT NULL,
|
|
||||||
`close` DOUBLE DEFAULT NULL,
|
|
||||||
volume DOUBLE DEFAULT NULL,
|
|
||||||
amount DOUBLE DEFAULT NULL,
|
|
||||||
turnover_rate DOUBLE DEFAULT NULL,
|
|
||||||
PRIMARY KEY (stock_code, bar_time),
|
|
||||||
INDEX idx_kline_5min_bar_time (bar_time)
|
|
||||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
|
|
||||||
""",
|
|
||||||
# ─── moneyflow (资金流,主力/大/中/小单净额) ──────────────────────────
|
|
||||||
"""
|
|
||||||
CREATE TABLE IF NOT EXISTS moneyflow (
|
|
||||||
stock_code VARCHAR(10) NOT NULL,
|
|
||||||
trade_date DATE NOT NULL,
|
|
||||||
main_net_inflow DOUBLE DEFAULT 0,
|
|
||||||
large_net_inflow DOUBLE DEFAULT 0,
|
|
||||||
medium_net_inflow DOUBLE DEFAULT 0,
|
|
||||||
small_net_inflow DOUBLE DEFAULT 0,
|
|
||||||
PRIMARY KEY (stock_code, trade_date),
|
|
||||||
INDEX idx_moneyflow_date (trade_date)
|
|
||||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
|
|
||||||
""",
|
|
||||||
# ─── share (股本快照) ────────────────────────────────────────────────
|
|
||||||
"""
|
|
||||||
CREATE TABLE IF NOT EXISTS share (
|
|
||||||
stock_code VARCHAR(10) NOT NULL,
|
|
||||||
trade_date DATE NOT NULL,
|
|
||||||
total_share DOUBLE NOT NULL,
|
|
||||||
float_share DOUBLE NOT NULL,
|
|
||||||
PRIMARY KEY (stock_code, trade_date)
|
|
||||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
|
|
||||||
""",
|
|
||||||
# ─── sectors (行业字典) ──────────────────────────────────────────────
|
|
||||||
"""
|
|
||||||
CREATE TABLE IF NOT EXISTS sectors (
|
|
||||||
sector_key VARCHAR(64) PRIMARY KEY,
|
|
||||||
sector_name VARCHAR(64) NOT NULL DEFAULT '',
|
|
||||||
taxonomy VARCHAR(64) DEFAULT '',
|
|
||||||
level VARCHAR(16) DEFAULT '',
|
|
||||||
source VARCHAR(32) DEFAULT '',
|
|
||||||
enabled INT DEFAULT 1,
|
|
||||||
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
|
||||||
INDEX idx_sectors_sector_name (sector_name)
|
|
||||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
|
|
||||||
""",
|
|
||||||
# ─── stock_sector_map (股票→行业映射) ─────────────────────────────────
|
|
||||||
"""
|
|
||||||
CREATE TABLE IF NOT EXISTS stock_sector_map (
|
|
||||||
stock_code VARCHAR(6) PRIMARY KEY,
|
|
||||||
sector_key VARCHAR(64) NOT NULL DEFAULT '',
|
|
||||||
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
|
||||||
INDEX idx_stock_sector_map_sector_key (sector_key)
|
|
||||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
|
|
||||||
""",
|
|
||||||
# ─── industry (股票-行业映射,datasource 实际写入用这张) ──────────────
|
|
||||||
"""
|
|
||||||
CREATE TABLE IF NOT EXISTS industry (
|
|
||||||
code VARCHAR(6) PRIMARY KEY,
|
|
||||||
industry_name VARCHAR(64) DEFAULT '',
|
|
||||||
industry_classification VARCHAR(32) DEFAULT '',
|
|
||||||
update_date DATE DEFAULT NULL,
|
|
||||||
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
|
||||||
INDEX idx_industry_industry_name (industry_name)
|
|
||||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
|
|
||||||
""",
|
|
||||||
# ─── sector_indices (行业聚合指数时序) ─────────────────────────────────
|
|
||||||
"""
|
|
||||||
CREATE TABLE IF NOT EXISTS sector_indices (
|
|
||||||
trade_date DATE NOT NULL,
|
|
||||||
sector_name VARCHAR(64) NOT NULL,
|
|
||||||
`close` DOUBLE DEFAULT 0,
|
|
||||||
sector_amplitude DOUBLE DEFAULT 0,
|
|
||||||
PRIMARY KEY (trade_date, sector_name),
|
|
||||||
INDEX idx_sector_indices_sector_name (sector_name)
|
|
||||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
|
|
||||||
""",
|
|
||||||
# ─── sector_features_daily (行业特征衍生) ─────────────────────────────
|
|
||||||
"""
|
|
||||||
CREATE TABLE IF NOT EXISTS sector_features_daily (
|
|
||||||
trade_date DATE NOT NULL,
|
|
||||||
sector_name VARCHAR(64) NOT NULL,
|
|
||||||
sector_ret DOUBLE DEFAULT 0,
|
|
||||||
sector_amplitude DOUBLE DEFAULT 0,
|
|
||||||
`close` DOUBLE DEFAULT 0,
|
|
||||||
ema10 DOUBLE DEFAULT 0,
|
|
||||||
ema20 DOUBLE DEFAULT 0,
|
|
||||||
ema200 DOUBLE DEFAULT 0,
|
|
||||||
score INT DEFAULT 0,
|
|
||||||
PRIMARY KEY (trade_date, sector_name),
|
|
||||||
INDEX idx_sector_features_sector_name (sector_name)
|
|
||||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
|
|
||||||
""",
|
|
||||||
# ─── market_regime_daily (市场情绪) ───────────────────────────────────
|
|
||||||
"""
|
|
||||||
CREATE TABLE IF NOT EXISTS market_regime_daily (
|
|
||||||
trade_date DATE PRIMARY KEY,
|
|
||||||
advancers DOUBLE DEFAULT 0,
|
|
||||||
decliners DOUBLE DEFAULT 0,
|
|
||||||
advance_ratio DOUBLE DEFAULT 0,
|
|
||||||
turnover DOUBLE DEFAULT 0,
|
|
||||||
turnover_avg_5d DOUBLE DEFAULT 0,
|
|
||||||
turnover_ratio_5d DOUBLE DEFAULT 0,
|
|
||||||
source VARCHAR(32) DEFAULT '',
|
|
||||||
is_extreme_panic INT DEFAULT 0,
|
|
||||||
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
|
||||||
INDEX idx_market_regime_daily_date (trade_date)
|
|
||||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
|
|
||||||
""",
|
|
||||||
# ─── registry (模型注册,dashboard 业务无关但库里有) ─────────────────
|
|
||||||
"""
|
|
||||||
CREATE TABLE IF NOT EXISTS registry (
|
|
||||||
model_name VARCHAR(64) NOT NULL,
|
|
||||||
role VARCHAR(32) NOT NULL,
|
|
||||||
version VARCHAR(16) DEFAULT NULL,
|
|
||||||
artifact_path VARCHAR(256) NOT NULL,
|
|
||||||
architecture VARCHAR(32) DEFAULT NULL,
|
|
||||||
enabled TINYINT(1) DEFAULT 1,
|
|
||||||
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
||||||
PRIMARY KEY (model_name, role)
|
|
||||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
|
|
||||||
""",
|
|
||||||
# ─── stock_scores (评分快照,dashboard 业务无关但库里有) ─────────────
|
|
||||||
"""
|
|
||||||
CREATE TABLE IF NOT EXISTS stock_scores (
|
|
||||||
`date` DATE NOT NULL,
|
|
||||||
stock_code VARCHAR(10) NOT NULL,
|
|
||||||
predicted_rounds DOUBLE DEFAULT 0,
|
|
||||||
stacking_probability DOUBLE DEFAULT 0,
|
|
||||||
meta_ranker_score DOUBLE DEFAULT NULL,
|
|
||||||
latest_close DOUBLE DEFAULT 0,
|
|
||||||
`rank` INT DEFAULT 0,
|
|
||||||
PRIMARY KEY (`date`, stock_code),
|
|
||||||
INDEX idx_stock_scores_date (`date`)
|
|
||||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
|
|
||||||
""",
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def ensure_all_tables(conn) -> None:
|
|
||||||
"""遍历 _DDL 列表,逐条执行 CREATE TABLE IF NOT EXISTS。"""
|
|
||||||
with conn.cursor() as cur:
|
|
||||||
for ddl in _DDL:
|
|
||||||
cur.execute(ddl)
|
|
||||||
conn.commit()
|
|
||||||
@@ -157,6 +157,22 @@ def update_job_status(key: str, status: str, message: str) -> None:
|
|||||||
_config_cache_ts = 0.0
|
_config_cache_ts = 0.0
|
||||||
|
|
||||||
|
|
||||||
|
def update_job_status_for_dataset(dataset_id: str, status: str, message: str) -> None:
|
||||||
|
"""按 dataset_id 找到对应 schedule config 并镜像 lastRun/lastStatus/lastMessage。
|
||||||
|
|
||||||
|
用法:SyncTask.run() 完成后对非 schedule 触发的运行(cli / runall_once / manual)
|
||||||
|
调用,让 schedule config 视图与实际跑过保持同步(fix Bug 2,2026-07-01)。
|
||||||
|
"""
|
||||||
|
for r in db_ops.fetch_configs_by_category("schedule"):
|
||||||
|
try:
|
||||||
|
obj = json.loads(r.get("value", "") or "{}")
|
||||||
|
except (json.JSONDecodeError, TypeError):
|
||||||
|
continue
|
||||||
|
if obj.get("job") == dataset_id:
|
||||||
|
update_job_status(r["key"], status, message)
|
||||||
|
return
|
||||||
|
|
||||||
|
|
||||||
# ── 调度循环 ──────────────────────────────────────────────────────────
|
# ── 调度循环 ──────────────────────────────────────────────────────────
|
||||||
|
|
||||||
def _loop() -> None:
|
def _loop() -> None:
|
||||||
@@ -300,12 +316,12 @@ DEFAULT_SCHEDULES: list[tuple[str, dict, str]] = [
|
|||||||
"schedule_moneyflow",
|
"schedule_moneyflow",
|
||||||
{
|
{
|
||||||
"name": "盘后资金流",
|
"name": "盘后资金流",
|
||||||
"time": "16:30",
|
"time": "21:35",
|
||||||
"condition": "trading_day",
|
"condition": "trading_day",
|
||||||
"job": "moneyflow",
|
"job": "moneyflow",
|
||||||
"enabled": True,
|
"enabled": True,
|
||||||
},
|
},
|
||||||
"每个交易日 16:30 拉取个股资金流",
|
"每个交易日 21:35 拉取个股资金流(mairui 21:30 发布)",
|
||||||
),
|
),
|
||||||
(
|
(
|
||||||
"schedule_industry_sector",
|
"schedule_industry_sector",
|
||||||
@@ -340,6 +356,17 @@ DEFAULT_SCHEDULES: list[tuple[str, dict, str]] = [
|
|||||||
},
|
},
|
||||||
"每个交易日 16:20 刷新个股总股本/流通股本快照",
|
"每个交易日 16:20 刷新个股总股本/流通股本快照",
|
||||||
),
|
),
|
||||||
|
(
|
||||||
|
"schedule_longhubang",
|
||||||
|
{
|
||||||
|
"name": "盘后龙虎榜",
|
||||||
|
"time": "22:00",
|
||||||
|
"condition": "trading_day",
|
||||||
|
"job": "longhubang",
|
||||||
|
"enabled": True,
|
||||||
|
},
|
||||||
|
"每个交易日 22:00 拉龙虎榜聚合层 + 席位层(akshare/东方财富 19:00~21:00 出齐)",
|
||||||
|
),
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
@@ -380,6 +407,7 @@ def register_sync_jobs() -> None:
|
|||||||
for dataset_id in [
|
for dataset_id in [
|
||||||
"stock_basic", "kline_daily", "kline_index", "kline_5min",
|
"stock_basic", "kline_daily", "kline_index", "kline_5min",
|
||||||
"moneyflow", "industry_sector", "share_snapshot", "market_regime",
|
"moneyflow", "industry_sector", "share_snapshot", "market_regime",
|
||||||
|
"longhubang",
|
||||||
]:
|
]:
|
||||||
|
|
||||||
def _make_job(did=dataset_id):
|
def _make_job(did=dataset_id):
|
||||||
|
|||||||
+33
-2
@@ -19,6 +19,7 @@ from app.core.sync.registry import (
|
|||||||
mark_sync_progress,
|
mark_sync_progress,
|
||||||
mark_sync_running,
|
mark_sync_running,
|
||||||
mark_sync_success,
|
mark_sync_success,
|
||||||
|
mark_sync_warning,
|
||||||
)
|
)
|
||||||
from app.core.utils.logging import get_logger
|
from app.core.utils.logging import get_logger
|
||||||
|
|
||||||
@@ -79,15 +80,20 @@ class SyncTask:
|
|||||||
error=result.get("error", message),
|
error=result.get("error", message),
|
||||||
status="blocked",
|
status="blocked",
|
||||||
)
|
)
|
||||||
else: # warning / partial
|
elif status == "warning":
|
||||||
|
# 部分失败:走专用 warning 路径,不写 last_failure_at / last_error
|
||||||
|
mark_sync_warning(self.dataset_id, message=message)
|
||||||
|
else: # 其他未识别状态默认走 failed
|
||||||
mark_sync_failed(
|
mark_sync_failed(
|
||||||
self.dataset_id,
|
self.dataset_id,
|
||||||
message=message,
|
message=message,
|
||||||
error=result.get("error", message),
|
error=result.get("error", message),
|
||||||
status="warning",
|
status="failed",
|
||||||
)
|
)
|
||||||
result["elapsed_sec"] = elapsed
|
result["elapsed_sec"] = elapsed
|
||||||
logger.info(f"[{self.dataset_id}] 完成 {status}: {message} ({elapsed}s)")
|
logger.info(f"[{self.dataset_id}] 完成 {status}: {message} ({elapsed}s)")
|
||||||
|
# 同步 schedule config 的 lastRun(fix Bug 2)
|
||||||
|
self._maybe_mirror_schedule_status(trigger_source, status, message)
|
||||||
return result
|
return result
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
elapsed = round(time.time() - t0, 1)
|
elapsed = round(time.time() - t0, 1)
|
||||||
@@ -139,3 +145,28 @@ class SyncTask:
|
|||||||
progress_total=total,
|
progress_total=total,
|
||||||
current_step=current_step,
|
current_step=current_step,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def _maybe_mirror_schedule_status(
|
||||||
|
self, trigger_source: str, status: str, message: str
|
||||||
|
) -> None:
|
||||||
|
"""把非 schedule 触发的运行结果同步到 schedule config 表(fix Bug 2)。
|
||||||
|
|
||||||
|
背景:bin/runall_once.py(15:30 systemd 触发)和 python -m app.entrypoints.cli sync
|
||||||
|
(手动触发)走的都是 task.run(trigger_source="cli" or "manual"),不会经过
|
||||||
|
app.core.scheduler 的 update_job_status。结果:schedule config 表的 lastRun /
|
||||||
|
lastStatus 一直停在 scheduler 真正跑过的那天(实际是 6/15 之后就没动过)。
|
||||||
|
UI 看到的就是"schedule 视图"和"实际跑过"脱节。
|
||||||
|
|
||||||
|
解法:每次 task.run() 完成后,lazy import scheduler,找到这个 dataset_id 对应的
|
||||||
|
schedule config 并镜像 lastRun/lastStatus/lastMessage。
|
||||||
|
|
||||||
|
scheduler 触发的(trigger_source="schedule")会自己 update_job_status,跳过即可。
|
||||||
|
"""
|
||||||
|
if trigger_source == "schedule":
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
from app.core.scheduler.scheduler import update_job_status_for_dataset
|
||||||
|
update_job_status_for_dataset(self.dataset_id, status, message)
|
||||||
|
except Exception as e:
|
||||||
|
# 镜像失败不影响主流程;只记 debug log
|
||||||
|
logger.debug(f"[_maybe_mirror_schedule_status] {self.dataset_id} 镜像失败: {e}")
|
||||||
|
|||||||
+64
-20
@@ -26,8 +26,8 @@ SYNC_DEFINITIONS: list[dict[str, Any]] = [
|
|||||||
"dataset_id": "stock_basic",
|
"dataset_id": "stock_basic",
|
||||||
"name": "股票基础信息(含最新股本)",
|
"name": "股票基础信息(含最新股本)",
|
||||||
"description": "全市场 A 股代码 / 名称 / 交易所 / 上市状态 + 股本快照(雪球)",
|
"description": "全市场 A 股代码 / 名称 / 交易所 / 上市状态 + 股本快照(雪球)",
|
||||||
"storage_uri": "MySQL market_data_sync_db.stocks",
|
"storage_uri": "PG market_data.stocks",
|
||||||
"storage_layer": "mysql",
|
"storage_layer": "pg",
|
||||||
"management_role": "基础字典",
|
"management_role": "基础字典",
|
||||||
"source": "Baostock query_stock_basic + 雪球 quote_detail",
|
"source": "Baostock query_stock_basic + 雪球 quote_detail",
|
||||||
"sync_script": "app.tasks.task_stocks_basic:run",
|
"sync_script": "app.tasks.task_stocks_basic:run",
|
||||||
@@ -38,8 +38,8 @@ SYNC_DEFINITIONS: list[dict[str, Any]] = [
|
|||||||
"dataset_id": "kline_daily",
|
"dataset_id": "kline_daily",
|
||||||
"name": "原始日K线数据",
|
"name": "原始日K线数据",
|
||||||
"description": "全市场 A 股日频 OHLCV,多源交叉校验(雪球/新浪/Baostock)",
|
"description": "全市场 A 股日频 OHLCV,多源交叉校验(雪球/新浪/Baostock)",
|
||||||
"storage_uri": "MySQL market_data_sync_db.kline_stock",
|
"storage_uri": "PG market_data.kline_stock",
|
||||||
"storage_layer": "mysql",
|
"storage_layer": "pg",
|
||||||
"management_role": "原始源",
|
"management_role": "原始源",
|
||||||
"source": "雪球主源 + 新浪/Baostock 复核",
|
"source": "雪球主源 + 新浪/Baostock 复核",
|
||||||
"sync_script": "app.tasks.task_kline_daily:run",
|
"sync_script": "app.tasks.task_kline_daily:run",
|
||||||
@@ -50,8 +50,8 @@ SYNC_DEFINITIONS: list[dict[str, Any]] = [
|
|||||||
"dataset_id": "kline_index",
|
"dataset_id": "kline_index",
|
||||||
"name": "六大指数日线",
|
"name": "六大指数日线",
|
||||||
"description": "上证 / 深证 / 创业板 / 沪深300 / 中证500 / 中证1000",
|
"description": "上证 / 深证 / 创业板 / 沪深300 / 中证500 / 中证1000",
|
||||||
"storage_uri": "MySQL market_data_sync_db.kline_index + indices",
|
"storage_uri": "PG market_data.kline_index + indices",
|
||||||
"storage_layer": "mysql",
|
"storage_layer": "pg",
|
||||||
"management_role": "原始源",
|
"management_role": "原始源",
|
||||||
"source": "新浪指数日K",
|
"source": "新浪指数日K",
|
||||||
"sync_script": "app.tasks.task_kline_index:run",
|
"sync_script": "app.tasks.task_kline_index:run",
|
||||||
@@ -62,22 +62,34 @@ SYNC_DEFINITIONS: list[dict[str, Any]] = [
|
|||||||
"dataset_id": "kline_5min",
|
"dataset_id": "kline_5min",
|
||||||
"name": "5 分钟 K 线",
|
"name": "5 分钟 K 线",
|
||||||
"description": "全市场 A 股 5 分钟 K 线(mairui 源)",
|
"description": "全市场 A 股 5 分钟 K 线(mairui 源)",
|
||||||
"storage_uri": "MySQL market_data_sync_db.kline_5min",
|
"storage_uri": "PG market_data.kline_5min",
|
||||||
"storage_layer": "mysql",
|
"storage_layer": "pg",
|
||||||
"management_role": "原始源",
|
"management_role": "原始源",
|
||||||
"source": "mairui stockMin",
|
"source": "mairui stockMin",
|
||||||
"sync_script": "app.tasks.task_kline_5min:run",
|
"sync_script": "app.tasks.task_kline_5min:run",
|
||||||
"dependency_ids": ["stock_basic"],
|
"dependency_ids": ["stock_basic"],
|
||||||
"sort_order": 40,
|
"sort_order": 40,
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"dataset_id": "tick_trade",
|
||||||
|
"name": "当天逐笔交易",
|
||||||
|
"description": "全市场 A 股当天逐笔成交(mairui hsrl/zbjy 源;每日 21:00 发布,仅当天数据)",
|
||||||
|
"storage_uri": "PG market_data.tick_trade",
|
||||||
|
"storage_layer": "pg",
|
||||||
|
"management_role": "原始源",
|
||||||
|
"source": "mairui hsrl/zbjy",
|
||||||
|
"sync_script": "app.tasks.task_tick_trade:run",
|
||||||
|
"dependency_ids": ["stock_basic"],
|
||||||
|
"sort_order": 45,
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"dataset_id": "moneyflow",
|
"dataset_id": "moneyflow",
|
||||||
"name": "资金流",
|
"name": "资金流",
|
||||||
"description": "个股资金流(主力/大/中/小单净额),mairui 源",
|
"description": "个股资金流(主力/大/中/小单净额),mairui 源(每日 21:30 发布,由 market-sync-moneyflow.timer 21:35 触发)",
|
||||||
"storage_uri": "MySQL market_data_sync_db.moneyflow",
|
"storage_uri": "PG market_data.moneyflow",
|
||||||
"storage_layer": "mysql",
|
"storage_layer": "pg",
|
||||||
"management_role": "原始源",
|
"management_role": "原始源",
|
||||||
"source": "mairui hsstock/history/transaction",
|
"source": "mairui hsstock/history/transaction (21:30 publish)",
|
||||||
"sync_script": "app.tasks.task_moneyflow:run",
|
"sync_script": "app.tasks.task_moneyflow:run",
|
||||||
"dependency_ids": ["stock_basic"],
|
"dependency_ids": ["stock_basic"],
|
||||||
"sort_order": 50,
|
"sort_order": 50,
|
||||||
@@ -86,8 +98,8 @@ SYNC_DEFINITIONS: list[dict[str, Any]] = [
|
|||||||
"dataset_id": "industry_sector",
|
"dataset_id": "industry_sector",
|
||||||
"name": "股票-行业映射",
|
"name": "股票-行业映射",
|
||||||
"description": "股票-行业映射 + 行业字典(Baostock)",
|
"description": "股票-行业映射 + 行业字典(Baostock)",
|
||||||
"storage_uri": "MySQL market_data_sync_db.industry + sectors + stock_sector_map",
|
"storage_uri": "PG market_data.industry + sectors + stock_sector_map",
|
||||||
"storage_layer": "mysql",
|
"storage_layer": "pg",
|
||||||
"management_role": "基础字典",
|
"management_role": "基础字典",
|
||||||
"source": "Baostock query_stock_industry",
|
"source": "Baostock query_stock_industry",
|
||||||
"sync_script": "app.tasks.task_industry_sector:run",
|
"sync_script": "app.tasks.task_industry_sector:run",
|
||||||
@@ -98,8 +110,8 @@ SYNC_DEFINITIONS: list[dict[str, Any]] = [
|
|||||||
"dataset_id": "sector_features",
|
"dataset_id": "sector_features",
|
||||||
"name": "行业聚合特征",
|
"name": "行业聚合特征",
|
||||||
"description": "由 kline_stock + industry 衍生:行业日收益 / 行业指数 close / EMA10-20-200 / score。纯本地计算,无外部 API。",
|
"description": "由 kline_stock + industry 衍生:行业日收益 / 行业指数 close / EMA10-20-200 / score。纯本地计算,无外部 API。",
|
||||||
"storage_uri": "MySQL market_data_sync_db.sector_indices + sector_features_daily",
|
"storage_uri": "PG market_data.sector_indices + sector_features_daily",
|
||||||
"storage_layer": "mysql",
|
"storage_layer": "pg",
|
||||||
"management_role": "衍生源",
|
"management_role": "衍生源",
|
||||||
"source": "本地计算(kline_stock + industry)",
|
"source": "本地计算(kline_stock + industry)",
|
||||||
"sync_script": "app.tasks.task_sector_features:run",
|
"sync_script": "app.tasks.task_sector_features:run",
|
||||||
@@ -110,8 +122,8 @@ SYNC_DEFINITIONS: list[dict[str, Any]] = [
|
|||||||
"dataset_id": "share_snapshot",
|
"dataset_id": "share_snapshot",
|
||||||
"name": "股本快照",
|
"name": "股本快照",
|
||||||
"description": "全市场 A 股最新股本(雪球 quote_detail,单位亿股)",
|
"description": "全市场 A 股最新股本(雪球 quote_detail,单位亿股)",
|
||||||
"storage_uri": "MySQL market_data_sync_db.stocks.total_share/float_share + share 表",
|
"storage_uri": "PG market_data.stocks.total_share/float_share + share 表",
|
||||||
"storage_layer": "mysql",
|
"storage_layer": "pg",
|
||||||
"management_role": "基础字典",
|
"management_role": "基础字典",
|
||||||
"source": "雪球 quote_detail",
|
"source": "雪球 quote_detail",
|
||||||
"sync_script": "app.tasks.task_share_snapshot:run",
|
"sync_script": "app.tasks.task_share_snapshot:run",
|
||||||
@@ -122,14 +134,26 @@ SYNC_DEFINITIONS: list[dict[str, Any]] = [
|
|||||||
"dataset_id": "market_regime",
|
"dataset_id": "market_regime",
|
||||||
"name": "市场情绪 (Market Regime)",
|
"name": "市场情绪 (Market Regime)",
|
||||||
"description": "基于本地日K线聚合的 advance_ratio / 恐慌标记(衍生源)",
|
"description": "基于本地日K线聚合的 advance_ratio / 恐慌标记(衍生源)",
|
||||||
"storage_uri": "MySQL market_data_sync_db.market_regime_daily",
|
"storage_uri": "PG market_data.market_regime_daily",
|
||||||
"storage_layer": "mysql",
|
"storage_layer": "pg",
|
||||||
"management_role": "衍生源",
|
"management_role": "衍生源",
|
||||||
"source": "本地日K线聚合(无外部接口)",
|
"source": "本地日K线聚合(无外部接口)",
|
||||||
"sync_script": "app.tasks.task_market_regime:run",
|
"sync_script": "app.tasks.task_market_regime:run",
|
||||||
"dependency_ids": ["kline_daily"],
|
"dependency_ids": ["kline_daily"],
|
||||||
"sort_order": 80,
|
"sort_order": 80,
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"dataset_id": "longhubang",
|
||||||
|
"name": "龙虎榜(聚合 + 席位)",
|
||||||
|
"description": "akshare 源(东方财富封装);每日 22:00 timer 触发,新增聚合层 + 席位层两张表",
|
||||||
|
"storage_uri": "PG market_data.longhubang_daily + longhubang_seat",
|
||||||
|
"storage_layer": "pg",
|
||||||
|
"management_role": "原始源",
|
||||||
|
"source": "akshare stock_lhb_detail_em + stock_lhb_stock_detail_em (东方财富)",
|
||||||
|
"sync_script": "app.tasks.task_longhubang:run",
|
||||||
|
"dependency_ids": ["stock_basic"],
|
||||||
|
"sort_order": 85,
|
||||||
|
},
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
@@ -236,6 +260,26 @@ def mark_sync_failed(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def mark_sync_warning(dataset_id: str, *, message: str = "") -> None:
|
||||||
|
"""部分失败的标记:成功 + 失败混合 → status=warning。
|
||||||
|
|
||||||
|
与 mark_sync_failed 的区别:warning 不写 last_failure_at / last_error,
|
||||||
|
也不触发 needs_resync=1(warning 状态下数据可能部分可用,不是硬错误)。
|
||||||
|
历史 bug(2026-07-01 修复):SyncTask.run() 把 warning 走 mark_sync_failed 路径
|
||||||
|
导致 UI 误显示"失败"(last_failure_at 有值、last_error 非空)。
|
||||||
|
"""
|
||||||
|
now = _now_ts()
|
||||||
|
db_ops.update_dataset_registry_state(
|
||||||
|
dataset_id,
|
||||||
|
status="warning",
|
||||||
|
finished_at=now,
|
||||||
|
last_success_at=now, # 部分成功也算成功一刻,记到 last_success
|
||||||
|
message=message or "部分失败",
|
||||||
|
current_step="",
|
||||||
|
updated_at=now,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def mark_sync_blocked(dataset_id: str, *, message: str) -> None:
|
def mark_sync_blocked(dataset_id: str, *, message: str) -> None:
|
||||||
mark_sync_failed(dataset_id, message=message, error=message, status="blocked")
|
mark_sync_failed(dataset_id, message=message, error=message, status="blocked")
|
||||||
|
|
||||||
|
|||||||
@@ -24,6 +24,7 @@ def main():
|
|||||||
p_sync.add_argument("--start", help="起始日期 YYYY-MM-DD", default=None)
|
p_sync.add_argument("--start", help="起始日期 YYYY-MM-DD", default=None)
|
||||||
p_sync.add_argument("--end", help="结束日期 YYYY-MM-DD", default=None)
|
p_sync.add_argument("--end", help="结束日期 YYYY-MM-DD", default=None)
|
||||||
p_sync.add_argument("--workers", type=int, default=10, help="并发数")
|
p_sync.add_argument("--workers", type=int, default=10, help="并发数")
|
||||||
|
p_sync.add_argument("--force", action="store_true", help="跳过时间门控(tick_trade 用)")
|
||||||
|
|
||||||
sub.add_parser("list", help="列出所有同步任务")
|
sub.add_parser("list", help="列出所有同步任务")
|
||||||
sub.add_parser("status", help="查看整体状态(调度器 / 数据源 / 同步任务)")
|
sub.add_parser("status", help="查看整体状态(调度器 / 数据源 / 同步任务)")
|
||||||
@@ -55,6 +56,8 @@ def main():
|
|||||||
kwargs["end"] = args.end
|
kwargs["end"] = args.end
|
||||||
if args.workers:
|
if args.workers:
|
||||||
kwargs["max_workers"] = args.workers
|
kwargs["max_workers"] = args.workers
|
||||||
|
if args.force:
|
||||||
|
kwargs["force"] = True
|
||||||
result = task.run(trigger_source="cli", **kwargs)
|
result = task.run(trigger_source="cli", **kwargs)
|
||||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||||
sys.exit(0 if result.get("status") in ("ok", "warning") else 2)
|
sys.exit(0 if result.get("status") in ("ok", "warning") else 2)
|
||||||
|
|||||||
@@ -0,0 +1,216 @@
|
|||||||
|
"""akshare 龙虎榜数据源(2026-07-01 引入)。
|
||||||
|
|
||||||
|
提供:
|
||||||
|
- 龙虎榜聚合层(`ak.stock_lhb_detail_em`):每日上榜股票汇总
|
||||||
|
- 龙虎榜席位层(`ak.stock_lhb_stock_detail_em`):单只上榜票的买卖前五营业部
|
||||||
|
|
||||||
|
设计原则:
|
||||||
|
- 严格遵循 [[no-akshare]] 当前规则:仅"无替代源"场景可用。
|
||||||
|
龙虎榜无合理替代(baostock / mairui / 雪球均无龙虎榜接口)→ 用 akshare。
|
||||||
|
- 速率限制:~0.5 RPS 主动 sleep(akshare 经验值;东财未明确限速)
|
||||||
|
- 失败处理:席位层对"未上榜"的股票会触发 akshare 内部 `'NoneType' object is not subscriptable`
|
||||||
|
→ 在外层 try/except 吞掉该异常,按"空数据"处理。
|
||||||
|
- 每次调用前显式 sleep 0.4s,避免被东财风控。
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
from typing import Any, Optional
|
||||||
|
|
||||||
|
import pandas as pd
|
||||||
|
|
||||||
|
from app.core.datasource.base import DataSource
|
||||||
|
from app.core.utils.logging import get_logger
|
||||||
|
|
||||||
|
logger = get_logger("akshare_lhb")
|
||||||
|
|
||||||
|
|
||||||
|
class AkshareLhbSource(DataSource):
|
||||||
|
"""akshare 龙虎榜数据源(基于东方财富私有 API 的 akshare 封装)。"""
|
||||||
|
|
||||||
|
key = "datasource_akshare_lhb"
|
||||||
|
name = "akshare 龙虎榜(东方财富封装)"
|
||||||
|
provides = ["longhubang_daily", "longhubang_seat"]
|
||||||
|
requires_credential = False
|
||||||
|
|
||||||
|
# 全局 RPS 限流(akshare → 东财私有 API)
|
||||||
|
# 设 2.5 RPS(0.4s 间隔)—— 经验值比 mairui 钻石档保守,避免被东财风控
|
||||||
|
_RPS_LIMIT = 2.5
|
||||||
|
_rate_lock = threading.Lock()
|
||||||
|
_last_request_ts = 0.0
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _wait_rps(cls) -> None:
|
||||||
|
"""全局 RPS 限流:每次 akshare 调用前 sleep。"""
|
||||||
|
with cls._rate_lock:
|
||||||
|
now = time.time()
|
||||||
|
elapsed = now - cls._last_request_ts
|
||||||
|
min_interval = 1.0 / cls._RPS_LIMIT
|
||||||
|
if elapsed < min_interval:
|
||||||
|
time.sleep(min_interval - elapsed)
|
||||||
|
cls._last_request_ts = time.time()
|
||||||
|
|
||||||
|
def is_available(self) -> tuple[bool, str]:
|
||||||
|
try:
|
||||||
|
import akshare # noqa: F401
|
||||||
|
return True, "akshare 已安装"
|
||||||
|
except ImportError as e:
|
||||||
|
return False, f"akshare 未安装: {e}"
|
||||||
|
|
||||||
|
def health_check(self) -> dict[str, Any]:
|
||||||
|
try:
|
||||||
|
import akshare as ak
|
||||||
|
# 用一次历史数据作烟测(取一个肯定有数据的工作日;2024-06-14 是周五)
|
||||||
|
self._wait_rps()
|
||||||
|
try:
|
||||||
|
df = ak.stock_lhb_detail_em(start_date="20240614", end_date="20240614")
|
||||||
|
except Exception as inner_e:
|
||||||
|
# akshare 在节假日可能返 None 然后内部崩,catch 后报失败
|
||||||
|
return {"success": False, "message": f"akshare 龙虎榜失败: {inner_e}"}
|
||||||
|
if df is None or df.empty:
|
||||||
|
return {"success": False, "message": "akshare 龙虎榜返回空(可能是节假日)"}
|
||||||
|
return {"success": True, "message": f"akshare 龙虎榜 OK,样例 {len(df)} 行"}
|
||||||
|
except Exception as e:
|
||||||
|
return {"success": False, "message": f"akshare 龙虎榜失败: {e}"}
|
||||||
|
|
||||||
|
# ── 聚合层 ───────────────────────────────────────────
|
||||||
|
|
||||||
|
def fetch_longhubang_daily(
|
||||||
|
self,
|
||||||
|
start_date: str,
|
||||||
|
end_date: str,
|
||||||
|
) -> pd.DataFrame:
|
||||||
|
"""龙虎榜每日上榜股票汇总。
|
||||||
|
|
||||||
|
Args:
|
||||||
|
start_date / end_date: 'YYYY-MM-DD' 或 'YYYYMMDD'(akshare 接受两种)
|
||||||
|
Returns:
|
||||||
|
DataFrame 字段:
|
||||||
|
序号 / 代码 / 名称 / 上榜日 / 解读 / 收盘价 / 涨跌幅 /
|
||||||
|
龙虎榜净买额 / 龙虎榜买入额 / 龙虎榜卖出额 / 龙虎榜成交额 /
|
||||||
|
市场总成交额 / 净买额占总成交比 / 成交额占总成交比 / 换手率 /
|
||||||
|
流通市值 / 上榜原因 / 上榜后1日 / 上榜后2日 / 上榜后5日 / 上榜后10日
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
import akshare as ak
|
||||||
|
except ImportError:
|
||||||
|
return pd.DataFrame()
|
||||||
|
# 兼容两种日期格式:YYYYMMDD 直接传,YYYY-MM-DD 剥横线
|
||||||
|
sd = start_date.replace("-", "")
|
||||||
|
ed = end_date.replace("-", "")
|
||||||
|
self._wait_rps()
|
||||||
|
try:
|
||||||
|
df = ak.stock_lhb_detail_em(start_date=sd, end_date=ed)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"[akshare] stock_lhb_detail_em({sd},{ed}) failed: {e}")
|
||||||
|
return pd.DataFrame()
|
||||||
|
if df is None or df.empty:
|
||||||
|
return pd.DataFrame()
|
||||||
|
# akshare 6 位代码不带交易所前缀;project 标准是 SH600000 / SZ000001 → 加前缀
|
||||||
|
df = df.copy()
|
||||||
|
df["stock_code"] = df["代码"].astype(str).str.zfill(6).apply(self._prefix_code)
|
||||||
|
df = df.rename(columns={
|
||||||
|
"名称": "stock_name",
|
||||||
|
"上榜日": "trade_date",
|
||||||
|
"解读": "comment",
|
||||||
|
"收盘价": "close",
|
||||||
|
"涨跌幅": "pct_chg",
|
||||||
|
"龙虎榜净买额": "lhb_net_buy",
|
||||||
|
"龙虎榜买入额": "lhb_buy_amt",
|
||||||
|
"龙虎榜卖出额": "lhb_sell_amt",
|
||||||
|
"龙虎榜成交额": "lhb_total_amt",
|
||||||
|
"市场总成交额": "market_total_amt",
|
||||||
|
"净买额占总成交比": "net_buy_ratio",
|
||||||
|
"成交额占总成交比": "total_amt_ratio",
|
||||||
|
"换手率": "turnover_rate",
|
||||||
|
"流通市值": "float_mv",
|
||||||
|
"上榜原因": "reason",
|
||||||
|
"上榜后1日": "post_1d_pct",
|
||||||
|
"上榜后2日": "post_2d_pct",
|
||||||
|
"上榜后5日": "post_5d_pct",
|
||||||
|
"上榜后10日": "post_10d_pct",
|
||||||
|
"序号": "rank_idx",
|
||||||
|
})
|
||||||
|
# 去重:akshare 同一天同一只票可能因多上榜原因(涨幅+换手率)返回多行,
|
||||||
|
# 数值相同只 reason 不同 → 按 (stock_code, trade_date) group by,
|
||||||
|
# reason 用 " | " 拼接保留所有原因,数值列 first()
|
||||||
|
agg_cols = {
|
||||||
|
"stock_name": "first",
|
||||||
|
"rank_idx": "min",
|
||||||
|
"comment": "first",
|
||||||
|
"close": "first",
|
||||||
|
"pct_chg": "first",
|
||||||
|
"lhb_net_buy": "first",
|
||||||
|
"lhb_buy_amt": "first",
|
||||||
|
"lhb_sell_amt": "first",
|
||||||
|
"lhb_total_amt": "first",
|
||||||
|
"market_total_amt": "first",
|
||||||
|
"net_buy_ratio": "first",
|
||||||
|
"total_amt_ratio": "first",
|
||||||
|
"turnover_rate": "first",
|
||||||
|
"float_mv": "first",
|
||||||
|
"reason": lambda s: " | ".join(sorted(set(v for v in s if v))),
|
||||||
|
"post_1d_pct": "first",
|
||||||
|
"post_2d_pct": "first",
|
||||||
|
"post_5d_pct": "first",
|
||||||
|
"post_10d_pct": "first",
|
||||||
|
}
|
||||||
|
df = (
|
||||||
|
df.groupby(["stock_code", "trade_date"], as_index=False)
|
||||||
|
.agg(agg_cols)
|
||||||
|
)
|
||||||
|
return df
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _prefix_code(code6: str) -> str:
|
||||||
|
"""6 位代码 → SH/SZ 前缀。"""
|
||||||
|
if code6.startswith(("5", "6", "9")):
|
||||||
|
return f"SH{code6}"
|
||||||
|
return f"SZ{code6}"
|
||||||
|
|
||||||
|
# ── 席位层 ───────────────────────────────────────────
|
||||||
|
|
||||||
|
def fetch_longhubang_seat(
|
||||||
|
self,
|
||||||
|
code6: str,
|
||||||
|
trade_date: str,
|
||||||
|
direction: str = "buy",
|
||||||
|
) -> pd.DataFrame:
|
||||||
|
"""单只上榜股票单日的买/卖前 5 营业部。
|
||||||
|
|
||||||
|
Args:
|
||||||
|
code6: 6 位股票代码(无交易所前缀)
|
||||||
|
trade_date: 'YYYY-MM-DD'(akshare 要求此格式)
|
||||||
|
direction: 'buy' / 'sell'(中文:'买入' / '卖出')
|
||||||
|
Returns:
|
||||||
|
DataFrame 字段:
|
||||||
|
序号 / 交易营业部名称 / 买入金额 / 买入金额-占总成交比例 /
|
||||||
|
卖出金额 / 卖出金额-占总成交比例 / 净额 / 类型
|
||||||
|
|
||||||
|
注意:若该股当日未上榜,akshare 内部抛 `'NoneType' object is not subscriptable`,
|
||||||
|
我们 try/except 吞掉返回空 DataFrame(正常情况,不是错)。
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
import akshare as ak
|
||||||
|
except ImportError:
|
||||||
|
return pd.DataFrame()
|
||||||
|
flag = "买入" if direction == "buy" else "卖出"
|
||||||
|
# akshare 的 stock_lhb_stock_detail_em 要 YYYYMMDD 格式(不带横线),
|
||||||
|
# YYYY-MM-DD 会返回 None 然后内部崩在 'NoneType' object is not subscriptable
|
||||||
|
date_yyyymmdd = str(trade_date).replace("-", "")
|
||||||
|
self._wait_rps()
|
||||||
|
try:
|
||||||
|
df = ak.stock_lhb_stock_detail_em(
|
||||||
|
symbol=str(code6).zfill(6),
|
||||||
|
date=date_yyyymmdd,
|
||||||
|
flag=flag,
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
# 'NoneType' object is not subscriptable = 股票未上榜(akshare bug)
|
||||||
|
# 其他真错也兜住,返回空
|
||||||
|
logger.debug(f"[akshare] stock_lhb_stock_detail_em({code6},{date_yyyymmdd},{flag}) → {type(e).__name__}: {e}")
|
||||||
|
return pd.DataFrame()
|
||||||
|
if df is None or df.empty:
|
||||||
|
return pd.DataFrame()
|
||||||
|
return df
|
||||||
+114
-11
@@ -5,6 +5,7 @@
|
|||||||
- 5 分钟 K 线(`hsstock/history/{code}.{ex}/5/n/{licence}`)
|
- 5 分钟 K 线(`hsstock/history/{code}.{ex}/5/n/{licence}`)
|
||||||
- 指数日 K 线(`hsindex/history/{code}.{ex}/d/{licence}`)
|
- 指数日 K 线(`hsindex/history/{code}.{ex}/d/{licence}`)
|
||||||
- 资金流向(`hsstock/history/transaction/{code}.{ex}/{licence}`)
|
- 资金流向(`hsstock/history/transaction/{code}.{ex}/{licence}`)
|
||||||
|
- 当天逐笔交易(`hsrl/zbjy/{code6}/{licence}`)
|
||||||
|
|
||||||
限速:1分钟300次(默认保守到 5 RPS = 1分钟300次)
|
限速:1分钟300次(默认保守到 5 RPS = 1分钟300次)
|
||||||
凭证:licence(无需登录态,从环境变量 MAIRUI_LICENCE 读取)
|
凭证:licence(无需登录态,从环境变量 MAIRUI_LICENCE 读取)
|
||||||
@@ -31,7 +32,7 @@ from app.core.datasource.utils import (
|
|||||||
class MairuiSource(DataSource):
|
class MairuiSource(DataSource):
|
||||||
key = "datasource_mairui"
|
key = "datasource_mairui"
|
||||||
name = "麦蕊智数(mairui.club)"
|
name = "麦蕊智数(mairui.club)"
|
||||||
provides = ["kline_daily", "kline_5min", "index_daily", "stock_basic", "moneyflow"]
|
provides = ["kline_daily", "kline_5min", "index_daily", "stock_basic", "moneyflow", "tick_trade"]
|
||||||
requires_credential = True
|
requires_credential = True
|
||||||
credential_key = "MAIRUI_LICENCE"
|
credential_key = "MAIRUI_LICENCE"
|
||||||
|
|
||||||
@@ -41,11 +42,20 @@ class MairuiSource(DataSource):
|
|||||||
}
|
}
|
||||||
_BASE_URL = "https://api.mairuiapi.com"
|
_BASE_URL = "https://api.mairuiapi.com"
|
||||||
|
|
||||||
# mairui 免费 licence 1 分钟 300 次 = 5 RPS(**单 licence**硬上限)。
|
# mairui licence 限速(按 tier):
|
||||||
# 5 workers × 1 RPS/worker = 5 RPS 总 = 刚好踩满上限,避免风控。
|
# 免费版: 1 min/300 次 = 5 RPS
|
||||||
|
# 体验版: 1 min/1千次 = 16.67 RPS
|
||||||
|
# 包年版: 1 min/3千次 = 50 RPS
|
||||||
|
# 钻石版: 1 min/6千次 = 100 RPS ← 当前项目用
|
||||||
|
#
|
||||||
|
# 用法:5 workers × _RPS_LIMIT RPS/worker = 总 RPS。
|
||||||
|
# 默认 10.0 → 5 workers = 50 RPS(钻石 100 RPS 的一半,留 50% buffer 防风控)。
|
||||||
|
# 如需调:env `MAIRUI_RPS_LIMIT=20` → 100 RPS (踩满钻石);
|
||||||
|
# `MAIRUI_RPS_LIMIT=1` → 5 RPS (回退到免费档兼容)。
|
||||||
|
#
|
||||||
# 用 thread-local 计时:每个 worker 独立计自己的 last_ts,
|
# 用 thread-local 计时:每个 worker 独立计自己的 last_ts,
|
||||||
# 避免 N 个 worker 串行抢一把锁退化成 1 worker。
|
# 避免 N 个 worker 串行抢一把锁退化成 1 worker。
|
||||||
_RPS_LIMIT = 1.0
|
_RPS_LIMIT = float(os.environ.get("MAIRUI_RPS_LIMIT", "10"))
|
||||||
_rps_local = threading.local()
|
_rps_local = threading.local()
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
@@ -87,7 +97,18 @@ class MairuiSource(DataSource):
|
|||||||
return {"success": False, "message": f"麦蕊连接失败: {e}"}
|
return {"success": False, "message": f"麦蕊连接失败: {e}"}
|
||||||
|
|
||||||
def _fetch(self, path: str, params: dict[str, Any], retry: int = 2) -> Any:
|
def _fetch(self, path: str, params: dict[str, Any], retry: int = 2) -> Any:
|
||||||
"""通用 GET,含限速 + 重试。"""
|
"""通用 GET,含限速 + 重试。
|
||||||
|
|
||||||
|
编码探测顺序 (2026-07-01 修复):
|
||||||
|
1) UTF-8 — 正常 JSON 响应
|
||||||
|
2) GBK — mairui 风控时的中文错误消息 ("接收数据异常,请稍后再试")
|
||||||
|
3) latin-1 兜底 — 防御性 fallback (历史 Python repr'd 字符串)
|
||||||
|
|
||||||
|
历史 bug (2026-06-16 之前): 无条件 `raw_bytes.decode("latin-1")`,
|
||||||
|
把 UTF-8 中文名全部 double-encoded (Latin-1 → UTF-8) → 写入 DB
|
||||||
|
后 stock_name 看起来正常但 hex 是错的双倍长度字节。
|
||||||
|
修复后正常 JSON 走 UTF-8,错误消息走 GBK,Latin-1 仅作兜底。
|
||||||
|
"""
|
||||||
self._wait_rps()
|
self._wait_rps()
|
||||||
qs = "&".join(f"{k}={v}" for k, v in params.items() if v)
|
qs = "&".join(f"{k}={v}" for k, v in params.items() if v)
|
||||||
url = f"{self._BASE_URL}{path}"
|
url = f"{self._BASE_URL}{path}"
|
||||||
@@ -98,16 +119,14 @@ class MairuiSource(DataSource):
|
|||||||
try:
|
try:
|
||||||
req = urllib.request.Request(url, headers=self._HEADERS)
|
req = urllib.request.Request(url, headers=self._HEADERS)
|
||||||
with urllib.request.urlopen(req, timeout=20) as resp:
|
with urllib.request.urlopen(req, timeout=20) as resp:
|
||||||
# 先按 gbk 试(中文乱码返回),再 fallback utf-8
|
|
||||||
raw_bytes = resp.read()
|
raw_bytes = resp.read()
|
||||||
# 用 latin-1 永不失败地把 bytes 转成字符串(每字节 1 字符)
|
# 编码探测: 优先 UTF-8, fallback GBK, 兜底 latin-1
|
||||||
raw = raw_bytes.decode("latin-1")
|
raw = self._decode_response(raw_bytes)
|
||||||
# mairui 风控时返回 "接收数据异常,请稍后再试"(gbk 编码)
|
# 风控提示 (GBK 编码的中文消息)
|
||||||
if "请稍后再试" in raw or "codec can't decode" in raw or raw.startswith("'utf-8'") or "Error -3 while decompressing" in raw:
|
if "请稍后再试" in raw or "codec can't decode" in raw or raw.startswith("'utf-8'") or "Error -3 while decompressing" in raw:
|
||||||
raise RuntimeError(f"mairui 返回风控提示: {raw[:80]}")
|
raise RuntimeError(f"mairui 返回风控提示: {raw[:80]}")
|
||||||
# 试 JSON parse;如果是 gbk 编码的中文提示,要先解码
|
# Python repr'd 字符串: "'接收数据异常,请稍后再试'" (GBK bytes 用 latin-1 解码后又被 Python repr)
|
||||||
if raw.startswith("'") and raw.endswith("'"):
|
if raw.startswith("'") and raw.endswith("'"):
|
||||||
# gbk 编码的 Python repr 字符串,如 "'接收数据异常,请稍后再试'"
|
|
||||||
try:
|
try:
|
||||||
decoded = raw[1:-1].encode("latin-1").decode("gbk")
|
decoded = raw[1:-1].encode("latin-1").decode("gbk")
|
||||||
if "请稍后再试" in decoded:
|
if "请稍后再试" in decoded:
|
||||||
@@ -123,6 +142,22 @@ class MairuiSource(DataSource):
|
|||||||
get_logger("mairui").debug(f"mairui fetch {url} failed: {last_err}")
|
get_logger("mairui").debug(f"mairui fetch {url} failed: {last_err}")
|
||||||
return []
|
return []
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _decode_response(raw_bytes: bytes) -> str:
|
||||||
|
"""探测响应字节流的编码,优先 UTF-8,fallback GBK,latin-1 兜底。"""
|
||||||
|
# 1) UTF-8: 99% 的情况 (正常 JSON 响应)
|
||||||
|
try:
|
||||||
|
return raw_bytes.decode("utf-8")
|
||||||
|
except UnicodeDecodeError:
|
||||||
|
pass
|
||||||
|
# 2) GBK: mairui 风控时返 GBK 编码的中文错误消息
|
||||||
|
try:
|
||||||
|
return raw_bytes.decode("gbk")
|
||||||
|
except UnicodeDecodeError:
|
||||||
|
pass
|
||||||
|
# 3) latin-1 兜底: 永不失败,但可能产生错乱字符
|
||||||
|
return raw_bytes.decode("latin-1")
|
||||||
|
|
||||||
# ── 股票列表 ───────────────────────────────────────────
|
# ── 股票列表 ───────────────────────────────────────────
|
||||||
|
|
||||||
def fetch_stock_list(self) -> list[dict]:
|
def fetch_stock_list(self) -> list[dict]:
|
||||||
@@ -377,3 +412,71 @@ class MairuiSource(DataSource):
|
|||||||
if end:
|
if end:
|
||||||
df = df[df["trade_date"] <= end]
|
df = df[df["trade_date"] <= end]
|
||||||
return df.sort_values("trade_date").reset_index(drop=True)
|
return df.sort_values("trade_date").reset_index(drop=True)
|
||||||
|
|
||||||
|
# ── 当天逐笔交易 ───────────────────────────────────────
|
||||||
|
|
||||||
|
def fetch_tick_trade(self, code6: str) -> pd.DataFrame:
|
||||||
|
"""当天逐笔交易(mairui 不支持历史回溯,仅当天数据)。
|
||||||
|
|
||||||
|
API: GET /hsrl/zbjy/{code6}/{licence} (无 st/et 参数)
|
||||||
|
文档:https://mairui.club/hsdata → "当天逐笔交易"
|
||||||
|
更新:每日 21:00。
|
||||||
|
排序:按时间倒序。
|
||||||
|
限速:1 min/300 次(与其它接口共享 licence 配额)。
|
||||||
|
|
||||||
|
返回字段:
|
||||||
|
d string 数据归属日期(yyyy-MM-dd)
|
||||||
|
t string 时间(HH:mm:dd,文档笔误,应为 HH:mm:ss)
|
||||||
|
v number 成交量(股)
|
||||||
|
p number 成交价(元)
|
||||||
|
ts number 交易方向 0=中性盘 / 1=买入 / 2=卖出
|
||||||
|
|
||||||
|
派生字段:
|
||||||
|
trade_time = d + 'T' + t(datetime64[ns, UTC],无 tz 直接当本地时区)
|
||||||
|
direction = 0/1/2 → 'neutral'/'buy'/'sell'
|
||||||
|
amount = price * volume(元)
|
||||||
|
stock_code = 外部传入的 6 位 code(保持与其它表一致)
|
||||||
|
"""
|
||||||
|
lic = self._get_licence()
|
||||||
|
if not lic:
|
||||||
|
return pd.DataFrame()
|
||||||
|
path = f"/hsrl/zbjy/{code6}/{lic}"
|
||||||
|
data = self._fetch(path, {}) # 200/200 —— 不需要 start/end
|
||||||
|
if not isinstance(data, list) or not data:
|
||||||
|
return pd.DataFrame()
|
||||||
|
|
||||||
|
direction_map = {0: "neutral", 1: "buy", 2: "sell"}
|
||||||
|
records = []
|
||||||
|
for item in data:
|
||||||
|
try:
|
||||||
|
d = str(item.get("d", "") or "").strip()
|
||||||
|
t = str(item.get("t", "") or "").strip()
|
||||||
|
if not d or not t:
|
||||||
|
continue
|
||||||
|
# mairui `t` 形如 "14:53:21",拼成 ISO 字符串让 pandas 解析
|
||||||
|
iso = f"{d}T{t}"
|
||||||
|
ts_val = pd.to_datetime(iso, errors="coerce")
|
||||||
|
if pd.isna(ts_val):
|
||||||
|
continue
|
||||||
|
price = float(item.get("p") or 0)
|
||||||
|
volume = float(item.get("v") or 0)
|
||||||
|
if price <= 0 or volume <= 0:
|
||||||
|
continue
|
||||||
|
ts_code = int(item.get("ts") or 0)
|
||||||
|
records.append({
|
||||||
|
"stock_code": code6,
|
||||||
|
"trade_date": d,
|
||||||
|
"trade_time": ts_val.to_pydatetime(),
|
||||||
|
"price": price,
|
||||||
|
"volume": volume,
|
||||||
|
"direction_code": ts_code,
|
||||||
|
"direction": direction_map.get(ts_code, ""),
|
||||||
|
"amount": round(price * volume, 2),
|
||||||
|
})
|
||||||
|
except (KeyError, ValueError, TypeError):
|
||||||
|
continue
|
||||||
|
if not records:
|
||||||
|
return pd.DataFrame()
|
||||||
|
df = pd.DataFrame(records)
|
||||||
|
# mairui 按时间倒序,统一升序便于入库
|
||||||
|
return df.sort_values("trade_time").reset_index(drop=True)
|
||||||
@@ -7,11 +7,13 @@ from app.tasks.task_industry_sector import SyncIndustrySector
|
|||||||
from app.tasks.task_kline_5min import SyncKline5Min
|
from app.tasks.task_kline_5min import SyncKline5Min
|
||||||
from app.tasks.task_kline_daily import SyncKlineDaily
|
from app.tasks.task_kline_daily import SyncKlineDaily
|
||||||
from app.tasks.task_kline_index import SyncKlineIndex
|
from app.tasks.task_kline_index import SyncKlineIndex
|
||||||
|
from app.tasks.task_longhubang import SyncLonghubang
|
||||||
from app.tasks.task_market_regime import SyncMarketRegime
|
from app.tasks.task_market_regime import SyncMarketRegime
|
||||||
from app.tasks.task_moneyflow import SyncMoneyflow
|
from app.tasks.task_moneyflow import SyncMoneyflow
|
||||||
from app.tasks.task_sector_features import SyncSectorFeatures
|
from app.tasks.task_sector_features import SyncSectorFeatures
|
||||||
from app.tasks.task_share_snapshot import SyncShareSnapshot
|
from app.tasks.task_share_snapshot import SyncShareSnapshot
|
||||||
from app.tasks.task_stocks_basic import SyncStocksBasic
|
from app.tasks.task_stocks_basic import SyncStocksBasic
|
||||||
|
from app.tasks.task_tick_trade import SyncTickTrade
|
||||||
|
|
||||||
TASKS: dict[str, type] = {
|
TASKS: dict[str, type] = {
|
||||||
cls.dataset_id: cls
|
cls.dataset_id: cls
|
||||||
@@ -20,11 +22,13 @@ TASKS: dict[str, type] = {
|
|||||||
SyncKlineDaily,
|
SyncKlineDaily,
|
||||||
SyncKlineIndex,
|
SyncKlineIndex,
|
||||||
SyncKline5Min,
|
SyncKline5Min,
|
||||||
|
SyncTickTrade,
|
||||||
SyncMoneyflow,
|
SyncMoneyflow,
|
||||||
SyncIndustrySector,
|
SyncIndustrySector,
|
||||||
SyncSectorFeatures,
|
SyncSectorFeatures,
|
||||||
SyncShareSnapshot,
|
SyncShareSnapshot,
|
||||||
SyncMarketRegime,
|
SyncMarketRegime,
|
||||||
|
SyncLonghubang,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -6,9 +6,13 @@ from __future__ import annotations
|
|||||||
import time
|
import time
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
|
from sqlalchemy import update
|
||||||
|
|
||||||
from app.core.db import ops as db_ops
|
from app.core.db import ops as db_ops
|
||||||
|
from app.core.db.models import Stock
|
||||||
from app.core.datasource.base import registry as ds_registry
|
from app.core.datasource.base import registry as ds_registry
|
||||||
from app.core.datasource.registry import is_source_ready
|
from app.core.datasource.registry import is_source_ready
|
||||||
|
from app.core.datasource.utils import to_hermes
|
||||||
from app.core.sync.base import SyncTask
|
from app.core.sync.base import SyncTask
|
||||||
from app.core.sync.registry import mark_sync_blocked
|
from app.core.sync.registry import mark_sync_blocked
|
||||||
from app.core.utils.logging import get_logger
|
from app.core.utils.logging import get_logger
|
||||||
@@ -65,9 +69,9 @@ class SyncIndustrySector(SyncTask):
|
|||||||
"source": "baostock_query_stock_industry",
|
"source": "baostock_query_stock_industry",
|
||||||
"enabled": 1,
|
"enabled": 1,
|
||||||
}
|
}
|
||||||
stock_sector_rows.append({"stock_code": code6, "sector_key": sector_key})
|
stock_sector_rows.append({"stock_code": to_hermes(code6), "sector_key": sector_key})
|
||||||
industry_rows.append({
|
industry_rows.append({
|
||||||
"code": code6,
|
"code": to_hermes(code6),
|
||||||
"industry_name": name,
|
"industry_name": name,
|
||||||
"industry_classification": classification,
|
"industry_classification": classification,
|
||||||
"update_date": r.get("update_date", ""),
|
"update_date": r.get("update_date", ""),
|
||||||
@@ -82,12 +86,9 @@ class SyncIndustrySector(SyncTask):
|
|||||||
return {"status": "error", "message": f"写库失败: {e}"}
|
return {"status": "error", "message": f"写库失败: {e}"}
|
||||||
|
|
||||||
# 回填 stocks.industry 字段(让 stocks 表也能直接看到)
|
# 回填 stocks.industry 字段(让 stocks 表也能直接看到)
|
||||||
# 使用批量 UPDATE ... CASE WHEN 方式一次性完成
|
# 走 ORM update(Stock) + IN (SHxxx, SZxxx, BJxxx, ...)
|
||||||
try:
|
try:
|
||||||
from app.core.db.connection import get_mysql
|
# 按 industry_name 分组,对每组用一个 IN 批量更新
|
||||||
from app.core.db.schema import ensure_all_tables
|
|
||||||
ensure_all_tables(get_mysql())
|
|
||||||
# 按 industry_name 分组,对每组用一个 IN 查询批量更新
|
|
||||||
by_industry: dict[str, list[str]] = {}
|
by_industry: dict[str, list[str]] = {}
|
||||||
for r in industry_rows:
|
for r in industry_rows:
|
||||||
name = r["industry_name"]
|
name = r["industry_name"]
|
||||||
@@ -95,19 +96,17 @@ class SyncIndustrySector(SyncTask):
|
|||||||
if name not in by_industry:
|
if name not in by_industry:
|
||||||
by_industry[name] = []
|
by_industry[name] = []
|
||||||
by_industry[name].append(code)
|
by_industry[name].append(code)
|
||||||
with get_mysql().cursor() as cur:
|
for ind_name, codes in by_industry.items():
|
||||||
for ind_name, codes in by_industry.items():
|
# 6 位 code × 3 个交易所前缀(SH/SZ/BJ)
|
||||||
# 构造 IN 子句:code IN (SHxxx, SZxxx, BJxxx)
|
code_variants = [
|
||||||
placeholders = []
|
f"{prefix}{c}" for c in codes for prefix in ("SH", "SZ", "BJ")
|
||||||
params = [ind_name]
|
]
|
||||||
for c in codes:
|
with db_ops.get_session() as s:
|
||||||
for prefix in ("SH", "SZ", "BJ"):
|
s.execute(
|
||||||
placeholders.append("%s")
|
update(Stock)
|
||||||
params.append(f"{prefix}{c}")
|
.where(Stock.code.in_(code_variants))
|
||||||
in_clause = ", ".join(placeholders)
|
.values(industry=ind_name)
|
||||||
sql = f"UPDATE stocks SET industry = %s WHERE code IN ({in_clause})"
|
)
|
||||||
cur.execute(sql, params)
|
|
||||||
get_mysql().commit()
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning(f"回填 stocks.industry 失败(不影响主任务): {e}")
|
logger.warning(f"回填 stocks.industry 失败(不影响主任务): {e}")
|
||||||
|
|
||||||
|
|||||||
@@ -18,13 +18,13 @@ from __future__ import annotations
|
|||||||
import os
|
import os
|
||||||
import time
|
import time
|
||||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||||
from datetime import datetime, timedelta
|
from datetime import datetime, timedelta, timezone
|
||||||
from typing import Any, Optional
|
from typing import Any, Optional
|
||||||
|
|
||||||
from app.core.db import ops as db_ops
|
from app.core.db import ops as db_ops
|
||||||
from app.core.datasource.base import registry as ds_registry
|
from app.core.datasource.base import registry as ds_registry
|
||||||
from app.core.datasource.registry import is_source_ready
|
from app.core.datasource.registry import is_source_ready
|
||||||
from app.core.datasource.utils import is_a_share_code, to_code6
|
from app.core.datasource.utils import is_a_share_code, to_code6, to_hermes
|
||||||
from app.core.sync.base import SyncTask
|
from app.core.sync.base import SyncTask
|
||||||
from app.core.sync.registry import mark_sync_blocked
|
from app.core.sync.registry import mark_sync_blocked
|
||||||
from app.core.utils.logging import get_logger
|
from app.core.utils.logging import get_logger
|
||||||
@@ -32,7 +32,7 @@ from app.core.utils.logging import get_logger
|
|||||||
logger = get_logger("sync.kline_5min")
|
logger = get_logger("sync.kline_5min")
|
||||||
|
|
||||||
# mairui 5min K 实测历史深度起点(实测 2023-06-14 才有数据,更早就 0)
|
# mairui 5min K 实测历史深度起点(实测 2023-06-14 才有数据,更早就 0)
|
||||||
MAIRUI_5MIN_FLOOR = datetime(2023, 6, 14)
|
MAIRUI_5MIN_FLOOR = datetime(2023, 6, 14, tzinfo=timezone.utc)
|
||||||
# 单次请求窗口大小:1 年(mairui 单次最多 ~11640 条 5min)
|
# 单次请求窗口大小:1 年(mairui 单次最多 ~11640 条 5min)
|
||||||
WINDOW_DAYS = 365
|
WINDOW_DAYS = 365
|
||||||
# 增量时往前多取的天数(防交易日历边界漏当天)
|
# 增量时往前多取的天数(防交易日历边界漏当天)
|
||||||
@@ -98,7 +98,8 @@ class SyncKline5Min(SyncTask):
|
|||||||
logger.info(f"[5min] DB 里已有 {len(snapshots)} 只的 kline_5min 数据")
|
logger.info(f"[5min] DB 里已有 {len(snapshots)} 只的 kline_5min 数据")
|
||||||
|
|
||||||
# ── 2) 内存算计划 ──
|
# ── 2) 内存算计划 ──
|
||||||
end = datetime.now()
|
# PG TIMESTAMPTZ 是 timezone-aware,所以 end 也必须 tz-aware 否则比较会炸
|
||||||
|
end = datetime.now(timezone.utc)
|
||||||
plans = self._plan(stock_codes, end, snapshots)
|
plans = self._plan(stock_codes, end, snapshots)
|
||||||
if not plans:
|
if not plans:
|
||||||
msg = f"5min K线 — 全部 {len(stock_codes)} 只都已最新(无需同步)"
|
msg = f"5min K线 — 全部 {len(stock_codes)} 只都已最新(无需同步)"
|
||||||
@@ -174,10 +175,11 @@ class SyncKline5Min(SyncTask):
|
|||||||
w_end.strftime("%Y-%m-%d"),
|
w_end.strftime("%Y-%m-%d"),
|
||||||
)
|
)
|
||||||
if df is not None and not df.empty:
|
if df is not None and not df.empty:
|
||||||
|
hermes = to_hermes(code6)
|
||||||
for _, r in df.iterrows():
|
for _, r in df.iterrows():
|
||||||
bar_time = r["bar_time"]
|
bar_time = r["bar_time"]
|
||||||
all_rows.append({
|
all_rows.append({
|
||||||
"stock_code": code6,
|
"stock_code": hermes,
|
||||||
"bar_time": bar_time.strftime("%Y-%m-%d %H:%M:%S")
|
"bar_time": bar_time.strftime("%Y-%m-%d %H:%M:%S")
|
||||||
if hasattr(bar_time, "strftime") else str(bar_time),
|
if hasattr(bar_time, "strftime") else str(bar_time),
|
||||||
"open": float(r.get("open") or 0),
|
"open": float(r.get("open") or 0),
|
||||||
|
|||||||
@@ -26,7 +26,7 @@ import pandas as pd
|
|||||||
|
|
||||||
from app.core.db import ops as db_ops
|
from app.core.db import ops as db_ops
|
||||||
from app.core.datasource.base import registry as ds_registry
|
from app.core.datasource.base import registry as ds_registry
|
||||||
from app.core.datasource.utils import is_a_share_code, to_code6
|
from app.core.datasource.utils import is_a_share_code, to_code6, to_hermes
|
||||||
from app.core.sync.base import SyncTask, _effective_sync_end
|
from app.core.sync.base import SyncTask, _effective_sync_end
|
||||||
from app.core.utils.logging import get_logger
|
from app.core.utils.logging import get_logger
|
||||||
|
|
||||||
@@ -145,11 +145,14 @@ class SyncKlineDaily(SyncTask):
|
|||||||
return (code6, None, str(e)[:120])
|
return (code6, None, str(e)[:120])
|
||||||
if df is None or df.empty:
|
if df is None or df.empty:
|
||||||
return (code6, None, "no data")
|
return (code6, None, "no data")
|
||||||
|
# 写库用 hermes(带 SH/SZ 前缀)—— 2026-07-01 重构:历史上 kline_stock 用
|
||||||
|
# 6位 code 与 stocks.code(SH600519) 格式不一致,跨表 JOIN 失败。现统一。
|
||||||
|
hermes = to_hermes(code6)
|
||||||
rows = []
|
rows = []
|
||||||
for _, r in df.iterrows():
|
for _, r in df.iterrows():
|
||||||
td = r["trade_date"]
|
td = r["trade_date"]
|
||||||
rows.append({
|
rows.append({
|
||||||
"stock_code": code6,
|
"stock_code": hermes,
|
||||||
"trade_date": td.strftime("%Y-%m-%d") if hasattr(td, "strftime") else str(td)[:10],
|
"trade_date": td.strftime("%Y-%m-%d") if hasattr(td, "strftime") else str(td)[:10],
|
||||||
"open": float(r["open"]),
|
"open": float(r["open"]),
|
||||||
"high": float(r["high"]),
|
"high": float(r["high"]),
|
||||||
@@ -226,11 +229,12 @@ class SyncKlineDaily(SyncTask):
|
|||||||
return {"status": "fail", "error": str(e)}
|
return {"status": "fail", "error": str(e)}
|
||||||
if df is None or df.empty:
|
if df is None or df.empty:
|
||||||
return {"status": "fail", "error": "no data"}
|
return {"status": "fail", "error": "no data"}
|
||||||
|
hermes = to_hermes(code6)
|
||||||
rows = []
|
rows = []
|
||||||
for _, r in df.iterrows():
|
for _, r in df.iterrows():
|
||||||
td = r["trade_date"]
|
td = r["trade_date"]
|
||||||
rows.append({
|
rows.append({
|
||||||
"stock_code": code6,
|
"stock_code": hermes,
|
||||||
"trade_date": td.strftime("%Y-%m-%d") if hasattr(td, "strftime") else str(td)[:10],
|
"trade_date": td.strftime("%Y-%m-%d") if hasattr(td, "strftime") else str(td)[:10],
|
||||||
"open": float(r["open"]),
|
"open": float(r["open"]),
|
||||||
"high": float(r["high"]),
|
"high": float(r["high"]),
|
||||||
|
|||||||
@@ -0,0 +1,245 @@
|
|||||||
|
"""同步任务:龙虎榜聚合层 + 席位层(akshare 源)。
|
||||||
|
|
||||||
|
数据源:
|
||||||
|
- 聚合层:`ak.stock_lhb_detail_em(start_date, end_date)` → `longhubang_daily` 表
|
||||||
|
- 席位层:`ak.stock_lhb_stock_detail_em(symbol, date, flag)` → `longhubang_seat` 表
|
||||||
|
|
||||||
|
设计要点:
|
||||||
|
1) 22:00 触发(mairui 21:30 跑 moneyflow;龙虎榜 19:00-21:00 陆续出齐,22:00 兜底)
|
||||||
|
2) 默认"今天"增量;CLI 支持 --backfill YYYYMMDD 单日;--backfill-range YYYYMMDD YYYYMMDD 日期范围
|
||||||
|
3) 聚合层一次性拉日期范围;席位层先抓聚合层取出 stock_code 集合再逐只抓(节省调用)
|
||||||
|
4) 串行执行(每日调用 ~3 次聚合层 + ~200 次席位层;3-5 分钟内完成)
|
||||||
|
5) akshare 对"未上榜"的股票会抛 `'NoneType' object is not subscriptable`
|
||||||
|
→ source 层 try/except 兜住,按"空数据"处理
|
||||||
|
6) 数据是历史回填友好(全量 4 年 ≈ 25 分钟,2026-07-01 烟测通过)
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import time
|
||||||
|
from datetime import datetime, timedelta
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from app.core.db import ops as db_ops
|
||||||
|
from app.core.datasource.base import registry as ds_registry
|
||||||
|
from app.core.datasource.registry import is_source_ready
|
||||||
|
from app.core.sync.base import SyncTask
|
||||||
|
from app.core.sync.registry import mark_sync_blocked
|
||||||
|
from app.core.utils.logging import get_logger
|
||||||
|
|
||||||
|
logger = get_logger("sync.longhubang")
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_yyyymmdd(s: str) -> str:
|
||||||
|
"""'YYYYMMDD' 或 'YYYY-MM-DD' → 'YYYY-MM-DD'。"""
|
||||||
|
s = str(s).strip()
|
||||||
|
if len(s) == 8 and s.isdigit():
|
||||||
|
return f"{s[:4]}-{s[4:6]}-{s[6:8]}"
|
||||||
|
return s
|
||||||
|
|
||||||
|
|
||||||
|
def _date_range(start: str, end: str) -> list[str]:
|
||||||
|
"""返回 start..end(含)期间所有日期字符串列表(YYYY-MM-DD)。"""
|
||||||
|
s = datetime.strptime(start, "%Y-%m-%d")
|
||||||
|
e = datetime.strptime(end, "%Y-%m-%d")
|
||||||
|
if e < s:
|
||||||
|
s, e = e, s
|
||||||
|
out = []
|
||||||
|
cur = s
|
||||||
|
while cur <= e:
|
||||||
|
out.append(cur.strftime("%Y-%m-%d"))
|
||||||
|
cur += timedelta(days=1)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
class SyncLonghubang(SyncTask):
|
||||||
|
"""龙虎榜聚合层 + 席位层同步。
|
||||||
|
|
||||||
|
CLI 用法:
|
||||||
|
python -m app.entrypoints.cli sync longhubang # 今天增量
|
||||||
|
... sync longhubang --backfill 20240614 # 单日
|
||||||
|
... sync longhubang --backfill-range 20240101 20240614 # 日期范围
|
||||||
|
... sync longhubang --start 20240101 --end 20240614 # 等价于 backfill-range
|
||||||
|
"""
|
||||||
|
dataset_id = "longhubang"
|
||||||
|
|
||||||
|
def _run(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
trigger_source: str = "manual",
|
||||||
|
backfill: str | None = None,
|
||||||
|
backfill_range: tuple[str, str] | None = None,
|
||||||
|
start: str | None = None,
|
||||||
|
end: str | None = None,
|
||||||
|
**kwargs,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
src = ds_registry.get("datasource_akshare_lhb")
|
||||||
|
if src is None:
|
||||||
|
return {"status": "error", "message": "akshare_lhb 数据源未注册"}
|
||||||
|
ok, reason = is_source_ready(src.key)
|
||||||
|
if not ok:
|
||||||
|
mark_sync_blocked(self.dataset_id, message=f"akshare_lhb 未就绪: {reason}")
|
||||||
|
return {"status": "blocked", "message": f"akshare_lhb 未就绪: {reason}"}
|
||||||
|
|
||||||
|
# ── 决定要拉哪几天 ──
|
||||||
|
if backfill_range:
|
||||||
|
s, e = _parse_yyyymmdd(backfill_range[0]), _parse_yyyymmdd(backfill_range[1])
|
||||||
|
elif start and end:
|
||||||
|
s, e = _parse_yyyymmdd(start), _parse_yyyymmdd(end)
|
||||||
|
elif backfill:
|
||||||
|
s = e = _parse_yyyymmdd(backfill)
|
||||||
|
else:
|
||||||
|
# 默认"今天"(盘后未收盘时 fallback 到昨天)
|
||||||
|
from app.core.sync.base import _effective_sync_end
|
||||||
|
s = e = _effective_sync_end()
|
||||||
|
|
||||||
|
dates = _date_range(s, e)
|
||||||
|
if not dates:
|
||||||
|
return {"status": "error", "message": f"日期范围无效: {s}..{e}"}
|
||||||
|
logger.info(f"[longhubang] 计划: {len(dates)} 天 ({s} ~ {e})")
|
||||||
|
|
||||||
|
# ── 逐日同步 ──
|
||||||
|
t0 = time.time()
|
||||||
|
daily_ok = daily_fail = 0
|
||||||
|
daily_total = seat_total = 0
|
||||||
|
for i, d in enumerate(dates, 1):
|
||||||
|
self._progress(
|
||||||
|
message=f"longhubang 进度 {i}/{len(dates)} 当前 {d}",
|
||||||
|
current=i, total=len(dates), current_step=d,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
d_rows, s_rows = self._sync_one_day(src, d)
|
||||||
|
daily_ok += 1
|
||||||
|
daily_total += d_rows
|
||||||
|
seat_total += s_rows
|
||||||
|
except Exception as e:
|
||||||
|
daily_fail += 1
|
||||||
|
logger.warning(f"[longhubang {d}] 失败: {e}")
|
||||||
|
|
||||||
|
elapsed = round(time.time() - t0, 1)
|
||||||
|
msg = (
|
||||||
|
f"龙虎榜 {len(dates)}天 daily_ok={daily_ok} daily_fail={daily_fail} "
|
||||||
|
f"聚合层={daily_total}行 席位层={seat_total}行, {elapsed}s"
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"status": "ok" if daily_fail == 0 else "warning",
|
||||||
|
"message": msg,
|
||||||
|
"ok": daily_ok, "fail": daily_fail,
|
||||||
|
"daily_rows": daily_total, "seat_rows": seat_total,
|
||||||
|
"elapsed_sec": elapsed,
|
||||||
|
}
|
||||||
|
|
||||||
|
# ── 单日同步(聚合 + 席位)──
|
||||||
|
|
||||||
|
def _sync_one_day(self, src, trade_date: str) -> tuple[int, int]:
|
||||||
|
"""返回 (聚合层行数, 席位层行数)。"""
|
||||||
|
# 1) 聚合层(1 调用/天)
|
||||||
|
df_daily = src.fetch_longhubang_daily(trade_date, trade_date)
|
||||||
|
daily_rows = 0
|
||||||
|
stock_codes: list[str] = []
|
||||||
|
if df_daily is not None and not df_daily.empty:
|
||||||
|
rows = []
|
||||||
|
for _, r in df_daily.iterrows():
|
||||||
|
rows.append({
|
||||||
|
"stock_code": str(r.get("stock_code") or "").strip(),
|
||||||
|
"stock_name": str(r.get("stock_name") or "").strip(),
|
||||||
|
"trade_date": str(r.get("trade_date") or trade_date)[:10],
|
||||||
|
"rank_idx": self._to_int(r.get("rank_idx")),
|
||||||
|
"comment": self._to_str(r.get("comment")),
|
||||||
|
"close": self._to_float(r.get("close")),
|
||||||
|
"pct_chg": self._to_float(r.get("pct_chg")),
|
||||||
|
"lhb_net_buy": self._to_float(r.get("lhb_net_buy")),
|
||||||
|
"lhb_buy_amt": self._to_float(r.get("lhb_buy_amt")),
|
||||||
|
"lhb_sell_amt": self._to_float(r.get("lhb_sell_amt")),
|
||||||
|
"lhb_total_amt": self._to_float(r.get("lhb_total_amt")),
|
||||||
|
"market_total_amt": self._to_float(r.get("market_total_amt")),
|
||||||
|
"net_buy_ratio": self._to_float(r.get("net_buy_ratio")),
|
||||||
|
"total_amt_ratio": self._to_float(r.get("total_amt_ratio")),
|
||||||
|
"turnover_rate": self._to_float(r.get("turnover_rate")),
|
||||||
|
"float_mv": self._to_float(r.get("float_mv")),
|
||||||
|
"reason": self._to_str(r.get("reason")),
|
||||||
|
"post_1d_pct": self._to_float(r.get("post_1d_pct")),
|
||||||
|
"post_2d_pct": self._to_float(r.get("post_2d_pct")),
|
||||||
|
"post_5d_pct": self._to_float(r.get("post_5d_pct")),
|
||||||
|
"post_10d_pct": self._to_float(r.get("post_10d_pct")),
|
||||||
|
})
|
||||||
|
if rows:
|
||||||
|
db_ops.upsert_longhubang_daily(rows)
|
||||||
|
daily_rows = len(rows)
|
||||||
|
# 取出 stock_code(去交易所前缀得 6 位,供席位层调用)
|
||||||
|
for r in rows:
|
||||||
|
code6 = str(r["stock_code"]).replace("SH", "").replace("SZ", "").replace("BJ", "")
|
||||||
|
if len(code6) == 6 and code6.isdigit():
|
||||||
|
stock_codes.append(code6)
|
||||||
|
logger.info(f"[longhubang {trade_date}] 聚合层 {daily_rows} 行, {len(stock_codes)} 只上榜")
|
||||||
|
|
||||||
|
# 2) 席位层(每只票 2 调用:buy + sell)
|
||||||
|
seat_rows_total = 0
|
||||||
|
if stock_codes:
|
||||||
|
seat_rows = []
|
||||||
|
for code6 in stock_codes:
|
||||||
|
for direction in ("buy", "sell"):
|
||||||
|
df_seat = src.fetch_longhubang_seat(code6, trade_date, direction)
|
||||||
|
if df_seat is None or df_seat.empty:
|
||||||
|
continue
|
||||||
|
for _, r in df_seat.iterrows():
|
||||||
|
# 字段映射(akshare 6 中文列名)
|
||||||
|
if direction == "buy":
|
||||||
|
buy_amt = self._to_float(r.get("买入金额"))
|
||||||
|
sell_amt = self._to_float(r.get("卖出金额"))
|
||||||
|
else:
|
||||||
|
buy_amt = self._to_float(r.get("买入金额"))
|
||||||
|
sell_amt = self._to_float(r.get("卖出金额"))
|
||||||
|
# 买方向的占比列叫"买入金额-占总成交比例",卖方向叫"卖出金额-..."
|
||||||
|
amt_ratio = (
|
||||||
|
self._to_float(r.get("买入金额-占总成交比例"))
|
||||||
|
if direction == "buy"
|
||||||
|
else self._to_float(r.get("卖出金额-占总成交比例"))
|
||||||
|
)
|
||||||
|
# amount=主金额:买方向=买入金额,卖方向=卖出金额
|
||||||
|
amt = buy_amt if direction == "buy" else sell_amt
|
||||||
|
seat_rows.append({
|
||||||
|
"stock_code": f"{'SH' if code6.startswith(('5','6','9')) else 'SZ'}{code6}",
|
||||||
|
"trade_date": trade_date,
|
||||||
|
"direction": direction,
|
||||||
|
"rank_idx": self._to_int(r.get("序号")) or 0,
|
||||||
|
"branch_name": self._to_str(r.get("交易营业部名称")),
|
||||||
|
"buy_amount": buy_amt,
|
||||||
|
"sell_amount": sell_amt,
|
||||||
|
"amount": amt,
|
||||||
|
"net_amount": self._to_float(r.get("净额")),
|
||||||
|
"amount_ratio": amt_ratio,
|
||||||
|
"explanation": self._to_str(r.get("类型")),
|
||||||
|
})
|
||||||
|
if seat_rows:
|
||||||
|
db_ops.upsert_longhubang_seat(seat_rows)
|
||||||
|
seat_rows_total = len(seat_rows)
|
||||||
|
logger.info(f"[longhubang {trade_date}] 席位层 {seat_rows_total} 行")
|
||||||
|
return daily_rows, seat_rows_total
|
||||||
|
|
||||||
|
# ── 类型转换 helper ──
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _to_float(v) -> float | None:
|
||||||
|
if v is None:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
f = float(v)
|
||||||
|
return f if f == f else None # NaN → None
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _to_int(v) -> int | None:
|
||||||
|
if v is None:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return int(v)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _to_str(v) -> str | None:
|
||||||
|
if v is None:
|
||||||
|
return None
|
||||||
|
s = str(v).strip()
|
||||||
|
return s if s else None
|
||||||
@@ -8,8 +8,10 @@ from typing import Any
|
|||||||
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
|
from sqlalchemy import text
|
||||||
|
|
||||||
from app.core.db import ops as db_ops
|
from app.core.db import ops as db_ops
|
||||||
|
from app.core.db.orm import engine as pg_engine
|
||||||
from app.core.sync.base import SyncTask
|
from app.core.sync.base import SyncTask
|
||||||
from app.core.utils.logging import get_logger
|
from app.core.utils.logging import get_logger
|
||||||
|
|
||||||
@@ -40,22 +42,19 @@ class SyncMarketRegime(SyncTask):
|
|||||||
if not codes:
|
if not codes:
|
||||||
return {"status": "error", "message": "stocks 表为空"}
|
return {"status": "error", "message": "stocks 表为空"}
|
||||||
|
|
||||||
# 2. 拉所有 K 线(按股票)- 这里用 SQL GROUP BY 一次性算 daily turnover
|
# 2. 拉所有 K 线(按股票)— 走 SA text() 走 PG
|
||||||
from app.core.db.connection import get_mysql
|
sql = text("""
|
||||||
|
|
||||||
sql = """
|
|
||||||
SELECT
|
SELECT
|
||||||
stock_code,
|
stock_code,
|
||||||
trade_date,
|
trade_date,
|
||||||
`close`,
|
"close",
|
||||||
volume
|
volume
|
||||||
FROM kline_stock
|
FROM market_data.kline_stock
|
||||||
WHERE trade_date BETWEEN %s AND %s
|
WHERE trade_date BETWEEN :start AND :end
|
||||||
ORDER BY stock_code, trade_date
|
ORDER BY stock_code, trade_date
|
||||||
"""
|
""")
|
||||||
with get_mysql().cursor() as cur:
|
with pg_engine.connect() as conn:
|
||||||
cur.execute(sql, (start, end))
|
rows = conn.execute(sql, {"start": start, "end": end}).mappings().all()
|
||||||
rows = cur.fetchall()
|
|
||||||
if not rows:
|
if not rows:
|
||||||
return {"status": "warning", "message": f"kline_stock 在 {start} ~ {end} 区间无数据"}
|
return {"status": "warning", "message": f"kline_stock 在 {start} ~ {end} 区间无数据"}
|
||||||
|
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ from typing import Any
|
|||||||
from app.core.db import ops as db_ops
|
from app.core.db import ops as db_ops
|
||||||
from app.core.datasource.base import registry as ds_registry
|
from app.core.datasource.base import registry as ds_registry
|
||||||
from app.core.datasource.registry import is_source_ready
|
from app.core.datasource.registry import is_source_ready
|
||||||
from app.core.datasource.utils import is_a_share_code, to_code6
|
from app.core.datasource.utils import is_a_share_code, to_code6, to_hermes
|
||||||
from app.core.sync.base import SyncTask
|
from app.core.sync.base import SyncTask
|
||||||
from app.core.sync.registry import mark_sync_blocked
|
from app.core.sync.registry import mark_sync_blocked
|
||||||
from app.core.utils.logging import get_logger
|
from app.core.utils.logging import get_logger
|
||||||
@@ -102,7 +102,7 @@ class SyncMoneyflow(SyncTask):
|
|||||||
return {"status": "fail", "error": "no data"}
|
return {"status": "fail", "error": "no data"}
|
||||||
rows = [
|
rows = [
|
||||||
{
|
{
|
||||||
"stock_code": code6,
|
"stock_code": to_hermes(code6),
|
||||||
"trade_date": str(r["trade_date"]),
|
"trade_date": str(r["trade_date"]),
|
||||||
"main_net_inflow": float(r.get("main_net_inflow") or 0),
|
"main_net_inflow": float(r.get("main_net_inflow") or 0),
|
||||||
"large_net_inflow": float(r.get("large_net_inflow") or 0),
|
"large_net_inflow": float(r.get("large_net_inflow") or 0),
|
||||||
|
|||||||
@@ -5,7 +5,7 @@
|
|||||||
- sector_indices :行业日线(trade_date × sector_name × close × sector_amplitude)
|
- sector_indices :行业日线(trade_date × sector_name × close × sector_amplitude)
|
||||||
- sector_features_daily :行业特征(+ sector_ret + ema10/20/200 + score)
|
- sector_features_daily :行业特征(+ sector_ret + ema10/20/200 + score)
|
||||||
|
|
||||||
算法(参考 dashboard/api/services/sync/sector.py 重写,持久层改为 MySQL):
|
算法(参考 dashboard/api/services/sync/sector.py 重写,持久层改为 PG):
|
||||||
1) 每只股票日涨跌幅 pct_chg = (close/prev_close - 1) * 100
|
1) 每只股票日涨跌幅 pct_chg = (close/prev_close - 1) * 100
|
||||||
2) 按行业聚合 sector_ret = 该行业所有股票当日 pct_chg 的均值
|
2) 按行业聚合 sector_ret = 该行业所有股票当日 pct_chg 的均值
|
||||||
3) 行业指数 close = 100 × ∏(1 + sector_ret/100) (基点 100,复合收益)
|
3) 行业指数 close = 100 × ∏(1 + sector_ret/100) (基点 100,复合收益)
|
||||||
@@ -26,7 +26,6 @@ from sqlalchemy import create_engine
|
|||||||
|
|
||||||
from app.core.config import settings
|
from app.core.config import settings
|
||||||
from app.core.db import ops as db_ops
|
from app.core.db import ops as db_ops
|
||||||
from app.core.db.connection import get_mysql
|
|
||||||
from app.core.sync.base import SyncTask
|
from app.core.sync.base import SyncTask
|
||||||
from app.core.utils.logging import get_logger
|
from app.core.utils.logging import get_logger
|
||||||
|
|
||||||
@@ -56,11 +55,11 @@ class SyncSectorFeatures(SyncTask):
|
|||||||
logger.info("[sector] 启动行业聚合特征计算")
|
logger.info("[sector] 启动行业聚合特征计算")
|
||||||
|
|
||||||
# ── 1) 拉 kline_stock(日线)只取需要的列 ──
|
# ── 1) 拉 kline_stock(日线)只取需要的列 ──
|
||||||
# 用 SQLAlchemy engine(pymysql 裸连接 + pd.read_sql 偶发只读 1 行的 bug)
|
# 走 SQLAlchemy engine + PG URL(pd.read_sql 一次性读 1.1kw 行 → pandas DataFrame)
|
||||||
engine = create_engine(settings.mysql_url())
|
engine = create_engine(settings.pg_sqlalchemy_url())
|
||||||
df_kline = pd.read_sql(
|
df_kline = pd.read_sql(
|
||||||
"SELECT stock_code, trade_date, open, high, low, `close`, volume "
|
'SELECT stock_code, trade_date, open, high, low, "close", volume '
|
||||||
"FROM kline_stock ORDER BY stock_code, trade_date",
|
'FROM market_data.kline_stock ORDER BY stock_code, trade_date',
|
||||||
engine,
|
engine,
|
||||||
)
|
)
|
||||||
if df_kline.empty:
|
if df_kline.empty:
|
||||||
@@ -72,7 +71,7 @@ class SyncSectorFeatures(SyncTask):
|
|||||||
|
|
||||||
# ── 2) 拉 industry(股票→行业映射)──
|
# ── 2) 拉 industry(股票→行业映射)──
|
||||||
df_ind = pd.read_sql(
|
df_ind = pd.read_sql(
|
||||||
"SELECT code, industry_name FROM industry WHERE industry_name IS NOT NULL",
|
"SELECT code, industry_name FROM market_data.industry WHERE industry_name IS NOT NULL",
|
||||||
engine,
|
engine,
|
||||||
)
|
)
|
||||||
if df_ind.empty:
|
if df_ind.empty:
|
||||||
|
|||||||
@@ -87,7 +87,10 @@ class SyncShareSnapshot(SyncTask):
|
|||||||
if r is None:
|
if r is None:
|
||||||
fail_cnt += 1
|
fail_cnt += 1
|
||||||
continue
|
continue
|
||||||
# 写 stocks
|
# 写 stocks + share 表 —— 两张表 stock_code 都用 hermes(带 SH/SZ 前缀)
|
||||||
|
# 与 stocks.code 保持一致,方便 JOIN。
|
||||||
|
# 历史 bug:之前 share 表写的是裸 6 位 c6,导致 share JOIN stocks 失败。
|
||||||
|
# 一次性 UPDATE(2026-07-01)已把存量 6.3 万行加上前缀。
|
||||||
hermes = self._to_hermes(c6)
|
hermes = self._to_hermes(c6)
|
||||||
db_ops.update_stock_share_snapshot(
|
db_ops.update_stock_share_snapshot(
|
||||||
code=hermes,
|
code=hermes,
|
||||||
@@ -95,9 +98,8 @@ class SyncShareSnapshot(SyncTask):
|
|||||||
float_share=r["float_share"],
|
float_share=r["float_share"],
|
||||||
trade_date=r.get("trade_date", today),
|
trade_date=r.get("trade_date", today),
|
||||||
)
|
)
|
||||||
# 也写 share 表
|
|
||||||
rows_to_share_table.append({
|
rows_to_share_table.append({
|
||||||
"stock_code": c6,
|
"stock_code": hermes,
|
||||||
"trade_date": r.get("trade_date", today),
|
"trade_date": r.get("trade_date", today),
|
||||||
"total_share": r["total_share"],
|
"total_share": r["total_share"],
|
||||||
"float_share": r["float_share"],
|
"float_share": r["float_share"],
|
||||||
|
|||||||
@@ -0,0 +1,168 @@
|
|||||||
|
"""同步任务:当天逐笔交易(mairui 源)。
|
||||||
|
|
||||||
|
数据源:mairui `hsrl/zbjy/{code6}/{licence}`
|
||||||
|
口径:单笔成交 — 时间 / 价 / 量 / 买卖方向
|
||||||
|
更新:每日 21:00(mairui 当天数据此时才发布;21:00 之前 API 返空或昨日数据)
|
||||||
|
|
||||||
|
设计要点:
|
||||||
|
1) 起步门控:21:00 之前(未传 force=True)→ 早返 warning
|
||||||
|
避免 0 行空跑(5000 只股票 × 1s 限速 ≈ 17 分钟浪费)
|
||||||
|
2) 5 worker × 1 RPS/worker = 5 RPS(守住 mairui 上限)
|
||||||
|
3) 不做"今日已同步"dedup —— mairui "当天"实际是"最近已发布交易日",
|
||||||
|
trade_date 可能比 wall-clock 早一天;重跑靠 PK + ON CONFLICT 幂等
|
||||||
|
4) 单只股票日均 5w-10w tick,CHUNK=2000 upsert 防 driver 撑爆
|
||||||
|
5) 数据是"当天 only",无 start/end 窗口;不存增量概念
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import time
|
||||||
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||||
|
from datetime import datetime
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from app.core.db import ops as db_ops
|
||||||
|
from app.core.datasource.base import registry as ds_registry
|
||||||
|
from app.core.datasource.registry import is_source_ready
|
||||||
|
from app.core.datasource.utils import is_a_share_code, to_code6, to_hermes
|
||||||
|
from app.core.sync.base import SyncTask
|
||||||
|
from app.core.sync.registry import mark_sync_blocked
|
||||||
|
from app.core.utils.logging import get_logger
|
||||||
|
|
||||||
|
logger = get_logger("sync.tick_trade")
|
||||||
|
|
||||||
|
# mairui 文档:「更新:每日 21:00」—— 早于此时 mairui 仍返昨日/空数据
|
||||||
|
MAIRUI_TICK_TRADE_PUBLISH_HOUR = 21
|
||||||
|
MAIRUI_TICK_TRADE_PUBLISH_MIN = 0
|
||||||
|
# 21:00 之后再跑(留 5 分钟缓冲,等 mairui 完整入库)
|
||||||
|
TICK_TRADE_RUN_GATE = (MAIRUI_TICK_TRADE_PUBLISH_HOUR, MAIRUI_TICK_TRADE_PUBLISH_MIN + 5)
|
||||||
|
|
||||||
|
|
||||||
|
def _passes_publish_gate(now: Optional[datetime] = None, *, force: bool = False) -> tuple[bool, str]:
|
||||||
|
"""判断当前时间是否已过 mairui 21:00 数据发布门控。
|
||||||
|
|
||||||
|
force=True 跳过门控(手动 / 测试用)。
|
||||||
|
"""
|
||||||
|
if force:
|
||||||
|
return True, "force=True 跳过门控"
|
||||||
|
n = now or datetime.now()
|
||||||
|
gate = n.replace(hour=TICK_TRADE_RUN_GATE[0], minute=TICK_TRADE_RUN_GATE[1], second=0, microsecond=0)
|
||||||
|
if n < gate:
|
||||||
|
return False, f"mairui 逐笔数据 {MAIRUI_TICK_TRADE_PUBLISH_HOUR:02d}:{MAIRUI_TICK_TRADE_PUBLISH_MIN:02d} 发布;当前 {n.strftime('%H:%M')} 早于门控,skip"
|
||||||
|
return True, "ok"
|
||||||
|
|
||||||
|
|
||||||
|
class SyncTickTrade(SyncTask):
|
||||||
|
dataset_id = "tick_trade"
|
||||||
|
|
||||||
|
def _run(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
trigger_source: str = "manual",
|
||||||
|
codes: list[str] | None = None,
|
||||||
|
max_workers: int = 5,
|
||||||
|
force: bool = False,
|
||||||
|
**kwargs,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
mr = ds_registry.get("datasource_mairui")
|
||||||
|
if mr is None:
|
||||||
|
return {"status": "error", "message": "mairui 数据源未注册"}
|
||||||
|
ok, reason = is_source_ready(mr.key)
|
||||||
|
if not ok:
|
||||||
|
mark_sync_blocked(self.dataset_id, message=f"mairui 未就绪: {reason}")
|
||||||
|
return {"status": "blocked", "message": f"mairui 未就绪: {reason}"}
|
||||||
|
|
||||||
|
# ── 21:00 门控(force=True 跳过)──
|
||||||
|
gate_ok, gate_msg = _passes_publish_gate(force=force)
|
||||||
|
if not gate_ok:
|
||||||
|
logger.info(f"[tick_trade] 门控未过: {gate_msg}")
|
||||||
|
return {"status": "warning", "message": gate_msg, "ok": 0, "fail": 0, "rows": 0, "elapsed_sec": 0.0}
|
||||||
|
|
||||||
|
# ── 取股票列表 ──
|
||||||
|
if codes:
|
||||||
|
stock_codes = [to_code6(c) for c in codes]
|
||||||
|
else:
|
||||||
|
stock_codes = [c for c in db_ops.iter_stock_codes(active_only=True) if is_a_share_code(c)]
|
||||||
|
limit_raw = os.environ.get("MARKET_DATA_STOCK_LIMIT", "").strip()
|
||||||
|
if limit_raw.isdigit() and int(limit_raw) > 0:
|
||||||
|
stock_codes = stock_codes[: int(limit_raw)]
|
||||||
|
if not stock_codes:
|
||||||
|
return {"status": "error", "message": "无股票代码"}
|
||||||
|
|
||||||
|
# 注:不做"今日已同步"dedup。原因:
|
||||||
|
# - mairui 的"当天"实际是"最近一个已发布的交易日"(21:00 之前是昨日),
|
||||||
|
# 用 today_str 做去重会失效(即使刚跑完,trade_date 还是昨日)。
|
||||||
|
# - PK 含 trade_date,重跑是 ON CONFLICT DO UPDATE,安全且幂等。
|
||||||
|
# - 17 分钟全市场开销可接受;非交易日用 `MARKET_DATA_STOCK_LIMIT=0` 跳。
|
||||||
|
|
||||||
|
total = len(stock_codes)
|
||||||
|
if total == 0:
|
||||||
|
return {"status": "ok", "message": "无股票需要同步", "ok": 0, "fail": 0, "rows": 0, "elapsed_sec": 0.0}
|
||||||
|
|
||||||
|
# ── 预估耗时:5 RPS 上限 ──
|
||||||
|
est_sec = total / 5.0
|
||||||
|
logger.info(f"[tick_trade] 计划: {total} 只, 预估 {est_sec:.0f}s ({est_sec/60:.1f}min)")
|
||||||
|
self._progress(
|
||||||
|
message=f"计划: {total} 只, 预估 {est_sec/60:.1f}min @5RPS",
|
||||||
|
current=0, total=total, current_step="planning",
|
||||||
|
)
|
||||||
|
|
||||||
|
# ── 执行 ──
|
||||||
|
t0 = time.time()
|
||||||
|
ok_cnt = fail_cnt = 0
|
||||||
|
rows_total = 0
|
||||||
|
with ThreadPoolExecutor(max_workers=max_workers) as pool:
|
||||||
|
futures = {pool.submit(self._sync_one, mr, c6): c6 for c6 in stock_codes}
|
||||||
|
for i, future in enumerate(as_completed(futures), 1):
|
||||||
|
c6 = futures[future]
|
||||||
|
try:
|
||||||
|
res = future.result()
|
||||||
|
if res["status"] == "ok":
|
||||||
|
ok_cnt += 1
|
||||||
|
rows_total += res["rows"]
|
||||||
|
else:
|
||||||
|
fail_cnt += 1
|
||||||
|
if res.get("error"):
|
||||||
|
logger.warning(f"[tick_trade {c6}] {res['error']}")
|
||||||
|
except Exception as e:
|
||||||
|
fail_cnt += 1
|
||||||
|
logger.warning(f"[tick_trade {c6}] {e}")
|
||||||
|
if i % 50 == 0 or i == total:
|
||||||
|
self._progress(
|
||||||
|
message=f"tick_trade 进度 {i}/{total} OK:{ok_cnt} FAIL:{fail_cnt} 行:{rows_total}",
|
||||||
|
current=i, total=total, current_step=c6,
|
||||||
|
)
|
||||||
|
elapsed = round(time.time() - t0, 1)
|
||||||
|
msg = f"逐笔 {ok_cnt}成 {fail_cnt}败 共{rows_total}行, {elapsed}s"
|
||||||
|
return {
|
||||||
|
"status": "ok" if fail_cnt == 0 else "warning",
|
||||||
|
"message": msg,
|
||||||
|
"ok": ok_cnt, "fail": fail_cnt, "rows": rows_total, "elapsed_sec": elapsed,
|
||||||
|
}
|
||||||
|
|
||||||
|
def _sync_one(self, mr, code6: str) -> dict:
|
||||||
|
try:
|
||||||
|
df = mr.fetch_tick_trade(code6)
|
||||||
|
except Exception as e:
|
||||||
|
return {"status": "fail", "error": str(e)}
|
||||||
|
if df is None or df.empty:
|
||||||
|
return {"status": "fail", "error": "no data (mairui 21:00 前可能为空)"}
|
||||||
|
try:
|
||||||
|
rows = [
|
||||||
|
{
|
||||||
|
"stock_code": to_hermes(code6),
|
||||||
|
"trade_date": str(r["trade_date"]),
|
||||||
|
"trade_time": r["trade_time"].strftime("%Y-%m-%d %H:%M:%S")
|
||||||
|
if hasattr(r["trade_time"], "strftime") else str(r["trade_time"]),
|
||||||
|
"price": float(r.get("price") or 0),
|
||||||
|
"volume": float(r.get("volume") or 0),
|
||||||
|
"direction_code": int(r.get("direction_code") or 0),
|
||||||
|
"direction": str(r.get("direction") or ""),
|
||||||
|
"amount": float(r.get("amount") or 0),
|
||||||
|
}
|
||||||
|
for _, r in df.iterrows()
|
||||||
|
]
|
||||||
|
db_ops.upsert_tick_trade(rows)
|
||||||
|
return {"status": "ok", "rows": len(rows)}
|
||||||
|
except Exception as e:
|
||||||
|
return {"status": "fail", "error": f"db write: {e}"}
|
||||||
@@ -0,0 +1,288 @@
|
|||||||
|
"""[DEPRECATED 2026-06-16] MySQL → PostgreSQL 数据迁移工具。
|
||||||
|
|
||||||
|
⚠️ 项目从 2026-06-16 起放弃 MySQL,只支持 PostgreSQL。
|
||||||
|
本脚本已归档到 ``bin/archive/``,仅作历史参考 / 应急回滚。
|
||||||
|
如果你想从 MySQL 备份恢复数据到 PG,可以临时把它移回 ``bin/`` 用一次。
|
||||||
|
|
||||||
|
同步目标:把 MySQL `market_data_sync_db` 库 15 张表的数据搬到 PG `market_data.schema` 下。
|
||||||
|
|
||||||
|
策略(不重新建表,建表由 pg_bootstrap 负责):
|
||||||
|
- 用 pymysql 读 MySQL 流式分页(避免 1 亿行一次占满内存)
|
||||||
|
- 用 psycopg2.copy_from 或 executemany 写 PG(按表选择最高效方式)
|
||||||
|
- 顺序:先小表(配置/字典)→ 后大表(kline_5min 等)
|
||||||
|
|
||||||
|
数据量预估(按当时 MySQL 实测,2026-06-16):
|
||||||
|
config: 20 行
|
||||||
|
dataset_registry: 15 行
|
||||||
|
industry: 5,207 行
|
||||||
|
sectors: 83 行
|
||||||
|
stock_sector_map: 5,207 行
|
||||||
|
indices: 6 行
|
||||||
|
market_regime_daily: 2,046 行
|
||||||
|
share: 6,141 行
|
||||||
|
stocks: 5,210 行
|
||||||
|
sector_indices: 103,084 行
|
||||||
|
sector_features_daily: 416,521 行
|
||||||
|
kline_index: 37,480 行
|
||||||
|
moneyflow: 3,188,301 行
|
||||||
|
kline_stock: 11,605,541 行
|
||||||
|
kline_5min: 121,945,849 行 ← 最大,约 1.2 亿
|
||||||
|
|
||||||
|
用法(应急时):
|
||||||
|
mv bin/archive/migrate_mysql_to_pg.py bin/migrate_mysql_to_pg.py
|
||||||
|
.venv/bin/python -m bin.migrate_mysql_to_pg
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
from io import StringIO
|
||||||
|
from typing import Iterable
|
||||||
|
|
||||||
|
import pymysql
|
||||||
|
import psycopg2
|
||||||
|
from dotenv import load_dotenv
|
||||||
|
|
||||||
|
load_dotenv(".env", override=False)
|
||||||
|
|
||||||
|
# 15 张表的列定义(MySQL → PG 列名映射)
|
||||||
|
# 大部分列名一致;少数 PG 用 JSONB(dataset_registry.dependency_ids)需要 JSON 解析
|
||||||
|
TABLES = [
|
||||||
|
# (table_name, mysql_order_by, list_of_column_names)
|
||||||
|
("config", "key", ["key", "value", "category", "description", "updated_at"]),
|
||||||
|
("dataset_registry", "dataset_id", ["dataset_id", "name", "description", "storage_uri", "storage_layer",
|
||||||
|
"management_role", "source", "sync_script", "dependency_ids",
|
||||||
|
"enabled", "sort_order", "status", "trigger_source",
|
||||||
|
"started_at", "finished_at", "last_success_at", "last_failure_at",
|
||||||
|
"message", "last_error", "needs_resync", "progress_current",
|
||||||
|
"progress_total", "current_step", "updated_at"]),
|
||||||
|
("indices", "index_code", ["index_code", "index_name", "market", "category", "source", "enabled", "updated_at"]),
|
||||||
|
("industry", "code", ["code", "industry_name", "industry_classification", "update_date", "updated_at"]),
|
||||||
|
("sectors", "sector_key", ["sector_key", "sector_name", "taxonomy", "level", "source", "enabled", "updated_at"]),
|
||||||
|
("stock_sector_map", "stock_code", ["stock_code", "sector_key", "updated_at"]),
|
||||||
|
("stocks", "code", ["code", "name", "exchange", "list_date", "listing_status", "industry",
|
||||||
|
"total_share", "float_share", "share_updated_at", "kline_synced_at",
|
||||||
|
"updated_at"]),
|
||||||
|
("market_regime_daily", "trade_date", ["trade_date", "advancers", "decliners", "advance_ratio", "turnover",
|
||||||
|
"turnover_avg_5d", "turnover_ratio_5d", "source", "is_extreme_panic",
|
||||||
|
"updated_at"]),
|
||||||
|
("share", "stock_code,trade_date", ["stock_code", "trade_date", "total_share", "float_share"]),
|
||||||
|
("sector_indices", "trade_date,sector_name", ["trade_date", "sector_name", "close", "sector_amplitude"]),
|
||||||
|
("sector_features_daily", "trade_date,sector_name", ["trade_date", "sector_name", "sector_ret", "sector_amplitude",
|
||||||
|
"close", "ema10", "ema20", "ema200", "score"]),
|
||||||
|
("kline_index", "index_code,trade_date", ["index_code", "trade_date", "open", "high", "low", "close", "volume"]),
|
||||||
|
("moneyflow", "stock_code,trade_date", ["stock_code", "trade_date", "main_net_inflow",
|
||||||
|
"large_net_inflow", "medium_net_inflow", "small_net_inflow"]),
|
||||||
|
("kline_stock", "stock_code,trade_date", ["stock_code", "trade_date", "open", "high", "low", "close", "volume"]),
|
||||||
|
("kline_5min", "stock_code,bar_time", ["stock_code", "bar_time", "open", "high", "low", "close",
|
||||||
|
"volume", "amount", "turnover_rate"]),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def mysql_conn():
|
||||||
|
return pymysql.connect(
|
||||||
|
host=__import__("os").environ.get("MYSQL_HOST", "127.0.0.1"),
|
||||||
|
port=int(__import__("os").environ.get("MYSQL_PORT", "3306")),
|
||||||
|
user=__import__("os").environ.get("MYSQL_USER", "root"),
|
||||||
|
password=__import__("os").environ.get("MYSQL_PASSWORD", ""),
|
||||||
|
database=__import__("os").environ.get("MYSQL_DATABASE", "market_data_sync_db"),
|
||||||
|
charset="utf8mb4",
|
||||||
|
cursorclass=pymysql.cursors.SSDictCursor, # 服务端 cursor,流式不撑内存
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def pg_conn():
|
||||||
|
import os as _os
|
||||||
|
pg_url = _os.environ.get("PG_URL", "").strip()
|
||||||
|
# SQLAlchemy 格式 → psycopg2 格式(剥掉 +psycopg2)
|
||||||
|
if pg_url.startswith("postgresql+psycopg2://"):
|
||||||
|
pg_url = pg_url.replace("postgresql+psycopg2://", "postgresql://", 1)
|
||||||
|
if not pg_url:
|
||||||
|
host = _os.environ.get("PG_HOST", "127.0.0.1")
|
||||||
|
port = _os.environ.get("PG_PORT", "5432")
|
||||||
|
user = _os.environ.get("PG_USER", "market_sync")
|
||||||
|
pw = _os.environ.get("PG_PASSWORD", "market_sync")
|
||||||
|
db = _os.environ.get("PG_DB_NAME", "market_data")
|
||||||
|
pg_url = f"postgresql://{user}:{pw}@{host}:{port}/{db}"
|
||||||
|
return psycopg2.connect(pg_url)
|
||||||
|
|
||||||
|
|
||||||
|
def _convert_value(col: str, v):
|
||||||
|
"""MySQL → PG 类型兼容转换。"""
|
||||||
|
if v is None:
|
||||||
|
return None
|
||||||
|
if col == "dependency_ids" and isinstance(v, str):
|
||||||
|
# MySQL 存的是 Python list repr(如 ['stock_info']),不是合法 JSON。
|
||||||
|
# 先后备方案:先 json.loads(合法 JSON),失败再 ast.literal_eval(Python repr)。
|
||||||
|
import ast
|
||||||
|
import json
|
||||||
|
if not v.strip():
|
||||||
|
return []
|
||||||
|
# 试 1: 当 JSON 解析
|
||||||
|
try:
|
||||||
|
return json.loads(v)
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
pass
|
||||||
|
# 试 2: 当 Python list/tuple repr 解析
|
||||||
|
try:
|
||||||
|
parsed = ast.literal_eval(v)
|
||||||
|
if isinstance(parsed, (list, tuple)):
|
||||||
|
return list(parsed)
|
||||||
|
except (ValueError, SyntaxError):
|
||||||
|
pass
|
||||||
|
# 试 3: 逗号分隔的纯字符串列表(兜底)
|
||||||
|
return [s.strip().strip("'\"") for s in v.strip("[]").split(",") if s.strip()]
|
||||||
|
if isinstance(v, str) and v == "":
|
||||||
|
return None
|
||||||
|
return v
|
||||||
|
|
||||||
|
|
||||||
|
def _copy_from_path(conn_pg, table_name: str, columns: list[str], tmp_path: str) -> int:
|
||||||
|
"""从已写好的 temp file COPY 到 PG。
|
||||||
|
|
||||||
|
temp file 由调用方在游标活跃时填充好。
|
||||||
|
"""
|
||||||
|
cur = conn_pg.cursor()
|
||||||
|
with open(tmp_path, "rb") as fb:
|
||||||
|
full_table = f"market_data.{table_name}"
|
||||||
|
cur.copy_expert(
|
||||||
|
f"COPY {full_table} ({','.join(columns)}) FROM STDIN WITH (FORMAT text, NULL '')",
|
||||||
|
fb,
|
||||||
|
)
|
||||||
|
conn_pg.commit()
|
||||||
|
return 0 # 实际行数已经在外层统计了
|
||||||
|
|
||||||
|
|
||||||
|
def _truncate_pg_table(conn_pg, table_name: str) -> None:
|
||||||
|
"""清空 PG 表。用 DELETE 而不是 TRUNCATE(TRUNCATE 需要额外权限)。
|
||||||
|
|
||||||
|
DELETE 在大表上慢,但仅迁移时用一次可以接受。
|
||||||
|
"""
|
||||||
|
cur = conn_pg.cursor()
|
||||||
|
cur.execute(f"DELETE FROM market_data.{table_name}")
|
||||||
|
conn_pg.commit()
|
||||||
|
|
||||||
|
|
||||||
|
def migrate_table(mysql, pg, table_name: str, order_by: str, columns: list[str],
|
||||||
|
chunk_size: int = 100_000) -> int:
|
||||||
|
"""单表迁移:先 TRUNCATE,再流式读 MySQL + COPY 写 PG。"""
|
||||||
|
t0 = time.time()
|
||||||
|
col_list = ", ".join(f"`{c}`" for c in columns)
|
||||||
|
|
||||||
|
with mysql.cursor() as cur:
|
||||||
|
cur.execute(f"SELECT COUNT(*) FROM {table_name}")
|
||||||
|
n_total = cur.fetchone()["COUNT(*)"]
|
||||||
|
|
||||||
|
if n_total == 0:
|
||||||
|
print(f" {table_name:<25} 0 行(跳过)")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
# 先清空 PG 表(确保是干净迁移,不重复)
|
||||||
|
_truncate_pg_table(pg, table_name)
|
||||||
|
|
||||||
|
# 读 MySQL(流式:SSCursor 不缓存全表)→ 写 PG 临时文件 → COPY
|
||||||
|
# 关键:迭代 + 写 temp file 必须在 with 块内完成(cur 关闭后再迭代会失败)
|
||||||
|
print(f" {table_name:<25} {n_total:>12,} 行 ...", end=" ", flush=True)
|
||||||
|
import json as _json
|
||||||
|
import tempfile
|
||||||
|
JSONB_COLS = {"dependency_ids"}
|
||||||
|
tmp = tempfile.NamedTemporaryFile(
|
||||||
|
mode="w", encoding="utf-8", prefix=f"migrate_{table_name}_", suffix=".tsv",
|
||||||
|
delete=False,
|
||||||
|
)
|
||||||
|
tmp_path = tmp.name
|
||||||
|
written = 0
|
||||||
|
try:
|
||||||
|
with mysql.cursor() as cur:
|
||||||
|
order_by_quoted = ", ".join(f"`{c.strip()}`" for c in order_by.split(","))
|
||||||
|
cur.execute(f"SELECT {col_list} FROM `{table_name}` ORDER BY {order_by_quoted}")
|
||||||
|
for r in cur: # SSCursor:一次一行,不缓存全表
|
||||||
|
cells = []
|
||||||
|
for col, v in zip(columns, (r.get(col) for col in columns)):
|
||||||
|
if v is None:
|
||||||
|
cells.append("")
|
||||||
|
elif col in JSONB_COLS and not isinstance(v, str):
|
||||||
|
cells.append(_json.dumps(v, ensure_ascii=False))
|
||||||
|
else:
|
||||||
|
s = str(v)
|
||||||
|
cells.append(s.replace("\\", "\\\\").replace("\t", "\\t").replace("\n", "\\n"))
|
||||||
|
tmp.write("\t".join(cells))
|
||||||
|
tmp.write("\n")
|
||||||
|
written += 1
|
||||||
|
# with 块结束,cur 已关闭。temp file 写完。关 file 准备 COPY。
|
||||||
|
tmp.flush()
|
||||||
|
tmp.close()
|
||||||
|
# COPY 从磁盘 temp file 读
|
||||||
|
with open(tmp_path, "rb") as fb:
|
||||||
|
cur_pg = pg.cursor()
|
||||||
|
full_table = f"market_data.{table_name}"
|
||||||
|
cur_pg.copy_expert(
|
||||||
|
f"COPY {full_table} ({','.join(columns)}) FROM STDIN WITH (FORMAT text, NULL '')",
|
||||||
|
fb,
|
||||||
|
)
|
||||||
|
pg.commit()
|
||||||
|
finally:
|
||||||
|
try:
|
||||||
|
import os as _os
|
||||||
|
_os.unlink(tmp_path)
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
elapsed = time.time() - t0
|
||||||
|
print(f"→ {written:>12,} 行 ({elapsed:.1f}s)")
|
||||||
|
return written
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
p = argparse.ArgumentParser()
|
||||||
|
p.add_argument("--only", help="逗号分隔的表名列表,只迁指定的")
|
||||||
|
p.add_argument("--chunk-size", type=int, default=100_000)
|
||||||
|
p.add_argument("--skip", help="逗号分隔要跳过的表")
|
||||||
|
args = p.parse_args()
|
||||||
|
|
||||||
|
only = set((args.only or "").split(",")) if args.only else None
|
||||||
|
skip = set((args.skip or "").split(",")) if args.skip else set()
|
||||||
|
|
||||||
|
targets = [
|
||||||
|
t for t in TABLES
|
||||||
|
if (only is None or t[0] in only) and t[0] not in skip
|
||||||
|
]
|
||||||
|
|
||||||
|
print(f"=== MySQL → PG 数据迁移 ===")
|
||||||
|
print(f"目标表: {len(targets)} 张")
|
||||||
|
print(f"源: MySQL market_data_sync_db (127.0.0.1:3306)")
|
||||||
|
print(f"目标: PG market_data 库 market_data schema")
|
||||||
|
print()
|
||||||
|
|
||||||
|
mysql = mysql_conn()
|
||||||
|
pg = pg_conn()
|
||||||
|
|
||||||
|
grand_total = 0
|
||||||
|
grand_t0 = time.time()
|
||||||
|
for table_name, order_by, columns in targets:
|
||||||
|
try:
|
||||||
|
n = migrate_table(mysql, pg, table_name, order_by, columns, args.chunk_size)
|
||||||
|
grand_total += n
|
||||||
|
except Exception as e:
|
||||||
|
print(f" ✗ {table_name} 失败: {e}")
|
||||||
|
grand_elapsed = time.time() - grand_t0
|
||||||
|
print()
|
||||||
|
print(f"=== 全部完成:{grand_total:,} 行 / 耗时 {grand_elapsed:.1f}s ===")
|
||||||
|
|
||||||
|
# 简单校验:每张表行数对比
|
||||||
|
print("\n=== 校验:MySQL vs PG 行数对比 ===")
|
||||||
|
with mysql.cursor() as cur, pg.cursor() as cur_pg:
|
||||||
|
for table_name, _, _ in targets:
|
||||||
|
cur.execute(f"SELECT COUNT(*) AS n FROM {table_name}")
|
||||||
|
n_mysql = cur.fetchone()["n"]
|
||||||
|
cur_pg.execute(f"SELECT COUNT(*) FROM market_data.{table_name}")
|
||||||
|
n_pg = cur_pg.fetchone()[0]
|
||||||
|
ok = "✓" if n_mysql == n_pg else "✗"
|
||||||
|
print(f" {ok} {table_name:<25} MySQL={n_mysql:>12,} PG={n_pg:>12,}")
|
||||||
|
|
||||||
|
mysql.close()
|
||||||
|
pg.close()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
Executable
+29
@@ -0,0 +1,29 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# systemd 调度的入口(专跑龙虎榜)
|
||||||
|
#
|
||||||
|
# 设计:
|
||||||
|
# - 与 market_sync_moneyflow_run.sh 同构,只跑 longhubang 一个 task
|
||||||
|
# - 22:00 触发(龙虎榜 19:00~21:00 陆续出齐;与 moneyflow 21:35 错开避免并发限速)
|
||||||
|
# - 单日调用 ~3 次聚合层 + ~200 次席位层,3-5 分钟内完成
|
||||||
|
# - 单独日志到 logs/lhb_<ts>.log
|
||||||
|
# - exit code 透传
|
||||||
|
set -u
|
||||||
|
|
||||||
|
PROJECT_ROOT="/home/gao/Development/quant_home/market_sync"
|
||||||
|
LOG_DIR="$PROJECT_ROOT/logs"
|
||||||
|
mkdir -p "$LOG_DIR"
|
||||||
|
|
||||||
|
TS=$(date +%Y%m%d_%H%M%S)
|
||||||
|
LOG="$LOG_DIR/lhb_${TS}.log"
|
||||||
|
|
||||||
|
echo "[market-sync-lhb] start ts=$TS log=$LOG" | tee -a "$LOG"
|
||||||
|
|
||||||
|
# 注:不做法定节假日过滤 —— 与其他 3 个 timer 一致的 fail-open 原则。
|
||||||
|
# 非交易日 akshare 仍可能返当周数据 → task 报 warning,不丢历史。
|
||||||
|
|
||||||
|
# ── 跑 longhubang ──
|
||||||
|
cd "$PROJECT_ROOT"
|
||||||
|
.venv/bin/python -m app.entrypoints.cli sync longhubang 2>&1 | tee -a "$LOG"
|
||||||
|
EXIT_CODE=${PIPESTATUS[0]}
|
||||||
|
echo "[market-sync-lhb] done exit_code=$EXIT_CODE" | tee -a "$LOG"
|
||||||
|
exit $EXIT_CODE
|
||||||
Executable
+29
@@ -0,0 +1,29 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# systemd 调度的入口(专跑 moneyflow)
|
||||||
|
#
|
||||||
|
# 设计:
|
||||||
|
# - 与 market_sync_tick_run.sh 同构,只跑 moneyflow 一个 task
|
||||||
|
# - mairui 文档:「更新:每日 21:30」—— systemd 21:35 触发,留 5min buffer
|
||||||
|
# - CLI 不传 --force:让 task 走 21:30 时间门控(moneyflow 暂无门控;为一致仍走通路径)
|
||||||
|
# - 单独日志到 logs/moneyflow_<ts>.log
|
||||||
|
# - exit code 透传
|
||||||
|
set -u
|
||||||
|
|
||||||
|
PROJECT_ROOT="/home/gao/Development/quant_home/market_sync"
|
||||||
|
LOG_DIR="$PROJECT_ROOT/logs"
|
||||||
|
mkdir -p "$LOG_DIR"
|
||||||
|
|
||||||
|
TS=$(date +%Y%m%d_%H%M%S)
|
||||||
|
LOG="$LOG_DIR/moneyflow_${TS}.log"
|
||||||
|
|
||||||
|
echo "[market-sync-moneyflow] start ts=$TS log=$LOG" | tee -a "$LOG"
|
||||||
|
|
||||||
|
# 注:不做法定节假日过滤 —— 与 market_sync_run.sh 同样的 fail-open 原则。
|
||||||
|
# 大不了非交易日 mairui 返回空 → task 报 warning,不丢历史。
|
||||||
|
|
||||||
|
# ── 跑 moneyflow ──
|
||||||
|
cd "$PROJECT_ROOT"
|
||||||
|
.venv/bin/python -m app.entrypoints.cli sync moneyflow 2>&1 | tee -a "$LOG"
|
||||||
|
EXIT_CODE=${PIPESTATUS[0]}
|
||||||
|
echo "[market-sync-moneyflow] done exit_code=$EXIT_CODE" | tee -a "$LOG"
|
||||||
|
exit $EXIT_CODE
|
||||||
Executable
+92
@@ -0,0 +1,92 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# systemd 调度的入口:跑一次全量同步 + 配套结构化 watch
|
||||||
|
#
|
||||||
|
# 设计:
|
||||||
|
# - 后台启动 runall_once.py(输出重定向到时间戳 log)
|
||||||
|
# - 同时后台启动 structured_watch.sh 盯进程(13 checkpoint × 递增间隔)
|
||||||
|
# - wrapper 自身 fast-poll runall PID(每 5s 一次)
|
||||||
|
# - runall 一退出就 SIGTERM watch(不靠 watch 自己的 checkpoint 轮询,避开 30min 长 sleep)
|
||||||
|
# - watch 自然结束后整个脚本退出
|
||||||
|
# - 退出码透传 runall_once.py 的 exit code,systemd 据此判断成功 / 失败
|
||||||
|
#
|
||||||
|
# 为什么不只用 structured_watch:
|
||||||
|
# - 看 [[background-watcher-pitfalls]] 记忆:先承诺后验证是三大坑之一
|
||||||
|
# - structured_watch.sh 的 schedule 在后期是 30min / 60min 间隔
|
||||||
|
# - runall 死后,watch 最坏要等 60min 才发现进程没了
|
||||||
|
# - wrapper fast-poll 让响应时间 < 5s
|
||||||
|
#
|
||||||
|
# 为什么不让 systemd TimeoutStartSec= 自动收尾:
|
||||||
|
# - 超时是 SIGTERM → SIGKILL,但 SIGKILL 不给 runall / watch 写日志的机会
|
||||||
|
# - 自己 fast-poll 能优雅收尾(先 SIGTERM watch 留出写日志时间,再 SIGKILL 兜底)
|
||||||
|
set -u
|
||||||
|
|
||||||
|
PROJECT_ROOT="/home/gao/Development/quant_home/market_sync"
|
||||||
|
LOG_DIR="$PROJECT_ROOT/logs"
|
||||||
|
mkdir -p "$LOG_DIR"
|
||||||
|
|
||||||
|
TS=$(date +%Y%m%d_%H%M%S)
|
||||||
|
RUNALL_LOG="$LOG_DIR/runall_${TS}.log"
|
||||||
|
WATCH_LOG="$LOG_DIR/watch_${TS}.log"
|
||||||
|
|
||||||
|
echo "[market-sync-run] start ts=$TS"
|
||||||
|
echo "[market-sync-run] runall_log=$RUNALL_LOG"
|
||||||
|
echo "[market-sync-run] watch_log=$WATCH_LOG"
|
||||||
|
|
||||||
|
# 注:不做法定节假日过滤。理由(fail-open 原则):
|
||||||
|
# - 节假日配置(TRADING_HOLIDAYS / config 表)一旦出错(误把今天加进去、
|
||||||
|
# 漏更新本年度、笔误等)= 直接漏一天数据,且没声音
|
||||||
|
# - 大不了非交易日跑出来 mairui/baostock 返回空数据,task 报 warning 但不丢历史
|
||||||
|
# - 周末过滤由 systemd timer `OnCalendar=Mon..Fri` 完成,足够
|
||||||
|
# ── 1) 后台启动 runall_once.py ──
|
||||||
|
cd "$PROJECT_ROOT"
|
||||||
|
.venv/bin/python bin/runall_once.py > "$RUNALL_LOG" 2>&1 &
|
||||||
|
RUNALL_PID=$!
|
||||||
|
echo "[market-sync-run] runall pid=$RUNALL_PID"
|
||||||
|
|
||||||
|
# ── 2) 后台启动 structured_watch.sh(独立进程,不阻塞 wrapper)
|
||||||
|
# PATTERN 不含 "python" → 触发 watch 内部 case 第二分支,前缀 [p]ython.* 拼出
|
||||||
|
"$PROJECT_ROOT/bin/structured_watch.sh" \
|
||||||
|
"runall_once" \
|
||||||
|
"$RUNALL_LOG" \
|
||||||
|
"$WATCH_LOG" \
|
||||||
|
"market-sync-full" > /dev/null 2>&1 &
|
||||||
|
WATCH_PID=$!
|
||||||
|
echo "[market-sync-run] watch pid=$WATCH_PID"
|
||||||
|
|
||||||
|
# ── 3) wrapper 自身 fast-poll runall PID(每 5s)
|
||||||
|
# 这是关键修复:runall 死后 < 5s 内就触发 watch 退出
|
||||||
|
RUNALL_EXIT_CODE=1
|
||||||
|
while kill -0 "$RUNALL_PID" 2>/dev/null; do
|
||||||
|
sleep 5
|
||||||
|
done
|
||||||
|
|
||||||
|
# runall 已退出,取 exit code
|
||||||
|
wait "$RUNALL_PID" 2>/dev/null
|
||||||
|
RUNALL_EXIT_CODE=$?
|
||||||
|
echo "[market-sync-run] runall 已退出 exit_code=$RUNALL_EXIT_CODE"
|
||||||
|
|
||||||
|
# ── 4) 通知 watch 退出(如果还在跑)──
|
||||||
|
if kill -0 "$WATCH_PID" 2>/dev/null; then
|
||||||
|
echo "[market-sync-run] 通知 watch 退出(pid=$WATCH_PID)"
|
||||||
|
kill -TERM "$WATCH_PID" 2>/dev/null || true
|
||||||
|
# 给 watch 30s 自然退出(不要立即 KILL,留它写最后日志)
|
||||||
|
for _ in 1 2 3 4 5 6; do
|
||||||
|
sleep 5
|
||||||
|
kill -0 "$WATCH_PID" 2>/dev/null || break
|
||||||
|
done
|
||||||
|
if kill -0 "$WATCH_PID" 2>/dev/null; then
|
||||||
|
echo "[market-sync-run] ⚠️ watch 30s 内未退出,强制 kill -9"
|
||||||
|
kill -KILL "$WATCH_PID" 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ── 5) 输出 summary ──
|
||||||
|
SUMMARY="$LOG_DIR/runall_summary.json"
|
||||||
|
if [ -f "$SUMMARY" ]; then
|
||||||
|
echo "[market-sync-run] === summary ==="
|
||||||
|
cat "$SUMMARY"
|
||||||
|
echo "[market-sync-run] === /summary ==="
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "[market-sync-run] done exit_code=$RUNALL_EXIT_CODE"
|
||||||
|
exit "$RUNALL_EXIT_CODE"
|
||||||
Executable
+30
@@ -0,0 +1,30 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# systemd 调度的入口(专跑 tick_trade)
|
||||||
|
#
|
||||||
|
# 设计:
|
||||||
|
# - 比 market_sync_run.sh 简单:只跑一个 task (tick_trade)
|
||||||
|
# - 21:00 门控由 task 内部处理;systemd 触发时间 21:05 留 5min buffer
|
||||||
|
# - CLI 加 --force 跳过 21:00 门控(systemd 触发时已是 21:05+,force 不会真正"跳过"
|
||||||
|
# 任何东西,但保持与 runall_once 同步任务的 force 语义一致)
|
||||||
|
# - 单独日志到 logs/tick_<ts>.log
|
||||||
|
# - exit code 透传(systemd 据此判成功 / 失败)
|
||||||
|
set -u
|
||||||
|
|
||||||
|
PROJECT_ROOT="/home/gao/Development/quant_home/market_sync"
|
||||||
|
LOG_DIR="$PROJECT_ROOT/logs"
|
||||||
|
mkdir -p "$LOG_DIR"
|
||||||
|
|
||||||
|
TS=$(date +%Y%m%d_%H%M%S)
|
||||||
|
LOG="$LOG_DIR/tick_${TS}.log"
|
||||||
|
|
||||||
|
echo "[market-sync-tick] start ts=$TS log=$LOG" | tee -a "$LOG"
|
||||||
|
|
||||||
|
# 注:不做法定节假日过滤 —— 与 market_sync_run.sh 同样的 fail-open 原则。
|
||||||
|
# 大不了非交易日 mairui 返回空 → task 报 warning,不丢历史。
|
||||||
|
|
||||||
|
# ── 跑 tick_trade(force=True 跳过 21:00 门控,因为 systemd 触发时间 21:05 已过门控)──
|
||||||
|
cd "$PROJECT_ROOT"
|
||||||
|
.venv/bin/python -m app.entrypoints.cli sync tick_trade --force 2>&1 | tee -a "$LOG"
|
||||||
|
EXIT_CODE=${PIPESTATUS[0]}
|
||||||
|
echo "[market-sync-tick] done exit_code=$EXIT_CODE" | tee -a "$LOG"
|
||||||
|
exit $EXIT_CODE
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
# 跑剩余三个任务:moneyflow → share_snapshot → market_regime
|
# 跑剩余任务:tick_trade → moneyflow → share_snapshot → market_regime
|
||||||
# 跳过 kline_5min(全量回填 6 年太慢,下次有空再补)
|
# 跳过 kline_5min(全量回填 6 年太慢,下次有空再补)
|
||||||
|
# tick_trade 用 --force 跳过 21:00 门控(人工补跑场景)
|
||||||
set -e
|
set -e
|
||||||
cd "$(dirname "$0")/.."
|
cd "$(dirname "$0")/.."
|
||||||
export PYTHONPATH="$(pwd)"
|
export PYTHONPATH="$(pwd)"
|
||||||
@@ -9,11 +10,12 @@ mkdir -p logs
|
|||||||
# 先清理上次可能留下的卡死记录(recover_interrupted_syncs 启动时自动处理,但显式更稳)
|
# 先清理上次可能留下的卡死记录(recover_interrupted_syncs 启动时自动处理,但显式更稳)
|
||||||
echo "=== 启动时间: $(date) ===" | tee logs/runall_remaining.log
|
echo "=== 启动时间: $(date) ===" | tee logs/runall_remaining.log
|
||||||
|
|
||||||
for TASK in moneyflow share_snapshot market_regime; do
|
for TASK in "tick_trade --force" moneyflow share_snapshot market_regime; do
|
||||||
echo "" | tee -a logs/runall_remaining.log
|
echo "" | tee -a logs/runall_remaining.log
|
||||||
echo "=== [$TASK] 开始 $(date) ===" | tee -a logs/runall_remaining.log
|
echo "=== [$TASK] 开始 $(date) ===" | tee -a logs/runall_remaining.log
|
||||||
T0=$(date +%s)
|
T0=$(date +%s)
|
||||||
if .venv/bin/python -m app.entrypoints.cli sync "$TASK" 2>&1 | tee -a logs/runall_remaining.log; then
|
# shellcheck disable=SC2086
|
||||||
|
if .venv/bin/python -m app.entrypoints.cli sync $TASK 2>&1 | tee -a logs/runall_remaining.log; then
|
||||||
T1=$(date +%s)
|
T1=$(date +%s)
|
||||||
echo "=== [$TASK] 完成,耗时 $((T1 - T0))s ===" | tee -a logs/runall_remaining.log
|
echo "=== [$TASK] 完成,耗时 $((T1 - T0))s ===" | tee -a logs/runall_remaining.log
|
||||||
else
|
else
|
||||||
|
|||||||
+14
-9
@@ -1,10 +1,14 @@
|
|||||||
"""一次性跑完所有 sync task(按依赖顺序)。
|
"""一次性跑完所有 sync task(按依赖顺序)。
|
||||||
|
|
||||||
跳过 industry_sector(已 ok)。
|
按 sort_order 触发:stock_basic → kline_daily → kline_index → kline_5min
|
||||||
按顺序触发:stock_basic → kline_daily → kline_index → kline_5min → moneyflow
|
→ industry_sector → sector_features
|
||||||
→ share_snapshot → market_regime
|
→ share_snapshot → market_regime
|
||||||
|
|
||||||
每个 task 跑完写日志 + 把 result 落到 summary.json。
|
每个 task 跑完写日志 + 把 result 落到 logs/runall_summary.json。
|
||||||
|
|
||||||
|
注:以下 task **不**在 runall 链中 —— 各自有独立的 systemd timer:
|
||||||
|
- tick_trade : mairui 21:00 发布 → market-sync-tick.timer (21:05)
|
||||||
|
- moneyflow : mairui 21:30 发布 → market-sync-moneyflow.timer (21:35)
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
@@ -28,20 +32,21 @@ seed_sync_registry()
|
|||||||
|
|
||||||
from app.tasks import get_task
|
from app.tasks import get_task
|
||||||
|
|
||||||
# 顺序执行,便于排查
|
# 顺序执行,按 sort_order 走(便于排查 + 避免外部 API 限流)
|
||||||
|
# 注:tick_trade / moneyflow 由独立 timer 跑(21:05 / 21:35)
|
||||||
TASKS = [
|
TASKS = [
|
||||||
"stock_basic",
|
"stock_basic",
|
||||||
"industry_sector", # 已 ok,但再跑一次全量也 OK (用 SKIP 标志)
|
|
||||||
"kline_index",
|
"kline_index",
|
||||||
"kline_daily",
|
"kline_daily",
|
||||||
"kline_5min",
|
"kline_5min",
|
||||||
"moneyflow",
|
"industry_sector",
|
||||||
|
"sector_features",
|
||||||
"share_snapshot",
|
"share_snapshot",
|
||||||
"market_regime",
|
"market_regime",
|
||||||
]
|
]
|
||||||
|
|
||||||
# industry_sector 跑过一次(291s 全量)已 ok,跳过
|
# 不跳任何 task — 全量跑
|
||||||
SKIP = {"industry_sector"}
|
SKIP: set[str] = set()
|
||||||
|
|
||||||
results = {}
|
results = {}
|
||||||
t_all = time.time()
|
t_all = time.time()
|
||||||
|
|||||||
@@ -0,0 +1,33 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Market data 龙虎榜 sync (akshare 源, daily 22:00)
|
||||||
|
Documentation=file:///home/gao/Development/quant_home/market_sync/bin/market_sync_lhb_run.sh
|
||||||
|
After=network-online.target market-sync.service market-sync-tick.service market-sync-moneyflow.service
|
||||||
|
Wants=network-online.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
WorkingDirectory=/home/gao/Development/quant_home/market_sync
|
||||||
|
User=gao
|
||||||
|
Group=gao
|
||||||
|
|
||||||
|
# 入口脚本:单独跑 longhubang 一个 task
|
||||||
|
# 龙虎榜数据 19:00~21:00 陆续出齐 → 22:00 触发(与 moneyflow 21:35 错开避免并发)
|
||||||
|
ExecStart=/home/gao/Development/quant_home/market_sync/bin/market_sync_lhb_run.sh
|
||||||
|
|
||||||
|
# 硬上限 1h(每日 ~200 次 akshare 调用 + DB upsert,3-5 分钟内完成,留 buffer)
|
||||||
|
TimeoutStartSec=3600
|
||||||
|
|
||||||
|
# 不要 Restart=(oneshot 失败就让 OnFailure= 发通知,别自动重跑——
|
||||||
|
# 重跑会撞外部 API 限流窗口,浪费资源)
|
||||||
|
# Restart=no 是 oneshot 默认值
|
||||||
|
|
||||||
|
# 日志走 journald(journalctl -u market-sync-lhb.service -f)
|
||||||
|
StandardOutput=journal
|
||||||
|
StandardError=journal
|
||||||
|
SyslogIdentifier=market-sync-lhb
|
||||||
|
|
||||||
|
# 环境(不读 /etc/environment,只带这几个;.env 由 cli 内部 load_dotenv)
|
||||||
|
Environment=PYTHONUNBUFFERED=1
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Schedule 龙虎榜 sync — Mon..Fri 22:00 Asia/Shanghai
|
||||||
|
# market-sync-lhb.service 是这个 timer 的执行单元
|
||||||
|
# 龙虎榜数据 19:00~21:00 陆续出齐 → 22:00 触发(与 moneyflow 21:35 错开)
|
||||||
|
|
||||||
|
[Timer]
|
||||||
|
# 龙虎榜出齐 → 22:00 触发
|
||||||
|
OnCalendar=Mon..Fri 22:00:00 Asia/Shanghai
|
||||||
|
# 系统关机 / 错过执行时,下次开机补跑一次(避免漏数据)
|
||||||
|
Persistent=true
|
||||||
|
# 单位(service)的精确名称
|
||||||
|
Unit=market-sync-lhb.service
|
||||||
|
# 不要 AccuracySec(默认 1min 漂移够用,避免 22:00:00 整点打堆)
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=timers.target
|
||||||
@@ -0,0 +1,33 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Market data moneyflow sync (mairui transaction, daily 21:35)
|
||||||
|
Documentation=file:///home/gao/Development/quant_home/market_sync/bin/market_sync_moneyflow_run.sh
|
||||||
|
After=network-online.target market-sync.service market-sync-tick.service
|
||||||
|
Wants=network-online.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
WorkingDirectory=/home/gao/Development/quant_home/market_sync
|
||||||
|
User=gao
|
||||||
|
Group=gao
|
||||||
|
|
||||||
|
# 入口脚本:单独跑 moneyflow 一个 task
|
||||||
|
# mairui 文档:「更新:每日 21:30」—— 21:35 触发留 5min buffer
|
||||||
|
ExecStart=/home/gao/Development/quant_home/market_sync/bin/market_sync_moneyflow_run.sh
|
||||||
|
|
||||||
|
# 硬上限 2h(5206 只 × 50 RPS = 104s 理论上限,留 buffer 给重试 / 慢响应)
|
||||||
|
TimeoutStartSec=7200
|
||||||
|
|
||||||
|
# 不要 Restart=(oneshot 失败就让 OnFailure= 发通知,别自动重跑——
|
||||||
|
# 重跑会撞外部 API 限流窗口,浪费 1.5h)
|
||||||
|
# Restart=no 是 oneshot 默认值
|
||||||
|
|
||||||
|
# 日志走 journald(journalctl -u market-sync-moneyflow.service -f)
|
||||||
|
StandardOutput=journal
|
||||||
|
StandardError=journal
|
||||||
|
SyslogIdentifier=market-sync-moneyflow
|
||||||
|
|
||||||
|
# 环境(不读 /etc/environment,只带这几个;.env 由 cli 内部 load_dotenv)
|
||||||
|
Environment=PYTHONUNBUFFERED=1
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Schedule moneyflow sync — Mon..Fri 21:35 Asia/Shanghai
|
||||||
|
# market-sync-moneyflow.service 是这个 timer 的执行单元
|
||||||
|
# mairui 文档:「更新:每日 21:30」—— 21:35 触发留 5 分钟 buffer
|
||||||
|
|
||||||
|
[Timer]
|
||||||
|
# mairui 21:30 发布数据 → 21:35 触发
|
||||||
|
OnCalendar=Mon..Fri 21:35:00 Asia/Shanghai
|
||||||
|
# 系统关机 / 错过执行时,下次开机补跑一次(避免漏数据)
|
||||||
|
Persistent=true
|
||||||
|
# 单位(service)的精确名称
|
||||||
|
Unit=market-sync-moneyflow.service
|
||||||
|
# 不要 AccuracySec(默认 1min 漂移够用,避免 21:35:00 整点打堆)
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=timers.target
|
||||||
@@ -0,0 +1,33 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Market data tick-by-tick trade sync (mairui hsrl/zbjy, daily 21:05)
|
||||||
|
Documentation=file:///home/gao/Development/quant_home/market_sync/bin/market_sync_tick_run.sh
|
||||||
|
After=network-online.target market-sync.service
|
||||||
|
Wants=network-online.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
WorkingDirectory=/home/gao/Development/quant_home/market_sync
|
||||||
|
User=gao
|
||||||
|
Group=gao
|
||||||
|
|
||||||
|
# 入口脚本:单独跑 tick_trade 一个 task(mairui 21:00 发布数据,
|
||||||
|
# 这个 service 在 21:05 触发,留 5min buffer;force=True 绕过 task 内部 21:00 门控)
|
||||||
|
ExecStart=/home/gao/Development/quant_home/market_sync/bin/market_sync_tick_run.sh
|
||||||
|
|
||||||
|
# 硬上限 2h(5000 只 × 1 RPS ≈ 17min,留 buffer 给 retry / 慢任务)
|
||||||
|
TimeoutStartSec=7200
|
||||||
|
|
||||||
|
# 不要 Restart=(oneshot 失败就让 OnFailure= 发通知,别自动重跑——
|
||||||
|
# 重跑会撞外部 API 限流窗口,浪费 1.5h)
|
||||||
|
# Restart=no 是 oneshot 默认值
|
||||||
|
|
||||||
|
# 日志走 journald(journalctl -u market-sync-tick.service -f)
|
||||||
|
StandardOutput=journal
|
||||||
|
StandardError=journal
|
||||||
|
SyslogIdentifier=market-sync-tick
|
||||||
|
|
||||||
|
# 环境(不读 /etc/environment,只带这几个;.env 由 cli 内部 load_dotenv)
|
||||||
|
Environment=PYTHONUNBUFFERED=1
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Schedule tick_trade sync — Mon..Fri 21:05 Asia/Shanghai
|
||||||
|
# market-sync-tick.service 是这个 timer 的执行单元
|
||||||
|
# 触发顺序:market-sync.service (15:30) 之后 market-sync-tick.service (21:05)
|
||||||
|
# 但 systemd timer 独立触发,After= 只是声明依赖关系(非强制)
|
||||||
|
|
||||||
|
[Timer]
|
||||||
|
# mairui 文档:「更新:每日 21:00」—— 21:05 触发留 5 分钟 buffer
|
||||||
|
OnCalendar=Mon..Fri 21:05:00 Asia/Shanghai
|
||||||
|
# 系统关机 / 错过执行时,下次开机补跑一次(避免漏数据)
|
||||||
|
Persistent=true
|
||||||
|
# 单位(service)的精确名称
|
||||||
|
Unit=market-sync-tick.service
|
||||||
|
# 不要 AccuracySec(默认 1min 漂移够用,避免 21:05:00 整点打堆)
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=timers.target
|
||||||
@@ -0,0 +1,32 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Market Data Full Sync (PostgreSQL, 9 sync tasks end-to-end)
|
||||||
|
Documentation=file:///home/gao/Development/quant_home/market_sync/bin/market_sync_run.sh
|
||||||
|
After=network-online.target
|
||||||
|
Wants=network-online.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
WorkingDirectory=/home/gao/Development/quant_home/market_sync
|
||||||
|
User=gao
|
||||||
|
Group=gao
|
||||||
|
|
||||||
|
# 入口脚本(runall_once.py + structured_watch.sh 二合一)
|
||||||
|
ExecStart=/home/gao/Development/quant_home/market_sync/bin/market_sync_run.sh
|
||||||
|
|
||||||
|
# 硬上限 2h(实际 ~80min,留 buffer 给 retry / 慢任务)
|
||||||
|
TimeoutStartSec=7200
|
||||||
|
|
||||||
|
# 不要 Restart=(oneshot 失败就让 OnFailure= 发通知,别自动重跑——
|
||||||
|
# 重跑会撞外部 API 限流窗口,浪费 1.5h)
|
||||||
|
# Restart=no 是 oneshot 默认值
|
||||||
|
|
||||||
|
# 日志走 journald(journalctl -u market-sync.service -f)
|
||||||
|
StandardOutput=journal
|
||||||
|
StandardError=journal
|
||||||
|
SyslogIdentifier=market-sync
|
||||||
|
|
||||||
|
# 环境(不读 /etc/environment,只带这几个;.env 由 runall_once.py 内部 load_dotenv)
|
||||||
|
Environment=PYTHONUNBUFFERED=1
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Schedule market data full sync — Mon..Fri 15:30 Asia/Shanghai
|
||||||
|
# market-sync.service 是这个 timer 的执行单元
|
||||||
|
|
||||||
|
[Timer]
|
||||||
|
# A 股收盘时间 15:00,30 分钟 buffer 让最后一笔 tick 落地
|
||||||
|
OnCalendar=Mon..Fri 15:30:00 Asia/Shanghai
|
||||||
|
# 系统关机 / 错过执行时,下次开机补跑一次(避免漏数据)
|
||||||
|
Persistent=true
|
||||||
|
# 单位(service)的精确名称
|
||||||
|
Unit=market-sync.service
|
||||||
|
# 不要 AccuracySec(默认 1min 漂移够用,避免 15:30:00 整点打堆)
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=timers.target
|
||||||
+4
-1
@@ -20,7 +20,10 @@ requests>=2.30.0
|
|||||||
tenacity>=8.0.0
|
tenacity>=8.0.0
|
||||||
|
|
||||||
# Datasources — A 股免费源
|
# Datasources — A 股免费源
|
||||||
# akshare 永远不使用(项目级决策)
|
# akshare 仅用于"无稳定替代源"的数据集(2026-07-01 决策)。
|
||||||
|
# 当前仅龙虎榜使用(聚合层 stock_lhb_detail_em + 席位层 stock_lhb_stock_detail_em)。
|
||||||
|
# 5min K / moneyflow / kline_daily 等已有 baostock / mairui 替代,禁 akshare。
|
||||||
|
akshare>=1.18.0
|
||||||
baostock>=0.9.0
|
baostock>=0.9.0
|
||||||
# 雪球非官方 SDK:需要 XUEQIU_TOKEN(环境变量)才能拿到股本 / 日K 复核
|
# 雪球非官方 SDK:需要 XUEQIU_TOKEN(环境变量)才能拿到股本 / 日K 复核
|
||||||
# 没装 → share / kline_daily 任务静默失败(ImportError 被 except 吞)
|
# 没装 → share / kline_daily 任务静默失败(ImportError 被 except 吞)
|
||||||
|
|||||||
+10
-2
@@ -57,6 +57,10 @@ case "${1:-all}" in
|
|||||||
4|kline_daily) run_step "4_kline_daily" "kline_daily" "--workers 20" ;;
|
4|kline_daily) run_step "4_kline_daily" "kline_daily" "--workers 20" ;;
|
||||||
5|share_snapshot) run_step "5_share_snapshot" "share_snapshot" "--workers 10" ;;
|
5|share_snapshot) run_step "5_share_snapshot" "share_snapshot" "--workers 10" ;;
|
||||||
6|market_regime) run_step "6_market_regime" "market_regime" ;;
|
6|market_regime) run_step "6_market_regime" "market_regime" ;;
|
||||||
|
7|kline_5min) run_step "7_kline_5min" "kline_5min" "--workers 10" ;;
|
||||||
|
8|moneyflow) run_step "8_moneyflow" "moneyflow" "--workers 10" ;;
|
||||||
|
9|sector_features) run_step "9_sector_features" "sector_features" ;;
|
||||||
|
10|tick_trade) run_step "10_tick_trade" "tick_trade" "--workers 5 --force" ;;
|
||||||
all)
|
all)
|
||||||
run_step "1_stock_basic" "stock_basic" || exit 1
|
run_step "1_stock_basic" "stock_basic" || exit 1
|
||||||
run_step "2_industry_sector" "industry_sector" || exit 1
|
run_step "2_industry_sector" "industry_sector" || exit 1
|
||||||
@@ -64,10 +68,14 @@ case "${1:-all}" in
|
|||||||
run_step "4_kline_daily" "kline_daily" "--workers 20" || exit 1
|
run_step "4_kline_daily" "kline_daily" "--workers 20" || exit 1
|
||||||
run_step "5_share_snapshot" "share_snapshot" "--workers 10" || exit 1
|
run_step "5_share_snapshot" "share_snapshot" "--workers 10" || exit 1
|
||||||
run_step "6_market_regime" "market_regime" || exit 1
|
run_step "6_market_regime" "market_regime" || exit 1
|
||||||
echo "$LOG_PREFIX ✓ 全部 6 步完成" | tee -a "$RUN_LOG"
|
run_step "7_kline_5min" "kline_5min" "--workers 10" || exit 1
|
||||||
|
run_step "8_moneyflow" "moneyflow" "--workers 10" || exit 1
|
||||||
|
run_step "9_sector_features" "sector_features" || exit 1
|
||||||
|
run_step "10_tick_trade" "tick_trade" "--workers 5 --force" || exit 1
|
||||||
|
echo "$LOG_PREFIX ✓ 全部 10 步完成" | tee -a "$RUN_LOG"
|
||||||
;;
|
;;
|
||||||
*)
|
*)
|
||||||
echo "usage: $0 [1|2|3|4|5|6|all]"
|
echo "usage: $0 [1|2|3|4|5|6|7|8|9|10|all]"
|
||||||
exit 1
|
exit 1
|
||||||
;;
|
;;
|
||||||
esac
|
esac
|
||||||
|
|||||||
@@ -0,0 +1,38 @@
|
|||||||
|
"""ORM 模型 metadata 一致性测试(PostgreSQL only,2026-06-16 重构后)。
|
||||||
|
|
||||||
|
历史背景:原 test_schema_models.py 还测了 schema.py vs models.py 的 DDL 漂移。
|
||||||
|
schema.py 在这次重构中已删除(MySQL DDL 不再需要),所以只保留 ORM metadata 完整性测试。
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
|
||||||
|
def test_orm_metadata_registers_all_tables():
|
||||||
|
"""ORMBase.metadata 应该注册 18 张 PG 表(项目主业务表)。"""
|
||||||
|
from app.core.db import models # 触发全部模型 import
|
||||||
|
from app.core.db.orm import ORMBase
|
||||||
|
|
||||||
|
# 注意:SA 2.x 的 MetaData.tables 既能按 bare name 也能按 schema-qualified name 索引
|
||||||
|
# (keyed 容器),所以用 values() 拿 Table 对象再用 .name 拿 bare 名
|
||||||
|
bare_names = {t.name for t in ORMBase.metadata.tables.values()}
|
||||||
|
assert len(bare_names) == 18, f"期望 18 张 ORM 表,实际 {len(bare_names)}: {bare_names}"
|
||||||
|
expected = {
|
||||||
|
"config", "dataset_registry", "stocks", "indices",
|
||||||
|
"kline_stock", "kline_index", "kline_5min", "moneyflow", "share",
|
||||||
|
"sectors", "stock_sector_map", "industry",
|
||||||
|
"sector_indices", "sector_features_daily", "market_regime_daily",
|
||||||
|
"tick_trade",
|
||||||
|
"longhubang_daily", "longhubang_seat", # 2026-07-01 龙虎榜
|
||||||
|
}
|
||||||
|
assert bare_names == expected, f"ORM 表名集合与期望不符: {bare_names ^ expected}"
|
||||||
|
|
||||||
|
|
||||||
|
def test_orm_tables_in_market_data_schema():
|
||||||
|
"""所有 ORM 表应该在 market_data PG schema 下(不是 public,不是 None)。"""
|
||||||
|
from app.core.db import models # 触发全部模型 import
|
||||||
|
from app.core.db.orm import ORMBase
|
||||||
|
|
||||||
|
bad = []
|
||||||
|
for table in ORMBase.metadata.tables.values():
|
||||||
|
if table.schema != "market_data":
|
||||||
|
bad.append((table.name, table.schema))
|
||||||
|
assert not bad, f"以下表 schema 不是 market_data: {bad}"
|
||||||
+18
-8
@@ -8,7 +8,7 @@ import pytest
|
|||||||
def test_import_config():
|
def test_import_config():
|
||||||
from app.core.config import settings
|
from app.core.config import settings
|
||||||
assert settings is not None
|
assert settings is not None
|
||||||
assert hasattr(settings, "mysql_host")
|
assert hasattr(settings, "pg_host")
|
||||||
|
|
||||||
|
|
||||||
def test_import_data_source_base():
|
def test_import_data_source_base():
|
||||||
@@ -35,9 +35,10 @@ def test_tasks_registry():
|
|||||||
from app.tasks import TASKS, get_task
|
from app.tasks import TASKS, get_task
|
||||||
assert isinstance(TASKS, dict)
|
assert isinstance(TASKS, dict)
|
||||||
# 注:每次新增 task 都要更新这里的数字
|
# 注:每次新增 task 都要更新这里的数字
|
||||||
# 当前 9 个:stock_basic, kline_daily, kline_index, kline_5min, moneyflow,
|
# 当前 11 个:stock_basic, kline_daily, kline_index, kline_5min, tick_trade,
|
||||||
# industry_sector, sector_features, share_snapshot, market_regime
|
# moneyflow, industry_sector, sector_features, share_snapshot, market_regime,
|
||||||
assert len(TASKS) == 9
|
# longhubang
|
||||||
|
assert len(TASKS) == 11
|
||||||
# get_task 应该返回实例
|
# get_task 应该返回实例
|
||||||
task = get_task("kline_daily")
|
task = get_task("kline_daily")
|
||||||
assert task.dataset_id == "kline_daily"
|
assert task.dataset_id == "kline_daily"
|
||||||
@@ -45,10 +46,18 @@ def test_tasks_registry():
|
|||||||
|
|
||||||
def test_sync_definitions():
|
def test_sync_definitions():
|
||||||
from app.core.sync.registry import SYNC_DEFINITIONS
|
from app.core.sync.registry import SYNC_DEFINITIONS
|
||||||
assert len(SYNC_DEFINITIONS) == 9
|
assert len(SYNC_DEFINITIONS) == 11
|
||||||
# 所有 dataset_id 应唯一
|
# 所有 dataset_id 应唯一
|
||||||
ids = [d["dataset_id"] for d in SYNC_DEFINITIONS]
|
ids = [d["dataset_id"] for d in SYNC_DEFINITIONS]
|
||||||
assert len(set(ids)) == 9
|
assert len(set(ids)) == 11
|
||||||
|
# tick_trade 应在其中且 sort_order=45
|
||||||
|
tt = next(d for d in SYNC_DEFINITIONS if d["dataset_id"] == "tick_trade")
|
||||||
|
assert tt["sort_order"] == 45
|
||||||
|
assert "stock_basic" in tt["dependency_ids"]
|
||||||
|
# longhubang 应在其中且 sort_order=85
|
||||||
|
lh = next(d for d in SYNC_DEFINITIONS if d["dataset_id"] == "longhubang")
|
||||||
|
assert lh["sort_order"] == 85
|
||||||
|
assert "stock_basic" in lh["dependency_ids"]
|
||||||
|
|
||||||
|
|
||||||
def test_build_default_registry():
|
def test_build_default_registry():
|
||||||
@@ -60,8 +69,9 @@ def test_build_default_registry():
|
|||||||
assert "datasource_xinlang" in keys
|
assert "datasource_xinlang" in keys
|
||||||
assert "datasource_baostock" in keys
|
assert "datasource_baostock" in keys
|
||||||
assert "datasource_mairui" in keys
|
assert "datasource_mairui" in keys
|
||||||
# akshare 已移除(项目级决策:永远不用)
|
# akshare:2026-07-01 决策更新——龙虎榜无替代源,烟测通过,可注册。
|
||||||
assert "datasource_akshare" not in keys
|
# 见 [[no-akshare]] 记忆:仅"无替代"+"akshare 烟测 OK"才允许用。
|
||||||
|
assert "datasource_akshare_lhb" in keys
|
||||||
assert "datasource_xueqiu" in keys
|
assert "datasource_xueqiu" in keys
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user