refactor: PG-only 迁移 + 龙虎榜/tick/moneyflow 同步 + stock_code 统一 + mairui 编码修复
## 1. PG-only 重构
- 删 app/core/db/connection.py + schema.py (MySQL 路径)
- 新 app/core/db/{orm,models,pg_bootstrap}.py — SQLAlchemy 2.x ORM 一键建表
- 16 张业务表全在 market_data schema,原生 TIMESTAMPTZ / JSONB / Float / TEXT
- requirements.txt 删 PyMySQL 路径,加 psycopg2
## 2. 同步任务扩展(3 个新 task)
- **tick_trade** (mairui hsrl/zbjy):当天逐笔交易,21:00 发布
- **moneyflow** (mairui hsstock/history/transaction):个股资金流,21:30 发布
- **longhubang** (akshare):龙虎榜聚合层 + 席位层(2 张新表)
- 长虎榜放宽 akshare 政策:仅"无替代源 + 烟测通过"场景允许
- data_eastmoney 私有 API 不需要(akshare 烟测通过)
- 3-timer 设计:
- 15:30 market-sync.service (8 base tasks via runall_once)
- 21:05 market-sync-tick.service (tick_trade)
- 21:35 market-sync-moneyflow.service (moneyflow)
- 22:00 market-sync-lhb.service (longhubang,新加)
- bin/systemd/ 新增 tick / moneyflow / lhb 各 1 对 service+timer
- bin/market_sync_*_run.sh wrapper 脚本(不做法定节假日过滤,fail-open)
## 3. stock_code 统一为带 SH/SZ/BJ 前缀
- 历史 bug:stocks.code 用 SH600519,但 kline/moneyflow/tick_trade/kline_5min
/stock_sector_map/industry 6 张表用纯 6 位 600519,跨表 JOIN 全部 0 行
- 新增 to_hermes() 工具:6位 / 9位(mairui `000001.SZ` 格式)→ 统一 SH000001
- 5 个 task 改写:用 to_hermes(code6) 写入 stock_code
- 一次性迁移 6 张表存量 154M 行(CASE WHEN 探测 + 去重 + 加前缀)
- ORM: stock_sector_map.stock_code / industry.code String(6)→String(10)
## 4. Bug 修复
- **share table stock_code 格式**:之前写 6 位不带前缀,与 stocks 不一致
→ 修 task_share_snapshot + 一次性 UPDATE 63,417 行加前缀
- **share_snapshot warning 状态错填 last_error**:
→ 加 mark_sync_warning() 走专用路径,不写 last_failure_at / last_error
- **schedule config lastRun 不同步**:
→ 加 update_job_status_for_dataset(),SyncTask.run() 完成后自动镜像
→ cli/runall 触发的 task 也能更新 schedule config
## 5. mairui UTF-8 编码修复
- 历史 bug:mairui.py:_fetch 用 latin-1 兜底解码,把所有 UTF-8 中文名
double-encoded 写入 stocks.name(如 `歌华有线` 变成 `æ\xad\x8cå\x8d\x8e...`)
- 加 _decode_response():UTF-8 → GBK → latin-1 兜底
- 一次性修复 stocks.name 5,213 行:
- 4,370 行 (encode('latin-1').decode('utf-8') 反向解码)
- 616 行 (含 fullwidth Ａ,宽松 printable 检查)
- 820 行 (mid-character 截断,重新从 mairui 拉)
## 6. 测试
- tests/test_smoke.py: TASKS 10→11, SYNC_DEFINITIONS 10→11
- tests/test_schema_models.py: 16→18 张表,新增 longhubang_daily/seat
- pytest 11/11 passed
## 验证
- 6 张表 0 残留无前缀行
- stocks JOIN kline_stock / kline_5min / moneyflow / tick_trade / stock_sector_map:88-100% 命中
- 5,213 stocks.name 全部正确 UTF-8 中文
- pytest 11/11 passed
This commit is contained in:
gao
@@ -1,4 +1,4 @@
|
||||
"""DB 子包：连接管理 + schema + 业务 CRUD。"""
|
||||
from app.core.db.connection import get_mysql, init_mysql_schema, close_mysql
|
||||
"""DB 子包：ORM + 业务 CRUD。"""
|
||||
from app.core.db.orm import get_session, engine, SessionLocal, ORMBase
|
||||
|
||||
__all__ = ["get_mysql", "init_mysql_schema", "close_mysql"]
|
||||
__all__ = ["get_session", "engine", "SessionLocal", "ORMBase"]
|
||||
|
||||
@@ -1,104 +0,0 @@
|
||||
"""MySQL 连接管理（thread-local，模式照搬 dashboard，但适配 market_data_sync 项目）。
|
||||
|
||||
要点：
|
||||
- pymysql 连接不是线程安全的，每个线程各持一份
|
||||
- 内网自签名证书场景：传 ssl_disabled=True 跳过 SSL 校验
|
||||
- DATE / DATETIME / TIMESTAMP 统一转字符串，避免下游类型处理
|
||||
- DictCursor：返回字典风格行，方便业务拼装
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
from typing import Optional
|
||||
|
||||
import pymysql
|
||||
import pymysql.constants.FIELD_TYPE
|
||||
from pymysql.converters import conversions as _pymysql_conv, convert_date, convert_datetime
|
||||
from pymysql.cursors import DictCursor
|
||||
|
||||
from app.core.config import settings
|
||||
|
||||
_local = threading.local()
|
||||
|
||||
|
||||
def _strdate(obj):
|
||||
d = convert_date(obj)
|
||||
return d.strftime("%Y-%m-%d") if d else None
|
||||
|
||||
|
||||
def _strdatetime(obj):
|
||||
d = convert_datetime(obj)
|
||||
return d.strftime("%Y-%m-%d %H:%M:%S") if d else None
|
||||
|
||||
|
||||
# 包装默认 converter，让 DATE / DATETIME / TIMESTAMP 一律出字符串
|
||||
_conv = _pymysql_conv.copy()
|
||||
_conv[pymysql.constants.FIELD_TYPE.DATE] = _strdate
|
||||
_conv[pymysql.constants.FIELD_TYPE.DATETIME] = _strdatetime
|
||||
_conv[pymysql.constants.FIELD_TYPE.TIMESTAMP] = _strdatetime
|
||||
|
||||
|
||||
def get_mysql(database: Optional[str] = None) -> pymysql.connections.Connection:
|
||||
"""取当前线程的 MySQL 连接。无则新建。
|
||||
|
||||
关键修复：高并发场景下，连接可能被服务端超时断开 (wait_timeout)。
|
||||
用 ping(reconnect=True) 让 pymysql 自动重连，避免 stale conn 导致
|
||||
2013 Lost connection 错误。
|
||||
"""
|
||||
attr = f"conn_{database or 'default'}"
|
||||
conn = getattr(_local, attr, None)
|
||||
if conn is not None:
|
||||
try:
|
||||
conn.ping(reconnect=True)
|
||||
except Exception:
|
||||
try:
|
||||
conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
conn = None
|
||||
if conn is None:
|
||||
kwargs = dict(
|
||||
host=settings.mysql_host,
|
||||
port=settings.mysql_port,
|
||||
user=settings.mysql_user,
|
||||
password=settings.mysql_password,
|
||||
database=database or settings.mysql_database,
|
||||
charset="utf8mb4",
|
||||
cursorclass=DictCursor,
|
||||
autocommit=False,
|
||||
conv=_conv,
|
||||
)
|
||||
# 内网自签名证书：禁用 SSL（用户场景）
|
||||
kwargs["ssl"] = None
|
||||
# 高并发时设较短 read_timeout / write_timeout
|
||||
kwargs["read_timeout"] = 30
|
||||
kwargs["write_timeout"] = 30
|
||||
conn = pymysql.connect(**kwargs)
|
||||
setattr(_local, attr, conn)
|
||||
return conn
|
||||
|
||||
|
||||
def close_mysql() -> None:
|
||||
"""关闭当前线程的所有连接。"""
|
||||
for attr in list(vars(_local).keys()):
|
||||
conn = getattr(_local, attr, None)
|
||||
if conn is not None:
|
||||
try:
|
||||
conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
delattr(_local, attr)
|
||||
except AttributeError:
|
||||
pass
|
||||
|
||||
|
||||
# ── Schema DDL 入口 ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def init_mysql_schema() -> None:
|
||||
"""集中创建/补齐所有表。被 db/ops.py 在第一次访问时调用。"""
|
||||
# 延后导入避免循环依赖
|
||||
from app.core.db.schema import ensure_all_tables
|
||||
|
||||
ensure_all_tables(get_mysql())
|
||||
@@ -0,0 +1,479 @@
|
||||
"""PostgreSQL ORM 模型（SQLAlchemy 2.x declarative）。
|
||||
|
||||
设计原则：
|
||||
- 全部 15 张表用 Python ORM 表达；`Base.metadata.create_all()` 一键建表
|
||||
- 所有表放在 `market_data` schema 下（PG-native）
|
||||
- 类型选择 PG-native：TIMESTAMPTZ / NUMERIC / JSONB / TEXT
|
||||
- 字段名与 MySQL 版尽量保持一致，便于将来数据迁移直接 INSERT ... SELECT
|
||||
|
||||
与原 MySQL 版的差异（按列类型一一对照）：
|
||||
- DATETIME → TIMESTAMPTZ
|
||||
- DOUBLE → Float（保持 double precision）
|
||||
- VARCHAR JSON 字段 → JSONB（dataset_registry.dependency_ids）
|
||||
- ON UPDATE CURRENT_TIMESTAMP → onupdate=func.now()（应用层处理，PG 没内置）
|
||||
- PrimaryKeyConstraint → 全部用主键约束表达；复合主键用 __table_args__
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date, datetime
|
||||
from typing import Optional
|
||||
|
||||
from sqlalchemy import (
|
||||
Date,
|
||||
DateTime,
|
||||
Float,
|
||||
Index,
|
||||
Integer,
|
||||
PrimaryKeyConstraint,
|
||||
SmallInteger,
|
||||
String,
|
||||
Text,
|
||||
func,
|
||||
)
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from app.core.db.orm import ORMBase
|
||||
|
||||
|
||||
# ──── 公共 updated_at 工厂 ────
|
||||
def _updated_at():
|
||||
"""`DEFAULT NOW() ON UPDATE NOW()` —— PG 应用层维护。"""
|
||||
return mapped_column(
|
||||
DateTime(timezone=True),
|
||||
server_default=func.now(),
|
||||
onupdate=func.now(),
|
||||
)
|
||||
|
||||
|
||||
# ──────────────────────── 1. config ────────────────────────
|
||||
class Config(ORMBase):
|
||||
"""键值配置表（.env / 数据源开关 / 调度参数等）。"""
|
||||
__tablename__ = "config"
|
||||
__table_args__ = {"schema": "market_data"}
|
||||
|
||||
key: Mapped[str] = mapped_column(String(128), primary_key=True)
|
||||
value: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
category: Mapped[str] = mapped_column(String(32), nullable=False, default="general")
|
||||
description: Mapped[Optional[str]] = mapped_column(String(255), default="")
|
||||
updated_at: Mapped[Optional[datetime]] = _updated_at()
|
||||
|
||||
|
||||
# ──────────────────────── 2. dataset_registry ────────────────────────
|
||||
class DatasetRegistry(ORMBase):
|
||||
"""同步任务状态机：idle → running → ok / warning / error / blocked。"""
|
||||
__tablename__ = "dataset_registry"
|
||||
__table_args__ = (
|
||||
Index("idx_dataset_registry_sort_order", "sort_order"),
|
||||
{"schema": "market_data"},
|
||||
)
|
||||
|
||||
dataset_id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(128), nullable=False, default="")
|
||||
description: Mapped[Optional[str]] = mapped_column(Text)
|
||||
storage_uri: Mapped[Optional[str]] = mapped_column(String(255), default="")
|
||||
storage_layer: Mapped[Optional[str]] = mapped_column(String(32), default="")
|
||||
management_role: Mapped[Optional[str]] = mapped_column(String(32), default="")
|
||||
source: Mapped[Optional[str]] = mapped_column(String(255), default="")
|
||||
sync_script: Mapped[Optional[str]] = mapped_column(String(255), default="")
|
||||
# 原 schema 是 VARCHAR(4096) DEFAULT '[]'（序列化的 JSON 字符串），PG 改为原生 JSONB
|
||||
dependency_ids: Mapped[Optional[list]] = mapped_column(JSONB, default=list)
|
||||
enabled: Mapped[int] = mapped_column(Integer, default=1)
|
||||
sort_order: Mapped[int] = mapped_column(Integer, default=0)
|
||||
status: Mapped[Optional[str]] = mapped_column(String(16), default="idle")
|
||||
trigger_source: Mapped[Optional[str]] = mapped_column(String(32), default="")
|
||||
started_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True))
|
||||
finished_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True))
|
||||
last_success_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True))
|
||||
last_failure_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True))
|
||||
message: Mapped[Optional[str]] = mapped_column(Text)
|
||||
last_error: Mapped[Optional[str]] = mapped_column(Text)
|
||||
needs_resync: Mapped[int] = mapped_column(Integer, default=0)
|
||||
progress_current: Mapped[int] = mapped_column(Integer, default=0)
|
||||
progress_total: Mapped[int] = mapped_column(Integer, default=0)
|
||||
current_step: Mapped[Optional[str]] = mapped_column(String(255), default="")
|
||||
updated_at: Mapped[Optional[datetime]] = _updated_at()
|
||||
|
||||
|
||||
# ──────────────────────── 3. stocks ────────────────────────
|
||||
class Stock(ORMBase):
|
||||
"""股票基础信息 + 最新股本快照（冗余缓存）。"""
|
||||
__tablename__ = "stocks"
|
||||
__table_args__ = {"schema": "market_data"}
|
||||
|
||||
code: Mapped[str] = mapped_column(String(10), primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(32), nullable=False, default="")
|
||||
exchange: Mapped[str] = mapped_column(String(8), nullable=False, default="")
|
||||
list_date: Mapped[Optional[date]] = mapped_column(Date)
|
||||
listing_status: Mapped[str] = mapped_column(String(16), nullable=False, default="normal")
|
||||
industry: Mapped[Optional[str]] = mapped_column(String(64), default="")
|
||||
total_share: Mapped[float] = mapped_column(Float, default=0)
|
||||
float_share: Mapped[float] = mapped_column(Float, default=0)
|
||||
share_updated_at: Mapped[Optional[date]] = mapped_column(Date)
|
||||
kline_synced_at: Mapped[Optional[date]] = mapped_column(Date)
|
||||
updated_at: Mapped[Optional[datetime]] = _updated_at()
|
||||
|
||||
|
||||
# ──────────────────────── 4. indices ────────────────────────
|
||||
# 类的 Python 名 `MarketIndex`（避免和 `from sqlalchemy import Index` 冲突），
|
||||
# 但 __tablename__ 仍是 "indices" 跟原 MySQL 一致。
|
||||
class MarketIndex(ORMBase):
|
||||
"""六大指数基础信息。"""
|
||||
__tablename__ = "indices"
|
||||
__table_args__ = {"schema": "market_data"}
|
||||
|
||||
index_code: Mapped[str] = mapped_column(String(10), primary_key=True)
|
||||
index_name: Mapped[str] = mapped_column(String(64), nullable=False, default="")
|
||||
market: Mapped[Optional[str]] = mapped_column(String(16), default="")
|
||||
category: Mapped[Optional[str]] = mapped_column(String(16), default="")
|
||||
source: Mapped[Optional[str]] = mapped_column(String(32), default="")
|
||||
enabled: Mapped[int] = mapped_column(Integer, default=1)
|
||||
updated_at: Mapped[Optional[datetime]] = _updated_at()
|
||||
|
||||
|
||||
# ──────────────────────── 5. kline_stock ────────────────────────
|
||||
class KlineStock(ORMBase):
|
||||
"""全市场 A 股日频 OHLCV。"""
|
||||
__tablename__ = "kline_stock"
|
||||
__table_args__ = (
|
||||
PrimaryKeyConstraint("stock_code", "trade_date"),
|
||||
Index("idx_kline_stock_date", "trade_date"),
|
||||
{"schema": "market_data"},
|
||||
)
|
||||
|
||||
stock_code: Mapped[str] = mapped_column(String(10), nullable=False)
|
||||
trade_date: Mapped[date] = mapped_column(Date, nullable=False)
|
||||
open: Mapped[float] = mapped_column(Float, nullable=False)
|
||||
high: Mapped[float] = mapped_column(Float, nullable=False)
|
||||
low: Mapped[float] = mapped_column(Float, nullable=False)
|
||||
close: Mapped[float] = mapped_column(Float, nullable=False)
|
||||
volume: Mapped[float] = mapped_column(Float, nullable=False)
|
||||
|
||||
|
||||
# ──────────────────────── 6. kline_index ────────────────────────
|
||||
class KlineIndex(ORMBase):
|
||||
"""六大指数日 K。"""
|
||||
__tablename__ = "kline_index"
|
||||
__table_args__ = (
|
||||
PrimaryKeyConstraint("index_code", "trade_date"),
|
||||
Index("idx_kline_index_date", "trade_date"),
|
||||
{"schema": "market_data"},
|
||||
)
|
||||
|
||||
index_code: Mapped[str] = mapped_column(String(10), nullable=False)
|
||||
trade_date: Mapped[date] = mapped_column(Date, nullable=False)
|
||||
open: Mapped[float] = mapped_column(Float, nullable=False)
|
||||
high: Mapped[float] = mapped_column(Float, nullable=False)
|
||||
low: Mapped[float] = mapped_column(Float, nullable=False)
|
||||
close: Mapped[float] = mapped_column(Float, nullable=False)
|
||||
volume: Mapped[float] = mapped_column(Float, nullable=False)
|
||||
|
||||
|
||||
# ──────────────────────── 7. kline_5min ────────────────────────
|
||||
class Kline5Min(ORMBase):
|
||||
"""全市场 A 股 5 分钟 K 线。"""
|
||||
__tablename__ = "kline_5min"
|
||||
__table_args__ = (
|
||||
PrimaryKeyConstraint("stock_code", "bar_time"),
|
||||
Index("idx_kline_5min_bar_time", "bar_time"),
|
||||
{"schema": "market_data"},
|
||||
)
|
||||
|
||||
stock_code: Mapped[str] = mapped_column(String(10), nullable=False)
|
||||
bar_time: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
open: Mapped[Optional[float]] = mapped_column(Float)
|
||||
high: Mapped[Optional[float]] = mapped_column(Float)
|
||||
low: Mapped[Optional[float]] = mapped_column(Float)
|
||||
close: Mapped[Optional[float]] = mapped_column(Float)
|
||||
volume: Mapped[Optional[float]] = mapped_column(Float)
|
||||
amount: Mapped[Optional[float]] = mapped_column(Float)
|
||||
turnover_rate: Mapped[Optional[float]] = mapped_column(Float)
|
||||
|
||||
|
||||
# ──────────────────────── 8. moneyflow ────────────────────────
|
||||
class Moneyflow(ORMBase):
|
||||
"""个股资金流（mairui 源：主力/大/中/小单 净额，单位元）。"""
|
||||
__tablename__ = "moneyflow"
|
||||
__table_args__ = (
|
||||
PrimaryKeyConstraint("stock_code", "trade_date"),
|
||||
Index("idx_moneyflow_date", "trade_date"),
|
||||
{"schema": "market_data"},
|
||||
)
|
||||
|
||||
stock_code: Mapped[str] = mapped_column(String(10), nullable=False)
|
||||
trade_date: Mapped[date] = mapped_column(Date, nullable=False)
|
||||
main_net_inflow: Mapped[float] = mapped_column(Float, default=0)
|
||||
large_net_inflow: Mapped[float] = mapped_column(Float, default=0)
|
||||
medium_net_inflow: Mapped[float] = mapped_column(Float, default=0)
|
||||
small_net_inflow: Mapped[float] = mapped_column(Float, default=0)
|
||||
|
||||
|
||||
# ──────────────────────── 9. share ────────────────────────
|
||||
class Share(ORMBase):
|
||||
"""个股股本快照（雪球 quote_detail，单位亿股）。"""
|
||||
__tablename__ = "share"
|
||||
__table_args__ = (
|
||||
PrimaryKeyConstraint("stock_code", "trade_date"),
|
||||
{"schema": "market_data"},
|
||||
)
|
||||
|
||||
stock_code: Mapped[str] = mapped_column(String(10), nullable=False)
|
||||
trade_date: Mapped[date] = mapped_column(Date, nullable=False)
|
||||
total_share: Mapped[float] = mapped_column(Float, nullable=False)
|
||||
float_share: Mapped[float] = mapped_column(Float, nullable=False)
|
||||
|
||||
|
||||
# ──────────────────────── 10. sectors ────────────────────────
|
||||
class Sectors(ORMBase):
|
||||
"""行业字典（baostock 行业分类 / 申万 / 中证 等多种 taxonomy）。"""
|
||||
__tablename__ = "sectors"
|
||||
__table_args__ = (
|
||||
Index("idx_sectors_sector_name", "sector_name"),
|
||||
{"schema": "market_data"},
|
||||
)
|
||||
|
||||
sector_key: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
sector_name: Mapped[str] = mapped_column(String(64), nullable=False, default="")
|
||||
taxonomy: Mapped[Optional[str]] = mapped_column(String(64), default="")
|
||||
level: Mapped[Optional[str]] = mapped_column(String(16), default="")
|
||||
source: Mapped[Optional[str]] = mapped_column(String(32), default="")
|
||||
enabled: Mapped[int] = mapped_column(Integer, default=1)
|
||||
updated_at: Mapped[Optional[datetime]] = _updated_at()
|
||||
|
||||
|
||||
# ──────────────────────── 11. stock_sector_map ────────────────────────
|
||||
class StockSectorMap(ORMBase):
|
||||
"""股票-行业多对多映射（一张股票可对应多个行业 / 概念板块）。"""
|
||||
__tablename__ = "stock_sector_map"
|
||||
__table_args__ = (
|
||||
Index("idx_stock_sector_map_sector_key", "sector_key"),
|
||||
{"schema": "market_data"},
|
||||
)
|
||||
|
||||
stock_code: Mapped[str] = mapped_column(String(10), primary_key=True)
|
||||
sector_key: Mapped[str] = mapped_column(String(64), nullable=False, default="")
|
||||
updated_at: Mapped[Optional[datetime]] = _updated_at()
|
||||
|
||||
|
||||
# ──────────────────────── 12. industry ────────────────────────
|
||||
class Industry(ORMBase):
|
||||
"""股票-行业映射（baostock 源，证监会分类标准）。"""
|
||||
__tablename__ = "industry"
|
||||
__table_args__ = (
|
||||
Index("idx_industry_industry_name", "industry_name"),
|
||||
{"schema": "market_data"},
|
||||
)
|
||||
|
||||
code: Mapped[str] = mapped_column(String(10), primary_key=True)
|
||||
industry_name: Mapped[Optional[str]] = mapped_column(String(64), default="")
|
||||
industry_classification: Mapped[Optional[str]] = mapped_column(String(32), default="")
|
||||
update_date: Mapped[Optional[date]] = mapped_column(Date)
|
||||
updated_at: Mapped[Optional[datetime]] = _updated_at()
|
||||
|
||||
|
||||
# ──────────────────────── 13. sector_indices ────────────────────────
|
||||
class SectorIndices(ORMBase):
|
||||
"""行业日线（基点 100，复合收益）。"""
|
||||
__tablename__ = "sector_indices"
|
||||
__table_args__ = (
|
||||
PrimaryKeyConstraint("trade_date", "sector_name"),
|
||||
Index("idx_sector_indices_sector_name", "sector_name"),
|
||||
{"schema": "market_data"},
|
||||
)
|
||||
|
||||
trade_date: Mapped[date] = mapped_column(Date, nullable=False)
|
||||
sector_name: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
close: Mapped[float] = mapped_column(Float, default=0)
|
||||
sector_amplitude: Mapped[float] = mapped_column(Float, default=0)
|
||||
|
||||
|
||||
# ──────────────────────── 14. sector_features_daily ────────────────────────
|
||||
class SectorFeaturesDaily(ORMBase):
|
||||
"""行业日特征：sector_ret / sector_amplitude / close / EMA / score。"""
|
||||
__tablename__ = "sector_features_daily"
|
||||
__table_args__ = (
|
||||
PrimaryKeyConstraint("trade_date", "sector_name"),
|
||||
Index("idx_sector_features_sector_name", "sector_name"),
|
||||
{"schema": "market_data"},
|
||||
)
|
||||
|
||||
trade_date: Mapped[date] = mapped_column(Date, nullable=False)
|
||||
sector_name: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
sector_ret: Mapped[float] = mapped_column(Float, default=0)
|
||||
sector_amplitude: Mapped[float] = mapped_column(Float, default=0)
|
||||
close: Mapped[float] = mapped_column(Float, default=0)
|
||||
ema10: Mapped[float] = mapped_column(Float, default=0)
|
||||
ema20: Mapped[float] = mapped_column(Float, default=0)
|
||||
ema200: Mapped[float] = mapped_column(Float, default=0)
|
||||
score: Mapped[int] = mapped_column(Integer, default=0)
|
||||
|
||||
|
||||
# ──────────────────────── 15. market_regime_daily ────────────────────────
|
||||
class MarketRegimeDaily(ORMBase):
|
||||
"""市场情绪衍生指标（基于本地 kline 聚合）。"""
|
||||
__tablename__ = "market_regime_daily"
|
||||
__table_args__ = (
|
||||
Index("idx_market_regime_daily_date", "trade_date"),
|
||||
{"schema": "market_data"},
|
||||
)
|
||||
|
||||
trade_date: Mapped[date] = mapped_column(Date, primary_key=True)
|
||||
advancers: Mapped[float] = mapped_column(Float, default=0)
|
||||
decliners: Mapped[float] = mapped_column(Float, default=0)
|
||||
advance_ratio: Mapped[float] = mapped_column(Float, default=0)
|
||||
turnover: Mapped[float] = mapped_column(Float, default=0)
|
||||
turnover_avg_5d: Mapped[float] = mapped_column(Float, default=0)
|
||||
turnover_ratio_5d: Mapped[float] = mapped_column(Float, default=0)
|
||||
source: Mapped[Optional[str]] = mapped_column(String(32), default="")
|
||||
is_extreme_panic: Mapped[int] = mapped_column(Integer, default=0)
|
||||
updated_at: Mapped[Optional[datetime]] = _updated_at()
|
||||
|
||||
|
||||
# ──────────────────────── 16. tick_trade ────────────────────────
|
||||
class TickTrade(ORMBase):
|
||||
"""个股当天逐笔成交（mairui 源）。
|
||||
|
||||
mairui API（`hsrl/zbjy/{code6}/{licence}`）不返唯一 trade_id；
|
||||
6 字段组合 `(stock_code, trade_date, trade_time, price, volume, direction_code)`
|
||||
作为复合主键保证幂等 upsert。
|
||||
|
||||
direction_code: 0=中性盘 / 1=买入 / 2=卖出（mairui 原始值）。
|
||||
direction: 派生 'neutral' / 'buy' / 'sell'（冗余存便于按文本查）。
|
||||
amount: 派生 price * volume（元；mairui 不返）。
|
||||
"""
|
||||
__tablename__ = "tick_trade"
|
||||
__table_args__ = (
|
||||
PrimaryKeyConstraint(
|
||||
"stock_code", "trade_date", "trade_time",
|
||||
"price", "volume", "direction_code",
|
||||
),
|
||||
Index("idx_tick_trade_date", "trade_date"),
|
||||
Index("idx_tick_trade_code_date", "stock_code", "trade_date"),
|
||||
{"schema": "market_data"},
|
||||
)
|
||||
|
||||
stock_code: Mapped[str] = mapped_column(String(10), nullable=False)
|
||||
trade_date: Mapped[date] = mapped_column(Date, nullable=False)
|
||||
trade_time: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
price: Mapped[float] = mapped_column(Float, nullable=False)
|
||||
volume: Mapped[float] = mapped_column(Float, nullable=False)
|
||||
direction_code: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
direction: Mapped[str] = mapped_column(String(8), nullable=False, default="")
|
||||
amount: Mapped[float] = mapped_column(Float, nullable=False, default=0)
|
||||
|
||||
|
||||
# ──────────────────────── 17. longhubang_daily ────────────────────────
|
||||
class LonghubangDaily(ORMBase):
|
||||
"""龙虎榜每日上榜股票汇总（聚合层，akshare 源）。
|
||||
|
||||
数据源：`ak.stock_lhb_detail_em(start_date, end_date)` —— 东方财富私有 API 封装。
|
||||
每个上榜日 × 每只上榜股票 = 1 行。
|
||||
|
||||
字段（21 列）：
|
||||
基础：rank_idx, stock_code, stock_name, trade_date, comment
|
||||
当日：close, pct_chg
|
||||
资金：lhb_net_buy, lhb_buy_amt, lhb_sell_amt, lhb_total_amt, market_total_amt
|
||||
比率：net_buy_ratio, total_amt_ratio, turnover_rate, float_mv
|
||||
原因：reason
|
||||
后效：post_1d_pct, post_2d_pct, post_5d_pct, post_10d_pct
|
||||
|
||||
注意：`stock_code` 含交易所前缀（SH600000 / SZ000001），与项目其他表一致。
|
||||
"""
|
||||
__tablename__ = "longhubang_daily"
|
||||
__table_args__ = (
|
||||
PrimaryKeyConstraint("stock_code", "trade_date"),
|
||||
Index("idx_lhb_daily_date", "trade_date"),
|
||||
{"schema": "market_data"},
|
||||
)
|
||||
|
||||
stock_code: Mapped[str] = mapped_column(String(10), nullable=False)
|
||||
stock_name: Mapped[str] = mapped_column(String(64), nullable=False, default="")
|
||||
trade_date: Mapped[date] = mapped_column(Date, nullable=False)
|
||||
rank_idx: Mapped[Optional[int]] = mapped_column(Integer, nullable=True)
|
||||
comment: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
|
||||
close: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
|
||||
pct_chg: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
|
||||
lhb_net_buy: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
|
||||
lhb_buy_amt: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
|
||||
lhb_sell_amt: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
|
||||
lhb_total_amt: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
|
||||
market_total_amt: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
|
||||
net_buy_ratio: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
|
||||
total_amt_ratio: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
|
||||
turnover_rate: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
|
||||
float_mv: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
|
||||
reason: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
|
||||
post_1d_pct: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
|
||||
post_2d_pct: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
|
||||
post_5d_pct: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
|
||||
post_10d_pct: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
|
||||
|
||||
|
||||
# ──────────────────────── 18. longhubang_seat ────────────────────────
|
||||
class LonghubangSeat(ORMBase):
|
||||
"""龙虎榜单只票买卖前五营业部（席位层，akshare 源）。
|
||||
|
||||
数据源：`ak.stock_lhb_stock_detail_em(symbol, date, flag)` —— 东方财富私有 API 封装。
|
||||
每只上榜票 × 每方向（buy/sell）= 5 行（共 10 行/票/日）。
|
||||
|
||||
字段（11 列 + 上榜原因）：
|
||||
基础：stock_code, trade_date, direction（buy/sell）, rank_idx
|
||||
营业部：branch_name（akshare 不返唯一 branch_code，用 name 作 PK 兜底）
|
||||
金额：buy_amount, sell_amount, amount, net_amount, amount_ratio
|
||||
当日：pct_chg, close, stock_total_amt
|
||||
原因：explanation
|
||||
|
||||
注意：akshare 的 `stock_lhb_stock_detail_em` 不返 `branch_code`，只用 `branch_name`
|
||||
作 PK 兜底（不同营业部名冲突概率极低；如未来需要可补 eastmoney 私有 API 抓 code）。
|
||||
"""
|
||||
__tablename__ = "longhubang_seat"
|
||||
__table_args__ = (
|
||||
PrimaryKeyConstraint("stock_code", "trade_date", "direction", "rank_idx"),
|
||||
Index("idx_lhb_seat_date", "trade_date"),
|
||||
Index("idx_lhb_seat_branch_name", "branch_name"),
|
||||
{"schema": "market_data"},
|
||||
)
|
||||
|
||||
stock_code: Mapped[str] = mapped_column(String(10), nullable=False)
|
||||
trade_date: Mapped[date] = mapped_column(Date, nullable=False)
|
||||
direction: Mapped[str] = mapped_column(String(8), nullable=False) # "buy" / "sell"
|
||||
rank_idx: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
branch_name: Mapped[Optional[str]] = mapped_column(String(128), nullable=True)
|
||||
buy_amount: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
|
||||
sell_amount: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
|
||||
amount: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
|
||||
net_amount: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
|
||||
amount_ratio: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
|
||||
pct_chg: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
|
||||
close: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
|
||||
stock_total_amt: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
|
||||
explanation: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
|
||||
|
||||
|
||||
__all__ = [
|
||||
# 1-2
|
||||
"Config",
|
||||
"DatasetRegistry",
|
||||
# 3-4
|
||||
"Stock",
|
||||
"MarketIndex",
|
||||
# 5-9
|
||||
"KlineStock",
|
||||
"KlineIndex",
|
||||
"Kline5Min",
|
||||
"Moneyflow",
|
||||
"Share",
|
||||
# 10-12
|
||||
"Sectors",
|
||||
"StockSectorMap",
|
||||
"Industry",
|
||||
# 13-15
|
||||
"SectorIndices",
|
||||
"SectorFeaturesDaily",
|
||||
"MarketRegimeDaily",
|
||||
# 16
|
||||
"TickTrade",
|
||||
# 17-18 (2026-07-01 龙虎榜)
|
||||
"LonghubangDaily",
|
||||
"LonghubangSeat",
|
||||
]
|
||||
+524
-440
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,59 @@
|
||||
"""SQLAlchemy 2.x ORM 基础设施（PostgreSQL only）。
|
||||
|
||||
2026-06-16 重构：项目放弃 MySQL，只保留 PG。
|
||||
所有模型 class 仍带 ``__table_args__ = {"schema": "market_data"}``，指向 PG schema。
|
||||
PG 数据库 / schema / role / 表 由 ``app.core.db.pg_bootstrap`` 一次性建好。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from contextlib import contextmanager
|
||||
from typing import Iterator
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.engine import Engine
|
||||
from sqlalchemy.orm import DeclarativeBase, Session, sessionmaker
|
||||
|
||||
from app.core.config import settings
|
||||
|
||||
|
||||
class ORMBase(DeclarativeBase):
|
||||
"""所有 ORM 模型的基类。"""
|
||||
|
||||
|
||||
def _build_engine() -> Engine:
|
||||
"""构造 PG 的 SQLAlchemy Engine。"""
|
||||
return create_engine(
|
||||
settings.pg_sqlalchemy_url(),
|
||||
pool_pre_ping=True,
|
||||
pool_recycle=3600,
|
||||
future=True,
|
||||
echo=False,
|
||||
)
|
||||
|
||||
|
||||
engine: Engine = _build_engine()
|
||||
|
||||
# sessionmaker 工厂
|
||||
SessionLocal = sessionmaker(
|
||||
bind=engine,
|
||||
expire_on_commit=False,
|
||||
autoflush=False,
|
||||
future=True,
|
||||
)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def get_session() -> Iterator[Session]:
|
||||
"""事务上下文：异常自动 rollback，正常自动 commit。"""
|
||||
s = SessionLocal()
|
||||
try:
|
||||
yield s
|
||||
s.commit()
|
||||
except Exception:
|
||||
s.rollback()
|
||||
raise
|
||||
finally:
|
||||
s.close()
|
||||
|
||||
|
||||
__all__ = ["ORMBase", "engine", "SessionLocal", "get_session"]
|
||||
@@ -0,0 +1,235 @@
|
||||
"""PostgreSQL bootstrap — 一键建库 / 建 schema / 建 role / 建表。
|
||||
|
||||
用法：
|
||||
.venv/bin/python -m app.core.db.pg_bootstrap
|
||||
|
||||
按顺序做：
|
||||
1) 以 superuser 身份连默认 'postgres' 库
|
||||
2) CREATE DATABASE market_data (IF NOT EXISTS)
|
||||
3) \\connect 到 market_data
|
||||
4) CREATE SCHEMA market_data, grid_seeker
|
||||
5) CREATE ROLE market_sync LOGIN PASSWORD ... (从 .env 读 PG_MARKET_SYNC_PASSWORD)
|
||||
6) GRANT 权限 + ALTER DEFAULT PRIVILEGES
|
||||
7) SQLAlchemy ORM Base.metadata.create_all() 一键建所有表
|
||||
|
||||
幂等：所有 DDL 都用 IF NOT EXISTS / OR REPLACE；可以重复跑。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
import psycopg2
|
||||
from psycopg2 import sql
|
||||
from sqlalchemy import create_engine, text
|
||||
from sqlalchemy.engine import URL
|
||||
|
||||
from app.core.db.orm import ORMBase as Base
|
||||
# 重要：必须显式 import models 模块，所有 @mapped_column 才会注册到 ORMBase.metadata
|
||||
# 漏了这一步 Base.metadata.tables 是空的，create_all() 不会建任何表
|
||||
# 改用 getattr 触发副作用，绕开 IDE "unused import" 警告
|
||||
_ = __import__("app.core.db.models", fromlist=["*"])
|
||||
|
||||
|
||||
# ──── 配置（从 .env 读） ────
|
||||
|
||||
def _env(key: str, default: str) -> str:
|
||||
return os.environ.get(key, default).strip() or default
|
||||
|
||||
|
||||
SUPERUSER = _env("PG_SUPERUSER", "postgres")
|
||||
SUPERPASS = _env("PG_SUPERUSER_PASSWORD", "postgres")
|
||||
SUPERHOST = _env("PG_SUPERHOST", "127.0.0.1")
|
||||
SUPERPORT = int(_env("PG_SUPERPORT", "5432"))
|
||||
|
||||
DB_NAME = _env("PG_DB_NAME", "market_data")
|
||||
SCHEMAS = _env("PG_SCHEMAS", "market_data,grid_seeker").split(",")
|
||||
|
||||
MARKET_SYNC_ROLE = _env("PG_MARKET_SYNC_ROLE", "market_sync")
|
||||
MARKET_SYNC_PASS = _env("PG_MARKET_SYNC_PASSWORD", "market_sync")
|
||||
|
||||
LOG_PREFIX = "[pg_bootstrap]"
|
||||
|
||||
|
||||
def log(msg: str) -> None:
|
||||
print(f"{LOG_PREFIX} {msg}", flush=True)
|
||||
|
||||
|
||||
def _connect(dbname: str):
|
||||
return psycopg2.connect(
|
||||
host=SUPERHOST, port=SUPERPORT,
|
||||
user=SUPERUSER, password=SUPERPASS,
|
||||
dbname=dbname, connect_timeout=10,
|
||||
)
|
||||
|
||||
|
||||
# ──── Step 1: 创建数据库 ────
|
||||
|
||||
def step1_create_database() -> None:
|
||||
log(f"Step 1: CREATE DATABASE {DB_NAME} (if not exists)")
|
||||
conn = _connect("postgres")
|
||||
conn.autocommit = True
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT 1 FROM pg_database WHERE datname = %s", (DB_NAME,))
|
||||
if cur.fetchone():
|
||||
log(f" ✓ {DB_NAME} 已存在，跳过")
|
||||
else:
|
||||
cur.execute(sql.SQL("CREATE DATABASE {}").format(sql.Identifier(DB_NAME)))
|
||||
log(f" ✓ {DB_NAME} 已创建")
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
# ──── Step 2: 创建 schema ────
|
||||
|
||||
def step2_create_schemas() -> None:
|
||||
log(f"Step 2: CREATE SCHEMA for {SCHEMAS}")
|
||||
conn = _connect(DB_NAME)
|
||||
conn.autocommit = True
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
for schema in SCHEMAS:
|
||||
schema = schema.strip()
|
||||
if not schema:
|
||||
continue
|
||||
cur.execute(
|
||||
sql.SQL("CREATE SCHEMA IF NOT EXISTS {}").format(sql.Identifier(schema))
|
||||
)
|
||||
log(f" ✓ schema '{schema}' 就绪")
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
# ──── Step 3: 创建角色 + 授权 ────
|
||||
|
||||
def step3_create_role() -> None:
|
||||
log(f"Step 3: CREATE ROLE {MARKET_SYNC_ROLE} + GRANT")
|
||||
conn = _connect(DB_NAME)
|
||||
conn.autocommit = True
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
# 角色（IF NOT EXISTS 9.0+ 支持，旧版会报错 — 我们用 try/except 兜底）
|
||||
cur.execute(
|
||||
"SELECT 1 FROM pg_roles WHERE rolname = %s",
|
||||
(MARKET_SYNC_ROLE,),
|
||||
)
|
||||
if cur.fetchone():
|
||||
log(f" ✓ role '{MARKET_SYNC_ROLE}' 已存在，跳过")
|
||||
else:
|
||||
cur.execute(
|
||||
sql.SQL("CREATE ROLE {} LOGIN PASSWORD %s").format(
|
||||
sql.Identifier(MARKET_SYNC_ROLE)
|
||||
),
|
||||
(MARKET_SYNC_PASS,),
|
||||
)
|
||||
log(f" ✓ role '{MARKET_SYNC_ROLE}' 已创建")
|
||||
|
||||
# 连接 DB 权限
|
||||
cur.execute(
|
||||
sql.SQL("GRANT CONNECT ON DATABASE {} TO {}").format(
|
||||
sql.Identifier(DB_NAME), sql.Identifier(MARKET_SYNC_ROLE)
|
||||
)
|
||||
)
|
||||
|
||||
# 4 个 schema 都有 USAGE
|
||||
for schema in SCHEMAS:
|
||||
schema = schema.strip()
|
||||
if not schema:
|
||||
continue
|
||||
cur.execute(
|
||||
sql.SQL("GRANT USAGE ON SCHEMA {} TO {}").format(
|
||||
sql.Identifier(schema), sql.Identifier(MARKET_SYNC_ROLE)
|
||||
)
|
||||
)
|
||||
|
||||
# market_data: RW（当前项目用的）
|
||||
cur.execute(
|
||||
sql.SQL("GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA market_data TO {}").format(
|
||||
sql.Identifier(MARKET_SYNC_ROLE)
|
||||
)
|
||||
)
|
||||
cur.execute(
|
||||
sql.SQL("GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA market_data TO {}").format(
|
||||
sql.Identifier(MARKET_SYNC_ROLE)
|
||||
)
|
||||
)
|
||||
# grid_seeker: 暂时也 RW（未来 grid_seeker 自己做权限时再收紧）
|
||||
cur.execute(
|
||||
sql.SQL("GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA grid_seeker TO {}").format(
|
||||
sql.Identifier(MARKET_SYNC_ROLE)
|
||||
)
|
||||
)
|
||||
cur.execute(
|
||||
sql.SQL("GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA grid_seeker TO {}").format(
|
||||
sql.Identifier(MARKET_SYNC_ROLE)
|
||||
)
|
||||
)
|
||||
|
||||
# 关键：未来新表自动继承（不然新加的表 market_sync 没权限）
|
||||
for schema in SCHEMAS:
|
||||
schema = schema.strip()
|
||||
if not schema:
|
||||
continue
|
||||
cur.execute(
|
||||
sql.SQL(
|
||||
"ALTER DEFAULT PRIVILEGES IN SCHEMA {s} "
|
||||
"GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO {r}"
|
||||
).format(s=sql.Identifier(schema), r=sql.Identifier(MARKET_SYNC_ROLE))
|
||||
)
|
||||
cur.execute(
|
||||
sql.SQL(
|
||||
"ALTER DEFAULT PRIVILEGES IN SCHEMA {s} "
|
||||
"GRANT USAGE, SELECT ON SEQUENCES TO {r}"
|
||||
).format(s=sql.Identifier(schema), r=sql.Identifier(MARKET_SYNC_ROLE))
|
||||
)
|
||||
log(f" ✓ {MARKET_SYNC_ROLE} 在所有 schema 上有 RW + 未来新表自动继承")
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
# ──── Step 4: SQLAlchemy ORM 建所有表 ────
|
||||
|
||||
def step4_create_tables() -> None:
|
||||
log("Step 4: SQLAlchemy Base.metadata.create_all() 建所有表")
|
||||
# 临时以 superuser 连接 market_data（建表后权限自动由 ALTER DEFAULT 接管）
|
||||
url = URL.create(
|
||||
drivername="postgresql+psycopg2",
|
||||
username=SUPERUSER, password=SUPERPASS,
|
||||
host=SUPERHOST, port=SUPERPORT,
|
||||
database=DB_NAME,
|
||||
)
|
||||
engine = create_engine(url, future=True, echo=False)
|
||||
try:
|
||||
Base.metadata.create_all(engine)
|
||||
with engine.connect() as c:
|
||||
rows = c.execute(
|
||||
text(
|
||||
"SELECT table_schema, table_name FROM information_schema.tables "
|
||||
"WHERE table_schema IN ('market_data', 'grid_seeker') "
|
||||
"ORDER BY table_schema, table_name"
|
||||
)
|
||||
).fetchall()
|
||||
log(f" ✓ 当前 schema 下的表（{len(rows)} 张）:")
|
||||
for schema, tbl in rows:
|
||||
log(f" - {schema}.{tbl}")
|
||||
finally:
|
||||
engine.dispose()
|
||||
|
||||
|
||||
# ──── main ────
|
||||
|
||||
def main() -> None:
|
||||
log(f"连接 superuser {SUPERUSER}@{SUPERHOST}:{SUPERPORT}")
|
||||
step1_create_database()
|
||||
step2_create_schemas()
|
||||
step3_create_role()
|
||||
step4_create_tables()
|
||||
log("✅ PG bootstrap 完成")
|
||||
log(f" DB: {DB_NAME}")
|
||||
log(f" 角色: {MARKET_SYNC_ROLE} (password 见 .env PG_MARKET_SYNC_PASSWORD)")
|
||||
log(f" schemas: {', '.join(SCHEMAS)}")
|
||||
log(f" 下一步：在 .env 设 PG_URL=postgresql+psycopg2://{MARKET_SYNC_ROLE}:{MARKET_SYNC_PASS}@{SUPERHOST}:{SUPERPORT}/{DB_NAME}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,264 +0,0 @@
|
||||
"""Schema DDL — 集中所有 CREATE TABLE IF NOT EXISTS。
|
||||
|
||||
参考 dashboard/api/services/mysql_conn.py::init_mysql_schema()，但补齐 dashboard 缺的几张表
|
||||
（kline_stock / kline_5min / kline_index / moneyflow / share / sector_features_daily / sector_indices
|
||||
/ stock_scores / registry），这些是 grid_seeker_model_base 生产库已存在但 mysql_conn.py 没建的表。
|
||||
|
||||
所有 DDL 写在同一个文件，方便审阅、diff 和补字段。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import pymysql.connections
|
||||
|
||||
|
||||
# DDL 列表（顺序无关，全部 IF NOT EXISTS）
|
||||
_DDL: list[str] = [
|
||||
# ─── config ─────────────────────────────────────────────────────────
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS config (
|
||||
`key` VARCHAR(128) PRIMARY KEY,
|
||||
`value` TEXT NOT NULL,
|
||||
category VARCHAR(32) NOT NULL DEFAULT 'general',
|
||||
description VARCHAR(255) DEFAULT '',
|
||||
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
|
||||
""",
|
||||
# ─── dataset_registry (同步状态机) ────────────────────────────────────
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS dataset_registry (
|
||||
dataset_id VARCHAR(64) PRIMARY KEY,
|
||||
name VARCHAR(128) NOT NULL DEFAULT '',
|
||||
description TEXT,
|
||||
storage_uri VARCHAR(255) DEFAULT '',
|
||||
storage_layer VARCHAR(32) DEFAULT '',
|
||||
management_role VARCHAR(32) DEFAULT '',
|
||||
source VARCHAR(255) DEFAULT '',
|
||||
sync_script VARCHAR(255) DEFAULT '',
|
||||
dependency_ids VARCHAR(4096) DEFAULT '[]',
|
||||
enabled INT DEFAULT 1,
|
||||
sort_order INT DEFAULT 0,
|
||||
status VARCHAR(16) DEFAULT 'idle',
|
||||
trigger_source VARCHAR(32) DEFAULT '',
|
||||
started_at DATETIME DEFAULT NULL,
|
||||
finished_at DATETIME DEFAULT NULL,
|
||||
last_success_at DATETIME DEFAULT NULL,
|
||||
last_failure_at DATETIME DEFAULT NULL,
|
||||
message TEXT,
|
||||
last_error TEXT,
|
||||
needs_resync INT DEFAULT 0,
|
||||
progress_current INT DEFAULT 0,
|
||||
progress_total INT DEFAULT 0,
|
||||
current_step VARCHAR(255) DEFAULT '',
|
||||
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
INDEX idx_dataset_registry_sort_order (sort_order)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
|
||||
""",
|
||||
# ─── stocks (基础信息 + 股本字段) ─────────────────────────────────────
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS stocks (
|
||||
code VARCHAR(10) PRIMARY KEY,
|
||||
name VARCHAR(32) NOT NULL DEFAULT '',
|
||||
exchange VARCHAR(8) NOT NULL DEFAULT '',
|
||||
list_date DATE DEFAULT NULL,
|
||||
listing_status VARCHAR(16) NOT NULL DEFAULT 'normal',
|
||||
industry VARCHAR(64) DEFAULT '',
|
||||
total_share DOUBLE DEFAULT 0,
|
||||
float_share DOUBLE DEFAULT 0,
|
||||
share_updated_at DATE DEFAULT NULL,
|
||||
kline_synced_at DATE DEFAULT NULL,
|
||||
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
|
||||
""",
|
||||
# ─── indices (指数字典) ───────────────────────────────────────────────
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS indices (
|
||||
index_code VARCHAR(10) PRIMARY KEY,
|
||||
index_name VARCHAR(64) NOT NULL DEFAULT '',
|
||||
market VARCHAR(16) DEFAULT '',
|
||||
category VARCHAR(16) DEFAULT '',
|
||||
source VARCHAR(32) DEFAULT '',
|
||||
enabled INT DEFAULT 1,
|
||||
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
|
||||
""",
|
||||
# ─── kline_stock (日 K 线，按股票) ────────────────────────────────────
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS kline_stock (
|
||||
stock_code VARCHAR(10) NOT NULL,
|
||||
trade_date DATE NOT NULL,
|
||||
`open` DOUBLE NOT NULL,
|
||||
`high` DOUBLE NOT NULL,
|
||||
`low` DOUBLE NOT NULL,
|
||||
`close` DOUBLE NOT NULL,
|
||||
volume DOUBLE NOT NULL,
|
||||
PRIMARY KEY (stock_code, trade_date),
|
||||
INDEX idx_kline_stock_date (trade_date)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
|
||||
""",
|
||||
# ─── kline_index (指数日 K 线) ────────────────────────────────────────
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS kline_index (
|
||||
index_code VARCHAR(10) NOT NULL,
|
||||
trade_date DATE NOT NULL,
|
||||
`open` DOUBLE NOT NULL,
|
||||
`high` DOUBLE NOT NULL,
|
||||
`low` DOUBLE NOT NULL,
|
||||
`close` DOUBLE NOT NULL,
|
||||
volume DOUBLE NOT NULL,
|
||||
PRIMARY KEY (index_code, trade_date),
|
||||
INDEX idx_kline_index_date (trade_date)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
|
||||
""",
|
||||
# ─── kline_5min (5 分钟 K 线) ────────────────────────────────────────
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS kline_5min (
|
||||
stock_code VARCHAR(10) NOT NULL,
|
||||
bar_time DATETIME NOT NULL,
|
||||
`open` DOUBLE DEFAULT NULL,
|
||||
`high` DOUBLE DEFAULT NULL,
|
||||
`low` DOUBLE DEFAULT NULL,
|
||||
`close` DOUBLE DEFAULT NULL,
|
||||
volume DOUBLE DEFAULT NULL,
|
||||
amount DOUBLE DEFAULT NULL,
|
||||
turnover_rate DOUBLE DEFAULT NULL,
|
||||
PRIMARY KEY (stock_code, bar_time),
|
||||
INDEX idx_kline_5min_bar_time (bar_time)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
|
||||
""",
|
||||
# ─── moneyflow (资金流，主力/大/中/小单净额) ──────────────────────────
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS moneyflow (
|
||||
stock_code VARCHAR(10) NOT NULL,
|
||||
trade_date DATE NOT NULL,
|
||||
main_net_inflow DOUBLE DEFAULT 0,
|
||||
large_net_inflow DOUBLE DEFAULT 0,
|
||||
medium_net_inflow DOUBLE DEFAULT 0,
|
||||
small_net_inflow DOUBLE DEFAULT 0,
|
||||
PRIMARY KEY (stock_code, trade_date),
|
||||
INDEX idx_moneyflow_date (trade_date)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
|
||||
""",
|
||||
# ─── share (股本快照) ────────────────────────────────────────────────
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS share (
|
||||
stock_code VARCHAR(10) NOT NULL,
|
||||
trade_date DATE NOT NULL,
|
||||
total_share DOUBLE NOT NULL,
|
||||
float_share DOUBLE NOT NULL,
|
||||
PRIMARY KEY (stock_code, trade_date)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
|
||||
""",
|
||||
# ─── sectors (行业字典) ──────────────────────────────────────────────
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS sectors (
|
||||
sector_key VARCHAR(64) PRIMARY KEY,
|
||||
sector_name VARCHAR(64) NOT NULL DEFAULT '',
|
||||
taxonomy VARCHAR(64) DEFAULT '',
|
||||
level VARCHAR(16) DEFAULT '',
|
||||
source VARCHAR(32) DEFAULT '',
|
||||
enabled INT DEFAULT 1,
|
||||
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
INDEX idx_sectors_sector_name (sector_name)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
|
||||
""",
|
||||
# ─── stock_sector_map (股票→行业映射) ─────────────────────────────────
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS stock_sector_map (
|
||||
stock_code VARCHAR(6) PRIMARY KEY,
|
||||
sector_key VARCHAR(64) NOT NULL DEFAULT '',
|
||||
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
INDEX idx_stock_sector_map_sector_key (sector_key)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
|
||||
""",
|
||||
# ─── industry (股票-行业映射，datasource 实际写入用这张) ──────────────
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS industry (
|
||||
code VARCHAR(6) PRIMARY KEY,
|
||||
industry_name VARCHAR(64) DEFAULT '',
|
||||
industry_classification VARCHAR(32) DEFAULT '',
|
||||
update_date DATE DEFAULT NULL,
|
||||
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
INDEX idx_industry_industry_name (industry_name)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
|
||||
""",
|
||||
# ─── sector_indices (行业聚合指数时序) ─────────────────────────────────
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS sector_indices (
|
||||
trade_date DATE NOT NULL,
|
||||
sector_name VARCHAR(64) NOT NULL,
|
||||
`close` DOUBLE DEFAULT 0,
|
||||
sector_amplitude DOUBLE DEFAULT 0,
|
||||
PRIMARY KEY (trade_date, sector_name),
|
||||
INDEX idx_sector_indices_sector_name (sector_name)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
|
||||
""",
|
||||
# ─── sector_features_daily (行业特征衍生) ─────────────────────────────
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS sector_features_daily (
|
||||
trade_date DATE NOT NULL,
|
||||
sector_name VARCHAR(64) NOT NULL,
|
||||
sector_ret DOUBLE DEFAULT 0,
|
||||
sector_amplitude DOUBLE DEFAULT 0,
|
||||
`close` DOUBLE DEFAULT 0,
|
||||
ema10 DOUBLE DEFAULT 0,
|
||||
ema20 DOUBLE DEFAULT 0,
|
||||
ema200 DOUBLE DEFAULT 0,
|
||||
score INT DEFAULT 0,
|
||||
PRIMARY KEY (trade_date, sector_name),
|
||||
INDEX idx_sector_features_sector_name (sector_name)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
|
||||
""",
|
||||
# ─── market_regime_daily (市场情绪) ───────────────────────────────────
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS market_regime_daily (
|
||||
trade_date DATE PRIMARY KEY,
|
||||
advancers DOUBLE DEFAULT 0,
|
||||
decliners DOUBLE DEFAULT 0,
|
||||
advance_ratio DOUBLE DEFAULT 0,
|
||||
turnover DOUBLE DEFAULT 0,
|
||||
turnover_avg_5d DOUBLE DEFAULT 0,
|
||||
turnover_ratio_5d DOUBLE DEFAULT 0,
|
||||
source VARCHAR(32) DEFAULT '',
|
||||
is_extreme_panic INT DEFAULT 0,
|
||||
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
INDEX idx_market_regime_daily_date (trade_date)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
|
||||
""",
|
||||
# ─── registry (模型注册，dashboard 业务无关但库里有) ─────────────────
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS registry (
|
||||
model_name VARCHAR(64) NOT NULL,
|
||||
role VARCHAR(32) NOT NULL,
|
||||
version VARCHAR(16) DEFAULT NULL,
|
||||
artifact_path VARCHAR(256) NOT NULL,
|
||||
architecture VARCHAR(32) DEFAULT NULL,
|
||||
enabled TINYINT(1) DEFAULT 1,
|
||||
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (model_name, role)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
|
||||
""",
|
||||
# ─── stock_scores (评分快照，dashboard 业务无关但库里有) ─────────────
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS stock_scores (
|
||||
`date` DATE NOT NULL,
|
||||
stock_code VARCHAR(10) NOT NULL,
|
||||
predicted_rounds DOUBLE DEFAULT 0,
|
||||
stacking_probability DOUBLE DEFAULT 0,
|
||||
meta_ranker_score DOUBLE DEFAULT NULL,
|
||||
latest_close DOUBLE DEFAULT 0,
|
||||
`rank` INT DEFAULT 0,
|
||||
PRIMARY KEY (`date`, stock_code),
|
||||
INDEX idx_stock_scores_date (`date`)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
|
||||
""",
|
||||
]
|
||||
|
||||
|
||||
def ensure_all_tables(conn) -> None:
|
||||
"""遍历 _DDL 列表，逐条执行 CREATE TABLE IF NOT EXISTS。"""
|
||||
with conn.cursor() as cur:
|
||||
for ddl in _DDL:
|
||||
cur.execute(ddl)
|
||||
conn.commit()
|
||||
Reference in New Issue
Block a user