模型，评分，修复网格策略市场状态监听
This commit is contained in:
@@ -0,0 +1 @@
|
||||
# grid_seeker v6.4 评分模块
|
||||
@@ -0,0 +1,165 @@
|
||||
"""
|
||||
grid_seeker v6.4 CLI 入口
|
||||
|
||||
Usage:
|
||||
python -m core.scoring.cli sync all # 按依赖顺序执行全部同步
|
||||
python -m core.scoring.cli sync kline # 仅同步个股+指数K线
|
||||
python -m core.scoring.cli sync stocks # 仅同步股票基础信息
|
||||
python -m core.scoring.cli sync industry # 仅同步行业映射
|
||||
python -m core.scoring.cli sync market # 仅计算市场状态
|
||||
python -m core.scoring.cli sync sector # 仅计算行业指数
|
||||
python -m core.scoring.cli score # 完整评分管道 (最新交易日)
|
||||
python -m core.scoring.cli score --date 20260615 # 指定日期
|
||||
python -m core.scoring.cli score --dry-run # 试运行 (不写库)
|
||||
python -m core.scoring.cli check 000001 # 检查单只股票数据充分性
|
||||
python -m core.scoring.cli list-candidates # 列出全部候选股
|
||||
"""
|
||||
import sys
|
||||
from datetime import date, datetime
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 2:
|
||||
_usage()
|
||||
return
|
||||
|
||||
cmd = sys.argv[1]
|
||||
|
||||
if cmd == 'sync':
|
||||
_cmd_sync()
|
||||
elif cmd == 'score':
|
||||
_cmd_score()
|
||||
elif cmd == 'check':
|
||||
_cmd_check()
|
||||
elif cmd == 'list-candidates':
|
||||
_cmd_list_candidates()
|
||||
else:
|
||||
print(f'未知命令: {cmd}')
|
||||
_usage()
|
||||
|
||||
|
||||
def _usage():
|
||||
print(__doc__)
|
||||
|
||||
|
||||
# ============================================================
|
||||
# sync 命令
|
||||
# ============================================================
|
||||
def _cmd_sync():
|
||||
target = sys.argv[2] if len(sys.argv) > 2 else 'all'
|
||||
from core.scoring.sync import (
|
||||
KlineStockSync, KlineIndexSync,
|
||||
StocksSync, IndustrySync,
|
||||
MarketRegimeSync, SectorFeaturesSync,
|
||||
)
|
||||
|
||||
syncs = {
|
||||
'kline': [KlineStockSync, KlineIndexSync],
|
||||
'stocks': [StocksSync],
|
||||
'industry': [IndustrySync],
|
||||
'market': [MarketRegimeSync],
|
||||
'sector': [SectorFeaturesSync],
|
||||
}
|
||||
|
||||
if target == 'all':
|
||||
# 按依赖顺序执行
|
||||
order = [
|
||||
('K线(个股)', KlineStockSync(count=300)),
|
||||
('K线(指数)', KlineIndexSync(count=300)),
|
||||
('股票信息', StocksSync()),
|
||||
('行业映射', IndustrySync()),
|
||||
('市场状态', MarketRegimeSync()),
|
||||
('行业指数', SectorFeaturesSync()),
|
||||
]
|
||||
for label, sync in order:
|
||||
print(f'\n===== {label} =====')
|
||||
sync.run()
|
||||
print('\n===== 全部同步完成 =====')
|
||||
elif target in syncs:
|
||||
for cls in syncs[target]:
|
||||
cls().run()
|
||||
else:
|
||||
print(f'未知同步目标: {target}')
|
||||
print(f'可用: all, {", ".join(syncs.keys())}')
|
||||
|
||||
|
||||
# ============================================================
|
||||
# score 命令
|
||||
# ============================================================
|
||||
def _cmd_score():
|
||||
dry_run = '--dry-run' in sys.argv
|
||||
date_str = None
|
||||
for i, arg in enumerate(sys.argv):
|
||||
if arg == '--date' and i + 1 < len(sys.argv):
|
||||
date_str = sys.argv[i + 1]
|
||||
break
|
||||
|
||||
if date_str:
|
||||
trade_date = datetime.strptime(date_str, '%Y%m%d').date()
|
||||
else:
|
||||
trade_date = date.today()
|
||||
|
||||
print(f'评分日期: {trade_date}')
|
||||
|
||||
from core.scoring.inference.scorer import GridSeekerPipeline
|
||||
|
||||
engine = GridSeekerPipeline()
|
||||
rankings = engine.run(trade_date)
|
||||
|
||||
if rankings.empty:
|
||||
print('无评分结果')
|
||||
return
|
||||
|
||||
if not dry_run:
|
||||
engine.persist(rankings, trade_date)
|
||||
print(f'结果已写入 ScoringResult ({len(rankings)} 条)')
|
||||
|
||||
# 打印 Top-20
|
||||
print('\n===== Top-20 =====')
|
||||
print(f'{"Rank":<6} {"Code":<10} {"Profit":>10} {"Rounds":>10} {"Prob":>10}')
|
||||
print('-' * 50)
|
||||
for code, row in rankings.head(20).iterrows():
|
||||
print(f'{int(row["score_rank"]):<6} {code:<10} '
|
||||
f'{row["stacking_probability"]:>10.4f} '
|
||||
f'{row.get("rank_predicted_rounds", 0):>10.2f} '
|
||||
f'{row.get("stacking_probability", 0):>10.4f}')
|
||||
|
||||
|
||||
# ============================================================
|
||||
# check 命令
|
||||
# ============================================================
|
||||
def _cmd_check():
|
||||
if len(sys.argv) < 3:
|
||||
print('Usage: python -m core.scoring.cli check <stock_code>')
|
||||
return
|
||||
|
||||
stock_code = sys.argv[2]
|
||||
from core.scoring.features.validator import _check_kline_sufficiency
|
||||
|
||||
ok, reason, close = _check_kline_sufficiency(stock_code, date.today())
|
||||
if ok:
|
||||
print(f'{stock_code}: ✅ 通过 (close={close:.2f})')
|
||||
else:
|
||||
print(f'{stock_code}: ❌ {reason}')
|
||||
|
||||
|
||||
# ============================================================
|
||||
# list-candidates 命令
|
||||
# ============================================================
|
||||
def _cmd_list_candidates():
|
||||
from core.scoring.features.validator import load_candidates
|
||||
|
||||
ctx = load_candidates(date.today())
|
||||
print(f'候选股总数: {len(ctx.candidates)}')
|
||||
print(f'排除: {len(ctx.excluded)}')
|
||||
print(f'\n候选股 (前100):')
|
||||
for i, code in enumerate(ctx.candidates[:100]):
|
||||
print(f' {i + 1}. {code}')
|
||||
if ctx.excluded:
|
||||
print(f'\n排除原因 (前20):')
|
||||
for i, (code, reason) in enumerate(list(ctx.excluded.items())[:20]):
|
||||
print(f' {code}: {reason}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,54 @@
|
||||
"""
|
||||
grid_seeker v6.6 评分配置常量
|
||||
"""
|
||||
from pathlib import Path
|
||||
import config as app_config
|
||||
|
||||
# ---- 网格交易参数 ----
|
||||
GRID_LOW = 1 # 网格下限
|
||||
GRID_HIGH = 11 # 网格上限
|
||||
GRID_STEP = 1 # 网格间距（整数格）
|
||||
|
||||
# ---- 候选股过滤 ----
|
||||
FILTER_MIN_CLOSE = 8 # 最低收盘价
|
||||
FILTER_MAX_CLOSE = 13 # 最高收盘价
|
||||
REQUIRE_DAYS = 120 # 最少交易日数
|
||||
|
||||
# ---- 窗口参数 ----
|
||||
WINDOW_180D = 180 # 长窗口（情绪特征）
|
||||
WINDOW_60D = 60 # 中窗口
|
||||
WINDOW_20D = 20 # 短窗口（独立性特征）
|
||||
ATR_PERIOD = 14 # ATR 周期
|
||||
|
||||
# ---- 指数 ----
|
||||
HS300_CODE = '000300' # 沪深300基准指数
|
||||
TRACKED_INDICES = [
|
||||
'000001', # 上证指数
|
||||
'000300', # 沪深300
|
||||
'000852', # 中证1000
|
||||
'000905', # 中证500
|
||||
'399001', # 深证成指
|
||||
'399006', # 创业板指
|
||||
]
|
||||
|
||||
# ---- 市场状态 ----
|
||||
PANIC_ADVANCE_RATIO = 0.20 # 恐慌日涨跌比阈值
|
||||
GREED_ADVANCE_RATIO = 0.55 # 贪婪日涨跌比阈值
|
||||
PANIC_TURNOVER_RATIO = 1.5 # 恐慌日量比阈值
|
||||
|
||||
# ---- 模型文件 ----
|
||||
def model_dir() -> Path:
|
||||
"""模型文件目录"""
|
||||
return app_config.app_dir() / 'models'
|
||||
|
||||
def get_model_path(name: str) -> Path:
|
||||
"""获取指定模型文件路径"""
|
||||
return model_dir() / f'{name}.pkl'
|
||||
|
||||
# 3 级模型文件名 (v6.6)
|
||||
RANK_MODEL = 'rank'
|
||||
TOP_MODEL = 'top'
|
||||
STACKING_MODEL = 'stacking'
|
||||
|
||||
# Stacking 选股阈值
|
||||
STACKING_THRESHOLD = 0.35
|
||||
@@ -0,0 +1,3 @@
|
||||
# 特征工程子包
|
||||
from core.scoring.features.validator import load_candidates, DataContext
|
||||
from core.scoring.features.pipeline import FeaturePipeline
|
||||
@@ -0,0 +1,147 @@
|
||||
"""
|
||||
情绪弹性特征 (4维) — calculate_relaxed_emotion_features(df, market_regime)
|
||||
180 日窗口，筛选 advance_ratio < 0.20 恐慌日 + advance_ratio > 0.55 贪婪日。
|
||||
"""
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from core.scoring.config import PANIC_ADVANCE_RATIO, GREED_ADVANCE_RATIO, WINDOW_180D
|
||||
|
||||
|
||||
def calculate_relaxed_emotion_features(ctx) -> pd.DataFrame:
|
||||
"""计算情绪弹性特征 (4维)"""
|
||||
kline = ctx.kline.copy()
|
||||
mkt = ctx.market_regime.copy() if ctx.market_regime is not None else pd.DataFrame()
|
||||
|
||||
if kline.empty or mkt.empty:
|
||||
return pd.DataFrame()
|
||||
|
||||
# 筛选恐慌日和贪婪日
|
||||
panic_dates = set()
|
||||
greed_dates = set()
|
||||
|
||||
for _, row in mkt.iterrows():
|
||||
td = row['trade_date']
|
||||
ar = row.get('advance_ratio', 0.5)
|
||||
if ar < PANIC_ADVANCE_RATIO:
|
||||
panic_dates.add(td.date() if hasattr(td, 'date') else td)
|
||||
if ar > GREED_ADVANCE_RATIO:
|
||||
greed_dates.add(td.date() if hasattr(td, 'date') else td)
|
||||
|
||||
kline = kline.sort_values(['stock_code', 'trade_date'])
|
||||
kline['td'] = (kline['trade_date'].dt.date
|
||||
if hasattr(kline['trade_date'], 'dt')
|
||||
else pd.to_datetime(kline['trade_date']).dt.date)
|
||||
|
||||
candidates = ctx.candidates
|
||||
|
||||
# 计算全市场平均振幅 (用于恐慌/贪婪 ratio)
|
||||
kline['amp'] = (kline['high'] - kline['low']) / np.where(
|
||||
kline['open'] > 0, kline['open'], 1
|
||||
)
|
||||
market_amp = kline.groupby('td')['amp'].mean().to_dict()
|
||||
|
||||
features = {}
|
||||
grouped = kline.groupby('stock_code')
|
||||
|
||||
for code, group in grouped:
|
||||
if code not in candidates:
|
||||
continue
|
||||
if len(group) < 20:
|
||||
continue
|
||||
|
||||
g = group.sort_values('trade_date').tail(WINDOW_180D)
|
||||
closes = g['close'].values
|
||||
opens = g['open'].values
|
||||
highs = g['high'].values
|
||||
lows = g['low'].values
|
||||
dates = g['td'].values
|
||||
amps = (highs - lows) / np.where(opens > 0, opens, 1)
|
||||
|
||||
feat = {}
|
||||
|
||||
# 恐慌日分析
|
||||
panic_indices = [i for i, d in enumerate(dates) if d in panic_dates]
|
||||
if panic_indices:
|
||||
panic_amp_ratios = []
|
||||
panic_drop_ratios = []
|
||||
panic_rebounds = []
|
||||
|
||||
for pi in panic_indices:
|
||||
td = dates[pi]
|
||||
mkt_amp_val = market_amp.get(td, amps[pi])
|
||||
if mkt_amp_val > 0:
|
||||
panic_amp_ratios.append(amps[pi] / mkt_amp_val)
|
||||
|
||||
# 恐慌日跌幅
|
||||
if pi > 0 and closes[pi - 1] > 0:
|
||||
drop = (closes[pi] - closes[pi - 1]) / closes[pi - 1]
|
||||
# 全市场跌幅: 用 advance_ratio 估算
|
||||
ar = _get_advance_ratio(mkt, td)
|
||||
market_drop = -0.02 if ar < PANIC_ADVANCE_RATIO else -0.005
|
||||
if market_drop != 0:
|
||||
panic_drop_ratios.append(drop / market_drop)
|
||||
|
||||
# 恐慌次日反弹
|
||||
if pi + 1 < len(g) and closes[pi] > 0:
|
||||
panic_rebounds.append(highs[pi + 1] / closes[pi] - 1)
|
||||
|
||||
# 64. relaxed_panic_amplitude_ratio
|
||||
feat['relaxed_panic_amplitude_ratio'] = (
|
||||
np.median(panic_amp_ratios) if panic_amp_ratios else 1.0
|
||||
)
|
||||
|
||||
# 65. relaxed_panic_rebound_strength
|
||||
feat['relaxed_panic_rebound_strength'] = (
|
||||
np.median(panic_rebounds) if panic_rebounds else 0.0
|
||||
)
|
||||
else:
|
||||
feat['relaxed_panic_amplitude_ratio'] = 1.0
|
||||
feat['relaxed_panic_rebound_strength'] = 0.0
|
||||
|
||||
# 贪婪日分析
|
||||
greed_indices = [i for i, d in enumerate(dates) if d in greed_dates]
|
||||
if greed_indices:
|
||||
greed_gains = []
|
||||
greed_amp_ratios = []
|
||||
|
||||
for gi in greed_indices:
|
||||
td = dates[gi]
|
||||
if gi > 0 and closes[gi - 1] > 0:
|
||||
gain = (closes[gi] - closes[gi - 1]) / closes[gi - 1]
|
||||
# 全市场收益
|
||||
ar = _get_advance_ratio(mkt, td)
|
||||
market_gain = 0.01 if ar > GREED_ADVANCE_RATIO else 0.003
|
||||
if market_gain > 0:
|
||||
greed_gains.append(gain / market_gain)
|
||||
|
||||
mkt_amp_val = market_amp.get(td, amps[gi])
|
||||
if mkt_amp_val > 0:
|
||||
greed_amp_ratios.append(amps[gi] / mkt_amp_val)
|
||||
|
||||
# 66. relaxed_greed_relative_gain
|
||||
feat['relaxed_greed_relative_gain'] = (
|
||||
np.median(greed_gains) if greed_gains else 0.0
|
||||
)
|
||||
|
||||
# 67. relaxed_greed_amplitude_ratio
|
||||
feat['relaxed_greed_amplitude_ratio'] = (
|
||||
np.median(greed_amp_ratios) if greed_amp_ratios else 1.0
|
||||
)
|
||||
else:
|
||||
feat['relaxed_greed_relative_gain'] = 0.0
|
||||
feat['relaxed_greed_amplitude_ratio'] = 1.0
|
||||
|
||||
features[code] = feat
|
||||
|
||||
return pd.DataFrame.from_dict(features, orient='index')
|
||||
|
||||
|
||||
def _get_advance_ratio(mkt_df, trade_date):
|
||||
"""获取指定日期的 advance_ratio"""
|
||||
if mkt_df is None or mkt_df.empty:
|
||||
return 0.5
|
||||
td = trade_date
|
||||
match = mkt_df[mkt_df['trade_date'] == td]
|
||||
if not match.empty:
|
||||
return match.iloc[0].get('advance_ratio', 0.5)
|
||||
return 0.5
|
||||
@@ -0,0 +1,97 @@
|
||||
"""
|
||||
大盘独立性特征 (3维) — calculate_independence_features(df)
|
||||
使用 HS300(000300) 作为 benchmark，最近 20 个交易日 OLS 回归。
|
||||
"""
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from core.scoring.config import WINDOW_20D
|
||||
from core.scoring.features.v3_2_features import _ols_slope
|
||||
|
||||
|
||||
def calculate_independence_features(ctx) -> pd.DataFrame:
|
||||
"""计算大盘独立性特征 (3维)"""
|
||||
kline = ctx.kline.copy()
|
||||
hs300 = ctx.hs300_kline.copy() if ctx.hs300_kline is not None else pd.DataFrame()
|
||||
|
||||
if kline.empty or hs300.empty:
|
||||
return pd.DataFrame()
|
||||
|
||||
# 准备 HS300 收益率序列
|
||||
hs300 = hs300.sort_values('trade_date')
|
||||
hs300['market_return'] = hs300['close'].pct_change()
|
||||
hs300['market_amplitude'] = (hs300['high'] - hs300['low']) / hs300['open']
|
||||
hs300 = hs300.dropna(subset=['market_return', 'market_amplitude'])
|
||||
|
||||
# 对齐日期
|
||||
hs300_dates = set(hs300['trade_date'].dt.date
|
||||
if hasattr(hs300['trade_date'], 'dt') else hs300['trade_date'])
|
||||
|
||||
kline = kline.sort_values(['stock_code', 'trade_date'])
|
||||
kline['trade_date_dt'] = (kline['trade_date'].dt.date
|
||||
if hasattr(kline['trade_date'], 'dt')
|
||||
else pd.to_datetime(kline['trade_date']).dt.date)
|
||||
|
||||
candidates = ctx.candidates
|
||||
|
||||
# 计算 HS300 最近20日平均振幅
|
||||
hs300_tail = hs300.tail(WINDOW_20D)
|
||||
hs300_avg_amp = hs300_tail['market_amplitude'].mean() if len(hs300_tail) > 0 else 0
|
||||
|
||||
features = {}
|
||||
grouped = kline.groupby('stock_code')
|
||||
|
||||
for code, group in grouped:
|
||||
if code not in candidates:
|
||||
continue
|
||||
if len(group) < 20:
|
||||
continue
|
||||
|
||||
g = group.sort_values('trade_date').tail(120)
|
||||
closes = g['close'].values
|
||||
|
||||
# 计算个股日收益率
|
||||
stock_rets = np.diff(closes) / np.where(closes[:-1] > 0, closes[:-1], 1)
|
||||
|
||||
# 对齐 HS300 收益率 (取对应日期)
|
||||
# 简化: 取最近 N 个交易日的数据点
|
||||
n = min(WINDOW_20D, len(stock_rets))
|
||||
|
||||
# 获取 HS300 最近 n 天的 market_return
|
||||
market_rets = hs300['market_return'].tail(n + 1).values
|
||||
if len(market_rets) < n:
|
||||
market_rets = hs300['market_return'].values[-n - 1:]
|
||||
|
||||
# 对齐长度
|
||||
min_len = min(n, len(market_rets) - 1, len(stock_rets))
|
||||
if min_len < 5: # 至少需要5个数据点做回归
|
||||
continue
|
||||
|
||||
stock_ret_window = stock_rets[-min_len:]
|
||||
market_ret_window = market_rets[-min_len:]
|
||||
|
||||
feat = {}
|
||||
|
||||
# OLS 回归: stock_ret ~ market_return
|
||||
try:
|
||||
slope, r_value = _ols_slope(market_ret_window, stock_ret_window)
|
||||
residuals = stock_ret_window - slope * market_ret_window
|
||||
|
||||
# 58. market_residual_volatility_20d: std(residuals) × √252
|
||||
feat['market_residual_volatility_20d'] = np.std(residuals, ddof=1) * np.sqrt(252)
|
||||
|
||||
# 59. market_independence_ratio_20d: 1 - R²
|
||||
feat['market_independence_ratio_20d'] = 1 - r_value ** 2
|
||||
|
||||
except Exception:
|
||||
feat['market_residual_volatility_20d'] = 0
|
||||
feat['market_independence_ratio_20d'] = 1
|
||||
|
||||
# 60. market_amplitude_deviation_20d: 个股平均振幅 - HS300 平均振幅
|
||||
g_amps = (g['high'].values[-20:] - g['low'].values[-20:]) / np.where(
|
||||
g['open'].values[-20:] > 0, g['open'].values[-20:], 1
|
||||
)
|
||||
feat['market_amplitude_deviation_20d'] = np.mean(g_amps) - hs300_avg_amp
|
||||
|
||||
features[code] = feat
|
||||
|
||||
return pd.DataFrame.from_dict(features, orient='index')
|
||||
@@ -0,0 +1,84 @@
|
||||
"""
|
||||
负向指标 (5维) — calculate_negative_features(df)
|
||||
注意: #53 trend_consistency_20d 与 #39 同名不同义，更新字典时会覆盖 #39
|
||||
"""
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
|
||||
def calculate_negative_features(ctx) -> pd.DataFrame:
|
||||
"""计算负向指标 (5维)"""
|
||||
kline = ctx.kline.copy()
|
||||
if kline.empty:
|
||||
return pd.DataFrame()
|
||||
|
||||
kline = kline.sort_values(['stock_code', 'trade_date'])
|
||||
candidates = ctx.candidates
|
||||
|
||||
features = {}
|
||||
grouped = kline.groupby('stock_code')
|
||||
|
||||
for code, group in grouped:
|
||||
if code not in candidates:
|
||||
continue
|
||||
g = group.tail(120)
|
||||
if len(g) < 20:
|
||||
continue
|
||||
|
||||
closes = g['close'].values
|
||||
opens = g['open'].values
|
||||
highs = g['high'].values
|
||||
lows = g['low'].values
|
||||
volumes = g['volume'].values
|
||||
|
||||
feat = {}
|
||||
|
||||
# 53. trend_consistency_20d (负向版本): |mean(return>0) - 0.5| × 100
|
||||
# 衡量偏离均衡的程度，越接近50%越弱
|
||||
rets_20 = np.diff(closes[-21:]) / np.where(closes[-21:-1] > 0, closes[-21:-1], 1)
|
||||
feat['trend_consistency_20d'] = abs(np.mean(rets_20 > 0) - 0.5) * 100
|
||||
|
||||
# 54. max_consecutive_direction_20d: 最大连续同向天数
|
||||
rets_sign = np.sign(np.diff(closes[-21:]))
|
||||
max_consec = 0
|
||||
curr_consec = 0
|
||||
curr_sign = 0
|
||||
for s in rets_sign:
|
||||
if s != 0 and s == curr_sign:
|
||||
curr_consec += 1
|
||||
elif s != 0:
|
||||
curr_sign = s
|
||||
curr_consec = 1
|
||||
else:
|
||||
curr_consec = 0
|
||||
max_consec = max(max_consec, curr_consec)
|
||||
feat['max_consecutive_direction_20d'] = max_consec
|
||||
|
||||
# 55. gap_risk_20d: mean(|open_t - close_{t-1}|/close_{t-1} > 0.02) × 100
|
||||
gap_count = 0
|
||||
n = 0
|
||||
for i in range(max(0, len(g) - 20), len(g)):
|
||||
if i > 0 and closes[i - 1] > 0:
|
||||
gap_pct = abs(opens[i] - closes[i - 1]) / closes[i - 1]
|
||||
if gap_pct > 0.02:
|
||||
gap_count += 1
|
||||
n += 1
|
||||
feat['gap_risk_20d'] = gap_count / n * 100 if n > 0 else 0
|
||||
|
||||
# 56. liquidity_drying_up_20d: min(vol_20d)/mean(vol_60d)
|
||||
vol_20 = volumes[-20:]
|
||||
vol_60 = volumes[-60:] if len(volumes) >= 60 else volumes
|
||||
feat['liquidity_drying_up_20d'] = (
|
||||
np.min(vol_20) / np.mean(vol_60) if np.mean(vol_60) > 0 else 1
|
||||
)
|
||||
|
||||
# 57. price_stagnation_20d: (max(high_20d)-min(low_20d))/close × 100
|
||||
h20 = np.max(highs[-20:])
|
||||
l20 = np.min(lows[-20:])
|
||||
feat['price_stagnation_20d'] = (
|
||||
(h20 - l20) / closes[-1] * 100 if closes[-1] > 0 else 0
|
||||
)
|
||||
|
||||
features[code] = feat
|
||||
|
||||
return pd.DataFrame.from_dict(features, orient='index')
|
||||
@@ -0,0 +1,106 @@
|
||||
"""
|
||||
特征工程编排器 — 串联全部特征组，输出完整特征 DataFrame
|
||||
"""
|
||||
import pandas as pd
|
||||
from datetime import date
|
||||
from core.scoring.features.validator import load_candidates, DataContext
|
||||
from core.scoring.features.v3_2_features import calculate_features_v3_2
|
||||
from core.scoring.features.v3_3_features import calculate_features_v3_3
|
||||
from core.scoring.features.v3_4_features import calculate_features_v3_4
|
||||
from core.scoring.features.negative_features import calculate_negative_features
|
||||
from core.scoring.features.independence_features import calculate_independence_features
|
||||
from core.scoring.features.sector_features import calculate_sector_independence_features
|
||||
from core.scoring.features.emotion_features import calculate_relaxed_emotion_features
|
||||
from core.logger import LogLevel, PrintLog
|
||||
|
||||
|
||||
class FeaturePipeline:
|
||||
"""
|
||||
特征工程管道 — 串联 8 组特征计算，输出完整特征矩阵。
|
||||
|
||||
Usage:
|
||||
pipeline = FeaturePipeline(trade_date=date.today())
|
||||
feature_df = pipeline.run() # DataFrame indexed by stock_code
|
||||
"""
|
||||
|
||||
def __init__(self, trade_date: date):
|
||||
self.trade_date = trade_date
|
||||
self.ctx: DataContext = None
|
||||
|
||||
def run(self) -> pd.DataFrame:
|
||||
"""
|
||||
执行完整特征工程管道。
|
||||
返回: DataFrame indexed by stock_code, columns = 全部特征
|
||||
"""
|
||||
# Stage 0: 加载候选股和数据
|
||||
PrintLog(LogLevel.INFO, '[pipeline] Stage 0: 加载候选股...')
|
||||
self.ctx = load_candidates(self.trade_date)
|
||||
if not self.ctx.candidates:
|
||||
PrintLog(LogLevel.WARNING, '[pipeline] 无候选股通过过滤')
|
||||
return pd.DataFrame()
|
||||
|
||||
PrintLog(LogLevel.INFO, f'[pipeline] 候选股: {len(self.ctx.candidates)}, '
|
||||
f'排除: {len(self.ctx.excluded)}')
|
||||
|
||||
# Stage 1: v3.2 基础特征 (20维)
|
||||
PrintLog(LogLevel.INFO, '[pipeline] Stage 1: v3.2 基础特征 (20维)...')
|
||||
df = calculate_features_v3_2(self.ctx)
|
||||
PrintLog(LogLevel.INFO, f'[pipeline] → {len(df)} stocks, {len(df.columns)} features')
|
||||
|
||||
# Stage 2: v3.3 扩展特征 (16维)
|
||||
PrintLog(LogLevel.INFO, '[pipeline] Stage 2: v3.3 扩展特征 (16维)...')
|
||||
df_v33 = calculate_features_v3_3(self.ctx)
|
||||
df = df.join(df_v33, how='inner', rsuffix='_v33')
|
||||
PrintLog(LogLevel.INFO, f'[pipeline] → {len(df)} stocks, {len(df.columns)} features')
|
||||
|
||||
# Stage 3: v3.4 扩展特征 (16维)
|
||||
PrintLog(LogLevel.INFO, '[pipeline] Stage 3: v3.4 扩展特征 (16维)...')
|
||||
df_v34 = calculate_features_v3_4(self.ctx)
|
||||
df = df.join(df_v34, how='inner', rsuffix='_v34')
|
||||
PrintLog(LogLevel.INFO, f'[pipeline] → {len(df)} stocks, {len(df.columns)} features')
|
||||
|
||||
# Stage 4: 负向指标 (5维) — 注意 #53 覆盖 #39
|
||||
PrintLog(LogLevel.INFO, '[pipeline] Stage 4: 负向指标 (5维)...')
|
||||
df_neg = calculate_negative_features(self.ctx)
|
||||
# 使用 update 模式: 负向指标的 trend_consistency_20d 覆盖 v3.4 版本
|
||||
common_cols = set(df.columns) & set(df_neg.columns)
|
||||
for col in common_cols:
|
||||
df[col] = df_neg[col] # 覆盖
|
||||
new_cols = set(df_neg.columns) - common_cols
|
||||
for col in new_cols:
|
||||
df[col] = df_neg[col]
|
||||
PrintLog(LogLevel.INFO, f'[pipeline] → {len(df)} stocks, {len(df.columns)} features')
|
||||
|
||||
# Stage 5: 大盘独立性 (3维)
|
||||
PrintLog(LogLevel.INFO, '[pipeline] Stage 5: 大盘独立性 (3维)...')
|
||||
df_ind = calculate_independence_features(self.ctx)
|
||||
df = df.join(df_ind, how='left')
|
||||
df[df_ind.columns] = df[df_ind.columns].fillna(0)
|
||||
PrintLog(LogLevel.INFO, f'[pipeline] → {len(df)} stocks, {len(df.columns)} features')
|
||||
|
||||
# Stage 6: 行业独立性 (3维)
|
||||
PrintLog(LogLevel.INFO, '[pipeline] Stage 6: 行业独立性 (3维)...')
|
||||
df_sec = calculate_sector_independence_features(self.ctx)
|
||||
df = df.join(df_sec, how='left')
|
||||
df[df_sec.columns] = df[df_sec.columns].fillna(0)
|
||||
PrintLog(LogLevel.INFO, f'[pipeline] → {len(df)} stocks, {len(df.columns)} features')
|
||||
|
||||
# Stage 7: 情绪弹性 (4维)
|
||||
PrintLog(LogLevel.INFO, '[pipeline] Stage 7: 情绪弹性 (4维)...')
|
||||
df_emo = calculate_relaxed_emotion_features(self.ctx)
|
||||
df = df.join(df_emo, how='left')
|
||||
df[df_emo.columns] = df[df_emo.columns].fillna(1.0)
|
||||
PrintLog(LogLevel.INFO, f'[pipeline] → {len(df)} stocks, {len(df.columns)} features')
|
||||
|
||||
# 添加 latest_close 列 (Meta Ranker 输入)
|
||||
df['latest_close'] = 0.0
|
||||
for code in df.index:
|
||||
g = self.ctx.kline[self.ctx.kline['stock_code'] == code]
|
||||
if not g.empty:
|
||||
g_sorted = g.sort_values('trade_date')
|
||||
df.at[code, 'latest_close'] = float(g_sorted['close'].iloc[-1])
|
||||
|
||||
PrintLog(LogLevel.INFO,
|
||||
f'[pipeline] 完成: {len(df)} 只股票, {len(df.columns)} 维特征')
|
||||
|
||||
return df
|
||||
@@ -0,0 +1,88 @@
|
||||
"""
|
||||
行业独立性特征 (3维) — calculate_sector_independence_features(df)
|
||||
匹配 sector_features_daily，最近 20 个交易日 OLS 回归。
|
||||
"""
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from core.scoring.config import WINDOW_20D
|
||||
from core.scoring.features.v3_2_features import _ols_slope
|
||||
|
||||
|
||||
def calculate_sector_independence_features(ctx) -> pd.DataFrame:
|
||||
"""计算行业独立性特征 (3维)"""
|
||||
kline = ctx.kline.copy()
|
||||
sector_df = ctx.sector_features.copy() if ctx.sector_features is not None else pd.DataFrame()
|
||||
industry_map = ctx.industry_map
|
||||
|
||||
if kline.empty or sector_df.empty or not industry_map:
|
||||
return pd.DataFrame()
|
||||
|
||||
sector_df = sector_df.sort_values(['sector_name', 'trade_date'])
|
||||
kline = kline.sort_values(['stock_code', 'trade_date'])
|
||||
|
||||
candidates = ctx.candidates
|
||||
|
||||
features = {}
|
||||
grouped = kline.groupby('stock_code')
|
||||
|
||||
for code, group in grouped:
|
||||
if code not in candidates:
|
||||
continue
|
||||
if len(group) < 20:
|
||||
continue
|
||||
|
||||
# 获取行业名称
|
||||
sector_name = industry_map.get(str(code), '')
|
||||
if not sector_name:
|
||||
continue
|
||||
|
||||
# 获取该行业的指数数据
|
||||
sector_data = sector_df[sector_df['sector_name'] == sector_name]
|
||||
if sector_data.empty or len(sector_data) < 5:
|
||||
continue
|
||||
|
||||
sector_data = sector_data.sort_values('trade_date').tail(120)
|
||||
sector_rets = sector_data['sector_ret'].values / 100 # 转为小数
|
||||
sector_amps = sector_data['sector_amplitude'].values / 100
|
||||
|
||||
g = group.sort_values('trade_date').tail(120)
|
||||
closes = g['close'].values
|
||||
|
||||
# 个股日收益率
|
||||
stock_rets = np.diff(closes) / np.where(closes[:-1] > 0, closes[:-1], 1)
|
||||
|
||||
# 对齐长度
|
||||
n = min(WINDOW_20D, len(stock_rets), len(sector_rets) - 1)
|
||||
if n < 5:
|
||||
continue
|
||||
|
||||
stock_ret_window = stock_rets[-n:]
|
||||
sector_ret_window = sector_rets[-n:]
|
||||
|
||||
feat = {}
|
||||
|
||||
# OLS: stock_ret ~ sector_ret
|
||||
try:
|
||||
slope, r_value = _ols_slope(sector_ret_window, stock_ret_window)
|
||||
residuals = stock_ret_window - slope * sector_ret_window
|
||||
|
||||
# 61. sector_residual_volatility_20d
|
||||
feat['sector_residual_volatility_20d'] = np.std(residuals, ddof=1) * np.sqrt(252)
|
||||
|
||||
# 62. sector_independence_ratio_20d: 1 - R²
|
||||
feat['sector_independence_ratio_20d'] = 1 - r_value ** 2
|
||||
|
||||
except Exception:
|
||||
feat['sector_residual_volatility_20d'] = 0
|
||||
feat['sector_independence_ratio_20d'] = 1
|
||||
|
||||
# 63. sector_amplitude_deviation_20d: 个股振幅 - 行业平均振幅
|
||||
g_amps = (g['high'].values[-20:] - g['low'].values[-20:]) / np.where(
|
||||
g['open'].values[-20:] > 0, g['open'].values[-20:], 1
|
||||
)
|
||||
sector_avg_amp = np.mean(sector_amps[-20:]) if len(sector_amps) >= 20 else np.mean(sector_amps)
|
||||
feat['sector_amplitude_deviation_20d'] = np.mean(g_amps) - sector_avg_amp
|
||||
|
||||
features[code] = feat
|
||||
|
||||
return pd.DataFrame.from_dict(features, orient='index')
|
||||
@@ -0,0 +1,203 @@
|
||||
"""
|
||||
基础特征 v3.2 (20维) — calculate_features(df, "v3.2")
|
||||
"""
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from core.scoring.config import GRID_LOW, GRID_HIGH
|
||||
|
||||
|
||||
def calculate_features_v3_2(ctx) -> pd.DataFrame:
|
||||
"""
|
||||
计算 v3.2 基础特征 (20维)。
|
||||
返回 DataFrame indexed by stock_code。
|
||||
"""
|
||||
kline = ctx.kline.copy()
|
||||
if kline.empty:
|
||||
return pd.DataFrame()
|
||||
|
||||
kline = kline.sort_values(['stock_code', 'trade_date'])
|
||||
candidates = ctx.candidates
|
||||
|
||||
features = {}
|
||||
grouped = kline.groupby('stock_code')
|
||||
|
||||
for code, group in grouped:
|
||||
if code not in candidates:
|
||||
continue
|
||||
g = group.tail(120) # 取最近120日用于大部分窗口
|
||||
if len(g) < 20:
|
||||
continue
|
||||
|
||||
closes = g['close'].values
|
||||
opens = g['open'].values
|
||||
highs = g['high'].values
|
||||
lows = g['low'].values
|
||||
volumes = g['volume'].values
|
||||
|
||||
feat = {}
|
||||
|
||||
# 1. rolling_grid_ratio_20d: mean(close∈[1,11]) × 100
|
||||
recent_20_close = closes[-20:]
|
||||
feat['rolling_grid_ratio_20d'] = (
|
||||
np.mean((recent_20_close >= GRID_LOW) & (recent_20_close <= GRID_HIGH)) * 100
|
||||
)
|
||||
|
||||
# 2. cross_freq_20d: 日高低区间穿越≥1条整数网格线的天数比例
|
||||
cross_count = 0
|
||||
for i in range(max(0, len(g) - 20), len(g)):
|
||||
h, l = highs[i], lows[i]
|
||||
if h > l:
|
||||
grid_low = int(np.ceil(l))
|
||||
grid_high = int(np.floor(h))
|
||||
if grid_high >= grid_low:
|
||||
cross_count += 1
|
||||
feat['cross_freq_20d'] = cross_count / min(20, len(g)) * 100
|
||||
|
||||
# 3. avg_daily_amp: mean((high-low)/open × 100)
|
||||
amps = (highs - lows) / np.where(opens > 0, opens, 1) * 100
|
||||
feat['avg_daily_amp'] = np.mean(amps[-20:])
|
||||
|
||||
# 4. high_amp_days: mean(振幅>2%) × 100
|
||||
feat['high_amp_days'] = np.mean(amps[-20:] > 2) * 100
|
||||
|
||||
# 5. atr_pct: mean(ATR_14)/close × 100
|
||||
atr = _compute_atr(highs, lows, closes, 14)
|
||||
feat['atr_pct'] = np.mean(atr[-14:]) / closes[-1] * 100 if closes[-1] > 0 else 0
|
||||
|
||||
# 6. volatility_20d: std(log_return) × √252 × 100
|
||||
log_rets = np.diff(np.log(np.maximum(closes, 1e-10)))
|
||||
vol_20 = np.std(log_rets[-20:], ddof=1) if len(log_rets) >= 20 else 0
|
||||
feat['volatility_20d'] = vol_20 * np.sqrt(252) * 100
|
||||
|
||||
# 7. price_cv: std(close)/mean(close) × 100
|
||||
feat['price_cv'] = np.std(closes) / np.mean(closes) * 100 if np.mean(closes) > 0 else 0
|
||||
|
||||
# 8. bb_width: (MA20+2σ - (MA20-2σ))/MA20 × 100
|
||||
ma20 = np.mean(closes[-20:])
|
||||
std20 = np.std(closes[-20:], ddof=1)
|
||||
feat['bb_width'] = (4 * std20) / ma20 * 100 if ma20 > 0 else 0
|
||||
|
||||
# 9. volume_ratio: mean(vol_20d)/mean(vol_60d)
|
||||
vol_20m = np.mean(volumes[-20:])
|
||||
vol_60m = np.mean(volumes[-60:]) if len(volumes) >= 60 else vol_20m
|
||||
feat['volume_ratio'] = vol_20m / vol_60m if vol_60m > 0 else 1
|
||||
|
||||
# 10. obv_slope: OBV序列最近20日线性回归斜率
|
||||
obv = _compute_obv(closes, volumes)
|
||||
feat['obv_slope'] = _ols_slope(np.arange(20), obv[-20:])[0] if len(obv) >= 20 else 0
|
||||
|
||||
# 11. amplitude_cv: std(振幅)/mean(振幅)
|
||||
feat['amplitude_cv'] = (np.std(amps[-20:]) / np.mean(amps[-20:])
|
||||
if np.mean(amps[-20:]) > 0 else 0)
|
||||
|
||||
# 12. price_entropy: Shannon熵 (10 bins)
|
||||
feat['price_entropy'] = _shannon_entropy(closes[-60:], bins=10)
|
||||
|
||||
# 13. intraday_trend_strength: mean(|close-open|/open) × 100
|
||||
intraday = np.abs(closes[-20:] - opens[-20:]) / np.where(opens[-20:] > 0, opens[-20:], 1) * 100
|
||||
feat['intraday_trend_strength'] = np.mean(intraday)
|
||||
|
||||
# 14-17. 交叉特征 (依赖前序特征)
|
||||
feat['amp_x_grid'] = feat['avg_daily_amp'] * feat['rolling_grid_ratio_20d'] / 100
|
||||
feat['amp_x_grid_vol'] = (feat['avg_daily_amp'] * feat['rolling_grid_ratio_20d'] / 100 *
|
||||
feat['volume_ratio'] / 10000)
|
||||
feat['amp_cv_x_entropy'] = feat['amplitude_cv'] * feat['price_entropy']
|
||||
feat['cross_freq_x_bb'] = feat['cross_freq_20d'] * feat['bb_width'] / 100
|
||||
|
||||
# 18. ln_float_mv: ln(close × total_share + 1)
|
||||
total_share = _get_total_share(ctx, code)
|
||||
float_mv = closes[-1] * total_share if total_share else closes[-1] * 1e8
|
||||
feat['ln_float_mv'] = np.log(float_mv + 1)
|
||||
|
||||
# 19. mv_vol_interact: ln_float_mv × volatility_20d/100
|
||||
feat['mv_vol_interact'] = feat['ln_float_mv'] * feat['volatility_20d'] / 100
|
||||
|
||||
# 20. small_cap_premium: 1/(浮动市值 + 1)
|
||||
feat['small_cap_premium'] = 1.0 / (float_mv + 1)
|
||||
|
||||
features[code] = feat
|
||||
|
||||
return pd.DataFrame.from_dict(features, orient='index')
|
||||
|
||||
|
||||
# ---- 辅助函数 ----
|
||||
|
||||
def _compute_atr(highs, lows, closes, period=14):
|
||||
"""计算 ATR"""
|
||||
n = len(closes)
|
||||
tr = np.zeros(n)
|
||||
for i in range(1, n):
|
||||
h_l = highs[i] - lows[i]
|
||||
h_c = abs(highs[i] - closes[i - 1])
|
||||
l_c = abs(lows[i] - closes[i - 1])
|
||||
tr[i] = max(h_l, h_c, l_c)
|
||||
atr = np.zeros(n)
|
||||
atr[:period] = np.mean(tr[:period])
|
||||
for i in range(period, n):
|
||||
atr[i] = (atr[i - 1] * (period - 1) + tr[i]) / period
|
||||
return atr
|
||||
|
||||
|
||||
def _compute_obv(closes, volumes):
|
||||
"""计算 OBV"""
|
||||
obv = np.zeros(len(closes))
|
||||
obv[0] = volumes[0]
|
||||
for i in range(1, len(closes)):
|
||||
if closes[i] > closes[i - 1]:
|
||||
obv[i] = obv[i - 1] + volumes[i]
|
||||
elif closes[i] < closes[i - 1]:
|
||||
obv[i] = obv[i - 1] - volumes[i]
|
||||
else:
|
||||
obv[i] = obv[i - 1]
|
||||
return obv
|
||||
|
||||
|
||||
def _shannon_entropy(values, bins=10):
|
||||
"""Shannon 熵"""
|
||||
if len(values) < bins:
|
||||
return 0.0
|
||||
hist, _ = np.histogram(values, bins=bins)
|
||||
hist = hist / hist.sum()
|
||||
hist = hist[hist > 0]
|
||||
return -np.sum(hist * np.log2(hist))
|
||||
|
||||
|
||||
def _get_total_share(ctx, code):
|
||||
"""从 StockInfo 获取总股本"""
|
||||
# 尝试各种前缀
|
||||
for prefix in ['', 'SH', 'SZ', 'BJ']:
|
||||
key = f'{code}.{prefix}' if prefix else code
|
||||
info = ctx.stock_info.get(key, {})
|
||||
ts = info.get('total_share', None)
|
||||
if ts and ts > 0:
|
||||
return ts
|
||||
return None
|
||||
|
||||
|
||||
def _ols_slope(x, y):
|
||||
"""
|
||||
纯 numpy OLS 线性回归。
|
||||
等价于 scipy.stats.linregress(x, y)，返回 (slope, r_value)。
|
||||
slope=0 且 r_value=0 表示计算失败（数据不足或方差为零）。
|
||||
"""
|
||||
if len(x) < 2 or len(y) < 2 or len(x) != len(y):
|
||||
return 0.0, 0.0
|
||||
x = np.asarray(x, dtype=float)
|
||||
y = np.asarray(y, dtype=float)
|
||||
x_mean = x.mean()
|
||||
y_mean = y.mean()
|
||||
num = np.sum((x - x_mean) * (y - y_mean))
|
||||
den = np.sum((x - x_mean) ** 2)
|
||||
if den < 1e-12:
|
||||
return 0.0, 0.0
|
||||
slope = num / den
|
||||
ss_xy = num
|
||||
ss_xx = np.sum((x - x_mean) ** 2)
|
||||
ss_yy = np.sum((y - y_mean) ** 2)
|
||||
if ss_xx < 1e-12 or ss_yy < 1e-12:
|
||||
r_value = 0.0
|
||||
else:
|
||||
r_value = ss_xy / (np.sqrt(ss_xx) * np.sqrt(ss_yy))
|
||||
return slope, r_value
|
||||
@@ -0,0 +1,130 @@
|
||||
"""
|
||||
扩展特征 v3.3 (16维)
|
||||
"""
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from core.scoring.features.v3_2_features import _ols_slope
|
||||
|
||||
|
||||
def calculate_features_v3_3(ctx) -> pd.DataFrame:
|
||||
"""计算 v3.3 扩展特征 (16维)"""
|
||||
kline = ctx.kline.copy()
|
||||
if kline.empty:
|
||||
return pd.DataFrame()
|
||||
|
||||
kline = kline.sort_values(['stock_code', 'trade_date'])
|
||||
candidates = ctx.candidates
|
||||
|
||||
features = {}
|
||||
grouped = kline.groupby('stock_code')
|
||||
|
||||
for code, group in grouped:
|
||||
if code not in candidates:
|
||||
continue
|
||||
g = group.tail(120)
|
||||
if len(g) < 20:
|
||||
continue
|
||||
|
||||
closes = g['close'].values
|
||||
highs = g['high'].values
|
||||
lows = g['low'].values
|
||||
volumes = g['volume'].values
|
||||
|
||||
feat = {}
|
||||
|
||||
# 21. grid_touch_count_20d: 高低区间触碰网格线总次数
|
||||
feat['grid_touch_count_20d'] = _grid_touch_count(highs[-20:], lows[-20:])
|
||||
|
||||
# 22. grid_touch_count_60d
|
||||
h60 = highs[-60:] if len(highs) >= 60 else highs
|
||||
l60 = lows[-60:] if len(lows) >= 60 else lows
|
||||
feat['grid_touch_count_60d'] = _grid_touch_count(h60, l60)
|
||||
|
||||
# 23. grid_cross_density_20d
|
||||
cross_count = 0
|
||||
w = min(20, len(g))
|
||||
for i in range(len(g) - w, len(g)):
|
||||
h, l = highs[i], lows[i]
|
||||
if h > l and int(np.floor(h)) >= int(np.ceil(l)):
|
||||
cross_count += 1
|
||||
feat['grid_cross_density_20d'] = cross_count / w
|
||||
|
||||
# 24. near_grid_line_ratio_20d
|
||||
recent_closes = closes[-20:]
|
||||
dist_to_int = np.abs(recent_closes - np.round(recent_closes))
|
||||
feat['near_grid_line_ratio_20d'] = np.mean(dist_to_int <= 0.15) * 100
|
||||
|
||||
# 25. trend_slope_20d
|
||||
feat['trend_slope_20d'] = _ols_slope(np.arange(20), closes[-20:])[0]
|
||||
|
||||
# 26. trend_slope_60d
|
||||
c60 = closes[-60:] if len(closes) >= 60 else closes
|
||||
feat['trend_slope_60d'] = _ols_slope(np.arange(len(c60)), c60)[0]
|
||||
|
||||
# 27. trend_abs_slope_20d
|
||||
feat['trend_abs_slope_20d'] = abs(feat['trend_slope_20d'])
|
||||
|
||||
# 28. range_position_60d
|
||||
h60_mx = np.max(highs[-60:]) if len(highs) >= 60 else np.max(highs)
|
||||
l60_mn = np.min(lows[-60:]) if len(lows) >= 60 else np.min(lows)
|
||||
feat['range_position_60d'] = (closes[-1] - l60_mn) / (h60_mx - l60_mn) * 100 \
|
||||
if h60_mx > l60_mn else 50
|
||||
|
||||
# 29. drawdown_60d
|
||||
c60_arr = closes[-60:] if len(closes) >= 60 else closes
|
||||
cummax = np.maximum.accumulate(c60_arr)
|
||||
dd = (1 - c60_arr / cummax) * 100
|
||||
feat['drawdown_60d'] = np.max(dd)
|
||||
|
||||
# 30. rebound_from_low_60d
|
||||
feat['rebound_from_low_60d'] = (closes[-1] - l60_mn) / l60_mn * 100 if l60_mn > 0 else 0
|
||||
|
||||
# 31. dist_to_grid_upper
|
||||
from core.scoring.config import GRID_HIGH
|
||||
feat['dist_to_grid_upper'] = max(0, (GRID_HIGH - closes[-1]) / 10 * 100)
|
||||
|
||||
# 32. dist_to_grid_lower
|
||||
from core.scoring.config import GRID_LOW
|
||||
feat['dist_to_grid_lower'] = max(0, (closes[-1] - GRID_LOW) / 10 * 100)
|
||||
|
||||
# 33. grid_room_balance
|
||||
upper = feat['dist_to_grid_upper']
|
||||
lower = feat['dist_to_grid_lower']
|
||||
feat['grid_room_balance'] = min(upper, lower) / (upper + lower) * 100 \
|
||||
if (upper + lower) > 0 else 50
|
||||
|
||||
# 34. amount_mean_20d
|
||||
amounts = closes[-20:] * volumes[-20:]
|
||||
feat['amount_mean_20d'] = np.mean(amounts)
|
||||
|
||||
# 35. amount_cv_20d
|
||||
feat['amount_cv_20d'] = np.std(amounts) / np.mean(amounts) * 100 \
|
||||
if np.mean(amounts) > 0 else 0
|
||||
|
||||
# 36. turnover_proxy_20d
|
||||
total_share = _get_total_share(ctx, code) or 1e8
|
||||
feat['turnover_proxy_20d'] = np.mean(volumes[-20:]) / (total_share * 1e8) * 100
|
||||
|
||||
features[code] = feat
|
||||
|
||||
return pd.DataFrame.from_dict(features, orient='index')
|
||||
|
||||
|
||||
def _get_total_share(ctx, code):
|
||||
for prefix in ['', 'SH', 'SZ', 'BJ']:
|
||||
key = f'{code}.{prefix}' if prefix else code
|
||||
info = ctx.stock_info.get(key, {})
|
||||
ts = info.get('total_share', None)
|
||||
if ts and ts > 0:
|
||||
return ts
|
||||
return None
|
||||
|
||||
|
||||
def _grid_touch_count(highs, lows):
|
||||
count = 0
|
||||
for h, l in zip(highs, lows):
|
||||
if h > l:
|
||||
grid_low = int(np.ceil(l))
|
||||
grid_high = int(np.floor(h))
|
||||
count += max(0, grid_high - grid_low + 1)
|
||||
return count
|
||||
@@ -0,0 +1,109 @@
|
||||
"""
|
||||
扩展特征 v3.4 (16维)
|
||||
"""
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from core.scoring.features.v3_2_features import _ols_slope
|
||||
from core.scoring.config import GRID_LOW, GRID_HIGH
|
||||
|
||||
|
||||
def calculate_features_v3_4(ctx) -> pd.DataFrame:
|
||||
"""计算 v3.4 扩展特征 (16维)"""
|
||||
kline = ctx.kline.copy()
|
||||
if kline.empty:
|
||||
return pd.DataFrame()
|
||||
|
||||
kline = kline.sort_values(['stock_code', 'trade_date'])
|
||||
candidates = ctx.candidates
|
||||
|
||||
features = {}
|
||||
grouped = kline.groupby('stock_code')
|
||||
|
||||
for code, group in grouped:
|
||||
if code not in candidates:
|
||||
continue
|
||||
g = group.tail(120)
|
||||
if len(g) < 20:
|
||||
continue
|
||||
|
||||
closes = g['close'].values
|
||||
opens = g['open'].values
|
||||
highs = g['high'].values
|
||||
lows = g['low'].values
|
||||
volumes = g['volume'].values
|
||||
|
||||
feat = {}
|
||||
|
||||
# 37. down_days_20d
|
||||
rets_20 = np.diff(closes[-21:]) / closes[-21:-1]
|
||||
feat['down_days_20d'] = np.mean(rets_20 < 0) * 100
|
||||
|
||||
# 38. up_days_20d
|
||||
feat['up_days_20d'] = np.mean(rets_20 > 0) * 100
|
||||
|
||||
# 39. trend_consistency_20d
|
||||
feat['trend_consistency_20d'] = max(feat['up_days_20d'], feat['down_days_20d'])
|
||||
|
||||
# 40. ma20_deviation_pct
|
||||
ma20 = np.mean(closes[-20:])
|
||||
feat['ma20_deviation_pct'] = (closes[-1] - ma20) / ma20 * 100 if ma20 > 0 else 0
|
||||
|
||||
# 41. ma60_deviation_pct
|
||||
ma60 = np.mean(closes[-60:]) if len(closes) >= 60 else np.mean(closes)
|
||||
feat['ma60_deviation_pct'] = (closes[-1] - ma60) / ma60 * 100 if ma60 > 0 else 0
|
||||
|
||||
# 42. ma20_ma60_gap_pct
|
||||
feat['ma20_ma60_gap_pct'] = (ma20 - ma60) / ma60 * 100 if ma60 > 0 else 0
|
||||
|
||||
# 43. usable_grid_count_upper
|
||||
feat['usable_grid_count_upper'] = sum(
|
||||
1 for g in range(GRID_LOW, GRID_HIGH + 1) if g > closes[-1])
|
||||
|
||||
# 44. usable_grid_count_lower
|
||||
feat['usable_grid_count_lower'] = sum(
|
||||
1 for g in range(GRID_LOW, GRID_HIGH + 1) if g < closes[-1])
|
||||
|
||||
# 45. near_upper_boundary_risk
|
||||
feat['near_upper_boundary_risk'] = max(0, min(100, (closes[-1] - 9) / 2 * 100))
|
||||
|
||||
# 46. near_lower_boundary_risk
|
||||
feat['near_lower_boundary_risk'] = max(0, min(100, (3 - closes[-1]) / 2 * 100))
|
||||
|
||||
# 47. volume_cv_20d
|
||||
vol_20 = volumes[-20:]
|
||||
feat['volume_cv_20d'] = np.std(vol_20) / np.mean(vol_20) * 100 \
|
||||
if np.mean(vol_20) > 0 else 0
|
||||
|
||||
# 48. amount_trend_20d
|
||||
amounts = closes[-20:] * volumes[-20:]
|
||||
feat['amount_trend_20d'] = _ols_slope(np.arange(len(amounts)), amounts)[0]
|
||||
|
||||
# 49. low_volume_days_20d
|
||||
mean_vol = np.mean(vol_20)
|
||||
feat['low_volume_days_20d'] = np.mean(vol_20 < 0.5 * mean_vol) * 100
|
||||
|
||||
# 50. close_reversal_count_20d
|
||||
rets_sign = np.sign(np.diff(closes[-21:]))
|
||||
reversals = sum(
|
||||
1 for i in range(1, len(rets_sign))
|
||||
if rets_sign[i] != 0 and rets_sign[i - 1] != 0 and rets_sign[i] != rets_sign[i - 1])
|
||||
feat['close_reversal_count_20d'] = reversals
|
||||
|
||||
# 51. range_compression_20d
|
||||
amps = (highs - lows) / np.where(closes > 0, closes, 1) * 100
|
||||
amp_20_mean = np.mean(amps[-20:])
|
||||
amp_60_mean = np.mean(amps[-60:]) if len(amps) >= 60 else amp_20_mean
|
||||
feat['range_compression_20d'] = amp_20_mean / amp_60_mean * 100 if amp_60_mean > 0 else 100
|
||||
|
||||
# 52. wick_ratio_20d
|
||||
upper_wick = highs[-20:] - np.maximum(opens[-20:], closes[-20:])
|
||||
lower_wick = np.minimum(opens[-20:], closes[-20:]) - lows[-20:]
|
||||
body = np.abs(closes[-20:] - opens[-20:])
|
||||
total_len = highs[-20:] - lows[-20:]
|
||||
wick_len = upper_wick + lower_wick
|
||||
feat['wick_ratio_20d'] = np.mean(
|
||||
wick_len / np.where(total_len > 0, total_len, 1)) * 100
|
||||
|
||||
features[code] = feat
|
||||
|
||||
return pd.DataFrame.from_dict(features, orient='index')
|
||||
@@ -0,0 +1,197 @@
|
||||
"""
|
||||
候选股过滤与数据加载
|
||||
"""
|
||||
import pandas as pd
|
||||
from datetime import date, timedelta
|
||||
from core.scoring.models import (
|
||||
KlineStock, StockInfo, IndustryMapping,
|
||||
KlineIndex, MarketRegimeDaily, SectorFeaturesDaily,
|
||||
)
|
||||
from core.scoring.config import (
|
||||
FILTER_MIN_CLOSE, FILTER_MAX_CLOSE, REQUIRE_DAYS,
|
||||
WINDOW_180D, HS300_CODE,
|
||||
)
|
||||
from core.logger import LogLevel, PrintLog
|
||||
|
||||
|
||||
class DataContext:
|
||||
"""特征计算所需的全部数据上下文"""
|
||||
|
||||
def __init__(self, trade_date: date):
|
||||
self.trade_date = trade_date
|
||||
self.require_days = REQUIRE_DAYS
|
||||
# 原始数据
|
||||
self.kline: pd.DataFrame = None # 候选股 K线 (180d)
|
||||
self.stock_info: dict = {} # code → StockInfo dict
|
||||
self.industry_map: dict = {} # code → industry_name
|
||||
self.hs300_kline: pd.DataFrame = None # HS300 K线 (180d)
|
||||
self.market_regime: pd.DataFrame = None # 市场状态 (180d)
|
||||
self.sector_features: pd.DataFrame = None # 行业指数 (180d)
|
||||
# 候选股列表
|
||||
self.candidates: list[str] = []
|
||||
self.excluded: dict[str, str] = {} # code → reason
|
||||
|
||||
|
||||
def _get_stock_codes_for_date(trade_date: date) -> list:
|
||||
"""获取评分日所有符合条件的股票代码（非ST/退市）"""
|
||||
rows = (StockInfo
|
||||
.select(StockInfo.code, StockInfo.listing_status)
|
||||
.where(StockInfo.listing_status.not_in(['delisted', 'ST']))
|
||||
.dicts())
|
||||
return [row['code'] for row in rows]
|
||||
|
||||
|
||||
def _check_kline_sufficiency(stock_code: str, trade_date: date) -> tuple[bool, str, float]:
|
||||
"""检查单只股票的K线数据是否满足评分条件"""
|
||||
# 查询最近 REQUIRE_DAYS + 30 (留缓冲) 个交易日
|
||||
start = trade_date - timedelta(days=(REQUIRE_DAYS + 60) * 2)
|
||||
rows = (KlineStock
|
||||
.select(KlineStock.trade_date, KlineStock.close)
|
||||
.where(
|
||||
(KlineStock.stock_code == stock_code) &
|
||||
(KlineStock.trade_date >= start) &
|
||||
(KlineStock.trade_date <= trade_date)
|
||||
)
|
||||
.order_by(KlineStock.trade_date.desc())
|
||||
.dicts())
|
||||
|
||||
if not rows:
|
||||
return False, '无K线数据', 0
|
||||
|
||||
# 过滤有效收盘价 (close > 0)
|
||||
valid_rows = [r for r in rows if r['close'] and r['close'] > 0]
|
||||
if len(valid_rows) < REQUIRE_DAYS:
|
||||
return False, f'交易天数不足({len(valid_rows)}<{REQUIRE_DAYS})', 0
|
||||
|
||||
latest_close = valid_rows[0]['close']
|
||||
|
||||
# 价格区间检查
|
||||
if latest_close < FILTER_MIN_CLOSE:
|
||||
return False, f'价格过低({latest_close:.2f}<{FILTER_MIN_CLOSE})', latest_close
|
||||
if latest_close > FILTER_MAX_CLOSE:
|
||||
return False, f'价格过高({latest_close:.2f}>{FILTER_MAX_CLOSE})', latest_close
|
||||
|
||||
return True, '', latest_close
|
||||
|
||||
|
||||
def load_candidates(trade_date: date) -> DataContext:
|
||||
"""
|
||||
加载评分日候选股及全部所需数据。
|
||||
返回 DataContext，包含候选股列表和预加载的原始数据。
|
||||
"""
|
||||
ctx = DataContext(trade_date)
|
||||
start_180 = trade_date - timedelta(days=365) # 取约1年数据覆盖180个交易日
|
||||
|
||||
# 1. 获取非ST/退市的全部股票
|
||||
all_codes = [r.split('.')[0] for r in _get_stock_codes_for_date(trade_date)]
|
||||
PrintLog(LogLevel.INFO, f'[validator] 全市场有效股票: {len(all_codes)} 只')
|
||||
|
||||
# 2. 批量加载 KlineStock (180d 窗口)
|
||||
raw_codes = all_codes # 使用纯数字代码查询
|
||||
kline_rows = (KlineStock
|
||||
.select()
|
||||
.where(
|
||||
(KlineStock.stock_code.in_(raw_codes)) &
|
||||
(KlineStock.trade_date >= start_180) &
|
||||
(KlineStock.trade_date <= trade_date)
|
||||
)
|
||||
.order_by(KlineStock.stock_code, KlineStock.trade_date)
|
||||
.dicts())
|
||||
|
||||
kline_df = pd.DataFrame(kline_rows)
|
||||
if kline_df.empty:
|
||||
PrintLog(LogLevel.WARNING, '[validator] KlineStock 无数据')
|
||||
return ctx
|
||||
|
||||
kline_df['trade_date'] = pd.to_datetime(kline_df['trade_date'])
|
||||
PrintLog(LogLevel.INFO, f'[validator] K线原始数据: {len(kline_df)} 行')
|
||||
|
||||
# 3. 逐股过滤
|
||||
candidates = []
|
||||
excluded = {}
|
||||
grouped = kline_df.groupby('stock_code')
|
||||
for code, group in grouped:
|
||||
g_sorted = group.sort_values('trade_date', ascending=False)
|
||||
valid_rows = g_sorted[g_sorted['close'].notna() & (g_sorted['close'] > 0)]
|
||||
if len(valid_rows) < REQUIRE_DAYS:
|
||||
excluded[code] = f'交易天数不足({len(valid_rows)}<{REQUIRE_DAYS})'
|
||||
continue
|
||||
latest_close = valid_rows.iloc[0]['close']
|
||||
if latest_close < FILTER_MIN_CLOSE:
|
||||
excluded[code] = f'价格过低({latest_close:.2f}<{FILTER_MIN_CLOSE})'
|
||||
continue
|
||||
if latest_close > FILTER_MAX_CLOSE:
|
||||
excluded[code] = f'价格过高({latest_close:.2f}>{FILTER_MAX_CLOSE})'
|
||||
continue
|
||||
candidates.append(code)
|
||||
|
||||
PrintLog(LogLevel.INFO,
|
||||
f'[validator] 候选: {len(candidates)} 通过, {len(excluded)} 排除')
|
||||
|
||||
# 4. 裁剪K线到只含候选股 (保留最近180日)
|
||||
kline_df = kline_df[kline_df['stock_code'].isin(candidates)].copy()
|
||||
cut_date = trade_date - timedelta(days=365)
|
||||
kline_df = kline_df[kline_df['trade_date'] >= pd.Timestamp(cut_date)]
|
||||
|
||||
ctx.kline = kline_df
|
||||
ctx.candidates = candidates
|
||||
ctx.excluded = excluded
|
||||
|
||||
# 5. 加载 StockInfo
|
||||
stock_rows = (StockInfo
|
||||
.select()
|
||||
.where(StockInfo.code.in_([f'{c}.SH' for c in candidates] +
|
||||
[f'{c}.SZ' for c in candidates] +
|
||||
[f'{c}.BJ' for c in candidates]))
|
||||
.dicts())
|
||||
ctx.stock_info = {r['code']: r for r in stock_rows}
|
||||
|
||||
# 6. 加载行业映射: code → industry_name
|
||||
ind_rows = (IndustryMapping
|
||||
.select()
|
||||
.where(IndustryMapping.code.in_(candidates))
|
||||
.dicts())
|
||||
ctx.industry_map = {r['code']: r['industry_name'] for r in ind_rows}
|
||||
PrintLog(LogLevel.INFO, f'[validator] 行业映射: {len(ctx.industry_map)} 只')
|
||||
|
||||
# 7. 加载 HS300 K线
|
||||
hs300_rows = (KlineIndex
|
||||
.select()
|
||||
.where(
|
||||
(KlineIndex.index_code == HS300_CODE) &
|
||||
(KlineIndex.trade_date >= start_180) &
|
||||
(KlineIndex.trade_date <= trade_date)
|
||||
)
|
||||
.order_by(KlineIndex.trade_date)
|
||||
.dicts())
|
||||
ctx.hs300_kline = pd.DataFrame(hs300_rows)
|
||||
if not ctx.hs300_kline.empty:
|
||||
ctx.hs300_kline['trade_date'] = pd.to_datetime(ctx.hs300_kline['trade_date'])
|
||||
|
||||
# 8. 加载市场状态 (180d)
|
||||
mkt_rows = (MarketRegimeDaily
|
||||
.select()
|
||||
.where(
|
||||
(MarketRegimeDaily.trade_date >= start_180) &
|
||||
(MarketRegimeDaily.trade_date <= trade_date)
|
||||
)
|
||||
.order_by(MarketRegimeDaily.trade_date)
|
||||
.dicts())
|
||||
ctx.market_regime = pd.DataFrame(mkt_rows)
|
||||
if not ctx.market_regime.empty:
|
||||
ctx.market_regime['trade_date'] = pd.to_datetime(ctx.market_regime['trade_date'])
|
||||
|
||||
# 9. 加载行业指数 (180d)
|
||||
sec_rows = (SectorFeaturesDaily
|
||||
.select()
|
||||
.where(
|
||||
(SectorFeaturesDaily.trade_date >= start_180) &
|
||||
(SectorFeaturesDaily.trade_date <= trade_date)
|
||||
)
|
||||
.order_by(SectorFeaturesDaily.trade_date)
|
||||
.dicts())
|
||||
ctx.sector_features = pd.DataFrame(sec_rows)
|
||||
if not ctx.sector_features.empty:
|
||||
ctx.sector_features['trade_date'] = pd.to_datetime(ctx.sector_features['trade_date'])
|
||||
|
||||
return ctx
|
||||
@@ -0,0 +1,2 @@
|
||||
# 模型推理子包
|
||||
from core.scoring.inference.scorer import GridSeekerPipeline
|
||||
@@ -0,0 +1,217 @@
|
||||
"""
|
||||
grid_seeker v6.6 三级模型推理管道
|
||||
Rank → Top → Stacking → stacking_probability (最终排序)
|
||||
"""
|
||||
import pickle
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from pathlib import Path
|
||||
from datetime import date
|
||||
from core.scoring.config import (
|
||||
get_model_path, RANK_MODEL, TOP_MODEL, STACKING_MODEL,
|
||||
STACKING_THRESHOLD,
|
||||
)
|
||||
from core.scoring.features.pipeline import FeaturePipeline
|
||||
from core.scoring.models import ScoringResult
|
||||
from core.database import db
|
||||
from core.logger import LogLevel, PrintLog
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Rank 模型输入特征 (52维, v3.4, 直接从模型文件的 selected_features 读取)
|
||||
# ============================================================
|
||||
def _get_rank_features() -> list:
|
||||
import pickle
|
||||
from core.scoring.config import get_model_path
|
||||
path = get_model_path(RANK_MODEL)
|
||||
with open(path, 'rb') as f:
|
||||
obj = pickle.load(f)
|
||||
if isinstance(obj, dict):
|
||||
sf = obj.get('selected_features', [])
|
||||
if sf:
|
||||
return sf
|
||||
raise RuntimeError("无法从 rank.pkl 读取 selected_features")
|
||||
|
||||
RANK_FEATURE_COLS = _get_rank_features()
|
||||
|
||||
|
||||
class GridSeekerPipeline:
|
||||
"""
|
||||
grid_seeker v6.6 三级模型评分管道。
|
||||
|
||||
Usage:
|
||||
engine = GridSeekerPipeline()
|
||||
rankings = engine.run(trade_date=date.today())
|
||||
# 返回 DataFrame: stock_code, stacking_probability, rank 等
|
||||
"""
|
||||
|
||||
def __init__(self, model_dir: Path = None):
|
||||
self._rank_model = None
|
||||
self._top_model = None
|
||||
self._stacking_model = None
|
||||
|
||||
# ---- 模型加载 ----
|
||||
|
||||
def _load_model(self, name: str):
|
||||
"""加载单个 .pkl 模型"""
|
||||
path = get_model_path(name)
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f'模型文件不存在: {path}')
|
||||
with open(path, 'rb') as f:
|
||||
obj = pickle.load(f)
|
||||
# 支持 dict 格式 {"model": lgbm_model, ...} 或直接返回模型对象
|
||||
if isinstance(obj, dict):
|
||||
return obj.get('model', obj)
|
||||
return obj
|
||||
|
||||
@property
|
||||
def rank_model(self):
|
||||
if self._rank_model is None:
|
||||
self._rank_model = self._load_model(RANK_MODEL)
|
||||
return self._rank_model
|
||||
|
||||
@property
|
||||
def top_model(self):
|
||||
if self._top_model is None:
|
||||
self._top_model = self._load_model(TOP_MODEL)
|
||||
return self._top_model
|
||||
|
||||
@property
|
||||
def stacking_model(self):
|
||||
if self._stacking_model is None:
|
||||
self._stacking_model = self._load_model(STACKING_MODEL)
|
||||
return self._stacking_model
|
||||
|
||||
# ---- 预测 ----
|
||||
|
||||
def _predict_with_model(self, model, X: pd.DataFrame, feature_cols: list) -> np.ndarray:
|
||||
"""
|
||||
使用模型预测。自动选择特征子集，兼容 sklearn API (predict/predict_proba)。
|
||||
"""
|
||||
available = [c for c in feature_cols if c in X.columns]
|
||||
missing = set(feature_cols) - set(available)
|
||||
if missing:
|
||||
PrintLog(LogLevel.WARNING,
|
||||
f'[scorer] 缺少特征列 ({len(missing)}): {list(missing)[:5]}...')
|
||||
|
||||
X_sub = X[available].fillna(0).values
|
||||
|
||||
try:
|
||||
if hasattr(model, 'predict_proba'):
|
||||
proba = model.predict_proba(X_sub)
|
||||
if proba.shape[1] >= 2:
|
||||
return proba[:, 1]
|
||||
return proba[:, 0]
|
||||
elif hasattr(model, 'predict'):
|
||||
return model.predict(X_sub)
|
||||
else:
|
||||
return model.predict(X_sub)
|
||||
except Exception as e:
|
||||
PrintLog(LogLevel.ERROR, f'[scorer] 模型预测失败: {e}')
|
||||
raise
|
||||
|
||||
# ---- 主流程 ----
|
||||
|
||||
def run(self, trade_date: date) -> pd.DataFrame:
|
||||
"""
|
||||
执行完整的 3 级评分管道。
|
||||
|
||||
Returns:
|
||||
DataFrame indexed by stock_code, 含 stacking_probability / rank 等列,
|
||||
按 stacking_probability 降序排列
|
||||
"""
|
||||
PrintLog(LogLevel.INFO, f'[scorer] ===== grid_seeker v6.6 评分开始 ({trade_date}) =====')
|
||||
|
||||
# 1. 特征工程
|
||||
pipeline = FeaturePipeline(trade_date)
|
||||
feature_df = pipeline.run()
|
||||
|
||||
if feature_df.empty:
|
||||
PrintLog(LogLevel.WARNING, '[scorer] 无股票通过特征工程, 终止')
|
||||
return pd.DataFrame()
|
||||
|
||||
PrintLog(LogLevel.INFO,
|
||||
f'[scorer] 特征工程完成: {len(feature_df)} stocks, '
|
||||
f'{len(feature_df.columns)} dims')
|
||||
|
||||
# 2. Stage 1: Rank 模型 → rank_predicted_rounds (52维)
|
||||
PrintLog(LogLevel.INFO, '[scorer] Stage 1/3: Rank 模型...')
|
||||
feature_df['rank_predicted_rounds'] = self._predict_with_model(
|
||||
self.rank_model, feature_df, RANK_FEATURE_COLS
|
||||
)
|
||||
|
||||
# 3. Stage 2: Top 模型 → top_elite_prob (53维 = 52 + rank_predicted_rounds)
|
||||
PrintLog(LogLevel.INFO, '[scorer] Stage 2/3: Top 模型...')
|
||||
top_cols = RANK_FEATURE_COLS + ['rank_predicted_rounds']
|
||||
feature_df['top_elite_prob'] = self._predict_with_model(
|
||||
self.top_model, feature_df, top_cols
|
||||
)
|
||||
|
||||
# 4. Stage 3: Stacking 模型 → stacking_probability (54维 = 52 + rank + top)
|
||||
PrintLog(LogLevel.INFO, '[scorer] Stage 3/3: Stacking 模型...')
|
||||
stk_cols = RANK_FEATURE_COLS + ['rank_predicted_rounds', 'top_elite_prob']
|
||||
feature_df['stacking_probability'] = self._predict_with_model(
|
||||
self.stacking_model, feature_df, stk_cols
|
||||
)
|
||||
|
||||
# 5. 排序（直接用 stacking_probability）
|
||||
feature_df['score_rank'] = feature_df['stacking_probability'].rank(
|
||||
ascending=False, method='min'
|
||||
).astype(int)
|
||||
feature_df['candidate_count'] = len(feature_df)
|
||||
feature_df = feature_df.sort_values('score_rank')
|
||||
|
||||
n_above = (feature_df['stacking_probability'] >= STACKING_THRESHOLD).sum()
|
||||
PrintLog(LogLevel.INFO,
|
||||
f'[scorer] 评分完成: {len(feature_df)} 只候选, '
|
||||
f'{n_above} 只高于阈值 {STACKING_THRESHOLD}')
|
||||
PrintLog(LogLevel.INFO,
|
||||
f'[scorer] Top-5: '
|
||||
f'{feature_df.head(5)[["stacking_probability", "rank_predicted_rounds"]].to_dict("index")}')
|
||||
|
||||
return feature_df
|
||||
|
||||
def persist(self, rankings: pd.DataFrame, trade_date: date):
|
||||
"""将评分结果持久化到 ScoringResult 表"""
|
||||
if rankings.empty:
|
||||
return
|
||||
|
||||
records = []
|
||||
for code, row in rankings.iterrows():
|
||||
records.append({
|
||||
'stock_code': str(code),
|
||||
'trade_date': trade_date,
|
||||
'predicted_profit': float(row.get('stacking_probability', 0)),
|
||||
'rank_predicted_rounds': float(row.get('rank_predicted_rounds', 0))
|
||||
if 'rank_predicted_rounds' in row else None,
|
||||
'top_elite_prob': float(row.get('top_elite_prob', 0))
|
||||
if 'top_elite_prob' in row else None,
|
||||
'stacking_probability': float(row.get('stacking_probability', 0))
|
||||
if 'stacking_probability' in row else None,
|
||||
'score_rank': int(row.get('score_rank', 0)),
|
||||
'candidate_count': int(row.get('candidate_count', 0)),
|
||||
})
|
||||
|
||||
with db.atomic():
|
||||
for batch in _chunked(records, 500):
|
||||
ScoringResult.insert_many(batch).on_conflict_replace().execute()
|
||||
|
||||
PrintLog(LogLevel.INFO,
|
||||
f'[scorer] 评分结果已持久化: {len(records)} 条')
|
||||
|
||||
def get_top_n(self, trade_date: date, n: int = 50) -> list[dict]:
|
||||
"""查询历史评分 Top-N"""
|
||||
rows = (ScoringResult
|
||||
.select()
|
||||
.where(
|
||||
(ScoringResult.trade_date == trade_date) &
|
||||
(ScoringResult.score_rank <= n)
|
||||
)
|
||||
.order_by(ScoringResult.score_rank)
|
||||
.dicts())
|
||||
return list(rows)
|
||||
|
||||
|
||||
def _chunked(lst: list, n: int):
|
||||
for i in range(0, len(lst), n):
|
||||
yield lst[i:i + n]
|
||||
@@ -0,0 +1,132 @@
|
||||
"""
|
||||
grid_seeker v6.4 数据库模型 — 6 张数据表 + 1 张评分结果表
|
||||
所有模型继承 core.database.BaseModel，复用现有 SQLite 连接。
|
||||
"""
|
||||
from peewee import (
|
||||
CharField, DateField, FloatField, IntegerField,
|
||||
CompositeKey, TextField,
|
||||
)
|
||||
from core.database import BaseModel, db
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 1. kline_stock — 个股日K线
|
||||
# ============================================================
|
||||
class KlineStock(BaseModel):
|
||||
stock_code = CharField(max_length=10) # 纯数字6位
|
||||
trade_date = DateField()
|
||||
open = FloatField()
|
||||
high = FloatField()
|
||||
low = FloatField()
|
||||
close = FloatField()
|
||||
volume = FloatField() # 成交量（股）
|
||||
|
||||
class Meta:
|
||||
primary_key = CompositeKey('stock_code', 'trade_date')
|
||||
indexes = (
|
||||
(('stock_code', 'trade_date'), False),
|
||||
)
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 2. stocks — 股票基础信息
|
||||
# ============================================================
|
||||
class StockInfo(BaseModel):
|
||||
code = CharField(max_length=12, primary_key=True) # 带 SH/SZ/BJ 前缀
|
||||
name = CharField(max_length=32)
|
||||
exchange = CharField(max_length=4) # SH / SZ / BJ
|
||||
list_date = DateField(null=True)
|
||||
listing_status = CharField(max_length=16, default='normal') # normal / delisted / ST
|
||||
total_share = FloatField(null=True) # 总股本（股）
|
||||
float_share = FloatField(null=True) # 流通股本（股）
|
||||
share_updated_at = DateField(null=True) # 股本数据同步时间
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 3. industry — 股票-行业映射
|
||||
# ============================================================
|
||||
class IndustryMapping(BaseModel):
|
||||
code = CharField(max_length=6, primary_key=True) # 纯数字6位
|
||||
industry_name = CharField(max_length=64, index=True)
|
||||
industry_classification = CharField(max_length=32) # 行业分类体系名称
|
||||
update_date = DateField()
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 4. kline_index — 指数日K线
|
||||
# ============================================================
|
||||
class KlineIndex(BaseModel):
|
||||
index_code = CharField(max_length=10) # 指数代码（6位数字）
|
||||
trade_date = DateField()
|
||||
open = FloatField()
|
||||
high = FloatField()
|
||||
low = FloatField()
|
||||
close = FloatField()
|
||||
volume = FloatField()
|
||||
|
||||
class Meta:
|
||||
primary_key = CompositeKey('index_code', 'trade_date')
|
||||
indexes = (
|
||||
(('index_code', 'trade_date'), False),
|
||||
)
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 5. market_regime_daily — 市场状态（本地计算）
|
||||
# ============================================================
|
||||
class MarketRegimeDaily(BaseModel):
|
||||
trade_date = DateField(primary_key=True)
|
||||
advancers = FloatField(default=0) # 当日上涨家数
|
||||
decliners = FloatField(default=0) # 当日下跌家数
|
||||
advance_ratio = FloatField(default=0) # 涨跌比 = advancers/(advancers+decliners)
|
||||
turnover = FloatField(default=0) # 全市场成交额
|
||||
turnover_avg_5d = FloatField(default=0) # 5日滚动均量
|
||||
turnover_ratio_5d = FloatField(default=0) # 量比 = turnover/turnover_avg_5d
|
||||
source = CharField(max_length=32, default='qmt')
|
||||
is_extreme_panic = IntegerField(default=0) # advance_ratio<0.2 且 turnover_ratio_5d>1.5
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 6. sector_features_daily — 行业聚合指数（本地计算）
|
||||
# ============================================================
|
||||
class SectorFeaturesDaily(BaseModel):
|
||||
trade_date = DateField()
|
||||
sector_name = CharField(max_length=64) # 与 industry.industry_name 对应
|
||||
sector_ret = FloatField(default=0) # 行业日收益率（均值）
|
||||
sector_amplitude = FloatField(default=0) # 行业平均振幅
|
||||
close = FloatField(default=100) # 行业指数（基值100）
|
||||
ema10 = FloatField(default=0)
|
||||
ema20 = FloatField(default=0)
|
||||
ema200 = FloatField(default=0)
|
||||
score = IntegerField(default=0) # 趋势评分 0/1/2
|
||||
|
||||
class Meta:
|
||||
primary_key = CompositeKey('trade_date', 'sector_name')
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 7. ScoringResult — 评分结果（模型输出写入表）
|
||||
# ============================================================
|
||||
class ScoringResult(BaseModel):
|
||||
stock_code = CharField(max_length=6) # 纯数字6位
|
||||
trade_date = DateField() # 评分日
|
||||
predicted_profit = FloatField(default=0) # 最终预测利润（=stacking_probability）
|
||||
rank_predicted_rounds = FloatField(null=True) # Rank 模型输出
|
||||
top_elite_prob = FloatField(null=True) # Top 模型输出
|
||||
stacking_probability = FloatField(null=True) # Stacking 模型输出
|
||||
score_rank = IntegerField(default=0) # 排名
|
||||
candidate_count = IntegerField(default=0) # 候选股总数
|
||||
|
||||
class Meta:
|
||||
primary_key = CompositeKey('stock_code', 'trade_date')
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 建表
|
||||
# ============================================================
|
||||
ALL_SCORING_TABLES = [
|
||||
KlineStock, StockInfo, IndustryMapping, KlineIndex,
|
||||
MarketRegimeDaily, SectorFeaturesDaily, ScoringResult,
|
||||
]
|
||||
|
||||
db.create_tables(ALL_SCORING_TABLES)
|
||||
@@ -0,0 +1,7 @@
|
||||
# 数据同步子包
|
||||
from core.scoring.sync.base import BaseSync
|
||||
from core.scoring.sync.kline_sync import KlineStockSync, KlineIndexSync
|
||||
from core.scoring.sync.stocks_sync import StocksSync
|
||||
from core.scoring.sync.industry_sync import IndustrySync
|
||||
from core.scoring.sync.market_regime import MarketRegimeSync
|
||||
from core.scoring.sync.sector_features import SectorFeaturesSync
|
||||
@@ -0,0 +1,41 @@
|
||||
"""
|
||||
数据同步抽象基类
|
||||
"""
|
||||
import abc
|
||||
from datetime import datetime
|
||||
from core.logger import LogLevel, PrintLog
|
||||
|
||||
|
||||
class BaseSync(abc.ABC):
|
||||
"""数据同步抽象基类，所有同步操作遵循 _fetch → _upsert 模式"""
|
||||
|
||||
def __init__(self):
|
||||
self.stats = {'inserted': 0, 'updated': 0, 'skipped': 0, 'errors': 0}
|
||||
|
||||
def run(self, **kwargs) -> dict:
|
||||
"""同步入口: 拉取数据 → 写入数据库 → 返回统计"""
|
||||
name = self.__class__.__name__
|
||||
PrintLog(LogLevel.INFO, f'[sync] {name} 开始同步...')
|
||||
t0 = datetime.now()
|
||||
try:
|
||||
data = self._fetch(**kwargs)
|
||||
self._upsert(data)
|
||||
elapsed = (datetime.now() - t0).total_seconds()
|
||||
PrintLog(
|
||||
LogLevel.INFO,
|
||||
f'[sync] {name} 完成 ({elapsed:.1f}s) — '
|
||||
f'insert={self.stats["inserted"]} update={self.stats["updated"]} '
|
||||
f'skip={self.stats["skipped"]} err={self.stats["errors"]}'
|
||||
)
|
||||
except Exception as e:
|
||||
PrintLog(LogLevel.ERROR, f'[sync] {name} 失败: {e}')
|
||||
raise
|
||||
return self.stats
|
||||
|
||||
@abc.abstractmethod
|
||||
def _fetch(self, **kwargs):
|
||||
"""从数据源拉取原始数据。子类实现。"""
|
||||
|
||||
@abc.abstractmethod
|
||||
def _upsert(self, data):
|
||||
"""将数据写入数据库。子类实现。"""
|
||||
@@ -0,0 +1,84 @@
|
||||
"""
|
||||
行业映射同步 — 从 QMT get_sector_list + get_stock_list_in_sector 获取
|
||||
"""
|
||||
from datetime import date
|
||||
from core.scoring.sync.base import BaseSync
|
||||
from core.scoring.models import IndustryMapping
|
||||
from core.database import db
|
||||
from core.logger import LogLevel, PrintLog
|
||||
|
||||
|
||||
class IndustrySync(BaseSync):
|
||||
"""行业映射同步 — QMT 行业板块 → IndustryMapping 表（全量替换）"""
|
||||
|
||||
def _fetch(self, **kwargs):
|
||||
"""从 QMT 拉取全部行业板块的成份股映射"""
|
||||
from xtquant import xtdata
|
||||
|
||||
all_sectors = xtdata.get_sector_list()
|
||||
PrintLog(LogLevel.INFO, f'[sync] Industry: 共 {len(all_sectors)} 个板块')
|
||||
|
||||
# 尝试使用 get_sector_info 过滤行业板块
|
||||
industry_sectors = []
|
||||
try:
|
||||
sector_info = xtdata.get_sector_info()
|
||||
if sector_info is not None and not sector_info.empty:
|
||||
for _, row in sector_info.iterrows():
|
||||
cat = row.get('category', '')
|
||||
if '行业' in str(cat):
|
||||
industry_sectors.append(row['sector'])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 如果 get_sector_info 无效，回退到名称过滤
|
||||
if not industry_sectors:
|
||||
for s in all_sectors:
|
||||
# 排除明显非行业的板块
|
||||
skip_markers = ['概念', '风格', '地域', '地区', '指数', '自定义',
|
||||
'ETF', 'LOF', '债券', '基金', '期货', '期权']
|
||||
if any(m in s for m in skip_markers):
|
||||
continue
|
||||
industry_sectors.append(s)
|
||||
|
||||
PrintLog(LogLevel.INFO, f'[sync] Industry: 筛选出 {len(industry_sectors)} 个行业板块')
|
||||
|
||||
# 构建 code → {industry_name, classification} 映射
|
||||
mapping = {} # code → (industry_name, classification)
|
||||
today = date.today()
|
||||
|
||||
for sector_name in industry_sectors:
|
||||
try:
|
||||
stocks = xtdata.get_stock_list_in_sector(sector_name)
|
||||
for full_code in stocks:
|
||||
code = full_code.split('.')[0]
|
||||
if code not in mapping:
|
||||
mapping[code] = {
|
||||
'code': code,
|
||||
'industry_name': sector_name,
|
||||
'industry_classification': 'QMT',
|
||||
'update_date': today,
|
||||
}
|
||||
except Exception:
|
||||
self.stats['errors'] += 1
|
||||
|
||||
self.stats['inserted'] = len(mapping)
|
||||
return list(mapping.values())
|
||||
|
||||
def _upsert(self, data: list):
|
||||
"""全量替换: 清空旧数据 → 批量插入新数据"""
|
||||
if not data:
|
||||
PrintLog(LogLevel.WARNING, '[sync] Industry: 无数据, 跳过')
|
||||
return
|
||||
|
||||
with db.atomic():
|
||||
IndustryMapping.delete().execute()
|
||||
for batch in _chunked(data, 500):
|
||||
IndustryMapping.insert_many(batch).execute()
|
||||
|
||||
PrintLog(LogLevel.INFO,
|
||||
f'[sync] Industry: 全量替换完成, {len(data)} 条映射')
|
||||
|
||||
|
||||
def _chunked(lst: list, n: int):
|
||||
for i in range(0, len(lst), n):
|
||||
yield lst[i:i + n]
|
||||
@@ -0,0 +1,222 @@
|
||||
"""
|
||||
K线数据同步 — 个股日K + 指数日K
|
||||
数据源: QMT xtdata
|
||||
增量同步: 只拉 max(trade_date) 之后的增量数据
|
||||
线程锁: KlineStockSync / KlineIndexSync 各自内部锁
|
||||
"""
|
||||
import pandas as pd
|
||||
from datetime import date, timedelta
|
||||
from core.scoring.sync.base import BaseSync
|
||||
from core.scoring.models import KlineStock, KlineIndex
|
||||
from core.scoring.config import TRACKED_INDICES
|
||||
from core.database import db
|
||||
from core.logger import LogLevel, PrintLog
|
||||
|
||||
BATCH_SIZE = 50
|
||||
DEFAULT_COUNT = 300
|
||||
|
||||
# 全局同步状态标记（字典引用传递，可被外部轮询）
|
||||
_sync_state = {"kline": False, "index": False, "stocks": False,
|
||||
"industry": False, "market": False, "sector": False}
|
||||
|
||||
|
||||
def is_syncing(key="kline") -> bool:
|
||||
return _sync_state.get(key, False)
|
||||
|
||||
|
||||
def _latest_date(model_cls) -> date | None:
|
||||
from peewee import fn
|
||||
row = model_cls.select(fn.MAX(model_cls.trade_date)).scalar()
|
||||
if isinstance(row, date):
|
||||
return row
|
||||
return None
|
||||
|
||||
|
||||
def _safe_get(df_dict, code, dt, default=0.0) -> float:
|
||||
"""安全获取 DataFrame 值"""
|
||||
if df_dict is None:
|
||||
return default
|
||||
df = df_dict.get(code)
|
||||
if df is None or code not in df.index:
|
||||
return default
|
||||
try:
|
||||
val = df.loc[code, dt]
|
||||
if pd.isna(val):
|
||||
return default
|
||||
return float(val)
|
||||
except Exception:
|
||||
return default
|
||||
|
||||
|
||||
class KlineStockSync(BaseSync):
|
||||
"""个股日K线同步 — 增量：只拉 max(trade_date) 之后的增量数据"""
|
||||
|
||||
def __init__(self, count: int = DEFAULT_COUNT):
|
||||
super().__init__()
|
||||
self.count = count
|
||||
|
||||
def _fetch(self, **kwargs):
|
||||
from xtquant import xtdata
|
||||
|
||||
# 增量判断
|
||||
latest = _latest_date(KlineStock)
|
||||
today = date.today()
|
||||
if latest is not None and latest >= today:
|
||||
PrintLog(LogLevel.INFO, f'[sync] KlineStock: 已最新 ({latest})，跳过')
|
||||
self.stats['skipped'] = 0
|
||||
return self.stats
|
||||
|
||||
# 增量起点
|
||||
start_date = (latest + timedelta(days=1)) if latest else None
|
||||
start_str = start_date.strftime('%Y%m%d') if start_date else ""
|
||||
PrintLog(LogLevel.INFO,
|
||||
f'[sync] KlineStock: 增量同步，起点={start_str or "全部"}')
|
||||
|
||||
all_stocks = xtdata.get_stock_list_in_sector("沪深A股")
|
||||
PrintLog(LogLevel.INFO, f'[sync] KlineStock: {len(all_stocks)} 只A股')
|
||||
field_list = ['open', 'high', 'low', 'close', 'volume']
|
||||
total = len(all_stocks)
|
||||
inserted = 0
|
||||
|
||||
for i, code in enumerate(all_stocks):
|
||||
if i > 0 and i % 50 == 0:
|
||||
PrintLog(LogLevel.INFO, f'[sync] KlineStock: {i}/{total} ({i*100//total}%)')
|
||||
|
||||
try:
|
||||
xtdata.download_history_data(code, period='1d', start_time=start_str)
|
||||
except Exception:
|
||||
self.stats['errors'] += 1
|
||||
continue
|
||||
|
||||
try:
|
||||
result = xtdata.get_market_data(
|
||||
field_list=field_list, stock_list=[code], period='1d',
|
||||
count=self.count, dividend_type='none', fill_data=False)
|
||||
inserted += self._upsert_incremental(code, result, start_date)
|
||||
except Exception:
|
||||
self.stats['errors'] += 1
|
||||
|
||||
self.stats['inserted'] = inserted
|
||||
PrintLog(LogLevel.INFO,
|
||||
f'[sync] KlineStock 完成: 新增={inserted} '
|
||||
f'跳过={self.stats["skipped"]} 错误={self.stats["errors"]}')
|
||||
return self.stats
|
||||
|
||||
def _upsert_incremental(self, full_code: str, result: dict, start_date) -> int:
|
||||
if not result:
|
||||
return 0
|
||||
close_df = result.get('close')
|
||||
if close_df is None or close_df.empty:
|
||||
return 0
|
||||
stock_code = full_code.split('.')[0]
|
||||
records = []
|
||||
for td in close_df.columns:
|
||||
td_date = td.date() if hasattr(td, 'date') else td
|
||||
if start_date is not None and td_date <= start_date:
|
||||
self.stats['skipped'] = self.stats.get('skipped', 0) + 1
|
||||
continue
|
||||
close_val = close_df.loc[full_code, td]
|
||||
if close_val is None or (isinstance(close_val, float) and pd.isna(close_val)):
|
||||
self.stats['skipped'] = self.stats.get('skipped', 0) + 1
|
||||
continue
|
||||
records.append({
|
||||
'stock_code': stock_code,
|
||||
'trade_date': td_date,
|
||||
'open': _safe_get(result.get('open'), full_code, td),
|
||||
'high': _safe_get(result.get('high'), full_code, td),
|
||||
'low': _safe_get(result.get('low'), full_code, td),
|
||||
'close': float(close_val),
|
||||
'volume': _safe_get(result.get('volume'), full_code, td),
|
||||
})
|
||||
if records:
|
||||
with db.atomic():
|
||||
for batch in _chunked(records, 500):
|
||||
KlineStock.insert_many(batch).on_conflict_replace().execute()
|
||||
return len(records)
|
||||
return 0
|
||||
|
||||
def _upsert(self, data):
|
||||
pass
|
||||
|
||||
|
||||
class KlineIndexSync(BaseSync):
|
||||
"""指数日K线同步 — 增量同步"""
|
||||
|
||||
def __init__(self, indices: list = None, count: int = DEFAULT_COUNT):
|
||||
super().__init__()
|
||||
self.indices = indices or TRACKED_INDICES
|
||||
self.count = count
|
||||
|
||||
def _fetch(self, **kwargs):
|
||||
from xtquant import xtdata
|
||||
|
||||
index_codes = []
|
||||
for code in self.indices:
|
||||
if code.startswith(('000', '001')):
|
||||
index_codes.append(f'{code}.SH')
|
||||
elif code.startswith('399'):
|
||||
index_codes.append(f'{code}.SZ')
|
||||
else:
|
||||
index_codes.append(f'{code}.SH')
|
||||
|
||||
latest = _latest_date(KlineIndex)
|
||||
today = date.today()
|
||||
if latest is not None and latest >= today:
|
||||
PrintLog(LogLevel.INFO, f'[sync] KlineIndex: 已最新 ({latest})，跳过')
|
||||
return {}
|
||||
|
||||
start_date = (latest + timedelta(days=1)) if latest else None
|
||||
start_str = start_date.strftime('%Y%m%d') if start_date else ""
|
||||
PrintLog(LogLevel.INFO,
|
||||
f'[sync] KlineIndex: 增量同步，起点={start_str or "全部"}')
|
||||
|
||||
for code in index_codes:
|
||||
try:
|
||||
xtdata.download_history_data(code, period='1d', start_time=start_str)
|
||||
except Exception:
|
||||
self.stats['errors'] += 1
|
||||
|
||||
field_list = ['open', 'high', 'low', 'close', 'volume']
|
||||
result = xtdata.get_market_data(
|
||||
field_list=field_list, stock_list=index_codes, period='1d',
|
||||
count=self.count, dividend_type='none', fill_data=False)
|
||||
return result or {}
|
||||
|
||||
def _upsert(self, data):
|
||||
if not data:
|
||||
return
|
||||
latest = _latest_date(KlineIndex)
|
||||
records = []
|
||||
close_df = data.get('close')
|
||||
if close_df is None or close_df.empty:
|
||||
return
|
||||
for full_code in close_df.index:
|
||||
index_code = full_code.split('.')[0]
|
||||
for td in close_df.columns:
|
||||
td_date = td.date() if hasattr(td, 'date') else td
|
||||
if latest is not None and td_date <= latest:
|
||||
self.stats['skipped'] = self.stats.get('skipped', 0) + 1
|
||||
continue
|
||||
close_val = close_df.loc[full_code, td]
|
||||
if close_val is None or (isinstance(close_val, float) and pd.isna(close_val)):
|
||||
self.stats['skipped'] = self.stats.get('skipped', 0) + 1
|
||||
continue
|
||||
records.append({
|
||||
'index_code': index_code,
|
||||
'trade_date': td_date,
|
||||
'open': _safe_get(data.get('open'), full_code, td),
|
||||
'high': _safe_get(data.get('high'), full_code, td),
|
||||
'low': _safe_get(data.get('low'), full_code, td),
|
||||
'close': float(close_val),
|
||||
'volume': _safe_get(data.get('volume'), full_code, td),
|
||||
})
|
||||
if records:
|
||||
with db.atomic():
|
||||
for batch in _chunked(records, 500):
|
||||
KlineIndex.insert_many(batch).on_conflict_replace().execute()
|
||||
self.stats['inserted'] = self.stats.get('inserted', 0) + len(records)
|
||||
|
||||
|
||||
def _chunked(lst: list, n: int):
|
||||
for i in range(0, len(lst), n):
|
||||
yield lst[i:i + n]
|
||||
@@ -0,0 +1,98 @@
|
||||
"""
|
||||
市场状态计算 — 从 kline_stock 聚合生成 market_regime_daily
|
||||
纯本地计算，不依赖外部数据源。
|
||||
"""
|
||||
import pandas as pd
|
||||
from peewee import fn, Case
|
||||
from core.scoring.sync.base import BaseSync
|
||||
from core.scoring.models import KlineStock, MarketRegimeDaily
|
||||
from core.scoring.config import PANIC_ADVANCE_RATIO, PANIC_TURNOVER_RATIO
|
||||
from core.database import db
|
||||
from core.logger import LogLevel, PrintLog
|
||||
|
||||
|
||||
class MarketRegimeSync(BaseSync):
|
||||
"""市场状态同步 — kline_stock 聚合 → market_regime_daily"""
|
||||
|
||||
def _fetch(self, **kwargs):
|
||||
"""从 KlineStock 逐日聚合涨跌家数和成交额"""
|
||||
PrintLog(LogLevel.INFO, '[sync] MarketRegime: 开始聚合全市场数据...')
|
||||
|
||||
# peewee 聚合查询: 逐日统计 advancers / decliners / turnover
|
||||
query = (KlineStock
|
||||
.select(
|
||||
KlineStock.trade_date,
|
||||
fn.SUM(
|
||||
Case(None, [(KlineStock.close > KlineStock.open, 1)], 0)
|
||||
).alias('advancers'),
|
||||
fn.SUM(
|
||||
Case(None, [(KlineStock.close < KlineStock.open, 1)], 0)
|
||||
).alias('decliners'),
|
||||
fn.SUM(KlineStock.close * KlineStock.volume).alias('turnover'),
|
||||
)
|
||||
.group_by(KlineStock.trade_date)
|
||||
.order_by(KlineStock.trade_date))
|
||||
|
||||
rows = list(query.dicts())
|
||||
if not rows:
|
||||
PrintLog(LogLevel.WARNING, '[sync] MarketRegime: KlineStock 表为空')
|
||||
return None
|
||||
|
||||
df = pd.DataFrame(rows)
|
||||
df['trade_date'] = pd.to_datetime(df['trade_date'])
|
||||
df = df.sort_values('trade_date').reset_index(drop=True)
|
||||
|
||||
# 计算涨跌比
|
||||
total = df['advancers'] + df['decliners']
|
||||
df['advance_ratio'] = (df['advancers'] / total.replace(0, 1)).round(4)
|
||||
|
||||
# 5日滚动均量
|
||||
df['turnover_avg_5d'] = (df['turnover']
|
||||
.rolling(window=5, min_periods=1)
|
||||
.mean()
|
||||
.round(2))
|
||||
|
||||
# 量比
|
||||
df['turnover_ratio_5d'] = (df['turnover'] /
|
||||
df['turnover_avg_5d'].replace(0, 1)).round(4)
|
||||
|
||||
# 极端恐慌标记
|
||||
df['is_extreme_panic'] = (
|
||||
(df['advance_ratio'] < PANIC_ADVANCE_RATIO) &
|
||||
(df['turnover_ratio_5d'] > PANIC_TURNOVER_RATIO)
|
||||
).astype(int)
|
||||
|
||||
PrintLog(LogLevel.INFO,
|
||||
f'[sync] MarketRegime: 聚合完成, {len(df)} 个交易日')
|
||||
|
||||
return df
|
||||
|
||||
def _upsert(self, df):
|
||||
"""写入 MarketRegimeDaily 表"""
|
||||
if df is None or df.empty:
|
||||
return
|
||||
|
||||
records = []
|
||||
for _, row in df.iterrows():
|
||||
records.append({
|
||||
'trade_date': row['trade_date'].date(),
|
||||
'advancers': int(row['advancers']),
|
||||
'decliners': int(row['decliners']),
|
||||
'advance_ratio': float(row['advance_ratio']),
|
||||
'turnover': float(row['turnover']),
|
||||
'turnover_avg_5d': float(row['turnover_avg_5d']),
|
||||
'turnover_ratio_5d': float(row['turnover_ratio_5d']),
|
||||
'source': 'qmt',
|
||||
'is_extreme_panic': int(row['is_extreme_panic']),
|
||||
})
|
||||
|
||||
if records:
|
||||
with db.atomic():
|
||||
for batch in _chunked(records, 500):
|
||||
MarketRegimeDaily.insert_many(batch).on_conflict_replace().execute()
|
||||
self.stats['inserted'] += len(records)
|
||||
|
||||
|
||||
def _chunked(lst: list, n: int):
|
||||
for i in range(0, len(lst), n):
|
||||
yield lst[i:i + n]
|
||||
@@ -0,0 +1,129 @@
|
||||
"""
|
||||
行业聚合指数计算 — 从 kline_stock + industry 生成 sector_features_daily
|
||||
纯本地计算，不依赖外部数据源。
|
||||
"""
|
||||
import pandas as pd
|
||||
from core.scoring.sync.base import BaseSync
|
||||
from core.scoring.models import KlineStock, IndustryMapping, SectorFeaturesDaily
|
||||
from core.database import db
|
||||
from core.logger import LogLevel, PrintLog
|
||||
|
||||
|
||||
class SectorFeaturesSync(BaseSync):
|
||||
"""行业聚合指数同步 — kline_stock + industry → sector_features_daily"""
|
||||
|
||||
def _fetch(self, **kwargs):
|
||||
"""从数据库加载原始数据, 计算行业指数特征"""
|
||||
PrintLog(LogLevel.INFO, '[sync] SectorFeatures: 加载原始数据...')
|
||||
|
||||
# 1. 加载行业映射: code → industry_name
|
||||
industries = list(IndustryMapping.select().dicts())
|
||||
if not industries:
|
||||
PrintLog(LogLevel.WARNING, '[sync] SectorFeatures: IndustryMapping 表为空, 请先执行 industry sync')
|
||||
return None
|
||||
code_to_industry = {row['code']: row['industry_name'] for row in industries}
|
||||
PrintLog(LogLevel.INFO, f'[sync] SectorFeatures: {len(code_to_industry)} 条行业映射')
|
||||
|
||||
# 2. 加载 K 线数据
|
||||
kline_rows = (KlineStock
|
||||
.select(
|
||||
KlineStock.stock_code,
|
||||
KlineStock.trade_date,
|
||||
KlineStock.open,
|
||||
KlineStock.high,
|
||||
KlineStock.low,
|
||||
KlineStock.close,
|
||||
)
|
||||
.order_by(KlineStock.stock_code, KlineStock.trade_date)
|
||||
.dicts())
|
||||
|
||||
if not kline_rows:
|
||||
PrintLog(LogLevel.WARNING, '[sync] SectorFeatures: KlineStock 表为空')
|
||||
return None
|
||||
|
||||
df = pd.DataFrame(kline_rows)
|
||||
df['trade_date'] = pd.to_datetime(df['trade_date'])
|
||||
PrintLog(LogLevel.INFO, f'[sync] SectorFeatures: {len(df)} 条K线数据')
|
||||
|
||||
# 3. 映射行业
|
||||
df['sector_name'] = df['stock_code'].map(code_to_industry)
|
||||
df = df.dropna(subset=['sector_name'])
|
||||
|
||||
# 4. 逐股计算日收益率和振幅
|
||||
df = df.sort_values(['stock_code', 'trade_date'])
|
||||
df['prev_close'] = df.groupby('stock_code')['close'].shift(1)
|
||||
df['pct_chg'] = (df['close'] - df['prev_close']) / df['prev_close'] * 100
|
||||
df['amplitude'] = (df['high'] - df['low']) / df['open'] * 100
|
||||
|
||||
# 清理无效值
|
||||
df = df.dropna(subset=['pct_chg', 'amplitude'])
|
||||
|
||||
# 5. 按行业+日期聚合
|
||||
agg = (df.groupby(['trade_date', 'sector_name'])
|
||||
.agg(
|
||||
sector_ret=('pct_chg', 'mean'),
|
||||
sector_amplitude=('amplitude', 'mean'),
|
||||
)
|
||||
.reset_index())
|
||||
|
||||
# 6. 构建行业指数 (基值=100)
|
||||
agg = agg.sort_values(['sector_name', 'trade_date'])
|
||||
agg['sector_index'] = agg.groupby('sector_name')['sector_ret'].transform(
|
||||
lambda x: (1 + x / 100).cumprod() * 100
|
||||
)
|
||||
# 重新基值=100 (每行业独立)
|
||||
for name, group in agg.groupby('sector_name'):
|
||||
idx = group.index
|
||||
first_val = group['sector_index'].iloc[0]
|
||||
agg.loc[idx, 'sector_index'] = group['sector_index'] / first_val * 100
|
||||
|
||||
# 7. 计算 EMA 均线
|
||||
agg['ema10'] = (agg.groupby('sector_name')['sector_index']
|
||||
.transform(lambda x: x.ewm(span=10, min_periods=1).mean()))
|
||||
agg['ema20'] = (agg.groupby('sector_name')['sector_index']
|
||||
.transform(lambda x: x.ewm(span=20, min_periods=1).mean()))
|
||||
agg['ema200'] = (agg.groupby('sector_name')['sector_index']
|
||||
.transform(lambda x: x.ewm(span=200, min_periods=1).mean()))
|
||||
|
||||
# 8. 趋势评分: close>ema200 得1分 + ema10>ema20 得1分
|
||||
agg['score'] = (
|
||||
(agg['sector_index'] > agg['ema200']).astype(int) +
|
||||
(agg['ema10'] > agg['ema20']).astype(int)
|
||||
)
|
||||
|
||||
PrintLog(LogLevel.INFO,
|
||||
f'[sync] SectorFeatures: 计算完成, {len(agg)} 行, '
|
||||
f'{agg["sector_name"].nunique()} 个行业')
|
||||
|
||||
return agg
|
||||
|
||||
def _upsert(self, agg):
|
||||
"""写入 SectorFeaturesDaily 表"""
|
||||
if agg is None or agg.empty:
|
||||
return
|
||||
|
||||
records = []
|
||||
for _, row in agg.iterrows():
|
||||
records.append({
|
||||
'trade_date': row['trade_date'].date()
|
||||
if hasattr(row['trade_date'], 'date') else row['trade_date'],
|
||||
'sector_name': str(row['sector_name']),
|
||||
'sector_ret': round(float(row['sector_ret']), 4),
|
||||
'sector_amplitude': round(float(row['sector_amplitude']), 4),
|
||||
'close': round(float(row['sector_index']), 4),
|
||||
'ema10': round(float(row['ema10']), 4),
|
||||
'ema20': round(float(row['ema20']), 4),
|
||||
'ema200': round(float(row['ema200']), 4),
|
||||
'score': int(row['score']),
|
||||
})
|
||||
|
||||
if records:
|
||||
with db.atomic():
|
||||
for batch in _chunked(records, 500):
|
||||
SectorFeaturesDaily.insert_many(batch).on_conflict_replace().execute()
|
||||
self.stats['inserted'] += len(records)
|
||||
|
||||
|
||||
def _chunked(lst: list, n: int):
|
||||
for i in range(0, len(lst), n):
|
||||
yield lst[i:i + n]
|
||||
@@ -0,0 +1,100 @@
|
||||
"""
|
||||
股票基础信息同步 — 从 QMT get_instrument_detail 获取
|
||||
"""
|
||||
from datetime import date
|
||||
from core.scoring.sync.base import BaseSync
|
||||
from core.scoring.models import StockInfo
|
||||
from core.database import db
|
||||
from core.logger import LogLevel, PrintLog
|
||||
|
||||
BATCH_SIZE = 200
|
||||
|
||||
|
||||
class StocksSync(BaseSync):
|
||||
"""股票基础信息同步 — QMT get_instrument_detail_list"""
|
||||
|
||||
def _fetch(self, **kwargs):
|
||||
"""从 QMT 拉取全部A股基础信息"""
|
||||
from xtquant import xtdata
|
||||
|
||||
all_stocks = xtdata.get_stock_list_in_sector("沪深A股")
|
||||
PrintLog(LogLevel.INFO, f'[sync] Stocks: 获取到 {len(all_stocks)} 只A股')
|
||||
|
||||
result = {}
|
||||
total = len(all_stocks)
|
||||
for i in range(0, total, BATCH_SIZE):
|
||||
batch = all_stocks[i:i + BATCH_SIZE]
|
||||
try:
|
||||
details = xtdata.get_instrument_detail_list(batch)
|
||||
result.update(details)
|
||||
except Exception as e:
|
||||
PrintLog(LogLevel.ERROR,
|
||||
f'[sync] Stocks batch {i}-{min(i + BATCH_SIZE, total)} failed: {e}')
|
||||
self.stats['errors'] += len(batch)
|
||||
|
||||
return result
|
||||
|
||||
def _upsert(self, data: dict):
|
||||
"""写入 StockInfo 表"""
|
||||
records = []
|
||||
today = date.today()
|
||||
|
||||
for full_code, inst in data.items():
|
||||
if not inst:
|
||||
self.stats['skipped'] += 1
|
||||
continue
|
||||
|
||||
code_num = full_code.split('.')[0]
|
||||
|
||||
# 判断交易所
|
||||
exchange = inst.get('ExchangeID', '')
|
||||
if not exchange:
|
||||
if '.SH' in full_code:
|
||||
exchange = 'SH'
|
||||
elif '.SZ' in full_code:
|
||||
exchange = 'SZ'
|
||||
elif '.BJ' in full_code:
|
||||
exchange = 'BJ'
|
||||
|
||||
# OpenDate 格式: 'YYYYMMDD' 或 int
|
||||
open_date = inst.get('OpenDate', '')
|
||||
if open_date and len(str(open_date)) == 8:
|
||||
list_date_val = f'{str(open_date)[:4]}-{str(open_date)[4:6]}-{str(open_date)[6:8]}'
|
||||
else:
|
||||
list_date_val = None
|
||||
|
||||
# InstrumentStatus 含义:
|
||||
# 0/1 = 正常股票（含已退市但仍在QMT列表的）
|
||||
# -1 = ST / *ST
|
||||
# 30/31 = *ST
|
||||
# 已退市股（名字含XD/退）K线不足120日，会在候选股过滤时被排除
|
||||
status = inst.get('InstrumentStatus', -1)
|
||||
if status in (-1, 30, 31):
|
||||
listing_status = 'ST'
|
||||
else:
|
||||
listing_status = 'normal'
|
||||
|
||||
total_share = inst.get('TotalVolume', None)
|
||||
float_share = inst.get('FloatVolume', None)
|
||||
|
||||
records.append({
|
||||
'code': full_code,
|
||||
'name': str(inst.get('InstrumentName', '')),
|
||||
'exchange': exchange,
|
||||
'list_date': list_date_val,
|
||||
'listing_status': listing_status,
|
||||
'total_share': float(total_share) if total_share else None,
|
||||
'float_share': float(float_share) if float_share else None,
|
||||
'share_updated_at': today,
|
||||
})
|
||||
|
||||
if records:
|
||||
with db.atomic():
|
||||
for batch in _chunked(records, 500):
|
||||
StockInfo.insert_many(batch).on_conflict_replace().execute()
|
||||
self.stats['inserted'] += len(records)
|
||||
|
||||
|
||||
def _chunked(lst: list, n: int):
|
||||
for i in range(0, len(lst), n):
|
||||
yield lst[i:i + n]
|
||||
Reference in New Issue
Block a user