模型，评分，修复网格策略市场状态监听
This commit is contained in:
@@ -0,0 +1,7 @@
|
||||
# 数据同步子包
|
||||
from core.scoring.sync.base import BaseSync
|
||||
from core.scoring.sync.kline_sync import KlineStockSync, KlineIndexSync
|
||||
from core.scoring.sync.stocks_sync import StocksSync
|
||||
from core.scoring.sync.industry_sync import IndustrySync
|
||||
from core.scoring.sync.market_regime import MarketRegimeSync
|
||||
from core.scoring.sync.sector_features import SectorFeaturesSync
|
||||
@@ -0,0 +1,41 @@
|
||||
"""
|
||||
数据同步抽象基类
|
||||
"""
|
||||
import abc
|
||||
from datetime import datetime
|
||||
from core.logger import LogLevel, PrintLog
|
||||
|
||||
|
||||
class BaseSync(abc.ABC):
|
||||
"""数据同步抽象基类，所有同步操作遵循 _fetch → _upsert 模式"""
|
||||
|
||||
def __init__(self):
|
||||
self.stats = {'inserted': 0, 'updated': 0, 'skipped': 0, 'errors': 0}
|
||||
|
||||
def run(self, **kwargs) -> dict:
|
||||
"""同步入口: 拉取数据 → 写入数据库 → 返回统计"""
|
||||
name = self.__class__.__name__
|
||||
PrintLog(LogLevel.INFO, f'[sync] {name} 开始同步...')
|
||||
t0 = datetime.now()
|
||||
try:
|
||||
data = self._fetch(**kwargs)
|
||||
self._upsert(data)
|
||||
elapsed = (datetime.now() - t0).total_seconds()
|
||||
PrintLog(
|
||||
LogLevel.INFO,
|
||||
f'[sync] {name} 完成 ({elapsed:.1f}s) — '
|
||||
f'insert={self.stats["inserted"]} update={self.stats["updated"]} '
|
||||
f'skip={self.stats["skipped"]} err={self.stats["errors"]}'
|
||||
)
|
||||
except Exception as e:
|
||||
PrintLog(LogLevel.ERROR, f'[sync] {name} 失败: {e}')
|
||||
raise
|
||||
return self.stats
|
||||
|
||||
@abc.abstractmethod
|
||||
def _fetch(self, **kwargs):
|
||||
"""从数据源拉取原始数据。子类实现。"""
|
||||
|
||||
@abc.abstractmethod
|
||||
def _upsert(self, data):
|
||||
"""将数据写入数据库。子类实现。"""
|
||||
@@ -0,0 +1,84 @@
|
||||
"""
|
||||
行业映射同步 — 从 QMT get_sector_list + get_stock_list_in_sector 获取
|
||||
"""
|
||||
from datetime import date
|
||||
from core.scoring.sync.base import BaseSync
|
||||
from core.scoring.models import IndustryMapping
|
||||
from core.database import db
|
||||
from core.logger import LogLevel, PrintLog
|
||||
|
||||
|
||||
class IndustrySync(BaseSync):
|
||||
"""行业映射同步 — QMT 行业板块 → IndustryMapping 表（全量替换）"""
|
||||
|
||||
def _fetch(self, **kwargs):
|
||||
"""从 QMT 拉取全部行业板块的成份股映射"""
|
||||
from xtquant import xtdata
|
||||
|
||||
all_sectors = xtdata.get_sector_list()
|
||||
PrintLog(LogLevel.INFO, f'[sync] Industry: 共 {len(all_sectors)} 个板块')
|
||||
|
||||
# 尝试使用 get_sector_info 过滤行业板块
|
||||
industry_sectors = []
|
||||
try:
|
||||
sector_info = xtdata.get_sector_info()
|
||||
if sector_info is not None and not sector_info.empty:
|
||||
for _, row in sector_info.iterrows():
|
||||
cat = row.get('category', '')
|
||||
if '行业' in str(cat):
|
||||
industry_sectors.append(row['sector'])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 如果 get_sector_info 无效，回退到名称过滤
|
||||
if not industry_sectors:
|
||||
for s in all_sectors:
|
||||
# 排除明显非行业的板块
|
||||
skip_markers = ['概念', '风格', '地域', '地区', '指数', '自定义',
|
||||
'ETF', 'LOF', '债券', '基金', '期货', '期权']
|
||||
if any(m in s for m in skip_markers):
|
||||
continue
|
||||
industry_sectors.append(s)
|
||||
|
||||
PrintLog(LogLevel.INFO, f'[sync] Industry: 筛选出 {len(industry_sectors)} 个行业板块')
|
||||
|
||||
# 构建 code → {industry_name, classification} 映射
|
||||
mapping = {} # code → (industry_name, classification)
|
||||
today = date.today()
|
||||
|
||||
for sector_name in industry_sectors:
|
||||
try:
|
||||
stocks = xtdata.get_stock_list_in_sector(sector_name)
|
||||
for full_code in stocks:
|
||||
code = full_code.split('.')[0]
|
||||
if code not in mapping:
|
||||
mapping[code] = {
|
||||
'code': code,
|
||||
'industry_name': sector_name,
|
||||
'industry_classification': 'QMT',
|
||||
'update_date': today,
|
||||
}
|
||||
except Exception:
|
||||
self.stats['errors'] += 1
|
||||
|
||||
self.stats['inserted'] = len(mapping)
|
||||
return list(mapping.values())
|
||||
|
||||
def _upsert(self, data: list):
|
||||
"""全量替换: 清空旧数据 → 批量插入新数据"""
|
||||
if not data:
|
||||
PrintLog(LogLevel.WARNING, '[sync] Industry: 无数据, 跳过')
|
||||
return
|
||||
|
||||
with db.atomic():
|
||||
IndustryMapping.delete().execute()
|
||||
for batch in _chunked(data, 500):
|
||||
IndustryMapping.insert_many(batch).execute()
|
||||
|
||||
PrintLog(LogLevel.INFO,
|
||||
f'[sync] Industry: 全量替换完成, {len(data)} 条映射')
|
||||
|
||||
|
||||
def _chunked(lst: list, n: int):
|
||||
for i in range(0, len(lst), n):
|
||||
yield lst[i:i + n]
|
||||
@@ -0,0 +1,222 @@
|
||||
"""
|
||||
K线数据同步 — 个股日K + 指数日K
|
||||
数据源: QMT xtdata
|
||||
增量同步: 只拉 max(trade_date) 之后的增量数据
|
||||
线程锁: KlineStockSync / KlineIndexSync 各自内部锁
|
||||
"""
|
||||
import pandas as pd
|
||||
from datetime import date, timedelta
|
||||
from core.scoring.sync.base import BaseSync
|
||||
from core.scoring.models import KlineStock, KlineIndex
|
||||
from core.scoring.config import TRACKED_INDICES
|
||||
from core.database import db
|
||||
from core.logger import LogLevel, PrintLog
|
||||
|
||||
BATCH_SIZE = 50
|
||||
DEFAULT_COUNT = 300
|
||||
|
||||
# 全局同步状态标记（字典引用传递，可被外部轮询）
|
||||
_sync_state = {"kline": False, "index": False, "stocks": False,
|
||||
"industry": False, "market": False, "sector": False}
|
||||
|
||||
|
||||
def is_syncing(key="kline") -> bool:
|
||||
return _sync_state.get(key, False)
|
||||
|
||||
|
||||
def _latest_date(model_cls) -> date | None:
|
||||
from peewee import fn
|
||||
row = model_cls.select(fn.MAX(model_cls.trade_date)).scalar()
|
||||
if isinstance(row, date):
|
||||
return row
|
||||
return None
|
||||
|
||||
|
||||
def _safe_get(df_dict, code, dt, default=0.0) -> float:
|
||||
"""安全获取 DataFrame 值"""
|
||||
if df_dict is None:
|
||||
return default
|
||||
df = df_dict.get(code)
|
||||
if df is None or code not in df.index:
|
||||
return default
|
||||
try:
|
||||
val = df.loc[code, dt]
|
||||
if pd.isna(val):
|
||||
return default
|
||||
return float(val)
|
||||
except Exception:
|
||||
return default
|
||||
|
||||
|
||||
class KlineStockSync(BaseSync):
|
||||
"""个股日K线同步 — 增量：只拉 max(trade_date) 之后的增量数据"""
|
||||
|
||||
def __init__(self, count: int = DEFAULT_COUNT):
|
||||
super().__init__()
|
||||
self.count = count
|
||||
|
||||
def _fetch(self, **kwargs):
|
||||
from xtquant import xtdata
|
||||
|
||||
# 增量判断
|
||||
latest = _latest_date(KlineStock)
|
||||
today = date.today()
|
||||
if latest is not None and latest >= today:
|
||||
PrintLog(LogLevel.INFO, f'[sync] KlineStock: 已最新 ({latest})，跳过')
|
||||
self.stats['skipped'] = 0
|
||||
return self.stats
|
||||
|
||||
# 增量起点
|
||||
start_date = (latest + timedelta(days=1)) if latest else None
|
||||
start_str = start_date.strftime('%Y%m%d') if start_date else ""
|
||||
PrintLog(LogLevel.INFO,
|
||||
f'[sync] KlineStock: 增量同步，起点={start_str or "全部"}')
|
||||
|
||||
all_stocks = xtdata.get_stock_list_in_sector("沪深A股")
|
||||
PrintLog(LogLevel.INFO, f'[sync] KlineStock: {len(all_stocks)} 只A股')
|
||||
field_list = ['open', 'high', 'low', 'close', 'volume']
|
||||
total = len(all_stocks)
|
||||
inserted = 0
|
||||
|
||||
for i, code in enumerate(all_stocks):
|
||||
if i > 0 and i % 50 == 0:
|
||||
PrintLog(LogLevel.INFO, f'[sync] KlineStock: {i}/{total} ({i*100//total}%)')
|
||||
|
||||
try:
|
||||
xtdata.download_history_data(code, period='1d', start_time=start_str)
|
||||
except Exception:
|
||||
self.stats['errors'] += 1
|
||||
continue
|
||||
|
||||
try:
|
||||
result = xtdata.get_market_data(
|
||||
field_list=field_list, stock_list=[code], period='1d',
|
||||
count=self.count, dividend_type='none', fill_data=False)
|
||||
inserted += self._upsert_incremental(code, result, start_date)
|
||||
except Exception:
|
||||
self.stats['errors'] += 1
|
||||
|
||||
self.stats['inserted'] = inserted
|
||||
PrintLog(LogLevel.INFO,
|
||||
f'[sync] KlineStock 完成: 新增={inserted} '
|
||||
f'跳过={self.stats["skipped"]} 错误={self.stats["errors"]}')
|
||||
return self.stats
|
||||
|
||||
def _upsert_incremental(self, full_code: str, result: dict, start_date) -> int:
|
||||
if not result:
|
||||
return 0
|
||||
close_df = result.get('close')
|
||||
if close_df is None or close_df.empty:
|
||||
return 0
|
||||
stock_code = full_code.split('.')[0]
|
||||
records = []
|
||||
for td in close_df.columns:
|
||||
td_date = td.date() if hasattr(td, 'date') else td
|
||||
if start_date is not None and td_date <= start_date:
|
||||
self.stats['skipped'] = self.stats.get('skipped', 0) + 1
|
||||
continue
|
||||
close_val = close_df.loc[full_code, td]
|
||||
if close_val is None or (isinstance(close_val, float) and pd.isna(close_val)):
|
||||
self.stats['skipped'] = self.stats.get('skipped', 0) + 1
|
||||
continue
|
||||
records.append({
|
||||
'stock_code': stock_code,
|
||||
'trade_date': td_date,
|
||||
'open': _safe_get(result.get('open'), full_code, td),
|
||||
'high': _safe_get(result.get('high'), full_code, td),
|
||||
'low': _safe_get(result.get('low'), full_code, td),
|
||||
'close': float(close_val),
|
||||
'volume': _safe_get(result.get('volume'), full_code, td),
|
||||
})
|
||||
if records:
|
||||
with db.atomic():
|
||||
for batch in _chunked(records, 500):
|
||||
KlineStock.insert_many(batch).on_conflict_replace().execute()
|
||||
return len(records)
|
||||
return 0
|
||||
|
||||
def _upsert(self, data):
|
||||
pass
|
||||
|
||||
|
||||
class KlineIndexSync(BaseSync):
|
||||
"""指数日K线同步 — 增量同步"""
|
||||
|
||||
def __init__(self, indices: list = None, count: int = DEFAULT_COUNT):
|
||||
super().__init__()
|
||||
self.indices = indices or TRACKED_INDICES
|
||||
self.count = count
|
||||
|
||||
def _fetch(self, **kwargs):
|
||||
from xtquant import xtdata
|
||||
|
||||
index_codes = []
|
||||
for code in self.indices:
|
||||
if code.startswith(('000', '001')):
|
||||
index_codes.append(f'{code}.SH')
|
||||
elif code.startswith('399'):
|
||||
index_codes.append(f'{code}.SZ')
|
||||
else:
|
||||
index_codes.append(f'{code}.SH')
|
||||
|
||||
latest = _latest_date(KlineIndex)
|
||||
today = date.today()
|
||||
if latest is not None and latest >= today:
|
||||
PrintLog(LogLevel.INFO, f'[sync] KlineIndex: 已最新 ({latest})，跳过')
|
||||
return {}
|
||||
|
||||
start_date = (latest + timedelta(days=1)) if latest else None
|
||||
start_str = start_date.strftime('%Y%m%d') if start_date else ""
|
||||
PrintLog(LogLevel.INFO,
|
||||
f'[sync] KlineIndex: 增量同步，起点={start_str or "全部"}')
|
||||
|
||||
for code in index_codes:
|
||||
try:
|
||||
xtdata.download_history_data(code, period='1d', start_time=start_str)
|
||||
except Exception:
|
||||
self.stats['errors'] += 1
|
||||
|
||||
field_list = ['open', 'high', 'low', 'close', 'volume']
|
||||
result = xtdata.get_market_data(
|
||||
field_list=field_list, stock_list=index_codes, period='1d',
|
||||
count=self.count, dividend_type='none', fill_data=False)
|
||||
return result or {}
|
||||
|
||||
def _upsert(self, data):
|
||||
if not data:
|
||||
return
|
||||
latest = _latest_date(KlineIndex)
|
||||
records = []
|
||||
close_df = data.get('close')
|
||||
if close_df is None or close_df.empty:
|
||||
return
|
||||
for full_code in close_df.index:
|
||||
index_code = full_code.split('.')[0]
|
||||
for td in close_df.columns:
|
||||
td_date = td.date() if hasattr(td, 'date') else td
|
||||
if latest is not None and td_date <= latest:
|
||||
self.stats['skipped'] = self.stats.get('skipped', 0) + 1
|
||||
continue
|
||||
close_val = close_df.loc[full_code, td]
|
||||
if close_val is None or (isinstance(close_val, float) and pd.isna(close_val)):
|
||||
self.stats['skipped'] = self.stats.get('skipped', 0) + 1
|
||||
continue
|
||||
records.append({
|
||||
'index_code': index_code,
|
||||
'trade_date': td_date,
|
||||
'open': _safe_get(data.get('open'), full_code, td),
|
||||
'high': _safe_get(data.get('high'), full_code, td),
|
||||
'low': _safe_get(data.get('low'), full_code, td),
|
||||
'close': float(close_val),
|
||||
'volume': _safe_get(data.get('volume'), full_code, td),
|
||||
})
|
||||
if records:
|
||||
with db.atomic():
|
||||
for batch in _chunked(records, 500):
|
||||
KlineIndex.insert_many(batch).on_conflict_replace().execute()
|
||||
self.stats['inserted'] = self.stats.get('inserted', 0) + len(records)
|
||||
|
||||
|
||||
def _chunked(lst: list, n: int):
|
||||
for i in range(0, len(lst), n):
|
||||
yield lst[i:i + n]
|
||||
@@ -0,0 +1,98 @@
|
||||
"""
|
||||
市场状态计算 — 从 kline_stock 聚合生成 market_regime_daily
|
||||
纯本地计算，不依赖外部数据源。
|
||||
"""
|
||||
import pandas as pd
|
||||
from peewee import fn, Case
|
||||
from core.scoring.sync.base import BaseSync
|
||||
from core.scoring.models import KlineStock, MarketRegimeDaily
|
||||
from core.scoring.config import PANIC_ADVANCE_RATIO, PANIC_TURNOVER_RATIO
|
||||
from core.database import db
|
||||
from core.logger import LogLevel, PrintLog
|
||||
|
||||
|
||||
class MarketRegimeSync(BaseSync):
|
||||
"""市场状态同步 — kline_stock 聚合 → market_regime_daily"""
|
||||
|
||||
def _fetch(self, **kwargs):
|
||||
"""从 KlineStock 逐日聚合涨跌家数和成交额"""
|
||||
PrintLog(LogLevel.INFO, '[sync] MarketRegime: 开始聚合全市场数据...')
|
||||
|
||||
# peewee 聚合查询: 逐日统计 advancers / decliners / turnover
|
||||
query = (KlineStock
|
||||
.select(
|
||||
KlineStock.trade_date,
|
||||
fn.SUM(
|
||||
Case(None, [(KlineStock.close > KlineStock.open, 1)], 0)
|
||||
).alias('advancers'),
|
||||
fn.SUM(
|
||||
Case(None, [(KlineStock.close < KlineStock.open, 1)], 0)
|
||||
).alias('decliners'),
|
||||
fn.SUM(KlineStock.close * KlineStock.volume).alias('turnover'),
|
||||
)
|
||||
.group_by(KlineStock.trade_date)
|
||||
.order_by(KlineStock.trade_date))
|
||||
|
||||
rows = list(query.dicts())
|
||||
if not rows:
|
||||
PrintLog(LogLevel.WARNING, '[sync] MarketRegime: KlineStock 表为空')
|
||||
return None
|
||||
|
||||
df = pd.DataFrame(rows)
|
||||
df['trade_date'] = pd.to_datetime(df['trade_date'])
|
||||
df = df.sort_values('trade_date').reset_index(drop=True)
|
||||
|
||||
# 计算涨跌比
|
||||
total = df['advancers'] + df['decliners']
|
||||
df['advance_ratio'] = (df['advancers'] / total.replace(0, 1)).round(4)
|
||||
|
||||
# 5日滚动均量
|
||||
df['turnover_avg_5d'] = (df['turnover']
|
||||
.rolling(window=5, min_periods=1)
|
||||
.mean()
|
||||
.round(2))
|
||||
|
||||
# 量比
|
||||
df['turnover_ratio_5d'] = (df['turnover'] /
|
||||
df['turnover_avg_5d'].replace(0, 1)).round(4)
|
||||
|
||||
# 极端恐慌标记
|
||||
df['is_extreme_panic'] = (
|
||||
(df['advance_ratio'] < PANIC_ADVANCE_RATIO) &
|
||||
(df['turnover_ratio_5d'] > PANIC_TURNOVER_RATIO)
|
||||
).astype(int)
|
||||
|
||||
PrintLog(LogLevel.INFO,
|
||||
f'[sync] MarketRegime: 聚合完成, {len(df)} 个交易日')
|
||||
|
||||
return df
|
||||
|
||||
def _upsert(self, df):
|
||||
"""写入 MarketRegimeDaily 表"""
|
||||
if df is None or df.empty:
|
||||
return
|
||||
|
||||
records = []
|
||||
for _, row in df.iterrows():
|
||||
records.append({
|
||||
'trade_date': row['trade_date'].date(),
|
||||
'advancers': int(row['advancers']),
|
||||
'decliners': int(row['decliners']),
|
||||
'advance_ratio': float(row['advance_ratio']),
|
||||
'turnover': float(row['turnover']),
|
||||
'turnover_avg_5d': float(row['turnover_avg_5d']),
|
||||
'turnover_ratio_5d': float(row['turnover_ratio_5d']),
|
||||
'source': 'qmt',
|
||||
'is_extreme_panic': int(row['is_extreme_panic']),
|
||||
})
|
||||
|
||||
if records:
|
||||
with db.atomic():
|
||||
for batch in _chunked(records, 500):
|
||||
MarketRegimeDaily.insert_many(batch).on_conflict_replace().execute()
|
||||
self.stats['inserted'] += len(records)
|
||||
|
||||
|
||||
def _chunked(lst: list, n: int):
|
||||
for i in range(0, len(lst), n):
|
||||
yield lst[i:i + n]
|
||||
@@ -0,0 +1,129 @@
|
||||
"""
|
||||
行业聚合指数计算 — 从 kline_stock + industry 生成 sector_features_daily
|
||||
纯本地计算，不依赖外部数据源。
|
||||
"""
|
||||
import pandas as pd
|
||||
from core.scoring.sync.base import BaseSync
|
||||
from core.scoring.models import KlineStock, IndustryMapping, SectorFeaturesDaily
|
||||
from core.database import db
|
||||
from core.logger import LogLevel, PrintLog
|
||||
|
||||
|
||||
class SectorFeaturesSync(BaseSync):
|
||||
"""行业聚合指数同步 — kline_stock + industry → sector_features_daily"""
|
||||
|
||||
def _fetch(self, **kwargs):
|
||||
"""从数据库加载原始数据, 计算行业指数特征"""
|
||||
PrintLog(LogLevel.INFO, '[sync] SectorFeatures: 加载原始数据...')
|
||||
|
||||
# 1. 加载行业映射: code → industry_name
|
||||
industries = list(IndustryMapping.select().dicts())
|
||||
if not industries:
|
||||
PrintLog(LogLevel.WARNING, '[sync] SectorFeatures: IndustryMapping 表为空, 请先执行 industry sync')
|
||||
return None
|
||||
code_to_industry = {row['code']: row['industry_name'] for row in industries}
|
||||
PrintLog(LogLevel.INFO, f'[sync] SectorFeatures: {len(code_to_industry)} 条行业映射')
|
||||
|
||||
# 2. 加载 K 线数据
|
||||
kline_rows = (KlineStock
|
||||
.select(
|
||||
KlineStock.stock_code,
|
||||
KlineStock.trade_date,
|
||||
KlineStock.open,
|
||||
KlineStock.high,
|
||||
KlineStock.low,
|
||||
KlineStock.close,
|
||||
)
|
||||
.order_by(KlineStock.stock_code, KlineStock.trade_date)
|
||||
.dicts())
|
||||
|
||||
if not kline_rows:
|
||||
PrintLog(LogLevel.WARNING, '[sync] SectorFeatures: KlineStock 表为空')
|
||||
return None
|
||||
|
||||
df = pd.DataFrame(kline_rows)
|
||||
df['trade_date'] = pd.to_datetime(df['trade_date'])
|
||||
PrintLog(LogLevel.INFO, f'[sync] SectorFeatures: {len(df)} 条K线数据')
|
||||
|
||||
# 3. 映射行业
|
||||
df['sector_name'] = df['stock_code'].map(code_to_industry)
|
||||
df = df.dropna(subset=['sector_name'])
|
||||
|
||||
# 4. 逐股计算日收益率和振幅
|
||||
df = df.sort_values(['stock_code', 'trade_date'])
|
||||
df['prev_close'] = df.groupby('stock_code')['close'].shift(1)
|
||||
df['pct_chg'] = (df['close'] - df['prev_close']) / df['prev_close'] * 100
|
||||
df['amplitude'] = (df['high'] - df['low']) / df['open'] * 100
|
||||
|
||||
# 清理无效值
|
||||
df = df.dropna(subset=['pct_chg', 'amplitude'])
|
||||
|
||||
# 5. 按行业+日期聚合
|
||||
agg = (df.groupby(['trade_date', 'sector_name'])
|
||||
.agg(
|
||||
sector_ret=('pct_chg', 'mean'),
|
||||
sector_amplitude=('amplitude', 'mean'),
|
||||
)
|
||||
.reset_index())
|
||||
|
||||
# 6. 构建行业指数 (基值=100)
|
||||
agg = agg.sort_values(['sector_name', 'trade_date'])
|
||||
agg['sector_index'] = agg.groupby('sector_name')['sector_ret'].transform(
|
||||
lambda x: (1 + x / 100).cumprod() * 100
|
||||
)
|
||||
# 重新基值=100 (每行业独立)
|
||||
for name, group in agg.groupby('sector_name'):
|
||||
idx = group.index
|
||||
first_val = group['sector_index'].iloc[0]
|
||||
agg.loc[idx, 'sector_index'] = group['sector_index'] / first_val * 100
|
||||
|
||||
# 7. 计算 EMA 均线
|
||||
agg['ema10'] = (agg.groupby('sector_name')['sector_index']
|
||||
.transform(lambda x: x.ewm(span=10, min_periods=1).mean()))
|
||||
agg['ema20'] = (agg.groupby('sector_name')['sector_index']
|
||||
.transform(lambda x: x.ewm(span=20, min_periods=1).mean()))
|
||||
agg['ema200'] = (agg.groupby('sector_name')['sector_index']
|
||||
.transform(lambda x: x.ewm(span=200, min_periods=1).mean()))
|
||||
|
||||
# 8. 趋势评分: close>ema200 得1分 + ema10>ema20 得1分
|
||||
agg['score'] = (
|
||||
(agg['sector_index'] > agg['ema200']).astype(int) +
|
||||
(agg['ema10'] > agg['ema20']).astype(int)
|
||||
)
|
||||
|
||||
PrintLog(LogLevel.INFO,
|
||||
f'[sync] SectorFeatures: 计算完成, {len(agg)} 行, '
|
||||
f'{agg["sector_name"].nunique()} 个行业')
|
||||
|
||||
return agg
|
||||
|
||||
def _upsert(self, agg):
|
||||
"""写入 SectorFeaturesDaily 表"""
|
||||
if agg is None or agg.empty:
|
||||
return
|
||||
|
||||
records = []
|
||||
for _, row in agg.iterrows():
|
||||
records.append({
|
||||
'trade_date': row['trade_date'].date()
|
||||
if hasattr(row['trade_date'], 'date') else row['trade_date'],
|
||||
'sector_name': str(row['sector_name']),
|
||||
'sector_ret': round(float(row['sector_ret']), 4),
|
||||
'sector_amplitude': round(float(row['sector_amplitude']), 4),
|
||||
'close': round(float(row['sector_index']), 4),
|
||||
'ema10': round(float(row['ema10']), 4),
|
||||
'ema20': round(float(row['ema20']), 4),
|
||||
'ema200': round(float(row['ema200']), 4),
|
||||
'score': int(row['score']),
|
||||
})
|
||||
|
||||
if records:
|
||||
with db.atomic():
|
||||
for batch in _chunked(records, 500):
|
||||
SectorFeaturesDaily.insert_many(batch).on_conflict_replace().execute()
|
||||
self.stats['inserted'] += len(records)
|
||||
|
||||
|
||||
def _chunked(lst: list, n: int):
|
||||
for i in range(0, len(lst), n):
|
||||
yield lst[i:i + n]
|
||||
@@ -0,0 +1,100 @@
|
||||
"""
|
||||
股票基础信息同步 — 从 QMT get_instrument_detail 获取
|
||||
"""
|
||||
from datetime import date
|
||||
from core.scoring.sync.base import BaseSync
|
||||
from core.scoring.models import StockInfo
|
||||
from core.database import db
|
||||
from core.logger import LogLevel, PrintLog
|
||||
|
||||
BATCH_SIZE = 200
|
||||
|
||||
|
||||
class StocksSync(BaseSync):
|
||||
"""股票基础信息同步 — QMT get_instrument_detail_list"""
|
||||
|
||||
def _fetch(self, **kwargs):
|
||||
"""从 QMT 拉取全部A股基础信息"""
|
||||
from xtquant import xtdata
|
||||
|
||||
all_stocks = xtdata.get_stock_list_in_sector("沪深A股")
|
||||
PrintLog(LogLevel.INFO, f'[sync] Stocks: 获取到 {len(all_stocks)} 只A股')
|
||||
|
||||
result = {}
|
||||
total = len(all_stocks)
|
||||
for i in range(0, total, BATCH_SIZE):
|
||||
batch = all_stocks[i:i + BATCH_SIZE]
|
||||
try:
|
||||
details = xtdata.get_instrument_detail_list(batch)
|
||||
result.update(details)
|
||||
except Exception as e:
|
||||
PrintLog(LogLevel.ERROR,
|
||||
f'[sync] Stocks batch {i}-{min(i + BATCH_SIZE, total)} failed: {e}')
|
||||
self.stats['errors'] += len(batch)
|
||||
|
||||
return result
|
||||
|
||||
def _upsert(self, data: dict):
|
||||
"""写入 StockInfo 表"""
|
||||
records = []
|
||||
today = date.today()
|
||||
|
||||
for full_code, inst in data.items():
|
||||
if not inst:
|
||||
self.stats['skipped'] += 1
|
||||
continue
|
||||
|
||||
code_num = full_code.split('.')[0]
|
||||
|
||||
# 判断交易所
|
||||
exchange = inst.get('ExchangeID', '')
|
||||
if not exchange:
|
||||
if '.SH' in full_code:
|
||||
exchange = 'SH'
|
||||
elif '.SZ' in full_code:
|
||||
exchange = 'SZ'
|
||||
elif '.BJ' in full_code:
|
||||
exchange = 'BJ'
|
||||
|
||||
# OpenDate 格式: 'YYYYMMDD' 或 int
|
||||
open_date = inst.get('OpenDate', '')
|
||||
if open_date and len(str(open_date)) == 8:
|
||||
list_date_val = f'{str(open_date)[:4]}-{str(open_date)[4:6]}-{str(open_date)[6:8]}'
|
||||
else:
|
||||
list_date_val = None
|
||||
|
||||
# InstrumentStatus 含义:
|
||||
# 0/1 = 正常股票（含已退市但仍在QMT列表的）
|
||||
# -1 = ST / *ST
|
||||
# 30/31 = *ST
|
||||
# 已退市股（名字含XD/退）K线不足120日，会在候选股过滤时被排除
|
||||
status = inst.get('InstrumentStatus', -1)
|
||||
if status in (-1, 30, 31):
|
||||
listing_status = 'ST'
|
||||
else:
|
||||
listing_status = 'normal'
|
||||
|
||||
total_share = inst.get('TotalVolume', None)
|
||||
float_share = inst.get('FloatVolume', None)
|
||||
|
||||
records.append({
|
||||
'code': full_code,
|
||||
'name': str(inst.get('InstrumentName', '')),
|
||||
'exchange': exchange,
|
||||
'list_date': list_date_val,
|
||||
'listing_status': listing_status,
|
||||
'total_share': float(total_share) if total_share else None,
|
||||
'float_share': float(float_share) if float_share else None,
|
||||
'share_updated_at': today,
|
||||
})
|
||||
|
||||
if records:
|
||||
with db.atomic():
|
||||
for batch in _chunked(records, 500):
|
||||
StockInfo.insert_many(batch).on_conflict_replace().execute()
|
||||
self.stats['inserted'] += len(records)
|
||||
|
||||
|
||||
def _chunked(lst: list, n: int):
|
||||
for i in range(0, len(lst), n):
|
||||
yield lst[i:i + n]
|
||||
Reference in New Issue
Block a user