完成模型更新

core/scoring/sync/kline_sync.py

@@ -5,7 +5,7 @@ K线数据同步 — 个股日K + 指数日K
 线程锁: KlineStockSync / KlineIndexSync 各自内部锁
 """
 import pandas as pd
-from datetime import date, timedelta
+from datetime import date, datetime, timedelta
 from core.scoring.sync.base import BaseSync
 from core.scoring.models import KlineStock, KlineIndex
 from core.scoring.config import TRACKED_INDICES
@@ -111,7 +111,11 @@ class KlineStockSync(BaseSync):
         stock_code = full_code.split('.')[0]
         records = []
         for td in close_df.columns:
-            td_date = td.date() if hasattr(td, 'date') else td
+            # xtdata 返回的列名可能是字符串 'YYYYMMDD' 或 datetime，需统一转成 date
+            if isinstance(td, str):
+                td_date = datetime.strptime(td, '%Y%m%d').date()
+            else:
+                td_date = td.date() if hasattr(td, 'date') else td
             if start_date is not None and td_date <= start_date:
                 self.stats['skipped'] = self.stats.get('skipped', 0) + 1
                 continue
@@ -193,7 +197,10 @@ class KlineIndexSync(BaseSync):
         for full_code in close_df.index:
             index_code = full_code.split('.')[0]
             for td in close_df.columns:
-                td_date = td.date() if hasattr(td, 'date') else td
+                if isinstance(td, str):
+                    td_date = datetime.strptime(td, '%Y%m%d').date()
+                else:
+                    td_date = td.date() if hasattr(td, 'date') else td
                 if latest is not None and td_date <= latest:
                     self.stats['skipped'] = self.stats.get('skipped', 0) + 1
                     continue

core/scoring/sync/sector_features.py

@@ -13,7 +13,7 @@ class SectorFeaturesSync(BaseSync):
     """行业聚合指数同步 — kline_stock + industry → sector_features_daily"""
 
     def _fetch(self, **kwargs):
-        """从数据库加载原始数据, 计算行业指数特征"""
+        """从数据库加载原始数据, 计算行业指数特征（分块处理避免内存溢出）"""
         PrintLog(LogLevel.INFO, '[sync] SectorFeatures: 加载原始数据...')
 
         # 1. 加载行业映射: code → industry_name
@@ -24,49 +24,81 @@ class SectorFeaturesSync(BaseSync):
         code_to_industry = {row['code']: row['industry_name'] for row in industries}
         PrintLog(LogLevel.INFO, f'[sync] SectorFeatures: {len(code_to_industry)} 条行业映射')
 
-        # 2. 加载 K 线数据
-        kline_rows = (KlineStock
-                      .select(
-                          KlineStock.stock_code,
-                          KlineStock.trade_date,
-                          KlineStock.open,
-                          KlineStock.high,
-                          KlineStock.low,
-                          KlineStock.close,
-                      )
-                      .order_by(KlineStock.stock_code, KlineStock.trade_date)
-                      .dicts())
+        # 2. 分块加载 K 线数据，避免内存溢出
+        # 聚合结果: {(trade_date, sector_name): [sum_pct_chg, sum_amp, count]}
+        sector_daily_agg = {}  # key: (date, sector) -> {'ret_sum': float, 'amp_sum': float, 'count': int}
+        CHUNK_SIZE = 50000
+        last_date_per_stock = {}  # stock_code -> prev_close
 
-        if not kline_rows:
-            PrintLog(LogLevel.WARNING, '[sync] SectorFeatures: KlineStock 表为空')
+        PrintLog(LogLevel.INFO, '[sync] SectorFeatures: 分块处理K线数据...')
+        chunk_num = 0
+        while True:
+            chunk_num += 1
+            rows = list(KlineStock
+                        .select(
+                            KlineStock.stock_code,
+                            KlineStock.trade_date,
+                            KlineStock.open,
+                            KlineStock.high,
+                            KlineStock.low,
+                            KlineStock.close,
+                        )
+                        .order_by(KlineStock.stock_code, KlineStock.trade_date)
+                        .offset((chunk_num - 1) * CHUNK_SIZE)
+                        .limit(CHUNK_SIZE)
+                        .dicts())
+            if not rows:
+                break
+
+            PrintLog(LogLevel.INFO, f'[sync] SectorFeatures: 处理块 {chunk_num} ({len(rows)} 行)...')
+
+            for row in rows:
+                code = str(row['stock_code'])
+                td = row['trade_date']
+                open_p = float(row['open'])
+                high = float(row['high'])
+                low = float(row['low'])
+                close = float(row['close'])
+
+                sector = code_to_industry.get(code)
+                if sector is None:
+                    continue
+
+                # 计算日收益率和振幅
+                prev_close = last_date_per_stock.get(code)
+                if prev_close is not None and prev_close > 0 and open_p > 0 and close > 0:
+                    pct_chg = (close - prev_close) / prev_close * 100
+                    amp = (high - low) / open_p * 100
+                    key = (td, sector)
+                    if key not in sector_daily_agg:
+                        sector_daily_agg[key] = {'ret_sum': 0.0, 'amp_sum': 0.0, 'count': 0}
+                    sector_daily_agg[key]['ret_sum'] += pct_chg
+                    sector_daily_agg[key]['amp_sum'] += amp
+                    sector_daily_agg[key]['count'] += 1
+
+                last_date_per_stock[code] = close
+
+        if not sector_daily_agg:
+            PrintLog(LogLevel.WARNING, '[sync] SectorFeatures: 无有效K线数据')
             return None
 
-        df = pd.DataFrame(kline_rows)
-        df['trade_date'] = pd.to_datetime(df['trade_date'])
-        PrintLog(LogLevel.INFO, f'[sync] SectorFeatures: {len(df)} 条K线数据')
+        PrintLog(LogLevel.INFO, f'[sync] SectorFeatures: 聚合完成, {len(sector_daily_agg)} 个行业-日组合')
 
-        # 3. 映射行业
-        df['sector_name'] = df['stock_code'].map(code_to_industry)
-        df = df.dropna(subset=['sector_name'])
+        # 3. 构建聚合 DataFrame
+        agg_data = []
+        for (td, sector), vals in sector_daily_agg.items():
+            agg_data.append({
+                'trade_date': td,
+                'sector_name': sector,
+                'sector_ret': vals['ret_sum'] / vals['count'],
+                'sector_amplitude': vals['amp_sum'] / vals['count'],
+            })
+        agg = pd.DataFrame(agg_data)
+        agg = agg.sort_values(['sector_name', 'trade_date'])
+        agg['trade_date'] = pd.to_datetime(agg['trade_date'])
+        PrintLog(LogLevel.INFO, f'[sync] SectorFeatures: {len(agg)} 行, {agg["sector_name"].nunique()} 个行业')
 
-        # 4. 逐股计算日收益率和振幅
-        df = df.sort_values(['stock_code', 'trade_date'])
-        df['prev_close'] = df.groupby('stock_code')['close'].shift(1)
-        df['pct_chg'] = (df['close'] - df['prev_close']) / df['prev_close'] * 100
-        df['amplitude'] = (df['high'] - df['low']) / df['open'] * 100
-
-        # 清理无效值
-        df = df.dropna(subset=['pct_chg', 'amplitude'])
-
-        # 5. 按行业+日期聚合
-        agg = (df.groupby(['trade_date', 'sector_name'])
-               .agg(
-                   sector_ret=('pct_chg', 'mean'),
-                   sector_amplitude=('amplitude', 'mean'),
-               )
-               .reset_index())
-
-        # 6. 构建行业指数 (基值=100)
+        # 4. 构建行业指数 (基值=100)
         agg = agg.sort_values(['sector_name', 'trade_date'])
         agg['sector_index'] = agg.groupby('sector_name')['sector_ret'].transform(
             lambda x: (1 + x / 100).cumprod() * 100
@@ -77,7 +109,7 @@ class SectorFeaturesSync(BaseSync):
             first_val = group['sector_index'].iloc[0]
             agg.loc[idx, 'sector_index'] = group['sector_index'] / first_val * 100
 
-        # 7. 计算 EMA 均线
+        # 5. 计算 EMA 均线
         agg['ema10'] = (agg.groupby('sector_name')['sector_index']
                         .transform(lambda x: x.ewm(span=10, min_periods=1).mean()))
         agg['ema20'] = (agg.groupby('sector_name')['sector_index']
@@ -85,7 +117,7 @@ class SectorFeaturesSync(BaseSync):
         agg['ema200'] = (agg.groupby('sector_name')['sector_index']
                          .transform(lambda x: x.ewm(span=200, min_periods=1).mean()))
 
-        # 8. 趋势评分: close>ema200 得1分 + ema10>ema20 得1分
+        # 6. 趋势评分: close>ema200 得1分 + ema10>ema20 得1分
         agg['score'] = (
             (agg['sector_index'] > agg['ema200']).astype(int) +
             (agg['ema10'] > agg['ema20']).astype(int)