kyugao/sfgrid
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
WIN-ALB39P9B6BU\Docker
2026-07-06 09:12:38 +08:00
parent fd5337f163
commit de83f97c05
6 changed files with 381 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/scoring/sync/kline_sync.py
+44 -23
View File
@@ -58,19 +58,18 @@ class KlineStockSync(BaseSync):
def _fetch(self, **kwargs):
from xtquant import xtdata
# 增量判断
# 增量判断: 以数据库最新一条记录为准
latest = _latest_date(KlineStock)
today = date.today()
if latest is not None and latest >= today:
PrintLog(LogLevel.INFO, f'[sync] KlineStock: 已最新 ({latest})，跳过')
self.stats['skipped'] = 0
return self.stats
# 增量起点
# 增量起点: last_db_date + 1; 截止: 昨天（盘中不能同步当天数据）
# 注意: 不能用 latest >= today 跳过，因为 latest 可能是错误的未收盘数据
start_date = (latest + timedelta(days=1)) if latest else None
end_date = today - timedelta(days=1) # 固定截止到昨天，收盘后同步昨天数据
start_str = start_date.strftime('%Y%m%d') if start_date else ""
end_str = end_date.strftime('%Y%m%d')
PrintLog(LogLevel.INFO,
f'[sync] KlineStock: 增量同步，起点={start_str or "全部"}')
f'[sync] KlineStock: 增量同步 {start_str} ~ {end_str}')
all_stocks = xtdata.get_stock_list_in_sector("沪深A股")
PrintLog(LogLevel.INFO, f'[sync] KlineStock: {len(all_stocks)} 只A股')
@@ -83,7 +82,7 @@ class KlineStockSync(BaseSync):
PrintLog(LogLevel.INFO, f'[sync] KlineStock: {i}/{total} ({i*100//total}%)')
try:
xtdata.download_history_data(code, period='1d', start_time=start_str)
xtdata.download_history_data(code, period='1d', start_time=start_str, end_time=end_str)
except Exception:
self.stats['errors'] += 1
continue
@@ -91,8 +90,9 @@ class KlineStockSync(BaseSync):
try:
result = xtdata.get_market_data(
field_list=field_list, stock_list=[code], period='1d',
count=self.count, dividend_type='none', fill_data=False)
inserted += self._upsert_incremental(code, result, start_date)
start_time=start_str, end_time=end_str,
dividend_type='none', fill_data=False)
inserted += self._upsert_incremental(code, result, start_date, end_date)
except Exception:
self.stats['errors'] += 1
@@ -102,7 +102,7 @@ class KlineStockSync(BaseSync):
f'跳过={self.stats["skipped"]} 错误={self.stats["errors"]}')
return self.stats
def _upsert_incremental(self, full_code: str, result: dict, start_date) -> int:
def _upsert_incremental(self, full_code: str, result: dict, start_date, end_date) -> int:
if not result:
return 0
close_df = result.get('close')
@@ -110,13 +110,23 @@ class KlineStockSync(BaseSync):
return 0
stock_code = full_code.split('.')[0]
records = []
vol_df = result.get('volume')
for td in close_df.columns:
# xtdata 返回的列名可能是字符串 'YYYYMMDD' 或 datetime，需统一转成 date
if isinstance(td, str):
td_date = datetime.strptime(td, '%Y%m%d').date()
else:
td_date = td.date() if hasattr(td, 'date') else td
if start_date is not None and td_date <= start_date:
# 过滤: 不在增量范围内的跳过 (start_date < td <= end_date)
if start_date is not None and td_date < start_date:
self.stats['skipped'] = self.stats.get('skipped', 0) + 1
continue
if end_date is not None and td_date > end_date:
self.stats['skipped'] = self.stats.get('skipped', 0) + 1
continue
# 跳过成交量为0的无效数据（盘中未结算数据）
vol = vol_df.loc[full_code, td] if vol_df is not None else None
if vol is None or (isinstance(vol, float) and pd.isna(vol)) or vol == 0:
self.stats['skipped'] = self.stats.get('skipped', 0) + 1
continue
close_val = close_df.loc[full_code, td]
@@ -130,7 +140,7 @@ class KlineStockSync(BaseSync):
'high': _safe_get(result.get('high'), full_code, td),
'low': _safe_get(result.get('low'), full_code, td),
'close': float(close_val),
'volume': _safe_get(result.get('volume'), full_code, td),
'volume': float(vol),
})
if records:
with db.atomic():
@@ -165,33 +175,35 @@ class KlineIndexSync(BaseSync):
latest = _latest_date(KlineIndex)
today = date.today()
if latest is not None and latest >= today:
PrintLog(LogLevel.INFO, f'[sync] KlineIndex: 已最新 ({latest})，跳过')
return {}
start_date = (latest + timedelta(days=1)) if latest else None
end_date = today - timedelta(days=1) # 截止到昨天
start_str = start_date.strftime('%Y%m%d') if start_date else ""
end_str = end_date.strftime('%Y%m%d')
PrintLog(LogLevel.INFO,
f'[sync] KlineIndex: 增量同步，起点={start_str or "全部"}')
f'[sync] KlineIndex: 增量同步 {start_str} ~ {end_str}')
for code in index_codes:
try:
xtdata.download_history_data(code, period='1d', start_time=start_str)
xtdata.download_history_data(code, period='1d', start_time=start_str, end_time=end_str)
except Exception:
self.stats['errors'] += 1
field_list = ['open', 'high', 'low', 'close', 'volume']
result = xtdata.get_market_data(
field_list=field_list, stock_list=index_codes, period='1d',
count=self.count, dividend_type='none', fill_data=False)
return result or {}
start_time=start_str, end_time=end_str,
dividend_type='none', fill_data=False)
def _upsert(self, data):
if not data:
return
latest = _latest_date(KlineIndex)
today = date.today()
start_date = (latest + timedelta(days=1)) if latest else None
end_date = today - timedelta(days=1)
records = []
close_df = data.get('close')
vol_df = data.get('volume')
if close_df is None or close_df.empty:
return
for full_code in close_df.index:
@@ -201,7 +213,16 @@ class KlineIndexSync(BaseSync):
td_date = datetime.strptime(td, '%Y%m%d').date()
else:
td_date = td.date() if hasattr(td, 'date') else td
if latest is not None and td_date <= latest:
# 增量范围过滤 (start_date < td <= end_date)
if start_date is not None and td_date < start_date:
self.stats['skipped'] = self.stats.get('skipped', 0) + 1
continue
if end_date is not None and td_date > end_date:
self.stats['skipped'] = self.stats.get('skipped', 0) + 1
continue
# 过滤成交量为0的无效数据
vol = vol_df.loc[full_code, td] if vol_df is not None else None
if vol is None or (isinstance(vol, float) and pd.isna(vol)) or vol == 0:
self.stats['skipped'] = self.stats.get('skipped', 0) + 1
continue
close_val = close_df.loc[full_code, td]
@@ -215,7 +236,7 @@ class KlineIndexSync(BaseSync):
'high': _safe_get(data.get('high'), full_code, td),
'low': _safe_get(data.get('low'), full_code, td),
'close': float(close_val),
'volume': _safe_get(data.get('volume'), full_code, td),
'volume': float(vol),
})
if records:
with db.atomic():