From d140fee9e7269fc0b1a294e9f73bb7795fd5f889 Mon Sep 17 00:00:00 2001 From: gao Date: Fri, 3 Jul 2026 10:50:25 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E6=97=A9=E7=9B=98=E5=89=8D=2009:00=20st?= =?UTF-8?q?ock=5Fbasic=20+=20industry=5Fsector=20=E6=B2=A1=E8=A7=A6?= =?UTF-8?q?=E5=8F=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit work #02 (2026-07-03)：daily_check 复跑仍 10 missed，细看时间窗口发现 新问题——09:00 / 09:30 该跑的 2 个早盘前 task today 没触发。 根因：worker.py 进程内 scheduler 是预期的主调度（含 09:00 / 09:30）， 但 worker.py 没人拉起；systemd timers 又只覆盖 15:30 / 21:xx，缺早盘档。 昨 18h 删 monitor 同时没补 worker.py → 早盘缺口暴露。 变更： - bin/market_sync_morning_run.sh: 轻量 morning wrapper，只跑 stock_basic + industry_sector 串行（~25min），不复用 runall_once.py 的 8 task - bin/systemd/market-sync-morning.{service,timer}: Mon..Fri 09:00 daily 触发，TimeoutStartSec=3600 (1h) - docs/works/2026-07-03-02-morning-no-trigger.md: 完整 work 记录 不在本 work 范围（更大重构）：把 worker.py 拉起作为 long-running， 与 systemd timer 收敛到单一调度器——见 dual-scheduler-overlap memory。 Co-Authored-By: Claude --- bin/market_sync_morning_run.sh | 46 ++++++++++++++ bin/systemd/market-sync-morning.service | 29 +++++++++ bin/systemd/market-sync-morning.timer | 14 +++++ .../works/2026-07-03-02-morning-no-trigger.md | 63 +++++++++++++++++++ 4 files changed, 152 insertions(+) create mode 100755 bin/market_sync_morning_run.sh create mode 100644 bin/systemd/market-sync-morning.service create mode 100644 bin/systemd/market-sync-morning.timer create mode 100644 docs/works/2026-07-03-02-morning-no-trigger.md diff --git a/bin/market_sync_morning_run.sh b/bin/market_sync_morning_run.sh new file mode 100755 index 0000000..b1f7e89 --- /dev/null +++ b/bin/market_sync_morning_run.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# 早盘前同步：stock_basic + industry_sector +# - stock_basic：雪球 quote_detail 拉全市场股本快照（~9min @ 5 RPS） +# - industry_sector：baostock 拉股票-行业映射（~16min @ 5 RPS） +# - 串行执行，industry_sector 依赖 stock_basic +# - 与 runall_once.py 区别：不拉 kline_daily / kline_5min 等耗时长任务 + +set -u + +PROJECT_ROOT="/home/gao/Development/quant_home/market_sync" +LOG_DIR="$PROJECT_ROOT/logs" +mkdir -p "$LOG_DIR" + +TS=$(date +%Y%m%d_%H%M%S) +LOG="$LOG_DIR/morning_run_${TS}.log" + +cd "$PROJECT_ROOT" +exec .venv/bin/python -c " +import sys, time +sys.path.insert(0, '$PROJECT_ROOT') + +from app.core.datasource.registry import build_default_registry +from app.core.sync.registry import seed_sync_registry +build_default_registry() +seed_sync_registry() + +from app.tasks import get_task + +results = [] +for tid in ['stock_basic', 'industry_sector']: + print(f'[morning-run] >>> 开始 {tid}', flush=True) + t0 = time.time() + try: + r = get_task(tid).run(trigger_source='morning_runall') + elapsed = round(time.time() - t0, 1) + results.append({'task': tid, 'status': r.get('status'), 'elapsed_sec': elapsed, 'message': r.get('message', '')}) + print(f'[morning-run] <<< {tid} status={r.get(\"status\")} elapsed={elapsed}s msg={r.get(\"message\", \"\")}', flush=True) + except Exception as e: + elapsed = round(time.time() - t0, 1) + results.append({'task': tid, 'status': 'error', 'elapsed_sec': elapsed, 'message': str(e)}) + print(f'[morning-run] !!! {tid} 异常: {e}', flush=True) + time.sleep(5) + +import json +print('[morning-run] summary:', json.dumps(results, ensure_ascii=False)) +" > "$LOG" 2>&1 \ No newline at end of file diff --git a/bin/systemd/market-sync-morning.service b/bin/systemd/market-sync-morning.service new file mode 100644 index 0000000..5c92fa2 --- /dev/null +++ b/bin/systemd/market-sync-morning.service @@ -0,0 +1,29 @@ +[Unit] +Description=Market sync morning pre-market — stock_basic + industry_sector +Documentation=file:///home/gao/Development/quant_home/market_sync/bin/market_sync_morning_run.sh +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +WorkingDirectory=/home/gao/Development/quant_home/market_sync +User=gao +Group=gao + +# 早盘前只跑 stock_basic + industry_sector（避免和 15:30 runall 重复） +ExecStart=/home/gao/Development/quant_home/market_sync/bin/market_sync_morning_run.sh + +# stock_basic ~9min + industry_sector ~16min + sleep 5s + buffer = 1h 够用 +TimeoutStartSec=3600 + +# 不要 Restart=（oneshot 失败就让 OnFailure= 发通知，别自动重跑） + +# 日志走 journald（journalctl -u market-sync-morning -f） +StandardOutput=journal +StandardError=journal +SyslogIdentifier=market-sync-morning + +Environment=PYTHONUNBUFFERED=1 + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/bin/systemd/market-sync-morning.timer b/bin/systemd/market-sync-morning.timer new file mode 100644 index 0000000..9a877a4 --- /dev/null +++ b/bin/systemd/market-sync-morning.timer @@ -0,0 +1,14 @@ +[Unit] +Description=Schedule market sync morning — Mon..Fri 09:00 Asia/Shanghai +# market-sync-morning.service 是这个 timer 的执行单元 +# 早盘前 09:00 触发：跑 stock_basic + industry_sector，给 09:30 开盘留 30min buffer + +[Timer] +# A 股开盘 09:30，09:00 跑 stock_basic (~9min) + industry_sector (~16min) = ~25min +OnCalendar=Mon..Fri 09:00:00 Asia/Shanghai +# 关机/错过时下次开机补跑 +Persistent=true +Unit=market-sync-morning.service + +[Install] +WantedBy=timers.target \ No newline at end of file diff --git a/docs/works/2026-07-03-02-morning-no-trigger.md b/docs/works/2026-07-03-02-morning-no-trigger.md new file mode 100644 index 0000000..c0d3509 --- /dev/null +++ b/docs/works/2026-07-03-02-morning-no-trigger.md @@ -0,0 +1,63 @@ +# 2026-07-03 — work #02: 早盘前 09:00 / 09:30 同步没触发 + +## 发现 + +work #01 commit `2951770` 后 18 分钟重跑 daily_check → 10 个 missed 状态没变（符合预期，因为 15:30 / 21:xx 还没到时间）。 + +但细看时间窗口发现**新问题**： +- 当前时间 10:49，09:00 / 09:30 应已跑完 stock_basic + industry_sector +- 两个 task last_success 还是 2026-07-02 → 今天**根本没触发** + +## 根因 + +调度链路里没有早盘触发器： + +| 调度器 | 时间点 | 早盘？ | +|---|---|---| +| systemd timers | 15:30 / 21:05 / 21:35 / 22:00 / Sat 11:30 / 23:00 | ❌ | +| worker.py 进程内 scheduler | 09:00 / 09:30 / 15:30 / ... | ✅ 有，但 worker.py 没运行 | + +`app/entrypoints/worker.py:57` 的 `start_scheduler()` 是预期的主调度器，但没人把 worker.py 作为 long-running 进程拉起： +- 之前应该是 monitor 守护的"逻辑分层"是：worker.py 长驻 + systemd timer 兜底 +- 昨 18h 删了 monitor，没补 worker.py → 早盘缺口暴露 + +## 解决 + +加 `bin/systemd/market-sync-morning.{service,timer}` @ 周一~五 09:00 daily： +- service 跑轻量 morning shell（`bin/market_sync_morning_run.sh`），只跑 stock_basic + industry_sector +- 不复用 runall_once.py（8 个 task 跑 80min 太长不适合早盘） +- TimeoutStartSec=3600 (1h) 够用（实测 2 个 task 串行 ~25min） + +## 与现有架构的关系 + +| 时间点 | systemd service | 跑什么 | +|---|---|---| +| 09:00 | market-sync-morning | stock_basic + industry_sector | +| 15:30 | market-sync (runall) | 8 task 全量 | +| 21:05 | market-sync-tick | tick_trade | +| 21:35 | market-sync-moneyflow | moneyflow | +| 22:00 | market-sync-lhb | longhubang | +| 23:00 | market-sync-daily-check | daily_check | +| Sat 11:30 | market-sync-stock-node | stock_node | + +**注意**：这次没启 worker.py。改 worker.py 长驻属于更大重构（与 systemd timer 收敛到一个调度器，见 [[dual-scheduler-overlap]]），不在本 work 范围。 + +## 关联 + +- [[dual-scheduler-overlap]]: 早盘缺口正是该 memory 预测的"worker.py 没在跑时 systemd 冗余"问题 +- [[market-data-overview]]: 调度时间表（本 work 后补全了 09:00 缺口） +- work #01: 2026-07-03-01-market-sync-timeout.md + +## 验收 + +```bash +sudo cp bin/market_sync_morning_run.sh /usr/local/bin/ # 可选；或保持 repo 内 +sudo cp bin/systemd/market-sync-morning.{service,timer} /etc/systemd/system/ +sudo systemctl daemon-reload +sudo systemctl enable market-sync-morning.timer +# 验证 timer 在 schedule 里 +systemctl list-timers | grep market-sync-morning +# 手动触发一次，验证脚本可用 +sudo systemctl start market-sync-morning.service +journalctl -u market-sync-morning -f +``` \ No newline at end of file