From 8f016f25df43a15ed00b9455d90c5b6f5ae263e9 Mon Sep 17 00:00:00 2001 From: gao Date: Tue, 21 Jul 2026 16:37:00 +0800 Subject: [PATCH] =?UTF-8?q?chore:=20baseline=20=E2=80=94=20ORM=20migration?= =?UTF-8?q?=20+=20systemd=20deploy=20+=20MCP=20server=20+=20daily=20check?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - ORM migration: models/ops 重构 - deploy: 标准化 systemd timer/service 部署体系 - MCP server: 5 个只读工具(任务/状态/数据集) - daily check: 数据一致性巡检 - watch: task_watch 后台监控 - kline_daily: 麦蕊优先,时间门禁15:00 - 删除: task_industry_sector, task_sector_features --- AGENTS.md | 302 +++++++++++++ Dockerfile | 64 +-- README.md | 127 ++++-- app/api/routes/dashboard.py | 13 +- app/core/config.py | 6 + app/core/datasource/utils.py | 133 +++++- app/core/db/models.py | 174 ++++---- app/core/db/ops.py | 251 ++++------- app/core/scheduler/scheduler.py | 37 +- app/core/sync/base.py | 14 +- app/core/sync/registry.py | 36 +- app/entrypoints/worker.py | 24 + app/mcp_server.py | 63 ++- app/sources/mairui.py | 64 ++- app/sources/xueqiu.py | 38 +- app/tasks/__init__.py | 6 +- app/tasks/task_industry_sector.py | 121 ------ app/tasks/task_kline_5min.py | 171 +++++++- app/tasks/task_kline_daily.py | 181 ++++++-- app/tasks/task_mairui_indicators.py | 261 +++++++++++ app/tasks/task_market_regime.py | 14 +- app/tasks/task_sector_features.py | 183 -------- app/tasks/task_share_snapshot.py | 13 +- app/tasks/task_stocks_basic.py | 11 +- app/tasks/task_tick_trade.py | 42 +- bin/backfill_share_history.py | 409 ++++++++++++++++++ bin/daily_sync_check.py | 136 +++++- bin/deploy_systemd.sh | 1 + bin/market_sync_kline_5min_run.sh | 17 + bin/market_sync_kline_daily_run.sh | 17 + bin/market_sync_kline_index_run.sh | 17 + bin/market_sync_mairui_indicators_run.sh | 25 ++ bin/market_sync_mairui_ma_daily_run.sh | 25 ++ bin/market_sync_market_regime_run.sh | 17 + bin/market_sync_morning_run.sh | 6 +- bin/market_sync_share_run.sh | 24 + bin/run_remaining.sh | 5 +- bin/runall_once.py | 149 +++++-- bin/service_run.sh | 9 + .../market-sync-mairui-indicators.service | 32 ++ .../market-sync-mairui-indicators.timer | 14 + .../market-sync-mairui-ma-daily.service | 32 ++ bin/systemd/market-sync-mairui-ma-daily.timer | 14 + bin/systemd/market-sync-morning.service | 6 +- bin/systemd/market-sync-morning.timer | 4 +- bin/systemd/market-sync-share.service | 31 ++ bin/systemd/market-sync-share.timer | 16 + bin/systemd/market-sync-stock-node.service | 2 +- bin/systemd/market-sync-worker.service | 25 ++ bin/systemd/market-sync.service | 9 +- bin/systemd_deploy.sh | 186 +------- bin/task_watch.py | 218 ++++++++++ deploy/docker/Dockerfile | 65 +++ deploy/docker/docker-compose.yml | 86 ++++ deploy/systemd/deploy.sh | 211 +++++++++ deploy/systemd/deploy_split.sh | 28 ++ deploy/systemd/scripts/deploy_systemd.sh | 59 +++ deploy/systemd/scripts/systemd_deploy.sh | 185 ++++++++ .../units/market-sync-daily-check.service | 31 ++ .../units/market-sync-daily-check.timer | 15 + .../units/market-sync-kline-5min.service | 20 + .../units/market-sync-kline-5min.timer | 10 + .../units/market-sync-kline-daily-early.timer | 11 + .../units/market-sync-kline-daily.service | 20 + .../units/market-sync-kline-daily.timer | 10 + .../units/market-sync-kline-index.service | 20 + .../units/market-sync-kline-index.timer | 10 + deploy/systemd/units/market-sync-lhb.service | 33 ++ deploy/systemd/units/market-sync-lhb.timer | 16 + .../market-sync-mairui-indicators.service | 32 ++ .../units/market-sync-mairui-indicators.timer | 14 + .../units/market-sync-mairui-ma-daily.service | 32 ++ .../units/market-sync-mairui-ma-daily.timer | 14 + .../units/market-sync-market-regime.service | 20 + .../units/market-sync-market-regime.timer | 10 + deploy/systemd/units/market-sync-mcp.service | 22 + .../units/market-sync-moneyflow.service | 33 ++ .../systemd/units/market-sync-moneyflow.timer | 16 + .../systemd/units/market-sync-morning.service | 29 ++ .../systemd/units/market-sync-morning.timer | 14 + .../systemd/units/market-sync-share.service | 31 ++ deploy/systemd/units/market-sync-share.timer | 16 + .../units/market-sync-stock-node.service | 33 ++ .../units/market-sync-stock-node.timer | 16 + deploy/systemd/units/market-sync-tick.service | 33 ++ deploy/systemd/units/market-sync-tick.timer | 17 + .../systemd/units/market-sync-worker.service | 25 ++ deploy/systemd/units/market-sync.service | 35 ++ deploy/systemd/units/market-sync.timer | 15 + docker-compose.yml | 85 +--- opencode.json | 10 + scripts/run_all_sync.sh | 95 ++-- tests/test_schema_models.py | 9 +- tests/test_smoke.py | 12 +- 94 files changed, 4117 insertions(+), 1176 deletions(-) create mode 100644 AGENTS.md mode change 100644 => 120000 Dockerfile delete mode 100644 app/tasks/task_industry_sector.py create mode 100644 app/tasks/task_mairui_indicators.py delete mode 100644 app/tasks/task_sector_features.py create mode 100755 bin/backfill_share_history.py create mode 120000 bin/deploy_systemd.sh create mode 100755 bin/market_sync_kline_5min_run.sh create mode 100755 bin/market_sync_kline_daily_run.sh create mode 100755 bin/market_sync_kline_index_run.sh create mode 100755 bin/market_sync_mairui_indicators_run.sh create mode 100755 bin/market_sync_mairui_ma_daily_run.sh create mode 100755 bin/market_sync_market_regime_run.sh create mode 100755 bin/market_sync_share_run.sh create mode 100644 bin/systemd/market-sync-mairui-indicators.service create mode 100644 bin/systemd/market-sync-mairui-indicators.timer create mode 100644 bin/systemd/market-sync-mairui-ma-daily.service create mode 100644 bin/systemd/market-sync-mairui-ma-daily.timer create mode 100644 bin/systemd/market-sync-share.service create mode 100644 bin/systemd/market-sync-share.timer create mode 100644 bin/systemd/market-sync-worker.service mode change 100755 => 120000 bin/systemd_deploy.sh create mode 100644 bin/task_watch.py create mode 100644 deploy/docker/Dockerfile create mode 100644 deploy/docker/docker-compose.yml create mode 100755 deploy/systemd/deploy.sh create mode 100644 deploy/systemd/deploy_split.sh create mode 100755 deploy/systemd/scripts/deploy_systemd.sh create mode 100755 deploy/systemd/scripts/systemd_deploy.sh create mode 100644 deploy/systemd/units/market-sync-daily-check.service create mode 100644 deploy/systemd/units/market-sync-daily-check.timer create mode 100644 deploy/systemd/units/market-sync-kline-5min.service create mode 100644 deploy/systemd/units/market-sync-kline-5min.timer create mode 100644 deploy/systemd/units/market-sync-kline-daily-early.timer create mode 100644 deploy/systemd/units/market-sync-kline-daily.service create mode 100644 deploy/systemd/units/market-sync-kline-daily.timer create mode 100644 deploy/systemd/units/market-sync-kline-index.service create mode 100644 deploy/systemd/units/market-sync-kline-index.timer create mode 100644 deploy/systemd/units/market-sync-lhb.service create mode 100644 deploy/systemd/units/market-sync-lhb.timer create mode 100644 deploy/systemd/units/market-sync-mairui-indicators.service create mode 100644 deploy/systemd/units/market-sync-mairui-indicators.timer create mode 100644 deploy/systemd/units/market-sync-mairui-ma-daily.service create mode 100644 deploy/systemd/units/market-sync-mairui-ma-daily.timer create mode 100644 deploy/systemd/units/market-sync-market-regime.service create mode 100644 deploy/systemd/units/market-sync-market-regime.timer create mode 100644 deploy/systemd/units/market-sync-mcp.service create mode 100644 deploy/systemd/units/market-sync-moneyflow.service create mode 100644 deploy/systemd/units/market-sync-moneyflow.timer create mode 100644 deploy/systemd/units/market-sync-morning.service create mode 100644 deploy/systemd/units/market-sync-morning.timer create mode 100644 deploy/systemd/units/market-sync-share.service create mode 100644 deploy/systemd/units/market-sync-share.timer create mode 100644 deploy/systemd/units/market-sync-stock-node.service create mode 100644 deploy/systemd/units/market-sync-stock-node.timer create mode 100644 deploy/systemd/units/market-sync-tick.service create mode 100644 deploy/systemd/units/market-sync-tick.timer create mode 100644 deploy/systemd/units/market-sync-worker.service create mode 100644 deploy/systemd/units/market-sync.service create mode 100644 deploy/systemd/units/market-sync.timer mode change 100644 => 120000 docker-compose.yml create mode 100644 opencode.json diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..c5a7b85 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,302 @@ +# Agent 工作记忆 + +## 活跃监视 + +task_watch 在后台运行（PID 见 `logs/task_watch.log`），持续跟踪运行中的 task： +- 查询 `dataset_registry` 中 status='running' 的任务 +- 解析 config 表 lastMessage 中的耗时数据作为历史基准 +- 自适应检查间隔（剩余时间 × 0.15，clamp 15s-300s，无历史默认 60s） + +## 待推进的优化（按优先级） + +### P0 - 高优 + +- **task_watch 耗时记录入库**：当前 watch 只读历史不做记录。任务完成后应将 `dataset_registry.message` 中的 elapsed_sec 写入 config 表（或新建 `run_history` 表），供下次估算 ETA 使用。 +- **task_mairui_indicators 外层超时**：与 kline_daily/kline_5min 对齐，给 `as_completed` 加上外层 timeout 兜底（`app/tasks/task_mairui_indicators.py:227`）。 + +### P1 - 中优 + +- **节假日自动更新**：`trading_calendar_holidays` 当前硬编码在 config 表。建议接入 https://github.com/shengshiyan/dataconf 或类似日历 API 自动拉取每年交易日历。 +- **`daily_sync_check` 失败告警**：昨日巡检因 `FAILURE_WEBHOOK_URL/SECRET` 未配置而 exit 1（期望的失败）。需确定是否要配 webhook，或改为非致命 warning。 +- **`kline_5min` mairui 偶发全 worker 挂起**：今天 recovery 4734 只失败发生在 mairui 单次 120s 全 hang。`_fetch` 已有 20s timeout + 重试，但仍抗不住 mairui 服务端全挂。建议监控 mairui 健康状态 + 失败股票重入队列。 + +### P2 - 低优 + +- **MySQL 残留配置**：`.env` 仍包含 `DB_BACKEND=mysql` 和 `MYSQL_*` 变量，config.py 已只支持 PG。可清理。 +- **`stock_node` 上次 warning**：上次运行 7/10 返回 warning（41/1164 节点失败），需关注是否偶发。 +- **`kline_5min` 大量 fail**：今日 recovery 4734 只失败，需单独排查 mairui 数据源。 + +## 设计决策记录 + +- **systemd timer 为主，进程内 scheduler 为辅**：宿主机直接部署用 systemd mode，Docker 部署用 docker mode。二者通过 `RUNTIME_MODE` 隔离。 +- **After= 只依赖 network-online.target**：service 间不设 After= 依赖链，避免一个任务阻塞后续所有任务。 +- **fail-open 策略**：节假日不过滤，让 task 在非交易日返回空数据报 warning 但不阻塞。周末由 systemd OnCalendar=Mon..Fri 保护。 + +## 数据集来源与依赖关系 + +> 数据来源与依赖关系是调度正确性的基础。原始源依赖外部 API；衍生源只读本地 PG，依赖上游原始源。 + +### 数据集清单 + +| dataset_id | 目标表 | 类型 | 来源 | 说明 | +|---|---|---|---|---| +| `stock_basic` | `market_data.stocks` | 原始源 | 麦蕊 `/hslt/list`（主）+ Baostock `query_stock_basic`（备） | 全市场 A 股代码/名称/交易所/上市状态；`stock_basic` 自身可附带雪球股本快照 | +| `kline_daily` | `market_data.kline_stock` | 原始源 | 雪球 `kline`（主）→ 麦蕊 `hsstock/history/*/d/n` → 新浪 `getKLineData`（备） | 全市场日 K OHLCV；支持 per-stock fallback | +| `kline_index` | `market_data.kline_index` + `indices` | 原始源 | 麦蕊 `hsindex/history`（主）→ 新浪指数日 K（备） | 上证/深证/创业板/沪深300/中证500/中证1000 | +| `kline_5min` | `market_data.kline_5min` | 原始源 | 麦蕊 `hsstock/history/*/5/n` | 全市场 5 分钟 K；历史深度 2023-06-14 起 | +| `tick_trade` | `market_data.tick_trade` | 原始源 | 麦蕊 `hsrl/zbjy` | 当天逐笔成交；每日 21:00 后发布 | +| `moneyflow` | `market_data.moneyflow` | 原始源 | 麦蕊 `hsstock/history/transaction` | 主力/大/中/小单净额；每日 21:30 后发布 | +| `longhubang` | `market_data.longhubang_daily` + `longhubang_seat` | 原始源 | akshare `stock_lhb_detail_em` / `stock_lhb_stock_detail_em`（东方财富封装） | 龙虎榜聚合层 + 席位层；每日 22:00 触发 | +| `stock_node` | `market_data.node_categories` + `nodes` + `stock_node_map` | 原始源 | 麦蕊 `/hszg/{list,gg}` | 股票-指数/行业/概念映射；每周六 11:30 | +| `mairui_indicators` | `market_data.kline_stock_macd_daily` + `kdj_daily` + `boll_daily` | 原始源 | 麦蕊 `hsstock/history/{macd,kdj,boll}` | 日 K MACD/KDJ/BOLL；增量拉取 | +| `share_snapshot` | `market_data.stocks` + `share` | 原始源 | 雪球 `quote_detail` | 最新总股本/流通股本；需 `XUEQIU_TOKEN` | +| `market_regime` | `market_data.market_regime_daily` | 衍生源 | 本地计算（`kline_stock`） | 涨跌家数/ advance_ratio / 恐慌标记 | +| `mairui_ma_daily` | `market_data.kline_stock_ma_daily` | 衍生源 | 本地计算（`kline_stock.close`） | MA5/10/20/60；未来可切麦蕊 `/d/maN`（需付费 licence） | + +### 依赖关系 + +``` +stock_basic ─┬─► kline_daily ─┬─► market_regime + │ └─► mairui_ma_daily + ├─► kline_5min + ├─► tick_trade + ├─► moneyflow + ├─► longhubang + ├─► stock_node + └─► share_snapshot + +kline_index （独立，无上游依赖） +``` + +- **根任务**：`stock_basic` 是大多数任务的根，提供股票代码列表。`kline_index` 独立运行。 +- **关键链**：`kline_daily` 是 2 个衍生源（`market_regime`、`mairui_ma_daily`）的实际数据基础。 +- **注意**：`mairui_ma_daily` 在 `app/core/sync/registry.py` 中声明依赖 `kline_daily`，但只有**调度顺序**约束；若 `kline_daily` 运行后数据又被回填（如 2026-07-15 场景），`mairui_ma_daily` 不会自动重算，需手动触发或等下次调度。 + +### 默认调度时间（工作日） + +| 时间 | dataset_id | 备注 | +|---|---|---| +| 09:00 | `stock_basic` | 开盘前刷新股票列表 | +| 15:30 | `kline_index` | 指数日 K | +| 15:40 | `kline_daily` | 个股日 K | +| 16:00 | `kline_5min` | 5 分钟 K | +| 15:15 | `market_regime` | 市场情绪 | +| 16:30 | `mairui_ma_daily` | 日 K MA（本地计算） | +| 16:40 | `mairui_indicators` | MACD/KDJ/BOLL | +| 21:35 | `moneyflow` | 资金流（等 21:30 发布） | +| 22:00 | `longhubang` | 龙虎榜（等 19:00-21:00 出齐） | +| 周六 11:30 | `stock_node` | 节点映射 | +| 周六 11:30 | `share_snapshot` | 股本快照（雪球，周度） | + +### systemd timer 与数据依赖对齐审核 + +总体：**基本对齐**，但存在 3 个需要注意的细节。 + +1. **`mairui_indicators.service` 的 `After=` 多余依赖** ✅ 已修复 + - 修复前：`After=network-online.target market-sync.service market-sync-mairui-ma-daily.service` + - 修复后：`After=network-online.target market-sync.service` + - 文件：`deploy/systemd/units/market-sync-mairui-indicators.service`、`bin/systemd/market-sync-mairui-indicators.service` + - 待部署：/etc/systemd/system/ 仍保留旧版，需 `sudo bash deploy/systemd/deploy.sh` + +2. **`stock_node.service` 未声明对 `stock_basic` 的依赖** ✅ 已修复 + - 修复前：`After=network-online.target` + - 修复后：`After=network-online.target market-sync-morning.service` + - 文件：`deploy/systemd/units/market-sync-stock-node.service`、`bin/systemd/market-sync-stock-node.service` + - 待部署：/etc/systemd/system/ 仍保留旧版，需 `sudo bash deploy/systemd/deploy.sh` + +3. **`After=` 只保证启动顺序，不保证上游成功** + - systemd 的 `After=` 不会让下游等待上游**成功完成**，仅控制 timer 触发后的排队顺序。 + - 如果 `market-sync.service` 在 15:30 启动后因 `kline_daily` 卡住而 3h 超时失败，`mairui_ma_daily` 仍会在 16:30 启动（因为 timer 已触发）。 + - 结果：`mairui_ma_daily` 可能基于不完整的 `kline_stock` 计算，产生类似 2026-07-15 的 3,404 行缺失。 + +### 数据回填级联规则 + +> **核心原则**：上游数据被回填或修复后，下游依赖它的衍生数据集必须手动/自动重跑，否则会出现“上游新、下游旧”的不一致。 + +| 上游任务/表 | 触发条件 | 必须重跑的下游任务 | +|---|---|---| +| `kline_daily` / `kline_stock` | 回填历史 K 线、修复错误日线、补充漏掉的股票/日期 | `market_regime`、`mairui_ma_daily` | +| `stock_basic` / `stocks` | 新上市/退市股票、代码变更、上市状态修正 | `kline_daily`、`kline_5min`、`tick_trade`、`moneyflow`、`longhubang`、`stock_node`、`share_snapshot` | +| `kline_stock`（任意修复） | close/volume 等核心字段修正 | `market_regime`、`mairui_ma_daily` | + +**操作建议：** + +- 单次少量回填（如几只股票、几天）：用 CLI 参数指定 `codes` / `start` / `end`，然后按上表手动触发下游。 +- 大量回填（如全市场、多月/多年历史）：先跑上游，再按依赖链顺序跑下游；必要时禁用当日独立 timer，避免与 runall 并发。 +- 每日巡检（`bin/daily_sync_check.py`）已增加数据一致性检查：对比 `kline_stock` 与 `kline_stock_ma_daily`、`market_regime_daily` 的最新日期与缺失行数，发现缺口即告警。 + +## 历史问题与解决思路 + +> 记录真实故障案例、排查路径和修复方法，供后续复察或参考。 + +### 案例 1：2026-07-15 `kline_stock_ma_daily` 缺失 3,404 行 + +**现象** +- `kline_stock` 11,728,408 行，`kline_stock_ma_daily` 只有 11,725,004 行，相差 3,404 行。 +- 差异集中在 2026-07-14（3,400 只股票）+ SH688287 的 4 天（2026-05-29/06-04/06-05/06-08）。 +- `dataset_registry` 中 `mairui_ma_daily` 状态为 `success`，`last_success_at=2026-07-14 17:11:48 UTC`。 + +**根因** +- `mairui_ma_daily` 在 2026-07-14 17:11 完成时，`kline_stock` 的 2026-07-14 数据还不完整（少 3,400 只）。 +- `kline_daily` 在 2026-07-15 10:04 又跑了一次，补充了这 3,400 只 2026-07-14 的 K 线。 +- `mairui_ma_daily` 没有自动感知上游变化，导致 MA 表缺了这 3,404 行。 + +**排查命令** +```bash +# 1. 对比行数 +SELECT COUNT(*) FROM market_data.kline_stock; +SELECT COUNT(*) FROM market_data.kline_stock_ma_daily; + +# 2. 查缺失日期分布 +SELECT trade_date, COUNT(*) FROM ( + SELECT stock_code, trade_date FROM market_data.kline_stock + EXCEPT + SELECT stock_code, trade_date FROM market_data.kline_stock_ma_daily +) t GROUP BY trade_date ORDER BY trade_date; + +# 3. 查 dataset_registry 时间线 +SELECT dataset_id, last_success_at, finished_at, message +FROM market_data.dataset_registry +WHERE dataset_id IN ('kline_daily', 'mairui_ma_daily'); +``` + +**修复方法** +```bash +# 手动重跑 mairui_ma_daily（全量重算 ~22min） +bin/market_sync_mairui_ma_daily_run.sh +``` + +**预防/改进** +- 在 `bin/daily_sync_check.py` 中增加数据一致性检查：每日 23:00 对比 `kline_stock` 与 `kline_stock_ma_daily` 的覆盖差异。 +- 上游回填历史数据后，按「数据回填级联规则」手动触发下游重跑。 + +--- + +### 案例 2：systemd `After=` 依赖设计缺陷 + +**现象** +- `market-sync-mairui-indicators.service` 写了 `After=market-sync-mairui-ma-daily.service`。 +- `market-sync-stock-node.service` 没有声明对 `stock_basic` 的依赖。 + +**根因** +- 早期设计时对数据依赖关系梳理不够，把无依赖的任务串起来，或遗漏了必要的依赖声明。 +- `systemd` 的 `After=` 只控制**启动顺序**，不检查上游任务是否**成功完成**。 + +**修复方法** +1. `mairui_indicators.service`：移除 `market-sync-mairui-ma-daily.service`，改为 `After=network-online.target market-sync.service`。 +2. `stock_node.service`：增加 `After=market-sync-morning.service`，确保 `stock_basic` 已更新。 +3. 同步修改 `deploy/systemd/units/` 和 `bin/systemd/` 下的对应文件，并执行： + ```bash + sudo bash deploy/systemd/deploy.sh + ``` + +**设计原则** +- `After=` 只声明最小必要依赖：network-online.target + 直接上游 service。 +- 不要为了让下游等上游成功而过度串 service；成功/失败应通过 `daily_sync_check` 和回填规则兜底。 + +--- + +### 案例 3：`dataset_registry` 状态卡在 `running` + +**现象** +- `mairui_ma_daily` 的 `sync_history` 已记录成功，但 `dataset_registry.status='running'`，`finished_at=NULL`。 +- `ps` 中无对应进程。 + +**根因** +- 状态更新与 `sync_history` 写入不在同一事务；极端情况下任务进程异常退出，导致 `mark_sync_success` 未执行。 +- 或并发/重入导致后一次 `mark_sync_running` 覆盖了前一次的成功状态。 + +**排查命令** +```bash +SELECT dataset_id, status, started_at, finished_at, last_success_at, message +FROM market_data.dataset_registry +WHERE dataset_id = 'mairui_ma_daily'; + +SELECT * FROM market_data.sync_history +WHERE dataset_id = 'mairui_ma_daily' +ORDER BY started_at DESC LIMIT 5; +``` + +**修复方法** +```python +from app.core.db import ops as db_ops +from datetime import datetime + +db_ops.update_dataset_registry_state( + 'mairui_ma_daily', + status='success', + finished_at='2026-07-15 12:02:21', + last_success_at='2026-07-15 12:02:21', + message='MA [5, 10, 20, 60] 共 11,728,408 行, 5511 只, 1349.2s', + last_error=None, + needs_resync=0, + progress_current=0, + progress_total=0, + current_step='', + updated_at=datetime.now().strftime('%Y-%m-%d %H:%M:%S'), +) +``` + +**改进建议** +- 启动 `runall_once.py` 时自动调用 `recover_interrupted_dataset_registry()` 清理卡死状态（已实现）。 +- `task_watch` 可升级为：发现任务 `running > 2× 历史平均耗时` 且无进程时，自动标记为 failed。 + +## Market Supervisor 工作流 + +> 本项目的 Agent 不仅是 code agent，也是 market data supervisor。修改代码/配置后必须完成部署与验证闭环，不能停留在 repo 层面。 + +### Agent 职责 + +1. **主动巡检**：定期/按需运行 `bin/daily_sync_check.py --report-only`，确认 dataset 状态、systemd drift、数据一致性。 +2. **问题闭环**：发现告警后，定位根因 → 修复代码/config → 部署到 /etc → 验证告警消除。 +3. **项目记忆同步**：把已确定的问题、根因、修复方法、验证命令写回 `AGENTS.md`，供后续复察。 +4. **数据依赖守护**：上游数据回填/修复后，按「数据回填级联规则」触发下游重跑，并验证一致性。 + +### systemd unit 修改后的标准流程 + +任何修改了 `deploy/systemd/units/` 或 `bin/systemd/` 下 unit 文件的操作，必须： + +1. **保持两边同步**：同时修改 `deploy/systemd/units/` 和 `bin/systemd/` 的对应文件（目前两者是镜像关系）。 +2. **执行 deploy**： + ```bash + cd /home/gao/Development/quant_home/market_sync + sudo bash deploy/systemd/deploy.sh + # 或跳过确认：sudo bash deploy/systemd/deploy.sh --yes + ``` +3. **验证 drift**： + ```bash + .venv/bin/python bin/daily_sync_check.py --report-only + ``` + 确保 `systemd_drift` 中 `has_drift=false`。 +4. **验证 unit 状态**： + ```bash + systemctl list-timers market-sync-* + systemctl status market-sync-mairui-indicators.service + systemctl status market-sync-stock-node.service + ``` + +### 部署验证 Checklist + +- [ ] repo unit 与 `/etc/systemd/system/` 内容一致（`daily_sync_check` 无 drift 告警） +- [ ] `systemctl daemon-reload` 已执行 +- [ ] 相关 timer 仍 enabled（`systemctl is-enabled .timer`） +- [ ] 修改后的 service 无语法错误（`systemd-analyze verify /etc/systemd/system/.service`） +- [ ] 数据一致性检查通过（`kline_stock` 与下游表日期对齐） + +## 已部署的优化（确认完成） + +| 项目 | 日期 | 说明 | +|------|------|------| +| `RUNTIME_MODE=systemd` 写入 `.env` | 07-13 | 防止双调度器冲突 | +| `market-sync-worker.service` mask | 07-13 | systemd 模式下禁用进程内 scheduler | +| mairui_ma_daily / mairui_indicators timer 部署 | 07-13 | 2 个缺失的 timer 补部署并启用 | +| `/etc` market-sync.service 漂移修复 | 07-13 | deploy.sh 同步 repo→etc，消除 drift | +| moneyflow/lhb `After=` 依赖链修复 | 07-13 | 去掉 `After=market-sync-tick.service`，任务不再串行阻塞 | +| `bin/task_watch.py` 创建 | 07-13 | 自适应间隔的任务跟踪监视器（nohup 后台运行） | +| `runall_once.py` logger f-string 修复 | 07-13 | 超时日志改为 f-string | +| `sync/base.py` docstring 更新 | 07-13 | `_effective_sync_end` 注释过期问题修复 | +| `call_with_timeout` 超时后线程阻塞修复 | 07-13 | 改用 daemon 线程替代 `ThreadPoolExecutor`，避免超时后 `shutdown(wait=True)` 阻塞调用方 | +| `tick_trade` 外层超时加固 | 07-13 | 加上 FETCH_HARD_TIMEOUT=120s + 手动 pool 管理，防止 mairui 挂起时永久卡住 | +| `kline_daily` per-stock fallback | 07-13 | 雪球失败自动降级到 mairui/新浪，`fallback_ok` 统计备胎救回数 | +| `kline_5min` 超时后重入队列 | 07-13 | mairui 全 hang 超时后冷却 30s 重试一次失败股票，救回临时故障 | +| MCP Server SSE 模式 + systemd 服务 | 07-13 | `market-sync-mcp.service` 开机自启，SSE 端点 `http://127.0.0.1:8101/sse` | diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index c63e383..0000000 --- a/Dockerfile +++ /dev/null @@ -1,63 +0,0 @@ -# syntax=docker/dockerfile:1.6 -# ───────────────────────────────────────────────────────────── -# market_sync 数据同步服务 -# 单镜像同时跑：FastAPI dashboard + 进程内 scheduler（worker） -# 部署：docker compose up -d -# ───────────────────────────────────────────────────────────── - -# --- builder stage: 装依赖到 venv --- -FROM python:3.11-slim AS builder - -ENV PYTHONUNBUFFERED=1 \ - PYTHONDONTWRITEBYTECODE=1 \ - PIP_NO_CACHE_DIR=1 \ - PIP_DISABLE_PIP_VERSION_CHECK=1 - -# 仅装编译期需要的系统包（很多 wheel 已预编译，但 psycopg2 / cryptography 可能要） -RUN apt-get update && apt-get install -y --no-install-recommends \ - build-essential gcc libpq-dev \ - && rm -rf /var/lib/apt/lists/* - -WORKDIR /app - -# 先 copy requirements 单独一层（利用 Docker 缓存：依赖不变就不重装） -COPY requirements.txt . -RUN python -m venv /app/.venv \ - && /app/.venv/bin/pip install --upgrade pip \ - && /app/.venv/bin/pip install -r requirements.txt - - -# --- runtime stage: 极简基础镜像 + 仅复制 venv 与代码 --- -FROM python:3.11-slim AS runtime - -ENV PYTHONUNBUFFERED=1 \ - PYTHONDONTWRITEBYTECODE=1 \ - PATH="/app/.venv/bin:${PATH}" \ - PYTHONPATH=/app \ - # service 模式默认监听 - API_HOST=0.0.0.0 \ - API_PORT=8100 - -# runtime 只需要 psycopg2 的运行时库 + tzdata（pandas tz aware 需要） -RUN apt-get update && apt-get install -y --no-install-recommends \ - libpq5 tzdata curl \ - && rm -rf /var/lib/apt/lists/* - -WORKDIR /app - -# 从 builder 复制 venv（已编译好的依赖） -COPY --from=builder /app/.venv /app/.venv - -# 复制应用代码 -COPY app/ ./app/ -COPY bin/ ./bin/ - -# 默认启动 service 入口（FastAPI + 进程内 scheduler） -# 也可覆盖为 cli / worker 单跑某个 task： -# docker compose run --rm market_sync python -m app.entrypoints.cli sync kline_daily -EXPOSE 8100 - -HEALTHCHECK --interval=60s --timeout=5s --start-period=30s --retries=3 \ - CMD curl -fsS http://localhost:8100/api/health || exit 1 - -ENTRYPOINT ["/app/bin/service_run.sh"] \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 120000 index 0000000..3e19319 --- /dev/null +++ b/Dockerfile @@ -0,0 +1 @@ +deploy/docker/Dockerfile \ No newline at end of file diff --git a/README.md b/README.md index a1ba5cc..07027f2 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ A 股市场数据**定时同步与管理**框架。 -把 `akshare` / `baostock` / 新浪 / 雪球 / 麦蕊智数 等多个数据源接入到 MySQL 库，提供： +把 `akshare` / `baostock` / 新浪 / 雪球 / 麦蕊智数 等多个数据源接入到 PostgreSQL 库，提供： - 后台线程调度器（time-based + interval-based，交易日感知） - 多数据源 + 自动降级 + 健康度监控 - `dataset_registry` 同步状态机（运行中 / 成功 / 失败 / 中断恢复） @@ -11,15 +11,69 @@ A 股市场数据**定时同步与管理**框架。 > 📖 详细架构说明见 [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md) -## 快速开始 +## 运行模式 + +项目支持两种互斥的运行模式，由环境变量 `RUNTIME_MODE` 控制： + +| 模式 | 值 | 适用场景 | 调度方式 | +|---|---|---|---| +| **Docker 模式** | `docker` | `docker compose up` 单容器部署 | 进程内 scheduler（uvicorn + daemon thread） | +| **systemd 模式** | `systemd` | Linux 宿主机长期运行 | systemd timer/service | + +> ⚠️ 两种模式**不能同时启用**，否则同一个 task 会被两边各触发一次，造成状态机抖动和重复 IO。 + +### Docker 模式 ```bash # 1. 准备环境（首次） -cp .env.example .env # 编辑 .env 填写真实 MySQL 凭据 + XUEQIU_TOKEN -chmod +x start.sh -./start.sh # 自动创建 .venv + 装依赖 + 启动 API（http://localhost:8100） +cp .env.example .env # 编辑 .env 填写真实 PG 凭据 + XUEQIU_TOKEN + MAIRUI_LICENCE -# 2. 常用命令 +# 2. 启动（包含 postgres + market_sync 服务） +docker compose up -d + +# 3. 查看日志 +docker compose logs -f market_sync + +# 4. 手动触发一次任务 +docker compose exec market_sync python -m app.entrypoints.cli sync kline_daily +``` + +Docker 模式下 `RUNTIME_MODE=docker` 已写在 `deploy/docker/docker-compose.yml` 中，容器内会启动进程内 scheduler 负责所有同步任务。 + +### systemd 模式 + +```bash +# 1. 准备环境（首次） +cp .env.example .env # 编辑 .env 填写真实 PG 凭据 +# 并在 .env 中加上 RUNTIME_MODE=systemd + +# 2. 部署 systemd units +sudo bash deploy/systemd/deploy.sh + +# 3. 查看 timer +systemctl list-timers market-sync* + +# 4. 查看 worker / 任务日志 +journalctl -u market-sync -f +journalctl -u market-sync-moneyflow -f +``` + +systemd 模式下： +- 所有 14 个 task 都由 systemd timer 触发 +- `market-sync-worker.service` 会被部署脚本自动 **stop / disable / mask**，避免进程内 scheduler 与 systemd timers 重叠 +- 每日 23:00 `market-sync-daily-check.timer` 跑巡检，失败时 POST webhook + +### 本地开发模式 + +```bash +# 1. 准备环境（首次） +cp .env.example .env # 编辑 .env 填写凭据 +chmod +x start.sh + +# 2. 启动 API + scheduler（默认 docker 行为，等价于本地单进程） +./start.sh # http://localhost:8100 + +# 3. 常用命令 ./start.sh worker # 仅启动调度器（无 web） ./start.sh list # 列出所有同步任务 ./start.sh sync kline_daily # 手动触发一次（dataset_id 即可） @@ -27,39 +81,54 @@ chmod +x start.sh ./start.sh datasources # 查看数据源健康度 ./start.sh reseed # 重新 seed 默认配置到 config 表 -# 3. 一键全量同步（首次接入） +# 4. 一键全量同步（首次接入） python bin/runall_once.py -# 4. 跑测试 +# 5. 跑测试 .venv/bin/pytest tests/ -v ``` ## 项目结构 ``` -app/ -├── core/ # 核心抽象（config / db / sync / scheduler / datasource base） -├── sources/ # 数据源实现（5 个） -├── tasks/ # 同步任务实现（8 个） -├── api/ # FastAPI 管理接口 -└── entrypoints/ # 进程入口（cli + worker） -bin/ # 一次性脚本 -docs/ # 架构文档 -tests/ # 单元测试 +app/ # 应用源码 +├── core/ # 核心抽象（config / db / sync / scheduler / datasource base） +├── sources/ # 数据源实现 +├── tasks/ # 同步任务实现（14 个） +├── api/ # FastAPI 管理接口 +└── entrypoints/ # 进程入口（cli + worker） +bin/ # 通用脚本（运行入口、CLI、一次性任务） +config/ # 配置文件、secrets +deploy/ # 部署相关 +├── docker/ # Dockerfile + docker-compose.yml +└── systemd/ # systemd units + 部署脚本 +│ ├── units/ # *.service / *.timer +│ └── deploy.sh # 一键部署脚本 +docs/ # 架构文档 +logs/ # 日志输出 +tests/ # 单元测试 +Dockerfile -> deploy/docker/Dockerfile +docker-compose.yml -> deploy/docker/docker-compose.yml +bin/deploy_systemd.sh -> deploy/systemd/deploy.sh +bin/systemd_deploy.sh -> deploy/systemd/deploy.sh ``` ## 内置同步任务 -| dataset_id | 触发时间 | 说明 | -|---|---|---| -| `stock_basic` | 交易日 09:00 | 全市场股票基础信息 + 股本 | -| `kline_daily` | 交易日 15:40 | 增量日 K 线（OHLCV，多源交叉） | -| `kline_index` | 交易日 15:50 | 六大指数日线 | -| `kline_5min` | 交易日 16:00 | 增量 5 分钟 K 线 | -| `moneyflow` | 交易日 16:30 | 资金流（主力/大/中/小单净额） | -| `industry_sector` | 周一 09:30 | 股票-行业映射（Baostock） | -| `share_snapshot` | 交易日 09:30 | 股本快照（雪球） | -| `market_regime` | 交易日 16:00 | 市场情绪（衍生源） | +| dataset_id | systemd 触发时间 | Docker scheduler 时间 | 说明 | +|---|---|---|---| +| `stock_basic` | 工作日 09:00 | 09:00 | 全市场股票基础信息 + 股本 | +| `kline_index` | 工作日 15:30 | 15:30 | 六大指数日线 | +| `kline_daily` | 工作日 15:30（runall）| 15:40 | 增量日 K 线（OHLCV，多源交叉） | +| `kline_5min` | 工作日 15:30（runall）| 16:00 | 增量 5 分钟 K 线 | +| `market_regime` | 工作日 15:30（runall）| 16:15 | 市场情绪（衍生源） | +| `share_snapshot` | 工作日 15:30（runall）| 16:20 | 股本快照（雪球） | +| `mairui_ma_daily` | 工作日 16:30 | 16:30 | 日 K MA5/10/20/60（本地派生） | +| `mairui_indicators` | 工作日 16:40 | 16:40 | 日 K MACD/KDJ/BOLL（mairui） | +| `moneyflow` | 工作日 21:35 | 21:35 | 资金流（主力/大/中/小单净额） | +| `tick_trade` | 工作日 21:05 | 21:05 | 当天逐笔成交（mairui） | +| `longhubang` | 工作日 22:00 | 22:00 | 龙虎榜（akshare / 东方财富） | +| `stock_node` | 周六 11:30 | 周六 11:30 | 股票-节点映射（mairui） | ## 添加新数据源 @@ -70,7 +139,9 @@ tests/ # 单元测试 1. 在 `app/tasks/` 新建 `task_xxx.py`，继承 `SyncTask`，设置 `dataset_id` 2. 在 `app/tasks/__init__.py` 的 `TASKS` 字典注册 -3. （可选）在 `app/core/sync/registry.py` 的 `SYNC_DEFINITIONS` 加条目 +3. 在 `app/core/sync/registry.py` 的 `SYNC_DEFINITIONS` 加条目 +4. 如果使用 systemd 模式，在 `deploy/systemd/units/` 新增对应的 `.service` 和 `.timer` +5. 如果使用 Docker 模式，在 `app/core/scheduler/scheduler.py` 的 `DEFAULT_SCHEDULES` 新增调度项 ## License diff --git a/app/api/routes/dashboard.py b/app/api/routes/dashboard.py index c5485c2..1f44996 100644 --- a/app/api/routes/dashboard.py +++ b/app/api/routes/dashboard.py @@ -84,14 +84,15 @@ def data_stats(): from sqlalchemy import func, select from app.core.db.models import ( KlineStock, KlineIndex, Kline5Min, Moneyflow, Share, TickTrade, - SectorIndices, SectorFeaturesDaily, MarketRegimeDaily, - LonghubangDaily, LonghubangSeat, Stocks, Industry, + MarketRegimeDaily, + LonghubangDaily, LonghubangSeat, Stock, NodeCategory, Node, StockNodeMap, KlineStockMADaily, + KlineStockMACDDaily, KlineStockKDJDaily, KlineStockBOLLDaily, ) from app.core.db.orm import SessionLocal targets = [ - ("stocks", Stocks), + ("stocks", Stock), ("kline_stock", KlineStock), ("kline_index", KlineIndex), ("kline_5min", Kline5Min), @@ -99,15 +100,15 @@ def data_stats(): ("moneyflow", Moneyflow), ("share", Share), ("tick_trade", TickTrade), - ("sector_indices", SectorIndices), - ("sector_features_daily", SectorFeaturesDaily), ("market_regime_daily", MarketRegimeDaily), ("longhubang_daily", LonghubangDaily), ("longhubang_seat", LonghubangSeat), - ("industry", Industry), ("node_categories", NodeCategory), ("nodes", Node), ("stock_node_map", StockNodeMap), + ("kline_stock_macd_daily", KlineStockMACDDaily), + ("kline_stock_kdj_daily", KlineStockKDJDaily), + ("kline_stock_boll_daily", KlineStockBOLLDaily), ] out = [] with SessionLocal() as s: diff --git a/app/core/config.py b/app/core/config.py index 6558f60..5031202 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -30,6 +30,12 @@ if _HAS_DOTENV: class Settings(BaseSettings): """应用配置。所有字段都可以通过环境变量或 .env 文件覆盖。""" + # Runtime mode: "docker" | "systemd" + # docker : 单容器运行，依赖进程内 scheduler 触发所有同步任务 + # systemd : Linux 宿主机运行，由 systemd timer/service 触发任务， + # 进程内 scheduler 必须关闭，避免双调度器重叠 + runtime_mode: str = "docker" + # Scheduler scheduler_tick_seconds: int = 5 scheduler_timezone: str = "Asia/Shanghai" diff --git a/app/core/datasource/utils.py b/app/core/datasource/utils.py index 6ebc739..0d70b96 100644 --- a/app/core/datasource/utils.py +++ b/app/core/datasource/utils.py @@ -1,15 +1,146 @@ """数据源公共工具：代码转换、K 线标准化、日期过滤。 -参考 dashboard/api/services/datasource/fetch_kline.py，但适配新项目。""" +参考 dashboard/api/services/datasource/fetch_kline.py，但适配新项目。 +""" from __future__ import annotations +import logging import re +import threading +from datetime import datetime, time as dtime, timedelta +from typing import Any, Callable, TypeVar import pandas as pd +logger = logging.getLogger("datasource.utils") + KLINE_COLS = ["trade_date", "open", "high", "low", "close", "volume"] KLINE_5MIN_COLS = ["bar_time", "open", "high", "low", "close", "volume", "amount", "turnover_rate"] +T = TypeVar("T") + +_THREAD_POOL: dict[int, threading.Thread] = {} + + +def call_with_timeout( + func: Callable[..., T], + *args: Any, + timeout: float = 20.0, + on_timeout: T | None = None, + description: str = "", + **kwargs: Any, +) -> T | None: + """用 daemon 线程跑 `func(*args, **kwargs)`,超时返回 `on_timeout`。 + + Why: 之前用 ThreadPoolExecutor，但超时后 `future.cancel()` 无法终止 + 已在运行的线程，pool 的 `shutdown(wait=True)` 会等待卡死的函数返回， + 把 20s 超时变成无限阻塞——这是 kline_daily 3406 只股票批量失败的根因。 + + 改用 daemon 线程：超时后直接返回，线程成为孤儿进程，不会阻塞任何调用方。 + daemon=True 确保进程退出时不会等待这些孤儿线程。 + + Args: + func: 要跑的同步函数 + *args/**kwargs: 透传给 func + timeout: 秒数,默认 20s(对标 mairui/sina 的 urlopen timeout) + on_timeout: 超时返回的占位值(默认 None,调用方需处理 None) + description: 日志里的调用描述(如 'xueqiu.kline SH600519') + + Returns: func 的返回值,或 on_timeout(超时时) + """ + result: list[T | None | BaseException] = [None] + done = threading.Event() + + def _wrapper() -> None: + try: + result[0] = func(*args, **kwargs) + except BaseException as e: + result[0] = e + finally: + done.set() + + t = threading.Thread( + target=_wrapper, + daemon=True, + name=f"ds-timeout-{description or func.__name__}", + ) + t.start() + + if not done.wait(timeout=timeout): + logger.warning( + "[%s] %s 超时 (>%ss), 返回 on_timeout", + description or func.__name__, description or func.__name__, timeout, + ) + return on_timeout + + if isinstance(result[0], BaseException): + raise result[0] # type: ignore[misc] + return result[0] + + +def effective_market_date( + now: datetime | None = None, + holidays: set[str] | None = None, +) -> str: + """返回"数据生效日"(最近已完成交易日)。 + + 15:30 是 A 股收盘时刻;之后今天的数据才完整,之前应取昨天。 + 然后向前回退直到找到一个交易日（跳过周末和法定假期）。 + 用于数据源给"无明确来源日期"的数据(snowball quote_detail 的股本快照等) + 打 trade_date 时用 — 永远不要用 datetime.now() 直接当 trade_date。 + + Why: 任务运行日不等于数据交易日。例如 7月9日 早盘 9:30 跑 share_snapshot, + 此时拿到的股本快照实际对应 7月8日 收盘,不能标 7月9日。早期代码用 + `datetime.now().strftime("%Y-%m-%d")` 导致 share 表里 7月9日 跑出来的 + 行标成 7月9日,与实际数据日不符,统计/回溯会出错。 + + 2026-07-12 增强: 加入交易日历回退逻辑。如果 15:30 后是法定节假日 + （如春节/国庆），会向前回退到最后一个交易日，不会把节假日当天当 trade_date。 + + Args: + now: 测试用注入点(默认 datetime.now()) + holidays: 已知的 A 股休市日集合（YYYY-MM-DD 格式）。 + 默认从 config 表 trading_calendar_holidays 加载。 + + Returns: 'YYYY-MM-DD' 字符串（永远是合法的交易日） + """ + t = now or datetime.now() + if t.time() >= dtime(15, 0): + candidate = t + else: + candidate = t - timedelta(days=1) + + # 加载交易日历（如果调用方未提供） + if holidays is None: + try: + from app.core.db import ops as db_ops + h_row = db_ops.fetch_config_by_key("trading_calendar_holidays") + if h_row and h_row.get("category") == "general": + import json + vals = json.loads(h_row.get("value", "") or "[]") + if isinstance(vals, list): + holidays = {h for h in vals if isinstance(h, str)} + except Exception: + pass + + if holidays is None: + holidays = set() + + # 向前回退直到找到交易日 + max_iter = 30 # 安全上限，避免死循环 + while max_iter > 0: + date_str = candidate.strftime("%Y-%m-%d") + if candidate.weekday() < 5 and date_str not in holidays: + return date_str + candidate -= timedelta(days=1) + max_iter -= 1 + + # 兜底（正常情况下不会走到这里） + t2 = now or datetime.now() + if t2.time() >= dtime(15, 30): + return t2.strftime("%Y-%m-%d") + return (t2 - timedelta(days=1)).strftime("%Y-%m-%d") + def to_code6(code: str) -> str: """'SH600000' / 'sh.600000' / '600000' / 'SH600000.SH' → '600000'""" diff --git a/app/core/db/models.py b/app/core/db/models.py index 52bb4c1..8e614d8 100644 --- a/app/core/db/models.py +++ b/app/core/db/models.py @@ -141,7 +141,6 @@ class Stock(ORMBase): exchange: Mapped[str] = mapped_column(String(8), nullable=False, default="") list_date: Mapped[Optional[date]] = mapped_column(Date) listing_status: Mapped[str] = mapped_column(String(16), nullable=False, default="normal") - industry: Mapped[Optional[str]] = mapped_column(String(64), default="") total_share: Mapped[float] = mapped_column(Float, default=0) float_share: Mapped[float] = mapped_column(Float, default=0) share_updated_at: Mapped[Optional[date]] = mapped_column(Date) @@ -258,92 +257,7 @@ class Share(ORMBase): float_share: Mapped[float] = mapped_column(Float, nullable=False) -# ──────────────────────── 10. sectors ──────────────────────── -class Sectors(ORMBase): - """行业字典（baostock 行业分类 / 申万 / 中证 等多种 taxonomy）。""" - __tablename__ = "sectors" - __table_args__ = ( - Index("idx_sectors_sector_name", "sector_name"), - {"schema": "market_data"}, - ) - - sector_key: Mapped[str] = mapped_column(String(64), primary_key=True) - sector_name: Mapped[str] = mapped_column(String(64), nullable=False, default="") - taxonomy: Mapped[Optional[str]] = mapped_column(String(64), default="") - level: Mapped[Optional[str]] = mapped_column(String(16), default="") - source: Mapped[Optional[str]] = mapped_column(String(32), default="") - enabled: Mapped[int] = mapped_column(Integer, default=1) - updated_at: Mapped[Optional[datetime]] = _updated_at() - - -# ──────────────────────── 11. stock_sector_map ──────────────────────── -class StockSectorMap(ORMBase): - """股票-行业多对多映射（一张股票可对应多个行业 / 概念板块）。""" - __tablename__ = "stock_sector_map" - __table_args__ = ( - Index("idx_stock_sector_map_sector_key", "sector_key"), - {"schema": "market_data"}, - ) - - stock_code: Mapped[str] = mapped_column(String(10), primary_key=True) - sector_key: Mapped[str] = mapped_column(String(64), nullable=False, default="") - updated_at: Mapped[Optional[datetime]] = _updated_at() - - -# ──────────────────────── 12. industry ──────────────────────── -class Industry(ORMBase): - """股票-行业映射（baostock 源，证监会分类标准）。""" - __tablename__ = "industry" - __table_args__ = ( - Index("idx_industry_industry_name", "industry_name"), - {"schema": "market_data"}, - ) - - code: Mapped[str] = mapped_column(String(10), primary_key=True) - industry_name: Mapped[Optional[str]] = mapped_column(String(64), default="") - industry_classification: Mapped[Optional[str]] = mapped_column(String(32), default="") - update_date: Mapped[Optional[date]] = mapped_column(Date) - updated_at: Mapped[Optional[datetime]] = _updated_at() - - -# ──────────────────────── 13. sector_indices ──────────────────────── -class SectorIndices(ORMBase): - """行业日线（基点 100，复合收益）。""" - __tablename__ = "sector_indices" - __table_args__ = ( - PrimaryKeyConstraint("trade_date", "sector_name"), - Index("idx_sector_indices_sector_name", "sector_name"), - {"schema": "market_data"}, - ) - - trade_date: Mapped[date] = mapped_column(Date, nullable=False) - sector_name: Mapped[str] = mapped_column(String(64), nullable=False) - close: Mapped[float] = mapped_column(Float, default=0) - sector_amplitude: Mapped[float] = mapped_column(Float, default=0) - - -# ──────────────────────── 14. sector_features_daily ──────────────────────── -class SectorFeaturesDaily(ORMBase): - """行业日特征：sector_ret / sector_amplitude / close / EMA / score。""" - __tablename__ = "sector_features_daily" - __table_args__ = ( - PrimaryKeyConstraint("trade_date", "sector_name"), - Index("idx_sector_features_sector_name", "sector_name"), - {"schema": "market_data"}, - ) - - trade_date: Mapped[date] = mapped_column(Date, nullable=False) - sector_name: Mapped[str] = mapped_column(String(64), nullable=False) - sector_ret: Mapped[float] = mapped_column(Float, default=0) - sector_amplitude: Mapped[float] = mapped_column(Float, default=0) - close: Mapped[float] = mapped_column(Float, default=0) - ema10: Mapped[float] = mapped_column(Float, default=0) - ema20: Mapped[float] = mapped_column(Float, default=0) - ema200: Mapped[float] = mapped_column(Float, default=0) - score: Mapped[int] = mapped_column(Integer, default=0) - - -# ──────────────────────── 15. market_regime_daily ──────────────────────── +# ──────────────────────── 12. market_regime_daily ──────────────────────── class MarketRegimeDaily(ORMBase): """市场情绪衍生指标（基于本地 kline 聚合）。""" __tablename__ = "market_regime_daily" @@ -599,6 +513,80 @@ class StockNodeMap(ORMBase): updated_at: Mapped[Optional[datetime]] = _updated_at() +# ──────────────────────── 22. kline_stock_macd_daily ──────────────────────── +class KlineStockMACDDaily(ORMBase): + """个股日 K 级别 MACD 指标（mairui /hsstock/history/macd 直拉）。 + + 字段（mairui 原始字段名，与 K 线口径一致）： + diff = DIF（快慢 EMA 差） + dea = DEA（DIF 的 9 日 EMA，即 signal line） + macd = MACD 柱（2 ×（DIF − DEA）） + ema12 / ema26 = 计算 DIF 用的两条 EMA + stock_code 用 hermes 格式（SH600519），与项目其他表一致。 + """ + __tablename__ = "kline_stock_macd_daily" + __table_args__ = ( + PrimaryKeyConstraint("stock_code", "trade_date"), + Index("idx_kline_stock_macd_date", "trade_date"), + {"schema": "market_data"}, + ) + + stock_code: Mapped[str] = mapped_column(String(10), nullable=False) + trade_date: Mapped[date] = mapped_column(Date, nullable=False) + diff: Mapped[Optional[float]] = mapped_column(Float) + dea: Mapped[Optional[float]] = mapped_column(Float) + macd: Mapped[Optional[float]] = mapped_column(Float) + ema12: Mapped[Optional[float]] = mapped_column(Float) + ema26: Mapped[Optional[float]] = mapped_column(Float) + source: Mapped[Optional[str]] = mapped_column(String(32), default="mairui") + updated_at: Mapped[Optional[datetime]] = _updated_at() + + +# ──────────────────────── 23. kline_stock_kdj_daily ──────────────────────── +class KlineStockKDJDaily(ORMBase): + """个股日 K 级别 KDJ 指标（mairui /hsstock/history/kdj 直拉）。 + + k / d / j 三条随机指标线。stock_code 用 hermes 格式（SH600519）。 + """ + __tablename__ = "kline_stock_kdj_daily" + __table_args__ = ( + PrimaryKeyConstraint("stock_code", "trade_date"), + Index("idx_kline_stock_kdj_date", "trade_date"), + {"schema": "market_data"}, + ) + + stock_code: Mapped[str] = mapped_column(String(10), nullable=False) + trade_date: Mapped[date] = mapped_column(Date, nullable=False) + k: Mapped[Optional[float]] = mapped_column(Float) + d: Mapped[Optional[float]] = mapped_column(Float) + j: Mapped[Optional[float]] = mapped_column(Float) + source: Mapped[Optional[str]] = mapped_column(String(32), default="mairui") + updated_at: Mapped[Optional[datetime]] = _updated_at() + + +# ──────────────────────── 24. kline_stock_boll_daily ──────────────────────── +class KlineStockBOLLDaily(ORMBase): + """个股日 K 级别 BOLL 布林带指标（mairui /hsstock/history/boll 直拉）。 + + mairui 原始字段：u=上轨, m=中轨, d=下轨。落库列名统一为 upper/mid/lower。 + stock_code 用 hermes 格式（SH600519）。 + """ + __tablename__ = "kline_stock_boll_daily" + __table_args__ = ( + PrimaryKeyConstraint("stock_code", "trade_date"), + Index("idx_kline_stock_boll_date", "trade_date"), + {"schema": "market_data"}, + ) + + stock_code: Mapped[str] = mapped_column(String(10), nullable=False) + trade_date: Mapped[date] = mapped_column(Date, nullable=False) + upper: Mapped[Optional[float]] = mapped_column(Float) + mid: Mapped[Optional[float]] = mapped_column(Float) + lower: Mapped[Optional[float]] = mapped_column(Float) + source: Mapped[Optional[str]] = mapped_column(String(32), default="mairui") + updated_at: Mapped[Optional[datetime]] = _updated_at() + + __all__ = [ # 1-2 "Config", @@ -612,13 +600,7 @@ __all__ = [ "Kline5Min", "Moneyflow", "Share", - # 10-12 - "Sectors", - "StockSectorMap", - "Industry", - # 13-15 - "SectorIndices", - "SectorFeaturesDaily", + # 10-11 "MarketRegimeDaily", # 16 "TickTrade", @@ -631,4 +613,8 @@ __all__ = [ "NodeCategory", "Node", "StockNodeMap", + # 22-24 (2026-07-08 mairui 技术指标 MACD/KDJ/BOLL - 日 K 级别) + "KlineStockMACDDaily", + "KlineStockKDJDaily", + "KlineStockBOLLDaily", ] diff --git a/app/core/db/ops.py b/app/core/db/ops.py index 6862d02..5c77842 100644 --- a/app/core/db/ops.py +++ b/app/core/db/ops.py @@ -22,10 +22,12 @@ from sqlalchemy.dialects.postgresql import insert as pg_insert from app.core.db.models import ( Config, DatasetRegistry, - Industry, Kline5Min, KlineIndex, KlineStock, + KlineStockBOLLDaily, + KlineStockKDJDaily, + KlineStockMACDDaily, KlineStockMADaily, LonghubangDaily, LonghubangSeat, @@ -34,13 +36,9 @@ from app.core.db.models import ( Moneyflow, Node, NodeCategory, - SectorFeaturesDaily, - SectorIndices, - Sectors, Share, Stock, StockNodeMap, - StockSectorMap, SyncHistory, TickTrade, ) @@ -451,19 +449,16 @@ def upsert_stock( exchange: str, list_date: str = "", listing_status: str = "normal", - industry: str = "", ) -> None: """upsert 一只股票基础信息（单条）。 注意：MySQL 9.7.0 在 ON DUPLICATE KEY UPDATE 阶段对 DATE 字段的 VALUES()/new.col - 求值存在 bug（会强制把空串塞进去，触发 1292 严格模式错误）。所以这里： - - list_date 字段在首次 INSERT 时写入；UPDATE 分支不更新（保持原值或 NULL） - - industry 字段也避开这个 bug + 求值存在 bug（会强制把空串塞进去，触发 1292 严格模式错误）。所以这里 + list_date 字段在首次 INSERT 时写入；UPDATE 分支不更新（保持原值或 NULL）。 批量写入请用 upsert_stocks_bulk()，性能高 10-20 倍。 """ del list_date # 显式不接受 list_date 更新（旧值保留） - del industry # 同上 with get_session() as s: stmt = _pg_upsert( Stock, @@ -477,7 +472,7 @@ def upsert_stock( def upsert_stocks_bulk(rows: list[dict[str, Any]], chunk_size: int = 500) -> int: """批量 upsert 股票基础信息。 - 每条 row 需有: code, name, exchange, listing_status（可选 list_date / industry） + 每条 row 需有: code, name, exchange, listing_status（可选 list_date） 性能：~5000 只股票从 ~50s 降到 ~3s。 """ if not rows: @@ -499,7 +494,7 @@ def upsert_stocks_bulk(rows: list[dict[str, Any]], chunk_size: int = 500) -> int stmt = _pg_upsert( Stock, values, conflict_keys=["code"], - update_cols=["name", "exchange", "list_date", "listing_status", "industry"], + update_cols=["name", "exchange", "list_date", "listing_status"], ) s.execute(stmt) total += len(chunk) @@ -507,15 +502,25 @@ def upsert_stocks_bulk(rows: list[dict[str, Any]], chunk_size: int = 500) -> int def update_stock_share_snapshot(code: str, total_share: float, float_share: float, trade_date: str = "") -> None: + """更新 stocks 表的股本字段。 + + Args: + trade_date: 数据源返回的股本快照"as-of"交易日(用于 share 表写入)。 + 注意:这个参数**不会**写到 stocks.share_updated_at 字段 — + 那个字段的语义是"我们什么时候拉到的"(运行日,date.today()), + 不是数据交易日。把 trade_date 写到 share_updated_at 会让 + task_share_snapshot / task_stocks_basic 的 dedup 检查 + (share_updated_at == today) 失效,见 task 层 fix。 + """ + del trade_date # 显式不接受 — 见 docstring with get_session() as s: - snap_date = _to_date_str(trade_date) if trade_date else None s.execute( update(Stock) .where(Stock.code == code) .values( total_share=float(total_share), float_share=float(float_share), - share_updated_at=snap_date, + share_updated_at=date.today(), # "拉取日",不是"数据日" ) ) @@ -543,7 +548,6 @@ def _stock_row_to_dict(r: Stock) -> dict[str, Any]: "exchange": r.exchange or "", "list_date": _to_date_str(r.list_date), "listing_status": r.listing_status or "normal", - "industry": r.industry or "", "total_share": r.total_share if r.total_share is not None else 0, "float_share": r.float_share if r.float_share is not None else 0, "share_updated_at": _to_date_str(r.share_updated_at), @@ -674,9 +678,13 @@ def get_kline_5min_snapshots() -> dict[str, Optional[datetime]]: 返回：{stock_code (6位): max(bar_time) 或 None} 用于 kline_5min 任务的"全量/增量"统一规划。 - DB 里 stock_code 形如 "000001.SZ"（mairui 写入格式），函数剥掉 - 交易所后缀统一为 6 位。bar_time 是 DATETIME 字段，SQLAlchemy 2.x - native 返回 datetime；如果是 str（方言边界情况）手动解析。 + DB 里 stock_code 是 hermes 格式 "SH600000"（带 SH/SZ/BJ 前缀，无点）， + 2026-07-09 之前误以为 mairui 原始格式 "000001.SZ"，用 split(".")[0] 剥 + 不到 → 全部股票被 skip → task 把整市场当"全量"重跑(实际是增量)。 + 修:用正则剥 SH/SZ/BJ 前缀。 + + bar_time 是 DATETIME 字段，SQLAlchemy 2.x native 返回 datetime；如果是 + str（方言边界情况）手动解析。 """ _ensure_schema() out: dict[str, Optional[datetime]] = {} @@ -686,7 +694,13 @@ def get_kline_5min_snapshots() -> dict[str, Optional[datetime]]: .group_by(Kline5Min.stock_code) ).all() for raw_code, latest in rows: - code6 = str(raw_code or "").strip().split(".")[0] + # 剥 SH/SZ/BJ 前缀(hermes 格式),容错 6 位裸码 + code6 = str(raw_code or "").strip() + for prefix in ("SH", "SZ", "BJ"): + if code6.startswith(prefix): + code6 = code6[len(prefix):] + break + code6 = code6.split(".")[0] # 兜底剥 "000001.SZ" 老格式 if len(code6) != 6 or not code6.isdigit(): continue if isinstance(latest, datetime): @@ -834,143 +848,6 @@ def replace_all_stock_node_map(rows: list[dict[str, Any]]) -> None: s.execute(stmt) -# ── 行业 / 概念板块 ───────────────────────────────────────────────────── - - -def replace_all_industries(rows: list[dict[str, Any]]) -> None: - """全量替换 industry 表。rows: code, industry_name, industry_classification, update_date""" - if not rows: - return - with get_session() as s: - s.execute(delete(Industry)) - values = [ - { - "code": str(r.get("code", "")).zfill(6), - "industry_name": r.get("industry_name") or None, - "industry_classification": r.get("industry_classification") or None, - "update_date": _to_date_str(r.get("update_date")), - } - for r in rows - ] - if values: - stmt = _pg_upsert(Industry, values, - conflict_keys=["code"], - update_cols=["industry_name", "industry_classification", "update_date"]) - s.execute(stmt) - - -def fetch_all_industries() -> list[dict[str, Any]]: - _ensure_schema() - with get_session() as s: - rows = s.execute( - select(Industry.code, Industry.industry_name, Industry.industry_classification, Industry.update_date) - ).all() - return [ - { - "code": r.code, - "industry_name": r.industry_name, - "industry_classification": r.industry_classification, - "update_date": _to_date_str(r.update_date), - } - for r in rows - ] - - -def replace_all_sectors(rows: list[dict[str, Any]]) -> None: - """rows: sector_key, sector_name, taxonomy, level, source, enabled""" - if not rows: - return - with get_session() as s: - s.execute(delete(Sectors)) - values = [ - { - "sector_key": r.get("sector_key", ""), - "sector_name": r.get("sector_name", ""), - "taxonomy": r.get("taxonomy", ""), - "level": r.get("level", ""), - "source": r.get("source", ""), - "enabled": int(r.get("enabled", 1)), - } - for r in rows - ] - if values: - stmt = _pg_upsert(Sectors, values, - conflict_keys=["sector_key"], - update_cols=["sector_name", "taxonomy", "level", "source", "enabled"]) - s.execute(stmt) - - -def replace_all_stock_sector_map(rows: list[dict[str, Any]]) -> None: - """rows: stock_code, sector_key""" - if not rows: - return - with get_session() as s: - s.execute(delete(StockSectorMap)) - values = [ - { - "stock_code": str(r.get("stock_code", "")).zfill(6), - "sector_key": r.get("sector_key", ""), - } - for r in rows - ] - if values: - stmt = _pg_upsert(StockSectorMap, values, - conflict_keys=["stock_code"], - update_cols=["sector_key"]) - s.execute(stmt) - - -# ── 行业聚合（衍生）───────────────────────────────────────────────────── - - -def replace_all_sector_indices(rows: list[dict[str, Any]]) -> None: - """rows: trade_date, sector_name, close, sector_amplitude""" - if not rows: - return - with get_session() as s: - values = [ - { - "trade_date": _to_date_str(r.get("trade_date")), - "sector_name": r.get("sector_name", ""), - "close": float(r.get("close") or 0), - "sector_amplitude": float(r.get("sector_amplitude") or 0), - } - for r in rows - ] - if values: - stmt = _pg_upsert(SectorIndices, values, - conflict_keys=["trade_date", "sector_name"], - update_cols=["close", "sector_amplitude"]) - s.execute(stmt) - - -def replace_all_sector_features(rows: list[dict[str, Any]]) -> None: - """rows: trade_date, sector_name, sector_ret, sector_amplitude, close, ema10, ema20, ema200, score""" - if not rows: - return - with get_session() as s: - values = [ - { - "trade_date": _to_date_str(r.get("trade_date")), - "sector_name": r.get("sector_name", ""), - "sector_ret": float(r.get("sector_ret") or 0), - "sector_amplitude": float(r.get("sector_amplitude") or 0), - "close": float(r.get("close") or 0), - "ema10": float(r.get("ema10") or 0), - "ema20": float(r.get("ema20") or 0), - "ema200": float(r.get("ema200") or 0), - "score": int(r.get("score") or 0), - } - for r in rows - ] - if values: - stmt = _pg_upsert(SectorFeaturesDaily, values, - conflict_keys=["trade_date", "sector_name"], - update_cols=["sector_ret", "sector_amplitude", "close", - "ema10", "ema20", "ema200", "score"]) - s.execute(stmt) - - # ── 市场情绪（衍生）───────────────────────────────────────────────────── @@ -1031,6 +908,51 @@ def upsert_kline_stock_ma_daily_rows(rows: list[dict[str, Any]]) -> None: s.execute(stmt) +def upsert_kline_stock_macd_daily_rows(rows: list[dict[str, Any]]) -> int: + """rows: stock_code, trade_date, diff, dea, macd, ema12, ema26, source""" + if not rows: + return 0 + with get_session() as s: + return _bulk_upsert_orm(s, KlineStockMACDDaily, rows) + + +def upsert_kline_stock_kdj_daily_rows(rows: list[dict[str, Any]]) -> int: + """rows: stock_code, trade_date, k, d, j, source""" + if not rows: + return 0 + with get_session() as s: + return _bulk_upsert_orm(s, KlineStockKDJDaily, rows) + + +def upsert_kline_stock_boll_daily_rows(rows: list[dict[str, Any]]) -> int: + """rows: stock_code, trade_date, upper, mid, lower, source""" + if not rows: + return 0 + with get_session() as s: + return _bulk_upsert_orm(s, KlineStockBOLLDaily, rows) + + +_INDICATOR_MODEL = { + "macd": KlineStockMACDDaily, + "kdj": KlineStockKDJDaily, + "boll": KlineStockBOLLDaily, +} + + +def get_indicator_max_date(indicator: str, code: str) -> Optional[str]: + """某只票某指标已入库的最大 trade_date（YYYY-MM-DD），无则 None。""" + model = _INDICATOR_MODEL.get(indicator.lower()) + if model is None: + raise ValueError(f"未知指标: {indicator!r}") + _ensure_schema() + with get_session() as s: + d = s.execute( + select(func.max(model.trade_date)) + .where(model.stock_code == code) + ).scalar() + return _to_date_str(d) if d else None + + def pd_isna(v: Any) -> bool: """避免直接 import pandas（开销大），手写 nan/None 检查。""" if v is None: @@ -1042,11 +964,26 @@ def pd_isna(v: Any) -> bool: def get_stock_kline_max_date(code: str) -> Optional[str]: + """某只票 kline_stock 已入库的最大 trade_date (YYYY-MM-DD),无则 None。 + + code 接受 6 位 ("600000") 或 hermes 格式 ("SH600000") — DB 里实际存的是 + hermes 格式,2026-07-09 之前用 `where KlineStock.stock_code == '600000'` + 永远查不到(hermes 有 SH/SZ/BJ 前缀),导致 incremental 把全市场当"无数据" + 重跑。修:同时查 6 位和 hermes 两种格式,取较新结果。 + """ _ensure_schema() + c6 = str(code or "").strip() + hermes = f"SH{c6}" if c6.startswith(("5", "6", "9")) else ( + f"SZ{c6}" if c6.startswith(("0", "2", "3")) else ( + f"BJ{c6}" if c6.startswith(("4", "8")) else c6 + )) + candidates = [c6] + if hermes != c6: + candidates.append(hermes) with get_session() as s: d = s.execute( select(func.max(KlineStock.trade_date)) - .where(KlineStock.stock_code == code) + .where(KlineStock.stock_code.in_(candidates)) ).scalar() return _to_date_str(d) if d else None diff --git a/app/core/scheduler/scheduler.py b/app/core/scheduler/scheduler.py index a586bc4..ac21ac2 100644 --- a/app/core/scheduler/scheduler.py +++ b/app/core/scheduler/scheduler.py @@ -323,17 +323,6 @@ DEFAULT_SCHEDULES: list[tuple[str, dict, str]] = [ }, "每个交易日 21:35 拉取个股资金流（mairui 21:30 发布）", ), - ( - "schedule_industry_sector", - { - "name": "周一行业映射", - "time": "09:30", - "condition": "trading_day", - "job": "industry_sector", - "enabled": True, - }, - "每个交易日 09:30 全量更新股票-行业映射（开销大，可改为 weekly）", - ), ( "schedule_market_regime", { @@ -345,17 +334,6 @@ DEFAULT_SCHEDULES: list[tuple[str, dict, str]] = [ }, "每个交易日 16:15 聚合市场情绪（基于本地日K线）", ), - ( - "schedule_share_snapshot", - { - "name": "盘后股本快照", - "time": "16:20", - "condition": "trading_day", - "job": "share_snapshot", - "enabled": True, - }, - "每个交易日 16:20 刷新个股总股本/流通股本快照", - ), ( "schedule_longhubang", { @@ -389,6 +367,17 @@ DEFAULT_SCHEDULES: list[tuple[str, dict, str]] = [ }, "每个交易日 16:30 基于 kline_stock 计算 MA5/10/20/60 (本地派生，~30s @5213 只)", ), + ( + "schedule_mairui_indicators", + { + "name": "日 K 技术指标 MACD/KDJ/BOLL", + "time": "16:40", + "condition": "trading_day", + "job": "mairui_indicators", + "enabled": True, + }, + "每个交易日 16:40 从 mairui 直拉 MACD/KDJ/BOLL 增量 (3 指标 × 全市场，增量约数分钟)", + ), ] @@ -428,8 +417,8 @@ def register_sync_jobs() -> None: for dataset_id in [ "stock_basic", "kline_daily", "kline_index", "kline_5min", - "moneyflow", "industry_sector", "share_snapshot", "market_regime", - "longhubang", "stock_node", "mairui_ma_daily", + "moneyflow", "share_snapshot", "market_regime", + "longhubang", "stock_node", "mairui_ma_daily", "mairui_indicators", ]: def _make_job(did=dataset_id): diff --git a/app/core/sync/base.py b/app/core/sync/base.py index fc5280d..7279f76 100644 --- a/app/core/sync/base.py +++ b/app/core/sync/base.py @@ -34,13 +34,13 @@ def _is_market_closed() -> bool: def _effective_sync_end() -> str: - """若盘后则今天，否则昨天。""" - today = datetime.now() - if today.time() >= dtime(15, 30): - return today.strftime("%Y-%m-%d") - # 昨天 - from datetime import timedelta - return (today - timedelta(days=1)).strftime("%Y-%m-%d") + """若盘后则今天，否则昨天。 + + 委派到 app.core.datasource.utils.effective_market_date() 统一实现， + 该函数加载交易日历，会跳过节假日返回最近交易日。 + """ + from app.core.datasource.utils import effective_market_date + return effective_market_date() class SyncTask: diff --git a/app/core/sync/registry.py b/app/core/sync/registry.py index 4f8808e..4804e20 100644 --- a/app/core/sync/registry.py +++ b/app/core/sync/registry.py @@ -94,30 +94,6 @@ SYNC_DEFINITIONS: list[dict[str, Any]] = [ "dependency_ids": ["stock_basic"], "sort_order": 50, }, - { - "dataset_id": "industry_sector", - "name": "股票-行业映射", - "description": "股票-行业映射 + 行业字典（Baostock）", - "storage_uri": "PG market_data.industry + sectors + stock_sector_map", - "storage_layer": "pg", - "management_role": "基础字典", - "source": "Baostock query_stock_industry", - "sync_script": "app.tasks.task_industry_sector:run", - "dependency_ids": ["stock_basic"], - "sort_order": 60, - }, - { - "dataset_id": "sector_features", - "name": "行业聚合特征", - "description": "由 kline_stock + industry 衍生：行业日收益 / 行业指数 close / EMA10-20-200 / score。纯本地计算，无外部 API。", - "storage_uri": "PG market_data.sector_indices + sector_features_daily", - "storage_layer": "pg", - "management_role": "衍生源", - "source": "本地计算（kline_stock + industry）", - "sync_script": "app.tasks.task_sector_features:run", - "dependency_ids": ["kline_daily", "industry_sector"], - "sort_order": 65, - }, { "dataset_id": "share_snapshot", "name": "股本快照", @@ -178,6 +154,18 @@ SYNC_DEFINITIONS: list[dict[str, Any]] = [ "dependency_ids": ["kline_daily"], "sort_order": 90, }, + { + "dataset_id": "mairui_indicators", + "name": "日 K 技术指标 MACD/KDJ/BOLL (mairui)", + "description": "mairui /hsstock/history/{macd,kdj,boll} 直拉，写 3 张表；每交易日 16:40 增量。", + "storage_uri": "PG market_data.kline_stock_macd_daily + kdj_daily + boll_daily", + "storage_layer": "pg", + "management_role": "原始源", + "source": "mairui /hsstock/history/{macd,kdj,boll}/{symbol}/d/n", + "sync_script": "app.tasks.task_mairui_indicators:run", + "dependency_ids": ["stock_basic"], + "sort_order": 91, + }, ] diff --git a/app/entrypoints/worker.py b/app/entrypoints/worker.py index df18f22..c24a86a 100644 --- a/app/entrypoints/worker.py +++ b/app/entrypoints/worker.py @@ -30,7 +30,15 @@ def start_scheduler_thread() -> None: 给 service_run.sh 用：在 uvicorn 同进程跑后台 scheduler，避免 docker 一个容器跑两个进程（supervisord / s6 那种方案）。 幂等：多次调用只启动一次。 + + 2026-07-12: 增加 RUNTIME_MODE 感知。systemd 模式下不启动 scheduler， + 因为同步由 systemd timer 触发，避免双调度器重叠。 """ + logger.info(f"[worker] runtime_mode={settings.runtime_mode}") + if settings.runtime_mode == "systemd": + logger.info("[worker] systemd 模式：不启动进程内 scheduler，同步由 systemd timer 触发") + return + # 1. 注册数据源 + seed config from app.core.datasource.registry import ( build_default_registry, @@ -74,6 +82,22 @@ def start_scheduler_thread() -> None: def main(): logger.info("[worker] market_data_sync worker 启动") + logger.info(f"[worker] runtime_mode={settings.runtime_mode}") + + if settings.runtime_mode == "systemd": + logger.info("[worker] systemd 模式：不启动进程内 scheduler，同步由 systemd timer 触发") + logger.info("[worker] 仅启动健康监控（数据源心跳），按 Ctrl+C 退出") + from app.core.datasource.registry import build_default_registry, seed_datasource_configs, start_health_monitor + build_default_registry() + seed_datasource_configs() + start_health_monitor() + try: + while True: + time.sleep(60) + except KeyboardInterrupt: + logger.info("[worker] 收到 SIGINT，退出") + sys.exit(0) + return # 1. 注册数据源 + seed config from app.core.datasource.registry import build_default_registry, seed_datasource_configs diff --git a/app/mcp_server.py b/app/mcp_server.py index cb9828f..5786626 100644 --- a/app/mcp_server.py +++ b/app/mcp_server.py @@ -34,6 +34,7 @@ if str(_PROJECT_ROOT) not in sys.path: # ── MCP server bootstrap ──────────────────────────────────────────── from mcp.server import Server +from mcp.server.sse import SseServerTransport from mcp.server.stdio import stdio_server from mcp.types import Tool, TextContent @@ -125,9 +126,10 @@ def list_datasets() -> list[dict]: from sqlalchemy import func, select from app.core.db.models import ( KlineStock, KlineIndex, Kline5Min, Moneyflow, Share, TickTrade, - SectorIndices, SectorFeaturesDaily, MarketRegimeDaily, - LonghubangDaily, LonghubangSeat, Stock, Industry, + MarketRegimeDaily, + LonghubangDaily, LonghubangSeat, Stock, NodeCategory, Node, StockNodeMap, KlineStockMADaily, + KlineStockMACDDaily, KlineStockKDJDaily, KlineStockBOLLDaily, ) from app.core.db.orm import SessionLocal @@ -140,15 +142,15 @@ def list_datasets() -> list[dict]: ("moneyflow", Moneyflow), ("share", Share), ("tick_trade", TickTrade), - ("sector_indices", SectorIndices), - ("sector_features_daily", SectorFeaturesDaily), ("market_regime_daily", MarketRegimeDaily), ("longhubang_daily", LonghubangDaily), ("longhubang_seat", LonghubangSeat), - ("industry", Industry), ("node_categories", NodeCategory), ("nodes", Node), ("stock_node_map", StockNodeMap), + ("kline_stock_macd_daily", KlineStockMACDDaily), + ("kline_stock_kdj_daily", KlineStockKDJDaily), + ("kline_stock_boll_daily", KlineStockBOLLDaily), ] out = [] with SessionLocal() as s: @@ -175,9 +177,10 @@ def get_dataset_info(table_name: str) -> dict: allowed = { "stocks", "kline_stock", "kline_index", "kline_5min", "kline_stock_ma_daily", "moneyflow", "share", "tick_trade", - "sector_indices", "sector_features_daily", "market_regime_daily", - "longhubang_daily", "longhubang_seat", "industry", + "market_regime_daily", + "longhubang_daily", "longhubang_seat", "node_categories", "nodes", "stock_node_map", + "kline_stock_macd_daily", "kline_stock_kdj_daily", "kline_stock_boll_daily", "dataset_registry", "sync_history", "config", } if table_name not in allowed: @@ -291,11 +294,51 @@ async def call_tool(name: str, arguments: dict): # ── main ──────────────────────────────────────────────────────────── -async def main(): - # 启动时注册数据源（让 datasource.health_check 之类方法可用） +async def main_sse(host: str = "127.0.0.1", port: int = 8101): + """启动 MCP 的 SSE (HTTP) 服务，供持久运行。 + + 客户端连接: http://{host}:{port}/sse + """ + sse_transport = SseServerTransport("/messages") + + async def app(scope, receive, send): + if scope["type"] != "http": + return + path = scope.get("path", "") + if path == "/sse": + async with sse_transport.connect_sse(scope, receive, send) as (read, write): + await server.run(read, write, server.create_initialization_options()) + elif path == "/messages" and scope.get("method") == "POST": + await sse_transport.handle_post_message(scope, receive, send) + + import uvicorn + from app.core.utils.logging import get_logger + logger = get_logger("mcp") + logger.info(f"MCP SSE server listening on http://{host}:{port}/sse") + config = uvicorn.Config(app, host=host, port=port, log_level="info") + srv = uvicorn.Server(config) + await srv.serve() + + +def main(): from app.core.datasource.registry import build_default_registry build_default_registry() + import sys + if "--sse" in sys.argv: + host = "127.0.0.1" + port = 8101 + for i, arg in enumerate(sys.argv): + if arg == "--host" and i + 1 < len(sys.argv): + host = sys.argv[i + 1] + if arg == "--port" and i + 1 < len(sys.argv): + port = int(sys.argv[i + 1]) + asyncio.run(main_sse(host, port)) + else: + asyncio.run(_stdio_main()) + + +async def _stdio_main(): async with stdio_server() as (read_stream, write_stream): await server.run( read_stream, @@ -305,4 +348,4 @@ async def main(): if __name__ == "__main__": - asyncio.run(main()) \ No newline at end of file + main() \ No newline at end of file diff --git a/app/sources/mairui.py b/app/sources/mairui.py index 9fb7082..914c109 100644 --- a/app/sources/mairui.py +++ b/app/sources/mairui.py @@ -32,7 +32,7 @@ from app.core.datasource.utils import ( class MairuiSource(DataSource): key = "datasource_mairui" name = "麦蕊智数（mairui.club）" - provides = ["kline_daily", "kline_5min", "index_daily", "stock_basic", "moneyflow", "tick_trade", "stock_node"] + provides = ["kline_daily", "kline_5min", "index_daily", "stock_basic", "moneyflow", "tick_trade", "stock_node", "indicator_daily"] requires_credential = True credential_key = "MAIRUI_LICENCE" @@ -285,6 +285,68 @@ class MairuiSource(DataSource): df = df[df["bar_time"] < pd.Timestamp(end) + pd.Timedelta(days=1)] return df + # ── 技术指标 fetch（MACD / KDJ / BOLL）───────────────────── + # mairui 端点：/hsstock/history/{indicator}/{symbol}/d/n/{licence} + # indicator ∈ {macd, kdj, boll}；日 K 级别 "/d/"；不复权 "/n/" + # 各指标字段（除公共 "t" 外）： + # macd → diff, dea, macd, ema12, ema26 + # kdj → k, d, j + # boll → u(上轨), d(下轨), m(中轨) ← 注意 mairui 用 u/d/m + _INDICATOR_FIELDS = { + "macd": ["diff", "dea", "macd", "ema12", "ema26"], + "kdj": ["k", "d", "j"], + "boll": ["u", "d", "m"], + } + + def fetch_indicator_daily( + self, indicator: str, code6: str, start: str = "", end: str = "" + ) -> pd.DataFrame: + """拉单只股票某个技术指标的日 K 序列。 + + 返回列：trade_date + 该指标字段（macd/kdj/boll 各自的原始字段名）。 + start/end 为 "YYYY-MM-DD"（可空 → 全历史）。 + """ + indicator = indicator.lower() + fields = self._INDICATOR_FIELDS.get(indicator) + if fields is None: + raise ValueError(f"未知指标: {indicator!r}，可选 {list(self._INDICATOR_FIELDS)}") + lic = self._get_licence() + if not lic: + return pd.DataFrame() + symbol = code6_to_mairui(code6) + path = f"/hsstock/history/{indicator}/{symbol}/d/n/{lic}" + params = { + "st": (start or "").replace("-", ""), + "et": (end or "").replace("-", ""), + } + data = self._fetch(path, params) + if not isinstance(data, list): + return pd.DataFrame() + records = [] + for item in data: + try: + t = item.get("t", "") + d = t.split(" ")[0] if isinstance(t, str) else "" + if not d: + continue + row = {"trade_date": d} + for f in fields: + v = item.get(f) + row[f] = None if v is None else float(v) + records.append(row) + except (KeyError, ValueError, TypeError): + continue + if not records: + return pd.DataFrame() + df = pd.DataFrame(records) + df["trade_date"] = pd.to_datetime(df["trade_date"], errors="coerce") + df = df.dropna(subset=["trade_date"]).sort_values("trade_date").reset_index(drop=True) + if start: + df = df[df["trade_date"] >= pd.Timestamp(start)] + if end: + df = df[df["trade_date"] <= pd.Timestamp(end)] + return df + def fetch_index_daily(self, index_code: str, start: str, end: str) -> pd.DataFrame: """指数日 K。index_code 用 mairui 格式（如 '000300.SH'）。""" lic = self._get_licence() diff --git a/app/sources/xueqiu.py b/app/sources/xueqiu.py index d6ecc53..c3e36f1 100644 --- a/app/sources/xueqiu.py +++ b/app/sources/xueqiu.py @@ -16,7 +16,7 @@ from typing import Any, Optional import pandas as pd from app.core.datasource.base import DataSource -from app.core.datasource.utils import code6_to_xueqiu +from app.core.datasource.utils import call_with_timeout, code6_to_xueqiu, effective_market_date logger = logging.getLogger("sync.xueqiu") @@ -94,7 +94,11 @@ class XueqiuSource(DataSource): try: ball = self._import_ball() self._set_token_once() - result = ball.quote_detail("SH600036") + result = call_with_timeout( + ball.quote_detail, "SH600036", + timeout=10.0, on_timeout=None, + description="xueqiu.health_check", + ) if result is None or result.get("error_code") != 0: return { "success": False, @@ -123,12 +127,23 @@ class XueqiuSource(DataSource): count = min(max((end_dt - start_dt).days + 60, 10), 5000) # 加重试：雪球风控偶尔返回空/错误 + # 用 call_with_timeout 包一层 — 2026-07-08 kline_daily 卡死 4h 根因 + # 就是 pysnowball ball.kline() 底层 requests 无 read timeout,雪球 + # 服务端卡住时这里永久阻塞。timeout=20s 与 mairui/sina 对齐。 result = None for attempt in range(3): - result = ball.kline(symbol, period="day", count=count) + result = call_with_timeout( + ball.kline, symbol, period="day", count=count, + timeout=20.0, on_timeout=None, + description=f"xueqiu.kline {symbol}", + ) if result and result.get("error_code") == 0: break - time.sleep(0.3 * (attempt + 1)) + if result is None and attempt < 2: + time.sleep(0.3 * (attempt + 1)) + continue + if result and result.get("error_code") != 0: + time.sleep(0.3 * (attempt + 1)) if not result or result.get("error_code") != 0: logger.warning( "[kline %s] 雪球 kline 失败: error_code=%s desc=%s", @@ -177,7 +192,12 @@ class XueqiuSource(DataSource): self._wait_rps() symbol = code6_to_xueqiu(code6) - result = ball.quote_detail(symbol) + # ball.quote_detail 同样有 pysnowball 无 read timeout 风险,包一层 + result = call_with_timeout( + ball.quote_detail, symbol, + timeout=15.0, on_timeout=None, + description=f"xueqiu.quote_detail {symbol}", + ) if not result or result.get("error_code") != 0: logger.warning( "[share %s] 雪球 quote_detail 失败: error_code=%s desc=%s", @@ -187,14 +207,18 @@ class XueqiuSource(DataSource): ) return None quote = result.get("data", {}).get("quote", {}) - today_str = datetime.now().strftime("%Y-%m-%d") + # 雪球 quote_detail 不返回明确"股本变动日",time 字段是 quote 当前 + # 时间,不是交易日。股本数据本质是慢变快照,这里标"最近已完成交易日" + # (15:30 前=昨天,15:30 后=今天) — 比 datetime.now() 精确,避免盘前 + # 拉到的快照被错标为"今天"。 + as_of = effective_market_date() total = round(float(quote.get("total_shares") or 0) / 1e8, 4) flt = round(float(quote.get("float_shares") or 0) / 1e8, 4) if total <= 0 or flt <= 0: logger.warning("[share %s] 雪球返回成功但 total/float 为 0", code6) return None return { - "trade_date": today_str, + "trade_date": as_of, "total_share": total, "float_share": flt, } diff --git a/app/tasks/__init__.py b/app/tasks/__init__.py index 021fe43..c505a43 100644 --- a/app/tasks/__init__.py +++ b/app/tasks/__init__.py @@ -3,15 +3,14 @@ 所有 `SyncTask` 子类在此集中注册，外部通过 `get_task(dataset_id)` 或 遍历 `TASKS` 字典使用。 """ -from app.tasks.task_industry_sector import SyncIndustrySector from app.tasks.task_kline_5min import SyncKline5Min from app.tasks.task_kline_daily import SyncKlineDaily from app.tasks.task_kline_index import SyncKlineIndex from app.tasks.task_longhubang import SyncLonghubang from app.tasks.task_mairui_ma_daily import SyncMairuiMADaily +from app.tasks.task_mairui_indicators import SyncMairuiIndicators from app.tasks.task_market_regime import SyncMarketRegime from app.tasks.task_moneyflow import SyncMoneyflow -from app.tasks.task_sector_features import SyncSectorFeatures from app.tasks.task_share_snapshot import SyncShareSnapshot from app.tasks.task_stock_node import SyncStockNode from app.tasks.task_stocks_basic import SyncStocksBasic @@ -26,13 +25,12 @@ TASKS: dict[str, type] = { SyncKline5Min, SyncTickTrade, SyncMoneyflow, - SyncIndustrySector, - SyncSectorFeatures, SyncShareSnapshot, SyncMarketRegime, SyncLonghubang, SyncStockNode, SyncMairuiMADaily, + SyncMairuiIndicators, ) } diff --git a/app/tasks/task_industry_sector.py b/app/tasks/task_industry_sector.py deleted file mode 100644 index 51dc54e..0000000 --- a/app/tasks/task_industry_sector.py +++ /dev/null @@ -1,121 +0,0 @@ -"""同步任务：股票-行业映射（Baostock）。 - -数据流：Baostock query_stock_industry() → 写 industry / sectors / stock_sector_map 三张表。""" -from __future__ import annotations - -import time -from typing import Any - -from sqlalchemy import update - -from app.core.db import ops as db_ops -from app.core.db.models import Stock -from app.core.datasource.base import registry as ds_registry -from app.core.datasource.registry import is_source_ready -from app.core.datasource.utils import to_hermes -from app.core.sync.base import SyncTask -from app.core.sync.registry import mark_sync_blocked -from app.core.utils.logging import get_logger - -logger = get_logger("sync.industry_sector") - - -def _build_sector_key(taxonomy: str, sector_name: str) -> str: - tax = (taxonomy or "").strip() or "default" - name = (sector_name or "").strip() - return f"{tax}::{name}" - - -class SyncIndustrySector(SyncTask): - dataset_id = "industry_sector" - - def _run(self, *, trigger_source: str = "manual", **kwargs) -> dict[str, Any]: - bs = ds_registry.get("datasource_baostock") - if bs is None: - return {"status": "error", "message": "Baostock 数据源未注册"} - ok, reason = is_source_ready(bs.key) - if not ok: - mark_sync_blocked(self.dataset_id, message=f"Baostock 未就绪: {reason}") - return {"status": "blocked", "message": f"Baostock 未就绪: {reason}"} - - self._progress(message="拉取股票-行业映射...") - t0 = time.time() - try: - raw_rows = bs.fetch_industry_map() - except Exception as e: - return {"status": "error", "message": f"拉取失败: {e}"} - - if not raw_rows: - return {"status": "warning", "message": "Baostock 未返回行业数据"} - - # 合并：sectors + stock_sector_map + industry - sectors_rows: dict[str, dict] = {} - stock_sector_rows: list[dict] = [] - industry_rows: list[dict] = [] - - for r in raw_rows: - code6 = str(r["code"]).zfill(6) - name = (r.get("industry_name") or "").strip() - if not name: - continue - classification = (r.get("industry_classification") or "").strip() - sector_key = _build_sector_key(classification, name) - - sectors_rows[sector_key] = { - "sector_key": sector_key, - "sector_name": name, - "taxonomy": classification, - "level": "", - "source": "baostock_query_stock_industry", - "enabled": 1, - } - stock_sector_rows.append({"stock_code": to_hermes(code6), "sector_key": sector_key}) - industry_rows.append({ - "code": to_hermes(code6), - "industry_name": name, - "industry_classification": classification, - "update_date": r.get("update_date", ""), - }) - - # 写库（全量替换） - try: - db_ops.replace_all_sectors(list(sectors_rows.values())) - db_ops.replace_all_stock_sector_map(stock_sector_rows) - db_ops.replace_all_industries(industry_rows) - except Exception as e: - return {"status": "error", "message": f"写库失败: {e}"} - - # 回填 stocks.industry 字段（让 stocks 表也能直接看到） - # 走 ORM update(Stock) + IN (SHxxx, SZxxx, BJxxx, ...) - try: - # 按 industry_name 分组，对每组用一个 IN 批量更新 - by_industry: dict[str, list[str]] = {} - for r in industry_rows: - name = r["industry_name"] - code = r["code"] - if name not in by_industry: - by_industry[name] = [] - by_industry[name].append(code) - for ind_name, codes in by_industry.items(): - # 6 位 code × 3 个交易所前缀（SH/SZ/BJ） - code_variants = [ - f"{prefix}{c}" for c in codes for prefix in ("SH", "SZ", "BJ") - ] - with db_ops.get_session() as s: - s.execute( - update(Stock) - .where(Stock.code.in_(code_variants)) - .values(industry=ind_name) - ) - except Exception as e: - logger.warning(f"回填 stocks.industry 失败（不影响主任务）: {e}") - - elapsed = round(time.time() - t0, 1) - msg = f"行业 {len(sectors_rows)}个 映射{len(stock_sector_rows)}条, {elapsed}s" - return { - "status": "ok", - "message": msg, - "sectors": len(sectors_rows), - "stock_sector_map": len(stock_sector_rows), - "elapsed_sec": elapsed, - } diff --git a/app/tasks/task_kline_5min.py b/app/tasks/task_kline_5min.py index f5799f8..f86e8bb 100644 --- a/app/tasks/task_kline_5min.py +++ b/app/tasks/task_kline_5min.py @@ -17,7 +17,7 @@ from __future__ import annotations import os import time -from concurrent.futures import ThreadPoolExecutor, as_completed +from concurrent.futures import ThreadPoolExecutor, as_completed, TimeoutError as FuturesTimeoutError from datetime import datetime, timedelta, timezone from typing import Any, Optional @@ -37,6 +37,10 @@ MAIRUI_5MIN_FLOOR = datetime(2023, 6, 14, tzinfo=timezone.utc) WINDOW_DAYS = 365 # 增量时往前多取的天数（防交易日历边界漏当天） INCREMENT_OVERLAP_DAYS = 2 +# 2026-07-08 教训: 数据源层有 20s timeout,task 层再硬兜底, +# 防 SDK 升级 / 网络层 bug 让单只股票卡住(7月7日 4.4只/秒 后突然 0 持续 24h) +# 7月9日 mairui/雪球 间歇性 "服务器连接失败" + Broken pipe,拉慢。给 120s 容忍。 +FETCH_HARD_TIMEOUT = 120.0 # 5min K 一只拉多年,单只允许更久 class SyncKline5Min(SyncTask): @@ -99,7 +103,12 @@ class SyncKline5Min(SyncTask): # ── 2) 内存算计划 ── # PG TIMESTAMPTZ 是 timezone-aware，所以 end 也必须 tz-aware 否则比较会炸 - end = datetime.now(timezone.utc) + # 2026-07-12 修复: end 改为上海时区最近交易日 15:00，避免 UTC 与上海时间混用 + from app.core.datasource.utils import effective_market_date + end_date_str = effective_market_date() + end = datetime.strptime(f"{end_date_str} 15:00:00", "%Y-%m-%d %H:%M:%S").replace( + tzinfo=timezone(timedelta(hours=8)) + ) plans = self._plan(stock_codes, end, snapshots) if not plans: msg = f"5min K线 — 全部 {len(stock_codes)} 只都已最新（无需同步）" @@ -129,12 +138,17 @@ class SyncKline5Min(SyncTask): t0 = time.time() ok_cnt = fail_cnt = 0 rows_total = 0 - with ThreadPoolExecutor(max_workers=max_workers) as pool: - futures = {pool.submit(self._sync_one, primary, p[0], p[1], p[2]): p[0] for p in plans} - for i, future in enumerate(as_completed(futures), 1): + # 不用 `with ThreadPoolExecutor(...) as pool:` — 它的 __exit__ 默认 wait=True, + # 一旦某个 worker 卡在 xueqiu IO,主线程会在 as_completed 触发 60s timeout 后 + # 仍被 __exit__ 阻塞等 worker 退出 → 进程挂死(2026-07-09 13:21 教训)。 + # 改成手动管理 + shutdown(wait=False),主线程能立刻退出。 + pool = ThreadPoolExecutor(max_workers=max_workers) + futures = {pool.submit(self._sync_one, primary, p[0], p[1], p[2]): p[0] for p in plans} + try: + for i, future in enumerate(as_completed(futures, timeout=FETCH_HARD_TIMEOUT), 1): c6 = futures[future] try: - res = future.result() + res = future.result(timeout=0.1) # 已被 as_completed 释放 if res["status"] == "ok": ok_cnt += 1 rows_total += res["rows"] @@ -142,6 +156,9 @@ class SyncKline5Min(SyncTask): fail_cnt += 1 if res.get("error"): logger.warning(f"[5min {c6}] {res['error']}") + except FuturesTimeoutError: + fail_cnt += 1 + logger.warning(f"[5min {c6}] 内部 race timeout, 记 fail") except Exception as e: fail_cnt += 1 logger.warning(f"[5min {c6}] {e}") @@ -150,6 +167,64 @@ class SyncKline5Min(SyncTask): message=f"5min 进度 {i}/{len(plans)} OK:{ok_cnt} FAIL:{fail_cnt}", current=i, total=len(plans), current_step=c6, ) + except FuturesTimeoutError: + stuck = [c6 for _, c6 in futures.items() if not _.done()] + logger.error( + "[5min] 所有 fetcher 卡死 (>%ss), %d 只股票未完成", + FETCH_HARD_TIMEOUT, len(stuck), + ) + for f, c6 in futures.items(): + try: + if not f.done() and hasattr(f, "cancel"): + f.cancel() + except Exception as e: + logger.warning(f"[5min] cancel future for {c6} 失败: {e}") + try: + pool.shutdown(wait=False) + except Exception as e: + logger.warning(f"[5min] pool.shutdown(wait=False) 失败: {e}") + + # ── 补偿：冷却后重入队列 ── + # mairui 偶发全 hang（中间件重启/网络抖动），等 30s 再试一次。 + # 如果还 hang 就放弃，下次调度增量重拉。 + if stuck: + cooldown = 30 + logger.warning(f"[5min] 冷却 {cooldown}s 后重试 {len(stuck)} 只…") + time.sleep(cooldown) + retry_ok = retry_fail = retry_rows = 0 + retry_pool = ThreadPoolExecutor(max_workers=max_workers) + plan_dict = {p[0]: (p[1], p[2]) for p in plans} + retry_futs = {} + for c6 in stuck: + if c6 in plan_dict: + s, e = plan_dict[c6] + # 重试直接用雪球 fallback（Mairui 已全 hang） + retry_futs[retry_pool.submit(self._fallback_xueqiu_5m_and_write, c6, s, e)] = c6 + if retry_futs: + for fut in as_completed(retry_futs, timeout=FETCH_HARD_TIMEOUT): + c6 = retry_futs[fut] + try: + res = fut.result(timeout=0.1) + if res["status"] == "ok": + retry_ok += 1 + retry_rows += res["rows"] + else: + retry_fail += 1 + except Exception: + retry_fail += 1 + try: + retry_pool.shutdown(wait=False) + except Exception: + pass + ok_cnt += retry_ok + fail_cnt -= retry_ok # 从 fail 挪到 ok + rows_total += retry_rows + logger.warning( + f"[5min] 重试结果: {retry_ok}成 {retry_fail}败 " + f"共{retry_rows}行, 救回 {retry_ok} 只" + ) + else: + pool.shutdown(wait=True) elapsed = round(time.time() - t0, 1) msg = ( f"5min K线 {ok_cnt}成 {fail_cnt}败 共{rows_total}行, " @@ -192,10 +267,16 @@ class SyncKline5Min(SyncTask): }) cur = w_end + timedelta(days=1) except Exception as e: - return {"status": "fail", "error": str(e)} + logger.warning(f"[5min {code6}] mairui 失败, 尝试雪球 5m fallback: {e}") + all_rows = self._fallback_xueqiu_5m(code6, start, end) if not all_rows: - return {"status": "fail", "error": "no data"} + # Mairui 无数据, 尝试雪球 fallback + logger.info(f"[5min {code6}] mairui 无数据, 尝试雪球 fallback") + all_rows = self._fallback_xueqiu_5m(code6, start, end) + + if not all_rows: + return {"status": "fail", "error": "no data (mairui + xueqiu fallback)"} try: CHUNK = 5000 for i in range(0, len(all_rows), CHUNK): @@ -203,3 +284,77 @@ class SyncKline5Min(SyncTask): except Exception as e: return {"status": "fail", "error": f"db write: {e}"} return {"status": "ok", "rows": len(all_rows)} + + @staticmethod + def _fallback_xueqiu_5m_and_write(code6: str, start: datetime, end: datetime) -> dict: + """雪球 5m fallback + 直接写库，对齐 _sync_one 返回格式供 retry 逻辑使用。""" + rows = SyncKline5Min._fallback_xueqiu_5m(code6, start, end) + if not rows: + return {"status": "fail", "error": "no data via xueqiu"} + try: + CHUNK = 5000 + for i in range(0, len(rows), CHUNK): + db_ops.upsert_kline_5min(rows[i: i + CHUNK]) + except Exception as e: + return {"status": "fail", "error": f"db write: {e}"} + return {"status": "ok", "rows": len(rows)} + + @staticmethod + def _fallback_xueqiu_5m(code6: str, start: datetime, end: datetime) -> list[dict]: + """通过雪球 5m K线 fallback 拉取数据，对齐 _sync_one 返回格式。""" + import os + from datetime import datetime as dt_mod + from app.core.datasource.utils import code6_to_xueqiu, to_hermes + + token_raw = os.environ.get("XUEQIU_TOKEN", "").strip() + if not token_raw: + # 从 .env 读 + try: + with open("/home/gao/Development/quant_home/market_sync/.env") as f: + for line in f: + if line.startswith("XUEQIU_TOKEN="): + token_raw = line.strip().split("=", 1)[1] + break + except Exception: + pass + if not token_raw: + logger.warning("[xueqiu_5m] 无 XUEQIU_TOKEN, 跳过 fallback") + return [] + + import pysnowball as ball + ball.set_token(token_raw) + + symbol = code6_to_xueqiu(code6) + # 计算需要多少根 5m bar (每天 48 根, 加 buffer) + days_needed = max((end - start).days + 2, 1) + count = days_needed * 48 + + try: + data = ball.kline(symbol, "5m", min(count, 5000)) + except Exception as e: + logger.warning(f"[xueqiu_5m {code6}] 请求失败: {e}") + return [] + + items = data.get("data", {}).get("item", []) + if not items: + return [] + + columns = data["data"]["column"] # [timestamp, volume, open, high, low, close, ...] + idx = {c: i for i, c in enumerate(columns)} + hermes = to_hermes(code6) + rows = [] + for item in items: + ts = item[idx["timestamp"]] / 1000 + bar_time = dt_mod.fromtimestamp(ts).strftime("%Y-%m-%d %H:%M:%S") + rows.append({ + "stock_code": hermes, + "bar_time": bar_time, + "open": float(item[idx["open"]]), + "high": float(item[idx["high"]]), + "low": float(item[idx["low"]]), + "close": float(item[idx["close"]]), + "volume": float(item[idx["volume"]]), + "amount": 0.0, + "turnover_rate": 0.0, + }) + return rows diff --git a/app/tasks/task_kline_daily.py b/app/tasks/task_kline_daily.py index 6d38ca7..3eec44a 100644 --- a/app/tasks/task_kline_daily.py +++ b/app/tasks/task_kline_daily.py @@ -19,6 +19,7 @@ import os import queue import threading import time +from concurrent.futures import ThreadPoolExecutor, as_completed, TimeoutError as FuturesTimeoutError from datetime import datetime, timedelta from typing import Any @@ -32,15 +33,19 @@ from app.core.utils.logging import get_logger logger = get_logger("sync.kline_daily") -# 主源优先级：mairui（5 RPS，免费 + 稳定 + 单接口含5min）→ 雪球 → Baostock → 新浪 -# 经验：sina 在单日窗口 (start=end=某天) 会返回空，雪球更可靠 -PRIMARY_PRIORITY = ["datasource_xueqiu", "datasource_mairui", "datasource_xinlang"] +# 源优先级：麦蕊 → 雪球 → 新浪 +# 2026-07-21 改: 麦蕊15:10已有数据,雪球要15:40后才齐,麦蕊为主源提速 +PRIMARY_PRIORITY = ["datasource_mairui", "datasource_xueqiu", "datasource_xinlang"] DEFAULT_START = "2018-01-01" # 写库批量：攒够 BATCH_SIZE 行就 executemany 一次 + commit BATCH_SIZE = 2000 # fetcher 并发：实测单线程最稳（5000 只 × 0.2s = 17 分钟，足够） # 雪球 token 限速 10 RPS，单线程 0.1s/只 已经打满。多 worker 会触发风控 hang DEFAULT_FETCH_WORKERS = 1 +# 2026-07-08 教训: 数据源层虽然加了 fetch timeout (20s), task 层也加硬上限兜底, +# 防止数据源 SDK 升级后又被绕过。60s 对应 xueqiu/mairui 的 15-20s 上限 + DB upsert +# / DataFrame 处理余量 + 7月9日 mairui 间歇慢请求的容忍窗口。 +FETCH_HARD_TIMEOUT = 180.0 class SyncKlineDaily(SyncTask): @@ -57,14 +62,15 @@ class SyncKlineDaily(SyncTask): max_workers: int = DEFAULT_FETCH_WORKERS, **kwargs, ) -> dict[str, Any]: - # 1. 选主源 - primary = self._pick_primary() - if primary is None: + # 1. 获取所有可用源（按优先级），支持 per-stock fallback + sources = self._get_available_sources() + if not sources: return {"status": "error", "message": "所有 K 线数据源均不可用"} + source_names = ", ".join(s.name for s in sources) self._progress( - message=f"使用 {primary.name} 同步日K线 (fetcher={max_workers}, batch={BATCH_SIZE})...", - current_step=primary.key, + message=f"使用 [{source_names}] 同步日K线 (fetcher={max_workers}, batch={BATCH_SIZE})...", + current_step=sources[0].key, ) # 2. 拉股票列表 @@ -108,7 +114,6 @@ class SyncKlineDaily(SyncTask): ) # 4. 并发 fetch + 攒 batch 写库（线程安全） - # mairui 等外部 API 无并发问题；DB 写串行加锁 from concurrent.futures import ThreadPoolExecutor, as_completed t0 = time.time() batch_lock = threading.Lock() @@ -116,6 +121,7 @@ class SyncKlineDaily(SyncTask): pending_codes: dict[str, str] = {} ok_lock = threading.Lock() ok_cnt = [0] + fallback_ok_cnt = [0] fail_cnt = [0] rows_cnt = [0] @@ -138,15 +144,7 @@ class SyncKlineDaily(SyncTask): except Exception as e: logger.warning(f"update synced_at 失败 {c6}: {e}") - def _fetch_one(code6: str, fetch_start: str) -> tuple[str, list[dict] | None, str | None]: - try: - df = primary.fetch_kline_daily(code6, fetch_start, _end) - except Exception as e: - return (code6, None, str(e)[:120]) - if df is None or df.empty: - return (code6, None, "no data") - # 写库用 hermes（带 SH/SZ 前缀）—— 2026-07-01 重构：历史上 kline_stock 用 - # 6位 code 与 stocks.code(SH600519) 格式不一致，跨表 JOIN 失败。现统一。 + def _rows_from_df(code6: str, df: pd.DataFrame) -> tuple[list[dict], str]: hermes = to_hermes(code6) rows = [] for _, r in df.iterrows(): @@ -162,20 +160,56 @@ class SyncKlineDaily(SyncTask): }) latest = df["trade_date"].max() latest = latest.strftime("%Y-%m-%d") if hasattr(latest, "strftime") else str(latest)[:10] - return (code6, rows, latest) + return (rows, latest) + + def _fetch_one(code6: str, fetch_start: str) -> tuple[str, list[dict] | None, str | None, bool]: + errors = [] + for src in sources: + try: + df = src.fetch_kline_daily(code6, fetch_start, _end) + except Exception as e: + errors.append(f"{src.key}: {e}") + continue + if df is None or df.empty: + errors.append(f"{src.key}: no data") + continue + rows, latest = _rows_from_df(code6, df) + is_fallback = src is not sources[0] + return (code6, rows, latest, is_fallback) + return (code6, None, "; ".join(errors)[:200], False) total = len(jobs) - with ThreadPoolExecutor(max_workers=max_workers) as pool: - futures = {pool.submit(_fetch_one, c6, fs): c6 for c6, fs in jobs} - done_cnt = 0 - for future in as_completed(futures): + # 不用 `with ThreadPoolExecutor(...) as pool:` — 它的 __exit__ 默认 wait=True, + # 一旦某个 worker 卡在 xueqiu IO,主线程会在 as_completed 触发 timeout 后 + # 仍被 __exit__ 阻塞等 worker 退出 → 进程挂死(2026-07-09 13:21 教训)。 + # 改成手动管理 + shutdown(wait=False),主线程能立刻退出。 + pool = ThreadPoolExecutor(max_workers=max_workers) + futures = {pool.submit(_fetch_one, c6, fs): c6 for c6, fs in jobs} + done_cnt = 0 + try: + for future in as_completed(futures, timeout=FETCH_HARD_TIMEOUT): done_cnt += 1 - code6, rows, latest = future.result() + code6 = futures[future] + try: + code6_r, rows, latest, is_fallback = future.result(timeout=0.1) + except FuturesTimeoutError: + with ok_lock: + fail_cnt[0] += 1 + logger.warning(f"[kline {code6}] 内部 race timeout, 记 fail") + continue + except Exception as e: + with ok_lock: + fail_cnt[0] += 1 + logger.warning(f"[kline {code6}] 异常: {e}") + continue if rows is None: with ok_lock: fail_cnt[0] += 1 logger.warning(f"[kline {code6}] {latest}") continue + if is_fallback: + with ok_lock: + fallback_ok_cnt[0] += 1 with batch_lock: batch_rows.extend(rows) pending_codes[code6] = latest @@ -194,32 +228,121 @@ class SyncKlineDaily(SyncTask): message=f"进度 {done_cnt}/{total} OK:{ok_cnt[0]} FAIL:{fail_cnt[0]} {rate:.1f}只/秒", current=done_cnt, total=total, current_step=code6, ) + except FuturesTimeoutError: + # 30s 内没 future 完成 → 所有 worker 都被卡死(数据源层 timeout 兜底失败) + # 取消 pending futures,标 fail。shutdown(wait=False) 让主线程立刻返回, + # 不被卡住的 worker 拖死。 + # 2026-07-11 修复: 防御性 cancel,避免异常处理链里变量被遮蔽导致 + # AttributeError: 'str' object has no attribute 'cancel'。 + stuck = [(f, c6) for f, c6 in futures.items() if not f.done()] + logger.error( + "所有 fetcher 卡死 (>%ss), %d 只股票未完成, 标 fail 后退出主循环", + FETCH_HARD_TIMEOUT, len(stuck), + ) + with ok_lock: + fail_cnt[0] += len(stuck) + for f, c6 in stuck: + try: + if hasattr(f, "cancel"): + f.cancel() + except Exception as cancel_err: + logger.warning(f"[kline] cancel future for {c6} 失败: {cancel_err}") + try: + pool.shutdown(wait=False) + except Exception as shutdown_err: + logger.warning(f"[kline] pool.shutdown(wait=False) 失败: {shutdown_err}") + + # ── 补偿：冷却后重入队列 ── + # 雪球/mairui 偶发全 hang（服务端限流/网络抖动），等 30s 再试一次。 + # 单线程拉取场景下，卡住通常意味着当前这只股票请求 hang 死； + # 冷却后把未完成的股票重新排队重试，救回临时故障。 + if stuck: + cooldown = 30 + logger.warning(f"[kline] 冷却 {cooldown}s 后重试 {len(stuck)} 只…") + time.sleep(cooldown) + retry_ok = 0 + retry_fb = 0 + retry_rows = 0 + retry_pool = ThreadPoolExecutor(max_workers=max_workers) + jobs_dict = {c6: fs for c6, fs in jobs} + retry_futs = {} + for _, c6 in stuck: + if c6 in jobs_dict: + retry_futs[retry_pool.submit(_fetch_one, c6, jobs_dict[c6])] = c6 + if retry_futs: + try: + for fut in as_completed(retry_futs, timeout=FETCH_HARD_TIMEOUT): + c6 = retry_futs[fut] + try: + code6_r, rows, latest, is_fallback = fut.result(timeout=0.1) + except Exception: + logger.warning(f"[kline {c6}] 重试异常") + continue + if rows is None: + logger.warning(f"[kline {c6}] 重试仍失败: {latest}") + continue + with batch_lock: + batch_rows.extend(rows) + pending_codes[c6] = latest + cur_size = len(batch_rows) + with ok_lock: + retry_ok += 1 + retry_rows += len(rows) + if is_fallback: + retry_fb += 1 + if cur_size >= BATCH_SIZE: + _flush() + except FuturesTimeoutError: + logger.error( + "[kline] 重试仍卡死 (>%ss), %d 只股票放弃", + FETCH_HARD_TIMEOUT, sum(1 for f in retry_futs if not f.done()), + ) + try: + retry_pool.shutdown(wait=False) + except Exception: + pass + with ok_lock: + ok_cnt[0] += retry_ok + fail_cnt[0] -= retry_ok + fallback_ok_cnt[0] += retry_fb + rows_cnt[0] += retry_rows + logger.warning( + f"[kline] 重试结果: {retry_ok}成 " + f"({retry_fb}只备胎) 共{retry_rows}行, 救回 {retry_ok} 只" + ) + else: + pool.shutdown(wait=True) # 收尾 flush _flush() elapsed = round(time.time() - t0, 1) - msg = f"日K线 {ok_cnt[0]}成 {fail_cnt[0]}败 {skipped}跳 共{rows_cnt[0]}行, {elapsed}s" + fb = fallback_ok_cnt[0] + fb_info = f" ({fb}只备胎)" if fb else "" + msg = f"日K线 {ok_cnt[0]}成 {fail_cnt[0]}败 {skipped}跳 共{rows_cnt[0]}行{fb_info}, {elapsed}s" return { "status": "ok" if fail_cnt[0] == 0 else "warning", "message": msg, "ok": ok_cnt[0], "fail": fail_cnt[0], + "fallback_ok": fb, "skip": skipped, "rows": rows_cnt[0], "elapsed_sec": elapsed, - "primary_source": primary.key, + "primary_source": sources[0].key if sources else "", + "sources": [s.key for s in sources], } - def _pick_primary(self): + def _get_available_sources(self): from app.core.datasource.registry import is_source_ready, run_health_check, get_health_status if not get_health_status(): run_health_check() + available = [] for key in PRIMARY_PRIORITY: ok, _ = is_source_ready(key) if ok: - return ds_registry.get(key) - return None + available.append(ds_registry.get(key)) + return available def _sync_one(self, primary, code6: str, start: str, end: str) -> dict: """保留这个方法以兼容外部调用（已不用，但单只测试可能用到）""" diff --git a/app/tasks/task_mairui_indicators.py b/app/tasks/task_mairui_indicators.py new file mode 100644 index 0000000..decbfb5 --- /dev/null +++ b/app/tasks/task_mairui_indicators.py @@ -0,0 +1,261 @@ +"""同步任务：mairui 日 K 级别技术指标 MACD / KDJ / BOLL。 + +数据源：mairui /hsstock/history/{indicator}/{symbol}/d/n/{licence} + - macd → market_data.kline_stock_macd_daily (diff, dea, macd, ema12, ema26) + - kdj → market_data.kline_stock_kdj_daily (k, d, j) + - boll → market_data.kline_stock_boll_daily (upper, mid, lower ← mairui u/m/d) + +设计（对齐 task_kline_daily）： + - 全市场 A 股循环，每只票分别拉 3 个指标 + - 增量：读各指标表 MAX(trade_date)，只补新增区间（回看 5 天防漏） + - 并发 fetch + 攒 BATCH 批量 upsert，写库串行加锁 + - stock_code 落库用 hermes 格式（SH600519），与项目其他表一致 + +参考实现：pre_limit_seeker/production/sync_mairui_indicators.py（parquet 版）。 +""" +from __future__ import annotations + +import os +import threading +import time +from concurrent.futures import ThreadPoolExecutor, as_completed +from datetime import datetime, timedelta +from typing import Any + +from app.core.db import ops as db_ops +from app.core.datasource.base import registry as ds_registry +from app.core.datasource.utils import is_a_share_code, to_code6, to_hermes +from app.core.sync.base import SyncTask, _effective_sync_end +from app.core.utils.logging import get_logger + +logger = get_logger("sync.mairui_indicators") + +DEFAULT_START = "2018-01-01" +BATCH_SIZE = 2000 +# mairui 钻石档 100 RPS；5 workers × 10 RPS/worker = 50 RPS，留 buffer +DEFAULT_FETCH_WORKERS = 5 + +# 指标 → (mairui 源字段, 落库列名, upsert 函数) +INDICATORS: dict[str, dict[str, Any]] = { + "macd": { + "src_fields": ["diff", "dea", "macd", "ema12", "ema26"], + "db_cols": ["diff", "dea", "macd", "ema12", "ema26"], + "upsert": db_ops.upsert_kline_stock_macd_daily_rows, + }, + "kdj": { + "src_fields": ["k", "d", "j"], + "db_cols": ["k", "d", "j"], + "upsert": db_ops.upsert_kline_stock_kdj_daily_rows, + }, + "boll": { + # mairui: u=上轨, m=中轨, d=下轨 → 落库 upper/mid/lower + "src_fields": ["u", "m", "d"], + "db_cols": ["upper", "mid", "lower"], + "upsert": db_ops.upsert_kline_stock_boll_daily_rows, + }, +} + + +class SyncMairuiIndicators(SyncTask): + dataset_id = "mairui_indicators" + + def _run( + self, + *, + trigger_source: str = "manual", + indicators: list[str] | None = None, + codes: list[str] | None = None, + start: str | None = None, + end: str | None = None, + incremental: bool = True, + max_workers: int = DEFAULT_FETCH_WORKERS, + **kwargs, + ) -> dict[str, Any]: + targets = [i.lower() for i in (indicators or ["macd", "kdj", "boll"])] + for i in targets: + if i not in INDICATORS: + return {"status": "error", "message": f"未知指标 {i!r}，可选 {list(INDICATORS)}"} + + source = self._pick_source() + if source is None: + return {"status": "error", "message": "mairui 数据源不可用（licence 未配置？）"} + + # 股票列表（hermes 格式） + if codes: + hermes_codes = [to_hermes(c) for c in codes] + else: + hermes_codes = [ + to_hermes(c) for c in db_ops.iter_stock_codes(active_only=True) + if is_a_share_code(c) + ] + limit_raw = os.environ.get("MARKET_DATA_STOCK_LIMIT", "").strip() + if limit_raw.isdigit() and int(limit_raw) > 0: + hermes_codes = hermes_codes[: int(limit_raw)] + if not hermes_codes: + return {"status": "error", "message": "无股票代码（stocks 表为空？）"} + + _end = end or _effective_sync_end() + end_date_obj = datetime.strptime(_end, "%Y-%m-%d").date() + + t0 = time.time() + agg_ok = 0 + agg_fail = 0 + agg_rows = 0 + per_indicator: dict[str, dict[str, int]] = {} + + for ind in targets: + self._progress(message=f"[{ind}] 规划增量区间...", current_step=ind) + jobs = self._plan_jobs(ind, hermes_codes, start, incremental, end_date_obj) + total = len(jobs) + skipped = len(hermes_codes) - total + logger.info(f"[mairui_indicators] {ind}: to_pull={total}, skip={skipped}") + self._progress( + message=f"[{ind}] 开始同步 {total} 只 (跳过 {skipped} 只已最新)", + current=0, total=total, current_step=ind, + ) + + ok, fail, rows = self._sync_indicator( + source, ind, jobs, _end, max_workers, + ) + per_indicator[ind] = {"ok": ok, "fail": fail, "skip": skipped, "rows": rows} + agg_ok += ok + agg_fail += fail + agg_rows += rows + + elapsed = round(time.time() - t0, 1) + parts = ", ".join( + f"{k}({v['ok']}成/{v['fail']}败/{v['rows']}行)" + for k, v in per_indicator.items() + ) + msg = f"指标 {parts}；共 {agg_rows} 行, {elapsed}s" + logger.info(f"[mairui_indicators] {msg}") + return { + "status": "ok" if agg_fail == 0 else "warning", + "message": msg, + "ok": agg_ok, + "fail": agg_fail, + "rows": agg_rows, + "elapsed_sec": elapsed, + "per_indicator": per_indicator, + } + + def _plan_jobs( + self, indicator: str, hermes_codes: list[str], + start: str | None, incremental: bool, end_date_obj, + ) -> list[tuple[str, str]]: + """返回 [(hermes_code, fetch_start), ...]。""" + jobs: list[tuple[str, str]] = [] + if incremental and not start: + for hc in hermes_codes: + last = db_ops.get_indicator_max_date(indicator, hc) + if last is None: + fetch_start = DEFAULT_START + elif datetime.strptime(last, "%Y-%m-%d").date() >= end_date_obj: + continue + else: + fetch_start = ( + datetime.strptime(last, "%Y-%m-%d").date() - timedelta(days=5) + ).strftime("%Y-%m-%d") + jobs.append((hc, fetch_start)) + else: + fetch_start = start or DEFAULT_START + jobs = [(hc, fetch_start) for hc in hermes_codes] + return jobs + + def _sync_indicator( + self, source, indicator: str, jobs: list[tuple[str, str]], + end: str, max_workers: int, + ) -> tuple[int, int, int]: + cfg = INDICATORS[indicator] + upsert_fn = cfg["upsert"] + src_fields = cfg["src_fields"] + db_cols = cfg["db_cols"] + + batch_lock = threading.Lock() + batch_rows: list[dict] = [] + ok_lock = threading.Lock() + ok_cnt = [0] + fail_cnt = [0] + rows_cnt = [0] + + def _flush(): + with batch_lock: + if not batch_rows: + return + to_write = list(batch_rows) + batch_rows.clear() + try: + upsert_fn(to_write) + except Exception as e: + logger.error(f"[{indicator}] batch upsert 失败 ({len(to_write)} 行): {e}") + + def _fetch_one(hermes_code: str, fetch_start: str): + code6 = to_code6(hermes_code) + try: + df = source.fetch_indicator_daily(indicator, code6, fetch_start, end) + except Exception as e: + return (hermes_code, None, str(e)[:120]) + if df is None or df.empty: + return (hermes_code, None, "no data") + rows = [] + for _, r in df.iterrows(): + td = r["trade_date"] + # 2026-07-11 修复: mairui 可能返回整行 NaN/NaT(如 688 早期无指标), + # float(NaT) 会抛 TypeError。用 pandas.isna 统一跳过无效值。 + row = { + "stock_code": hermes_code, + "trade_date": td.strftime("%Y-%m-%d") if hasattr(td, "strftime") else str(td)[:10], + "source": "mairui", + } + for src_f, db_c in zip(src_fields, db_cols): + v = r.get(src_f) + if v is None or (hasattr(v, "isna") and v.isna()) or (isinstance(v, float) and v != v): + row[db_c] = None + else: + try: + row[db_c] = float(v) + except Exception: + row[db_c] = None + rows.append(row) + return (hermes_code, rows, None) + + total = len(jobs) + t0 = time.time() + with ThreadPoolExecutor(max_workers=max_workers) as pool: + futures = {pool.submit(_fetch_one, hc, fs): hc for hc, fs in jobs} + done = 0 + for fut in as_completed(futures): + done += 1 + hermes_code, rows, err = fut.result() + if rows is None: + with ok_lock: + fail_cnt[0] += 1 + continue + with batch_lock: + batch_rows.extend(rows) + cur = len(batch_rows) + with ok_lock: + ok_cnt[0] += 1 + rows_cnt[0] += len(rows) + if cur >= BATCH_SIZE: + _flush() + if done % 100 == 0 or done == total: + el = time.time() - t0 + rate = done / el if el > 0 else 0 + self._progress( + message=f"[{indicator}] {done}/{total} OK:{ok_cnt[0]} FAIL:{fail_cnt[0]} {rate:.1f}只/秒", + current=done, total=total, current_step=indicator, + ) + _flush() + return ok_cnt[0], fail_cnt[0], rows_cnt[0] + + def _pick_source(self): + from app.core.datasource.registry import ( + get_health_status, is_source_ready, run_health_check, + ) + if not get_health_status(): + run_health_check() + ok, _ = is_source_ready("datasource_mairui") + if ok: + return ds_registry.get("datasource_mairui") + return None diff --git a/app/tasks/task_market_regime.py b/app/tasks/task_market_regime.py index 4f3646b..05b6067 100644 --- a/app/tasks/task_market_regime.py +++ b/app/tasks/task_market_regime.py @@ -3,7 +3,6 @@ 基于本地 kline_stock 数据聚合：advancers / decliners / turnover / advance_ratio。""" from __future__ import annotations -from datetime import datetime from typing import Any import numpy as np @@ -12,7 +11,7 @@ from sqlalchemy import text from app.core.db import ops as db_ops from app.core.db.orm import engine as pg_engine -from app.core.sync.base import SyncTask +from app.core.sync.base import SyncTask, _effective_sync_end from app.core.utils.logging import get_logger logger = get_logger("sync.market_regime") @@ -34,7 +33,16 @@ class SyncMarketRegime(SyncTask): dataset_id = "market_regime" def _run(self, *, trigger_source: str = "manual", start: str = DEFAULT_START, **kwargs) -> dict[str, Any]: - end = datetime.now().strftime("%Y-%m-%d") + # 2026-07-12 修复: end 用 kline_stock 最大 trade_date，而不是 datetime.now() + # 避免节假日/周末跑时把非交易日当 end，导致 SQL 条件不精确 + try: + with pg_engine.connect() as conn: + max_date = conn.execute( + text("SELECT MAX(trade_date) FROM market_data.kline_stock") + ).scalar() + end = str(max_date) if max_date else _effective_sync_end() + except Exception: + end = _effective_sync_end() self._progress(message=f"构建 market_regime {start} ~ {end}...") # 1. 取所有非退市股票代码 diff --git a/app/tasks/task_sector_features.py b/app/tasks/task_sector_features.py deleted file mode 100644 index 4e681a6..0000000 --- a/app/tasks/task_sector_features.py +++ /dev/null @@ -1,183 +0,0 @@ -"""同步任务：行业聚合特征（衍生计算）。 - -输入：kline_stock（日 K 线）+ industry（股票→行业映射） -输出： - - sector_indices ：行业日线（trade_date × sector_name × close × sector_amplitude） - - sector_features_daily ：行业特征（+ sector_ret + ema10/20/200 + score） - -算法（参考 dashboard/api/services/sync/sector.py 重写，持久层改为 PG）： - 1) 每只股票日涨跌幅 pct_chg = (close/prev_close - 1) * 100 - 2) 按行业聚合 sector_ret = 该行业所有股票当日 pct_chg 的均值 - 3) 行业指数 close = 100 × ∏(1 + sector_ret/100) （基点 100，复合收益） - 4) 行业振幅 sector_amplitude = 该行业所有股票当日 (high-low)/close 的均值 - 5) EMA10/20/200 = close 的指数移动平均（adjust=False） - 6) score: 0/1/2 — close>ema200 + 1，ema10>ema20 + 1 - -无外部 API 调用，纯本地计算；增量逻辑以"全量重算"实现（数据量小 ~75K 行）。 -""" -from __future__ import annotations - -import time -from datetime import datetime, timedelta -from typing import Any - -import pandas as pd -from sqlalchemy import create_engine - -from app.core.config import settings -from app.core.db import ops as db_ops -from app.core.sync.base import SyncTask -from app.core.utils.logging import get_logger - -logger = get_logger("sync.sector_features") - - -# 起算点：DB 里 kline_stock 最早一天 往前 5 年，或固定一个默认起点 -# （不用写死 2021-04-24，让数据自己决定；下面自动算） -DEFAULT_START_YEARS_BACK = 5 - -# 输出基点（行业指数 close 从多少开始） -INDEX_BASE = 100.0 - - -class SyncSectorFeatures(SyncTask): - dataset_id = "sector_features" - - def _run( - self, - *, - trigger_source: str = "manual", - codes: list[str] | None = None, - max_workers: int = 1, # 本任务纯本地计算，单线程足够 - **kwargs, - ) -> dict[str, Any]: - t0 = time.time() - logger.info("[sector] 启动行业聚合特征计算") - - # ── 1) 拉 kline_stock（日线）只取需要的列 ── - # 走 SQLAlchemy engine + PG URL（pd.read_sql 一次性读 1.1kw 行 → pandas DataFrame） - engine = create_engine(settings.pg_sqlalchemy_url()) - df_kline = pd.read_sql( - 'SELECT stock_code, trade_date, open, high, low, "close", volume ' - 'FROM market_data.kline_stock ORDER BY stock_code, trade_date', - engine, - ) - if df_kline.empty: - return {"status": "error", "message": "kline_stock 为空"} - df_kline["stock_code"] = df_kline["stock_code"].astype(str).str.zfill(6) - df_kline["trade_date"] = pd.to_datetime(df_kline["trade_date"], errors="coerce") - logger.info(f"[sector] kline_stock: {len(df_kline):,} 行, {df_kline['stock_code'].nunique()} 只, " - f"{df_kline['trade_date'].min().date()} ~ {df_kline['trade_date'].max().date()}") - - # ── 2) 拉 industry（股票→行业映射）── - df_ind = pd.read_sql( - "SELECT code, industry_name FROM market_data.industry WHERE industry_name IS NOT NULL", - engine, - ) - if df_ind.empty: - return {"status": "error", "message": "industry 表为空"} - df_ind["code"] = df_ind["code"].astype(str).str.zfill(6) - df_ind = df_ind.drop_duplicates(subset=["code"], keep="first") - logger.info(f"[sector] industry 映射: {len(df_ind):,} 行, {df_ind['industry_name'].nunique()} 个行业") - - # ── 3) 算每只股票日涨跌幅 + 振幅 ── - df = df_kline.merge(df_ind, left_on="stock_code", right_on="code", how="inner") - df = df.sort_values(["stock_code", "trade_date"]) - df["prev_close"] = df.groupby("stock_code")["close"].shift(1) - df["stock_amplitude"] = (df["high"] - df["low"]) / df["close"] - df = df.dropna(subset=["prev_close", "industry_name"]) - df["pct_chg"] = (df["close"] / df["prev_close"] - 1.0) * 100.0 - logger.info(f"[sector] 合并后: {len(df):,} 行, 行业 {df['industry_name'].nunique()} 个") - - # ── 4) 按行业 + 日期聚合 ── - sector_daily = ( - df.groupby(["industry_name", "trade_date"], as_index=False).agg( - sector_ret=("pct_chg", "mean"), - sector_amplitude=("stock_amplitude", "mean"), - ) - .rename(columns={"industry_name": "sector_name"}) - .sort_values(["sector_name", "trade_date"]) - .reset_index(drop=True) - ) - logger.info(f"[sector] 行业日聚合: {len(sector_daily):,} 行") - - # ── 5) 算行业指数 close（基点 100，复合）── - sector_index = self._build_index(sector_daily) - - # ── 6) EMA + score ── - sector_index = self._calc_ema(sector_index) - sector_index["score"] = sector_index.apply( - lambda r: self._calc_score(r["close"], r["ema10"], r["ema20"], r["ema200"]), - axis=1, - ) - - # 整理列名 - out_cols = ["trade_date", "sector_name", "sector_ret", "sector_amplitude", - "close", "ema10", "ema20", "ema200", "score"] - sector_index = sector_index[out_cols] - sector_index["trade_date"] = sector_index["trade_date"].dt.strftime("%Y-%m-%d") - # NaN → None（让 MySQL 接受 NULL） - sector_index = sector_index.where(pd.notnull(sector_index), None) - - # ── 7) 写入两张表 ── - # 7a. sector_indices (4 列) - si_rows = sector_index[["trade_date", "sector_name", "close", "sector_amplitude"]].to_dict("records") - db_ops.replace_all_sector_indices(si_rows) - # 7b. sector_features_daily (9 列) - sf_rows = sector_index.to_dict("records") - db_ops.replace_all_sector_features(sf_rows) - - elapsed = round(time.time() - t0, 1) - msg = ( - f"行业聚合 {sector_index['sector_name'].nunique()} 个行业 " - f"× {sector_index['trade_date'].nunique()} 天, " - f"共 {len(sector_index):,} 行, {elapsed}s" - ) - logger.info(f"[sector] {msg}") - return { - "status": "ok", - "message": msg, - "sectors": int(sector_index["sector_name"].nunique()), - "days": int(sector_index["trade_date"].nunique()), - "rows": int(len(sector_index)), - "elapsed_sec": elapsed, - } - - @staticmethod - def _build_index(sector_daily: pd.DataFrame) -> pd.DataFrame: - """行业指数 close = INDEX_BASE × ∏(1 + sector_ret/100)""" - out = [] - for sector_name, group in sector_daily.groupby("sector_name", sort=False): - g = group.sort_values("trade_date").copy() - close_vals = [] - cur_base = INDEX_BASE - for ret in g["sector_ret"].to_numpy(dtype=float): - if pd.isna(ret): - close_vals.append(cur_base) - else: - cur_base = cur_base * (1 + float(ret) / 100.0) - close_vals.append(cur_base) - g["close"] = close_vals - out.append(g) - return pd.concat(out, ignore_index=True) - - @staticmethod - def _calc_ema(sector_index: pd.DataFrame) -> pd.DataFrame: - """每个行业分别算 EMA10/20/200""" - out = [] - for sector_name, group in sector_index.groupby("sector_name", sort=False): - g = group.sort_values("trade_date").copy() - g["ema10"] = g["close"].ewm(span=10, adjust=False).mean() - g["ema20"] = g["close"].ewm(span=20, adjust=False).mean() - g["ema200"] = g["close"].ewm(span=200, adjust=False).mean() - out.append(g) - return pd.concat(out, ignore_index=True) - - @staticmethod - def _calc_score(close, ema10, ema20, ema200) -> int: - score = 0 - if pd.notna(close) and pd.notna(ema200) and close > ema200: - score += 1 - if pd.notna(ema10) and pd.notna(ema20) and ema10 > ema20: - score += 1 - return int(score) diff --git a/app/tasks/task_share_snapshot.py b/app/tasks/task_share_snapshot.py index a2e1e13..f72e0b8 100644 --- a/app/tasks/task_share_snapshot.py +++ b/app/tasks/task_share_snapshot.py @@ -91,16 +91,25 @@ class SyncShareSnapshot(SyncTask): # 与 stocks.code 保持一致，方便 JOIN。 # 历史 bug：之前 share 表写的是裸 6 位 c6，导致 share JOIN stocks 失败。 # 一次性 UPDATE（2026-07-01）已把存量 6.3 万行加上前缀。 + # trade_date 必须用源数据返回的交易日(数据源层负责"最近已完成交易日" + # 兜底),不允许用 today 兜底 — 7月9日 早盘跑任务时 today=7月9日, + # 但股本快照实际对应 7月8日 收盘。早期代码用 today 会让 share 表里 + # 同一股本被重复写多行,统计/回溯出错。 + trade_date = r.get("trade_date") + if not trade_date: + fail_cnt += 1 + logger.warning(f"[share {c6}] 源数据未返回 trade_date,跳过(避免用运行日兜底)") + continue hermes = self._to_hermes(c6) db_ops.update_stock_share_snapshot( code=hermes, total_share=r["total_share"], float_share=r["float_share"], - trade_date=r.get("trade_date", today), + trade_date=trade_date, ) rows_to_share_table.append({ "stock_code": hermes, - "trade_date": r.get("trade_date", today), + "trade_date": trade_date, "total_share": r["total_share"], "float_share": r["float_share"], }) diff --git a/app/tasks/task_stocks_basic.py b/app/tasks/task_stocks_basic.py index 2e242e1..285607c 100644 --- a/app/tasks/task_stocks_basic.py +++ b/app/tasks/task_stocks_basic.py @@ -104,7 +104,6 @@ class SyncStocksBasic(SyncTask): exchange=old.get("exchange", ""), list_date=old.get("list_date", ""), listing_status="delisted", - industry=old.get("industry", ""), ) delisted += 1 @@ -197,11 +196,19 @@ class SyncStocksBasic(SyncTask): if r is None: fail += 1 else: + # trade_date 必须是源数据返回的交易日(数据源层用 effective_market_date + # 兜底为"最近已完成交易日"),绝不允许用 today 兜底 — 7月9日 早盘跑 + # 时 today=7月9日,但股本快照实际对应 7月8日 收盘。 + trade_date = r.get("trade_date") + if not trade_date: + fail += 1 + logger.warning(f"[share {c6}] 源数据未返回 trade_date,跳过(避免用运行日兜底)") + continue db_ops.update_stock_share_snapshot( code=self._to_hermes(c6), total_share=r["total_share"], float_share=r["float_share"], - trade_date=r.get("trade_date", today), + trade_date=trade_date, ) ok += 1 if done % 100 == 0: diff --git a/app/tasks/task_tick_trade.py b/app/tasks/task_tick_trade.py index d8dacec..01a425a 100644 --- a/app/tasks/task_tick_trade.py +++ b/app/tasks/task_tick_trade.py @@ -12,12 +12,15 @@ trade_date 可能比 wall-clock 早一天；重跑靠 PK + ON CONFLICT 幂等 4) 单只股票日均 5w-10w tick，CHUNK=2000 upsert 防 driver 撑爆 5) 数据是"当天 only"，无 start/end 窗口；不存增量概念 + 6) 2026-07-13 加固: 加上外层 FETCH_HARD_TIMEOUT,避免 mairui 挂起时 + as_completed 无 timeout 导致整个 task 永久卡住(systemd 2h 超时杀)。 + 改用手动 pool 管理 + shutdown(wait=False),与 kline_daily 对齐。 """ from __future__ import annotations import os import time -from concurrent.futures import ThreadPoolExecutor, as_completed +from concurrent.futures import ThreadPoolExecutor, as_completed, TimeoutError as FuturesTimeoutError from datetime import datetime from typing import Any @@ -36,6 +39,10 @@ MAIRUI_TICK_TRADE_PUBLISH_HOUR = 21 MAIRUI_TICK_TRADE_PUBLISH_MIN = 0 # 21:00 之后再跑（留 5 分钟缓冲，等 mairui 完整入库） TICK_TRADE_RUN_GATE = (MAIRUI_TICK_TRADE_PUBLISH_HOUR, MAIRUI_TICK_TRADE_PUBLISH_MIN + 5) +# 外层超时: mairui _fetch 有 20s timeout,但 as_completed 无 timeout 的话 +# 若所有 worker 同时被 mairui 挂起(罕见),task 会永久卡住等 systemd 2h 杀。 +# 120s 内无任何 future 完成 → 判全部失败退出。 +FETCH_HARD_TIMEOUT = 120.0 def _passes_publish_gate(now: Optional[datetime] = None, *, force: bool = False) -> tuple[bool, str]: @@ -108,15 +115,18 @@ class SyncTickTrade(SyncTask): ) # ── 执行 ── + # 不用 `with ThreadPoolExecutor` — 与外层 as_completed timeout 配合, + # 超时后 `with` 的 shutdown(wait=True) 会阻塞等卡死线程 → 进程挂死(同 kline_daily 教训)。 t0 = time.time() ok_cnt = fail_cnt = 0 rows_total = 0 - with ThreadPoolExecutor(max_workers=max_workers) as pool: - futures = {pool.submit(self._sync_one, mr, c6): c6 for c6 in stock_codes} - for i, future in enumerate(as_completed(futures), 1): + pool = ThreadPoolExecutor(max_workers=max_workers) + futures = {pool.submit(self._sync_one, mr, c6): c6 for c6 in stock_codes} + try: + for i, future in enumerate(as_completed(futures, timeout=FETCH_HARD_TIMEOUT), 1): c6 = futures[future] try: - res = future.result() + res = future.result(timeout=0.1) if res["status"] == "ok": ok_cnt += 1 rows_total += res["rows"] @@ -124,6 +134,9 @@ class SyncTickTrade(SyncTask): fail_cnt += 1 if res.get("error"): logger.warning(f"[tick_trade {c6}] {res['error']}") + except FuturesTimeoutError: + fail_cnt += 1 + logger.warning(f"[tick_trade {c6}] 内部 race timeout, 记 fail") except Exception as e: fail_cnt += 1 logger.warning(f"[tick_trade {c6}] {e}") @@ -132,6 +145,25 @@ class SyncTickTrade(SyncTask): message=f"tick_trade 进度 {i}/{total} OK:{ok_cnt} FAIL:{fail_cnt} 行:{rows_total}", current=i, total=total, current_step=c6, ) + except FuturesTimeoutError: + stuck = [(f, c6) for f, c6 in futures.items() if not f.done()] + logger.error( + "[tick_trade] 所有 fetcher 卡死 (>%ss), %d 只股票未完成, 标 fail 后退出", + FETCH_HARD_TIMEOUT, len(stuck), + ) + fail_cnt += len(stuck) + for f, c6 in stuck: + try: + if hasattr(f, "cancel"): + f.cancel() + except Exception as e: + logger.warning(f"[tick_trade] cancel future for {c6} 失败: {e}") + try: + pool.shutdown(wait=False) + except Exception as e: + logger.warning(f"[tick_trade] pool.shutdown(wait=False) 失败: {e}") + else: + pool.shutdown(wait=True) elapsed = round(time.time() - t0, 1) msg = f"逐笔 {ok_cnt}成 {fail_cnt}败 共{rows_total}行, {elapsed}s" return { diff --git a/bin/backfill_share_history.py b/bin/backfill_share_history.py new file mode 100755 index 0000000..be9bed3 --- /dev/null +++ b/bin/backfill_share_history.py @@ -0,0 +1,409 @@ +#!/usr/bin/env python3 +"""历史股本回填脚本。 + +目标：把 2023-01-01 ~ 2026-07-15 期间每只 A 股的股本，按“每周一条” +写入 market_data.share 表。 + +数据源： + 1. akshare stock_share_change_cninfo（巨潮资讯-公司股本变动） + 覆盖 2023 及更早的历史股本变动记录，单位：万股。 + 2. 现有 market_data.share 快照（2026 年雪球 quote_detail 已写入） + 作为 2024 年底之后最新股本的补充。 + +策略： + - 每周五作为本周代表日（若周五休市则仍取周五，因为股本变动是事件日， + 不依赖当天是否交易；最后一周如果结束日不是周五，额外补一个结束日）。 + - 对每只股票，把历史股本变动 + 现有快照合并成一条时间线，按日期前向填充， + 得到每个采样日的 total_share / float_share。 + - 最终批量 upsert 到 share 表。 + - 默认使用巨潮“总股本 / 已流通股份”口径；加 --a-share-only 时只取 A 股部分 + （total_share = 人民币普通股，float_share = 人民币普通股 - 流通受限股份）， + 与雪球 quote_detail 的 float_shares 口径更一致。 + +用法： + .venv/bin/python bin/backfill_share_history.py + .venv/bin/python bin/backfill_share_history.py --dry-run --limit 50 + .venv/bin/python bin/backfill_share_history.py --max-workers 10 --start 2023-01-01 --end 2026-07-15 + .venv/bin/python bin/backfill_share_history.py --a-share-only +""" +from __future__ import annotations + +import argparse +import os +import sys +import time +from concurrent.futures import ThreadPoolExecutor, as_completed +from datetime import date, datetime, timedelta +from typing import Any, Optional + +import pandas as pd +from sqlalchemy import text + +# 项目根目录导入 +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from app.core.db import ops as db_ops +from app.core.db.orm import get_session +from app.core.datasource.utils import is_a_share_code, to_hermes +from app.core.utils.logging import get_logger + +logger = get_logger("backfill_share_history") + +# 默认回填区间 +DEFAULT_START = "2023-01-01" +DEFAULT_END = "2026-07-15" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="回填历史股本到 market_data.share") + parser.add_argument("--start", default=DEFAULT_START, help="开始日期 (YYYY-MM-DD)") + parser.add_argument("--end", default=DEFAULT_END, help="结束日期 (YYYY-MM-DD)") + parser.add_argument("--max-workers", type=int, default=5, help="并发 worker 数") + parser.add_argument("--limit", type=int, default=0, help="仅处理前 N 只股票（测试用）") + parser.add_argument( + "--codes", + default="", + help="指定股票代码，逗号分隔（如 000001,600519）；空则处理全市场", + ) + parser.add_argument("--dry-run", action="store_true", help="只统计不写入数据库") + parser.add_argument( + "--a-share-only", + action="store_true", + help="只取 A 股股本口径（去掉 H 股 / B 股）", + ) + parser.add_argument( + "--upsert-chunk", + type=int, + default=2000, + help="每次 upsert 行数", + ) + return parser.parse_args() + + +def load_stock_codes(limit: int = 0, codes_str: str = "") -> list[str]: + """返回 6 位代码列表（已过滤 A 股）。""" + if codes_str: + raw = [c.strip() for c in codes_str.split(",") if c.strip()] + return [c for c in raw if is_a_share_code(c)] + + codes = [ + c + for c in db_ops.iter_stock_codes(active_only=True) + if is_a_share_code(c) + ] + if limit > 0: + codes = codes[:limit] + return codes + + +def load_existing_share_snapshots(code6s: list[str]) -> dict[str, dict[str, float]]: + """读取每只股票的最新 share 快照（ trade_date -> total_share/float_share ）。 + + 返回：{code6: {trade_date_str: {"total_share": float, "float_share": float}}} + """ + if not code6s: + return {} + hermes_codes = [to_hermes(c) for c in code6s] + out: dict[str, dict[str, dict[str, float]]] = {c: {} for c in code6s} + + # 按 hermes 前缀分组查询，避免 SQL IN 子句过长 + batch_size = 500 + for i in range(0, len(hermes_codes), batch_size): + batch = hermes_codes[i : i + batch_size] + placeholders = ",".join([f"'{c}'" for c in batch]) + sql = f""" + SELECT stock_code, trade_date, total_share, float_share + FROM market_data.share + WHERE stock_code IN ({placeholders}) + AND trade_date >= '2023-01-01' + ORDER BY stock_code, trade_date + """ + with get_session() as s: + rows = s.execute(text(sql)).fetchall() + for r in rows: + hermes = r[0] + code6 = hermes[2:] if hermes[:2] in ("SH", "SZ", "BJ") else hermes + if code6 not in out: + out[code6] = {} + out[code6][r[1].strftime("%Y-%m-%d")] = { + "total_share": float(r[2] or 0), + "float_share": float(r[3] or 0), + } + return out + + +def fetch_cninfo_changes(code6: str, a_share_only: bool = False) -> pd.DataFrame: + """调用 akshare 巨潮接口，返回标准化后的股本变动 DataFrame。 + + 列：change_date, total_share, float_share（单位：亿股） + + Args: + a_share_only: True 时只取 A 股口径： + total_share = 人民币普通股（缺失时用 总股本 - H股 - B股） + float_share = 人民币普通股 - 流通受限股份 + """ + try: + import akshare as ak + + df = ak.stock_share_change_cninfo(symbol=code6) + except Exception as e: + logger.warning("[%s] 巨潮接口调用失败: %s", code6, e) + return pd.DataFrame() + + if df is None or df.empty: + return pd.DataFrame() + + # 只保留我们需要的列 + if a_share_only: + rename_map = { + "变动日期": "change_date", + "总股本": "total_share_all", + "人民币普通股": "a_share_total", + "流通受限股份": "restricted", + "境外上市外资股-H股": "h_share", + "境内上市外资股-B股": "b_share", + } + else: + rename_map = { + "变动日期": "change_date", + "总股本": "total_share", + "已流通股份": "float_share", + } + + missing = [k for k in rename_map if k not in df.columns] + if missing: + logger.warning("[%s] 巨潮返回缺少字段: %s", code6, missing) + return pd.DataFrame() + + df = df[list(rename_map.keys())].rename(columns=rename_map) + df["change_date"] = pd.to_datetime(df["change_date"], errors="coerce") + for col in rename_map.values(): + if col != "change_date": + df[col] = pd.to_numeric(df[col], errors="coerce") + + # 万股 -> 亿股 + for col in rename_map.values(): + if col != "change_date": + df[col] = df[col] / 10000.0 + + if a_share_only: + # A 股总股本：优先用“人民币普通股”，缺失时从总股本中扣除 H 股 / B 股 + a_total = df["a_share_total"].where( + df["a_share_total"].notna(), + df["total_share_all"] - df["h_share"].fillna(0) - df["b_share"].fillna(0), + ) + # A 股流通股本：A 股总股本 - 流通受限股份 + a_float = a_total - df["restricted"].fillna(0) + df["total_share"] = a_total.round(4) + df["float_share"] = a_float.round(4) + + df = df.dropna(subset=["change_date", "total_share", "float_share"]) + if df.empty: + return pd.DataFrame() + + # 剔除异常：A 股流通股本不能为负，也不能超过 A 股总股本 + df = df[df["float_share"] >= 0] + df = df[df["float_share"] <= df["total_share"] * 1.001] + if df.empty: + return pd.DataFrame() + + df = df[["change_date", "total_share", "float_share"]].sort_values("change_date").reset_index(drop=True) + return df + + +def build_weekly_snapshots( + code6: str, + cninfo_df: pd.DataFrame, + existing: dict[str, dict[str, float]], + sample_dates: list[date], +) -> list[dict[str, Any]]: + """合并历史变动 + 现有快照，生成该股票每周股本记录。""" + # 构建时间线：change_date -> 最新股本 + timeline: dict[str, dict[str, float]] = {} + + # 1) 巨潮历史变动 + for _, row in cninfo_df.iterrows(): + d = row["change_date"].strftime("%Y-%m-%d") + timeline[d] = { + "total_share": float(row["total_share"]), + "float_share": float(row["float_share"]), + } + + # 2) 现有 share 快照（仅用于补充巨潮未覆盖的 2025 年及之后最新日期， + # 避免旧口径数据覆盖 2024 及更早的历史回填） + if existing: + for d, v in existing.items(): + if d >= "2025-01-01": + timeline[d] = v + + if not timeline: + return [] + + # 按日期排序并做前向填充 + dates = sorted(timeline.keys()) + rows = [] + for sd in sample_dates: + sd_str = sd.strftime("%Y-%m-%d") + # 找到 <= sd 的最新一条 + latest = None + for d in dates: + if d > sd_str: + break + latest = d + if latest is None: + continue + v = timeline[latest] + if v["total_share"] <= 0 or v["float_share"] <= 0: + continue + rows.append( + { + "stock_code": to_hermes(code6), + "trade_date": sd_str, + "total_share": v["total_share"], + "float_share": v["float_share"], + } + ) + return rows + + +def generate_sample_dates(start: str, end: str) -> list[date]: + """生成每周五采样日，若结束日不是周五则额外包含结束日。""" + start_dt = datetime.strptime(start, "%Y-%m-%d").date() + end_dt = datetime.strptime(end, "%Y-%m-%d").date() + + # pandas 每周五 + fridays = pd.date_range(start=start, end=end, freq="W-FRI") + dates = [d.date() for d in fridays] + + # 若结束日不是周五，补一个结束日本身 + if end_dt not in dates: + dates.append(end_dt) + dates.sort() + + return dates + + +def process_one_stock( + code6: str, + existing_map: dict[str, dict[str, float]], + sample_dates: list[date], + a_share_only: bool = False, +) -> list[dict[str, Any]]: + cninfo_df = fetch_cninfo_changes(code6, a_share_only=a_share_only) + existing = existing_map.get(code6, {}) + return build_weekly_snapshots(code6, cninfo_df, existing, sample_dates) + + +def main() -> int: + args = parse_args() + + logger.info("开始回填历史股本: %s ~ %s", args.start, args.end) + logger.info( + "参数: workers=%s, dry_run=%s, limit=%s, a_share_only=%s", + args.max_workers, args.dry_run, args.limit, args.a_share_only, + ) + + # 1. 股票代码 + code6s = load_stock_codes(limit=args.limit, codes_str=args.codes) + if not code6s: + logger.error("无可用股票代码") + return 1 + logger.info("共 %s 只股票待处理", len(code6s)) + + # 2. 采样日期 + sample_dates = generate_sample_dates(args.start, args.end) + logger.info("采样日期: %s 个（%s ~ %s）", len(sample_dates), sample_dates[0], sample_dates[-1]) + + # 3. 预读现有 share 快照 + logger.info("预读现有 share 快照...") + t0 = time.time() + existing_map = load_existing_share_snapshots(code6s) + logger.info("预读完成，耗时 %.1fs", time.time() - t0) + + # 4. 多线程拉取 + 合并 + all_rows: list[dict[str, Any]] = [] + ok = fail = 0 + t0 = time.time() + with ThreadPoolExecutor(max_workers=args.max_workers) as pool: + futures = { + pool.submit( + process_one_stock, c, existing_map, sample_dates, args.a_share_only + ): c + for c in code6s + } + for i, future in enumerate(as_completed(futures), 1): + c6 = futures[future] + try: + rows = future.result() + except Exception as e: + fail += 1 + logger.warning("[%s] 处理异常: %s", c6, e) + continue + if rows: + all_rows.extend(rows) + ok += 1 + else: + fail += 1 + if i % 100 == 0 or i == len(code6s): + logger.info( + "进度 %s/%s, OK=%s, FAIL=%s, rows=%s", + i, + len(code6s), + ok, + fail, + len(all_rows), + ) + + elapsed = time.time() - t0 + logger.info( + "拉取完成: OK=%s, FAIL=%s, 总条数=%s, 耗时=%.1fs", + ok, + fail, + len(all_rows), + elapsed, + ) + + if not all_rows: + logger.warning("无数据可写入") + return 0 + + # 5. 写入 + if args.dry_run: + logger.info("DRY RUN: 本应写入 %s 行", len(all_rows)) + # 打印几只样本（首尾各 3 行，方便看后段是否被现有快照覆盖） + df = pd.DataFrame(all_rows) + sample_codes = df["stock_code"].unique()[:3] + for sc in sample_codes: + sub = df[df["stock_code"] == sc] + preview = pd.concat([sub.head(3), sub.tail(3)]) + logger.info("样本 %s:\n%s", sc, preview.to_string(index=False)) + return 0 + + # 5. 写入前先清理目标区间内非采样日的旧记录，保证“每周一条” + keep_dates = {d.strftime("%Y-%m-%d") for d in sample_dates} + keep_str = ",".join([f"'{d}'" for d in keep_dates]) + delete_sql = f""" + DELETE FROM market_data.share + WHERE trade_date BETWEEN '{args.start}' AND '{args.end}' + AND trade_date NOT IN ({keep_str}) + """ + logger.info("清理非采样日旧记录...") + with get_session() as s: + deleted = s.execute(text(delete_sql)).rowcount + s.commit() + logger.info("清理完成: 删除 %s 行", deleted) + + logger.info("开始写入 share 表，chunk=%s", args.upsert_chunk) + t0 = time.time() + total_written = 0 + for i in range(0, len(all_rows), args.upsert_chunk): + chunk = all_rows[i : i + args.upsert_chunk] + total_written += db_ops.upsert_share(chunk) + logger.info("已写入 %s/%s", total_written, len(all_rows)) + logger.info("写入完成: %s 行, 耗时 %.1fs", total_written, time.time() - t0) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bin/daily_sync_check.py b/bin/daily_sync_check.py index 6343c8f..3643690 100644 --- a/bin/daily_sync_check.py +++ b/bin/daily_sync_check.py @@ -29,6 +29,7 @@ from pathlib import Path from typing import Any, Optional import requests +import sqlalchemy as sa # ── 路径与日志 ───────────────────────────────────────────────────────────── @@ -42,21 +43,43 @@ SECRETS_FILE = PROJECT_ROOT / "config" / "daily_check_secrets.env" LOG = logging.getLogger("daily-sync-check") +# ── 运行模式 ─────────────────────────────────────────────────────────────── +def _runtime_mode() -> str: + """读取当前运行模式：docker | systemd。 + + 优先读环境变量 RUNTIME_MODE，未设置时尝试从 .env 读取，默认 docker。 + """ + mode = os.environ.get("RUNTIME_MODE", "") + if not mode: + env_file = PROJECT_ROOT / ".env" + if env_file.exists(): + try: + for line in env_file.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if line.startswith("RUNTIME_MODE="): + mode = line.split("=", 1)[1].strip().strip('"').strip("'") + break + except Exception: + pass + return (mode or "docker").lower() + + # ── 预期调度表 ───────────────────────────────────────────────────────────── # 与 [[market-data-overview]] 保持一致；同步触发点（HH:MM）按 Asia/Shanghai 解释。 # weekend_only=True 表示只在周六/周日期望运行（实际仅 stock_node 周六一次）。 SCHEDULE: dict[str, dict[str, Any]] = { "stock_basic": {"window_end": "09:30", "weekend_only": False}, - "industry_sector": {"window_end": "09:45", "weekend_only": False}, "kline_index": {"window_end": "15:35", "weekend_only": False}, "kline_daily": {"window_end": "15:45", "weekend_only": False}, "kline_5min": {"window_end": "16:05", "weekend_only": False}, "market_regime": {"window_end": "16:20", "weekend_only": False}, - "share_snapshot": {"window_end": "16:25", "weekend_only": False}, + "share_snapshot": {"window_end": "12:00", "weekend_only": True}, # 周六 + "moneyflow": {"window_end": "21:40", "weekend_only": False}, "longhubang": {"window_end": "22:05", "weekend_only": False}, "stock_node": {"window_end": "12:00", "weekend_only": True}, # 周六 "mairui_ma_daily": {"window_end": "17:00", "weekend_only": False}, # 本地派生 + "mairui_indicators": {"window_end": "17:10", "weekend_only": False}, # mairui MACD/KDJ/BOLL } @@ -245,25 +268,29 @@ def _classify_task( # ── systemd unit drift 检测 ──────────────────────────────────────────────── -SYSTEMD_REPO_DIR = PROJECT_ROOT / "bin" / "systemd" +SYSTEMD_REPO_DIR = PROJECT_ROOT / "deploy" / "systemd" / "units" +SYSTEMD_LEGACY_REPO_DIR = PROJECT_ROOT / "bin" / "systemd" SYSTEMD_ETC_DIR = Path("/etc/systemd/system") def _check_systemd_drift() -> dict[str, Any]: - """对比 bin/systemd/ 与 /etc/systemd/system/ 下的 market-sync* unit。 + """对比 deploy/systemd/units/ 与 /etc/systemd/system/ 下的 market-sync* unit。 + + 兼容旧路径 bin/systemd/：如果 deploy/systemd/units/ 不存在则回退。 返回： missing_in_etc: repo 有但 /etc 没装（drift 1：未部署） drifted: 两边都有但内容不一致（drift 2：部署的版本过期） orphan_in_etc: /etc 有但 repo 没有（drift 3：临时/僵尸 unit） """ + repo_dir = SYSTEMD_REPO_DIR if SYSTEMD_REPO_DIR.exists() else SYSTEMD_LEGACY_REPO_DIR missing_in_etc: list[str] = [] drifted: list[dict[str, str]] = [] orphan_in_etc: list[str] = [] repo_units: set[str] = set() - if SYSTEMD_REPO_DIR.exists(): - for f in SYSTEMD_REPO_DIR.iterdir(): + if repo_dir.exists(): + for f in repo_dir.iterdir(): if f.suffix in {".service", ".timer"} and f.name.startswith("market-sync"): repo_units.add(f.name) @@ -279,8 +306,12 @@ def _check_systemd_drift() -> dict[str, Any]: # drift 2: 两边都有但内容不一致 for name in sorted(repo_units & etc_units): - repo_path = SYSTEMD_REPO_DIR / name + repo_path = repo_dir / name etc_path = SYSTEMD_ETC_DIR / name + # 跳过被 mask 的 unit：/etc 下是 /dev/null 的符号链接，表示 systemd 故意禁用 + # 这是 deploy.sh 对 market-sync-worker.service 的标准操作，不应报 drift + if etc_path.is_symlink() and etc_path.resolve() == Path("/dev/null"): + continue try: if repo_path.read_bytes() != etc_path.read_bytes(): drifted.append({ @@ -308,6 +339,87 @@ def _check_systemd_drift() -> dict[str, Any]: } +# ── 数据一致性检查 ───────────────────────────────────────────────────────── + +def _check_data_consistency() -> dict[str, Any]: + """检查上游 kline_stock 与下游衍生表是否同步。 + + 当前覆盖： + - kline_stock_ma_daily 是否比 kline_stock 缺行/缺日期 + - market_regime_daily 是否比 kline_stock 缺最近交易日 + + 返回： + alerts: 可读告警列表 + details: 原始指标 + """ + from app.core.config import settings + + alerts: list[dict[str, str]] = [] + details: dict[str, Any] = {"checked": False} + + try: + engine = sa.create_engine(settings.pg_sqlalchemy_url()) + with engine.connect() as conn: + # 1) kline_stock vs kline_stock_ma_daily + kline_max = conn.execute( + sa.text("SELECT MAX(trade_date) FROM market_data.kline_stock") + ).scalar() + ma_max = conn.execute( + sa.text("SELECT MAX(trade_date) FROM market_data.kline_stock_ma_daily") + ).scalar() + missing_rows = conn.execute( + sa.text(""" + SELECT COUNT(*) FROM ( + SELECT stock_code, trade_date FROM market_data.kline_stock + EXCEPT + SELECT stock_code, trade_date FROM market_data.kline_stock_ma_daily + ) t + """) + ).scalar() or 0 + + details["kline_stock_max_date"] = str(kline_max) if kline_max else None + details["kline_stock_ma_daily_max_date"] = str(ma_max) if ma_max else None + details["kline_ma_missing_rows"] = missing_rows + + if kline_max and ma_max and ma_max < kline_max: + alerts.append({ + "dataset_id": "data_consistency/kline_ma_daily", + "status": "stale", + "reason": f"kline_stock_ma_daily 最新日期 {ma_max} 落后于 kline_stock {kline_max}", + }) + elif missing_rows and missing_rows > 0: + alerts.append({ + "dataset_id": "data_consistency/kline_ma_daily", + "status": "gap", + "reason": f"kline_stock 有 {missing_rows} 行未覆盖到 kline_stock_ma_daily", + }) + + # 2) kline_stock vs market_regime_daily（最近 3 个交易日） + regime_max = conn.execute( + sa.text("SELECT MAX(trade_date) FROM market_data.market_regime_daily") + ).scalar() + + details["market_regime_daily_max_date"] = str(regime_max) if regime_max else None + + if kline_max and regime_max and regime_max < kline_max: + alerts.append({ + "dataset_id": "data_consistency/market_regime", + "status": "stale", + "reason": f"market_regime_daily 最新日期 {regime_max} 落后于 kline_stock {kline_max}", + }) + + details["checked"] = True + except Exception as e: + details["error"] = str(e) + alerts.append({ + "dataset_id": "data_consistency/check_failed", + "status": "error", + "reason": f"数据一致性检查异常: {e}", + }) + + return {"alerts": alerts, "details": details} + + # ── 报告生成 ─────────────────────────────────────────────────────────────── def build_report(check_date: date) -> dict[str, Any]: """生成当日巡检报告。""" @@ -338,7 +450,7 @@ def build_report(check_date: date) -> dict[str, Any]: alerts.append({ "dataset_id": f"systemd/{u}", "status": "not_deployed", - "reason": f"unit {u} 在 bin/systemd/ 有但 /etc/systemd/system/ 没装", + "reason": f"unit {u} 在 deploy/systemd/units/ 有但 /etc/systemd/system/ 没装", }) if drift["drifted"]: for d in drift["drifted"]: @@ -355,6 +467,10 @@ def build_report(check_date: date) -> dict[str, Any]: "reason": f"unit {u} 在 /etc/systemd/system/ 但 repo 没维护", }) + # 数据一致性检查（上游回填后下游未跟进） + consistency = _check_data_consistency() + alerts.extend(consistency["alerts"]) + overall = "ok" if not alerts else ( "warning" if any(a["status"] in {"missed", "never_run", "not_deployed", "drifted"} for a in alerts) else "error" ) @@ -363,11 +479,13 @@ def build_report(check_date: date) -> dict[str, Any]: "schema_version": 2, "generated_at": now.strftime("%Y-%m-%d %H:%M:%S"), "check_date": check_date.isoformat(), + "runtime_mode": _runtime_mode(), "overall": overall, "counts": counts, "alerts": alerts, "tasks": classified, "systemd_drift": drift, + "data_consistency": consistency["details"], } @@ -383,7 +501,7 @@ def write_report(report: dict[str, Any]) -> Path: def print_summary(report: dict[str, Any]) -> None: """stdout 一眼看懂的汇总。""" print("=" * 78) - print(f"每日同步巡检 date={report['check_date']} overall={report['overall']}") + print(f"每日同步巡检 date={report['check_date']} runtime_mode={report.get('runtime_mode', 'unknown')} overall={report['overall']}") print("=" * 78) print(f"counts: {report['counts']}") if report["alerts"]: diff --git a/bin/deploy_systemd.sh b/bin/deploy_systemd.sh new file mode 120000 index 0000000..0b14086 --- /dev/null +++ b/bin/deploy_systemd.sh @@ -0,0 +1 @@ +deploy/systemd/deploy.sh \ No newline at end of file diff --git a/bin/market_sync_kline_5min_run.sh b/bin/market_sync_kline_5min_run.sh new file mode 100755 index 0000000..2e385be --- /dev/null +++ b/bin/market_sync_kline_5min_run.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# systemd 调度入口：单跑 kline_5min（评分核心依赖） +set -u + +PROJECT_ROOT="/home/gao/Development/quant_home/market_sync" +LOG_DIR="$PROJECT_ROOT/logs" +mkdir -p "$LOG_DIR" + +TS=$(date +%Y%m%d_%H%M%S) +LOG="$LOG_DIR/kline_5min_${TS}.log" + +echo "[market-sync-kline-5min] start ts=$TS log=$LOG" | tee -a "$LOG" +cd "$PROJECT_ROOT" +.venv/bin/python -m app.entrypoints.cli sync kline_5min 2>&1 | tee -a "$LOG" +EXIT_CODE=${PIPESTATUS[0]} +echo "[market-sync-kline-5min] done exit_code=$EXIT_CODE" | tee -a "$LOG" +exit $EXIT_CODE diff --git a/bin/market_sync_kline_daily_run.sh b/bin/market_sync_kline_daily_run.sh new file mode 100755 index 0000000..c330641 --- /dev/null +++ b/bin/market_sync_kline_daily_run.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# systemd 调度入口：单跑 kline_daily（评分核心依赖） +set -u + +PROJECT_ROOT="/home/gao/Development/quant_home/market_sync" +LOG_DIR="$PROJECT_ROOT/logs" +mkdir -p "$LOG_DIR" + +TS=$(date +%Y%m%d_%H%M%S) +LOG="$LOG_DIR/kline_daily_${TS}.log" + +echo "[market-sync-kline-daily] start ts=$TS log=$LOG" | tee -a "$LOG" +cd "$PROJECT_ROOT" +.venv/bin/python -m app.entrypoints.cli sync kline_daily 2>&1 | tee -a "$LOG" +EXIT_CODE=${PIPESTATUS[0]} +echo "[market-sync-kline-daily] done exit_code=$EXIT_CODE" | tee -a "$LOG" +exit $EXIT_CODE diff --git a/bin/market_sync_kline_index_run.sh b/bin/market_sync_kline_index_run.sh new file mode 100755 index 0000000..7c7a27f --- /dev/null +++ b/bin/market_sync_kline_index_run.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# systemd 调度入口：单跑 kline_index +set -u + +PROJECT_ROOT="/home/gao/Development/quant_home/market_sync" +LOG_DIR="$PROJECT_ROOT/logs" +mkdir -p "$LOG_DIR" + +TS=$(date +%Y%m%d_%H%M%S) +LOG="$LOG_DIR/kline_index_${TS}.log" + +echo "[market-sync-kline-index] start ts=$TS log=$LOG" | tee -a "$LOG" +cd "$PROJECT_ROOT" +.venv/bin/python -m app.entrypoints.cli sync kline_index 2>&1 | tee -a "$LOG" +EXIT_CODE=${PIPESTATUS[0]} +echo "[market-sync-kline-index] done exit_code=$EXIT_CODE" | tee -a "$LOG" +exit $EXIT_CODE diff --git a/bin/market_sync_mairui_indicators_run.sh b/bin/market_sync_mairui_indicators_run.sh new file mode 100755 index 0000000..2514b6d --- /dev/null +++ b/bin/market_sync_mairui_indicators_run.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# systemd 调度的入口（专跑 mairui_indicators） +# +# 设计： +# - 交易日 16:40 触发，从 mairui 拉 MACD/KDJ/BOLL 增量 +# - 数据量大（3 指标 × 全市场），单独跑避免拖垮 runall 链 +# - 单独日志到 logs/mairui_indicators_.log +# - exit code 透传 +set -u + +PROJECT_ROOT="/home/gao/Development/quant_home/market_sync" +LOG_DIR="$PROJECT_ROOT/logs" +mkdir -p "$LOG_DIR" + +TS=$(date +%Y%m%d_%H%M%S) +LOG="$LOG_DIR/mairui_indicators_${TS}.log" + +echo "[market-sync-mairui-indicators] start ts=$TS log=$LOG" | tee -a "$LOG" + +# ── 跑 mairui_indicators ── +cd "$PROJECT_ROOT" +.venv/bin/python -m app.entrypoints.cli sync mairui_indicators 2>&1 | tee -a "$LOG" +EXIT_CODE=${PIPESTATUS[0]} +echo "[market-sync-mairui-indicators] done exit_code=$EXIT_CODE" | tee -a "$LOG" +exit $EXIT_CODE diff --git a/bin/market_sync_mairui_ma_daily_run.sh b/bin/market_sync_mairui_ma_daily_run.sh new file mode 100755 index 0000000..9a1975d --- /dev/null +++ b/bin/market_sync_mairui_ma_daily_run.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# systemd 调度的入口（专跑 mairui_ma_daily） +# +# 设计： +# - 交易日 16:30 触发，基于 kline_stock 计算全市场 MA5/10/20/60 +# - 本地派生任务，无外部 API 依赖 +# - 单独日志到 logs/mairui_ma_daily_.log +# - exit code 透传 +set -u + +PROJECT_ROOT="/home/gao/Development/quant_home/market_sync" +LOG_DIR="$PROJECT_ROOT/logs" +mkdir -p "$LOG_DIR" + +TS=$(date +%Y%m%d_%H%M%S) +LOG="$LOG_DIR/mairui_ma_daily_${TS}.log" + +echo "[market-sync-mairui-ma-daily] start ts=$TS log=$LOG" | tee -a "$LOG" + +# ── 跑 mairui_ma_daily ── +cd "$PROJECT_ROOT" +.venv/bin/python -m app.entrypoints.cli sync mairui_ma_daily 2>&1 | tee -a "$LOG" +EXIT_CODE=${PIPESTATUS[0]} +echo "[market-sync-mairui-ma-daily] done exit_code=$EXIT_CODE" | tee -a "$LOG" +exit $EXIT_CODE diff --git a/bin/market_sync_market_regime_run.sh b/bin/market_sync_market_regime_run.sh new file mode 100755 index 0000000..6a30219 --- /dev/null +++ b/bin/market_sync_market_regime_run.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# systemd 调度入口：单跑 market_regime（本地计算，基于 kline_stock） +set -u + +PROJECT_ROOT="/home/gao/Development/quant_home/market_sync" +LOG_DIR="$PROJECT_ROOT/logs" +mkdir -p "$LOG_DIR" + +TS=$(date +%Y%m%d_%H%M%S) +LOG="$LOG_DIR/market_regime_${TS}.log" + +echo "[market-sync-market-regime] start ts=$TS log=$LOG" | tee -a "$LOG" +cd "$PROJECT_ROOT" +.venv/bin/python -m app.entrypoints.cli sync market_regime 2>&1 | tee -a "$LOG" +EXIT_CODE=${PIPESTATUS[0]} +echo "[market-sync-market-regime] done exit_code=$EXIT_CODE" | tee -a "$LOG" +exit $EXIT_CODE diff --git a/bin/market_sync_morning_run.sh b/bin/market_sync_morning_run.sh index b1f7e89..436f36c 100755 --- a/bin/market_sync_morning_run.sh +++ b/bin/market_sync_morning_run.sh @@ -1,8 +1,6 @@ #!/bin/bash -# 早盘前同步：stock_basic + industry_sector +# 早盘前同步：stock_basic（仅） # - stock_basic：雪球 quote_detail 拉全市场股本快照（~9min @ 5 RPS） -# - industry_sector：baostock 拉股票-行业映射（~16min @ 5 RPS） -# - 串行执行，industry_sector 依赖 stock_basic # - 与 runall_once.py 区别：不拉 kline_daily / kline_5min 等耗时长任务 set -u @@ -27,7 +25,7 @@ seed_sync_registry() from app.tasks import get_task results = [] -for tid in ['stock_basic', 'industry_sector']: +for tid in ['stock_basic']: print(f'[morning-run] >>> 开始 {tid}', flush=True) t0 = time.time() try: diff --git a/bin/market_sync_share_run.sh b/bin/market_sync_share_run.sh new file mode 100755 index 0000000..0451daf --- /dev/null +++ b/bin/market_sync_share_run.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# systemd 调度的入口（专跑股本快照） +# +# 设计： +# - 与 market_sync_stock_node_run.sh 同构,只跑 share_snapshot 一个 task +# - 每周六 11:30 触发（周末收盘后股本数据稳定，与 stock_node 同时段） +# - 单独日志到 logs/share_.log +# - exit code 透传 +set -u + +PROJECT_ROOT="/home/gao/Development/quant_home/market_sync" +LOG_DIR="$PROJECT_ROOT/logs" +mkdir -p "$LOG_DIR" + +TS=$(date +%Y%m%d_%H%M%S) +LOG="$LOG_DIR/share_${TS}.log" + +echo "[market-sync-share] start ts=$TS log=$LOG" | tee -a "$LOG" + +cd "$PROJECT_ROOT" +.venv/bin/python -m app.entrypoints.cli sync share_snapshot 2>&1 | tee -a "$LOG" +EXIT_CODE=${PIPESTATUS[0]} +echo "[market-sync-share] done exit_code=$EXIT_CODE" | tee -a "$LOG" +exit $EXIT_CODE diff --git a/bin/run_remaining.sh b/bin/run_remaining.sh index d96597c..d530f18 100755 --- a/bin/run_remaining.sh +++ b/bin/run_remaining.sh @@ -1,5 +1,6 @@ #!/bin/bash -# 跑剩余任务：tick_trade → moneyflow → share_snapshot → market_regime +# 跑剩余任务：tick_trade → moneyflow → market_regime +# share_snapshot 已改为每周单独 timer（market-sync-share.timer），不在此处触发 # 跳过 kline_5min（全量回填 6 年太慢，下次有空再补） # tick_trade 用 --force 跳过 21:00 门控（人工补跑场景） set -e @@ -10,7 +11,7 @@ mkdir -p logs # 先清理上次可能留下的卡死记录（recover_interrupted_syncs 启动时自动处理，但显式更稳） echo "=== 启动时间: $(date) ===" | tee logs/runall_remaining.log -for TASK in "tick_trade --force" moneyflow share_snapshot market_regime; do +for TASK in "tick_trade --force" moneyflow market_regime; do echo "" | tee -a logs/runall_remaining.log echo "=== [$TASK] 开始 $(date) ===" | tee -a logs/runall_remaining.log T0=$(date +%s) diff --git a/bin/runall_once.py b/bin/runall_once.py index 7144389..5aaa18a 100644 --- a/bin/runall_once.py +++ b/bin/runall_once.py @@ -9,13 +9,26 @@ 注：以下 task **不**在 runall 链中 —— 各自有独立的 systemd timer： - tick_trade : mairui 21:00 发布 → market-sync-tick.timer (21:05) - moneyflow : mairui 21:30 发布 → market-sync-moneyflow.timer (21:35) + +2026-07-08 教训: 之前 runall 是顺序串行,任一 task 永久卡住(kline_daily 在 +5200/5204 卡死 4h)→ 后面 5 个 task 全部没机会跑,runall 进程被 systemd 4h +超时杀, dataset_registry 卡在 running 状态无人清理。修复: + 1. 启动时调 recover_interrupted_dataset_registry() — 清上次被中断的卡死 + 2. 每个 task 用 multiprocessing.Process 跑 + 硬上限 PER_TASK_TIMEOUT_SEC, + 超时直接 kill 子进程,继续下一个 task (不卡整条 runall) + 3. 数据源/任务层也加了 timeout(xueqiu 20s + kline_daily as_completed 30s), + 这里是最后一道防线 """ from __future__ import annotations import json +import multiprocessing as mp +import os +import signal import sys import time from pathlib import Path +from typing import Any _PROJECT_ROOT = Path(__file__).resolve().parent.parent if str(_PROJECT_ROOT) not in sys.path: @@ -30,8 +43,19 @@ from app.core.sync.registry import seed_sync_registry build_default_registry() seed_sync_registry() +from app.core.db import ops as db_ops from app.tasks import get_task +# 启动时清理上一轮被中断的卡死状态(2026-07-08 教训) +# 注意: 这里清的是"上次 runall 被杀时的 running",如果当前 runall 自己卡死, +# 子进程被 kill 后 recover 仍要再调一次 — 见末尾的 finally。 +try: + n_recovered = db_ops.recover_interrupted_dataset_registry() + if n_recovered: + logger.warning("[runall] 启动时清掉 %d 个上轮卡死的 running 任务", n_recovered) +except Exception as e: + logger.warning("[runall] 启动清理失败(非致命): %s", e) + # 顺序执行，按 sort_order 走（便于排查 + 避免外部 API 限流） # 注：tick_trade / moneyflow 由独立 timer 跑（21:05 / 21:35） TASKS = [ @@ -39,48 +63,111 @@ TASKS = [ "kline_index", "kline_daily", "kline_5min", - "industry_sector", - "sector_features", - "share_snapshot", "market_regime", ] # 不跳任何 task — 全量跑 SKIP: set[str] = set() -results = {} -t_all = time.time() -for tid in TASKS: - if tid in SKIP: - results[tid] = {"status": "skipped", "message": "已 ok"} - logger.info(f"[runall] {tid} 跳过（已 ok）") - continue - logger.info(f"[runall] >>> 开始 {tid}") - t0 = time.time() +# 每个 task 的硬上限（秒）。覆盖数据源/任务层都失败的最坏情况: +# kline_daily: 35 min (正常 18 min, 留余量) +# kline_5min: 75 min (全量首次跑很久, 增量 ~30 min) +# share_snapshot: 15 min +# 其他小表: 10 min +PER_TASK_TIMEOUT_SEC = { + "kline_daily": 35 * 60, + "kline_5min": 75 * 60, + "share_snapshot": 15 * 60, + "market_regime": 5 * 60, + "stock_basic": 15 * 60, + "kline_index": 5 * 60, +} +DEFAULT_TASK_TIMEOUT = 20 * 60 + + +def _run_task_in_subprocess(tid: str, result_queue: mp.Queue) -> None: + """子进程入口: 跑 task.run() 并把 result 通过 queue 返回。 + + 单独进程是为了父进程能用 SIGKILL 干掉它(timeout 时)而不污染主 runall 状态。 + """ try: task = get_task(tid) - # 大表 task 限小批量股票，避免跑爆；其他 task 跑全量 kwargs = {"max_workers": 5} - if tid in {"kline_daily", "kline_5min", "moneyflow", "share_snapshot"}: - # 这些 task 支持 MARKET_DATA_STOCK_LIMIT 环境变量，但通过 cli 跑可 --codes 限 - # 测全量费时，先跑全量 - pass r = task.run(trigger_source="runall", **kwargs) - elapsed = round(time.time() - t0, 1) - r["elapsed_sec"] = elapsed - results[tid] = r - logger.info(f"[runall] <<< {tid} 完成 status={r.get('status')} elapsed={elapsed}s msg={r.get('message')}") + result_queue.put(r) except Exception as e: + result_queue.put({"status": "error", "message": f"{type(e).__name__}: {e}"}) + + +results: dict[str, Any] = {} +t_all = time.time() + +# 注意: 主执行块必须在 __name__ == '__main__' 内部，否则 multiprocessing spawn +# 子进程重新 import 本模块时会再次执行进程启动逻辑，导致 RuntimeError: +# "An attempt has been made to start a new process before the current process +# has finished its bootstrapping phase." (2026-07-13 15:30 runall 全部失败) +def _run_tasks(): + global results + for tid in TASKS: + if tid in SKIP: + results[tid] = {"status": "skipped", "message": "已 ok"} + logger.info(f"[runall] {tid} 跳过（已 ok）") + continue + timeout_sec = PER_TASK_TIMEOUT_SEC.get(tid, DEFAULT_TASK_TIMEOUT) + logger.info(f"[runall] >>> 开始 {tid} (hard timeout {timeout_sec}s)") + t0 = time.time() + # 子进程跑 task,父进程用 .join(timeout) 守门 + ctx = mp.get_context("spawn") + result_queue: mp.Queue = ctx.Queue() + proc = ctx.Process(target=_run_task_in_subprocess, args=(tid, result_queue), name=f"runall-{tid}") + proc.start() + proc.join(timeout=timeout_sec) elapsed = round(time.time() - t0, 1) - logger.exception(f"[runall] !!! {tid} 异常: {e}") - results[tid] = {"status": "error", "message": str(e), "elapsed_sec": elapsed} - # 每个 task 之间 sleep 5s 让健康监控跑 - time.sleep(5) + if proc.is_alive(): + logger.error( + f"[runall] !!! {tid} 超时 (>{timeout_sec}s, elapsed={elapsed}s), 强制 kill" + ) + proc.terminate() + proc.join(5) + if proc.is_alive(): + proc.kill() + proc.join(2) + results[tid] = { + "status": "error", + "message": f"runall timeout (>={timeout_sec}s), killed", + "elapsed_sec": elapsed, + } + try: + db_ops.recover_interrupted_dataset_registry() + except Exception as e: + logger.warning("[runall] 超时后清理 dataset_registry 失败: %s", e) + else: + try: + r = result_queue.get(timeout=5) + except Exception as e: + r = {"status": "error", "message": f"无法获取子进程结果: {e}"} + r["elapsed_sec"] = elapsed + results[tid] = r + logger.info( + f"[runall] <<< {tid} 完成 status={r.get('status')} elapsed={elapsed}s msg={r.get('message')}" + ) -elapsed_total = round(time.time() - t_all, 1) -summary = {"total_elapsed_sec": elapsed_total, "tasks": results} -out = _PROJECT_ROOT / "logs" / "runall_summary.json" -out.write_text(json.dumps(summary, ensure_ascii=False, indent=2)) -logger.info(f"[runall] 全部完成 total={elapsed_total}s summary={out}") -print(json.dumps(summary, ensure_ascii=False, indent=2)) \ No newline at end of file + time.sleep(5) + + +if __name__ == '__main__': + _run_tasks() + + # 全部完成清一次 stuck 状态 + try: + db_ops.recover_interrupted_dataset_registry() + except Exception as e: + logger.warning("[runall] 末尾清理失败(非致命): %s", e) + + elapsed_total = round(time.time() - t_all, 1) + summary: dict[str, Any] = {"total_elapsed_sec": elapsed_total, "tasks": results} + out = _PROJECT_ROOT / "logs" / "runall_summary.json" + out.write_text(json.dumps(summary, ensure_ascii=False, indent=2)) + logger.info(f"[runall] 全部完成 total={elapsed_total}s summary={out}") + print(json.dumps(summary, ensure_ascii=False, indent=2)) \ No newline at end of file diff --git a/bin/service_run.sh b/bin/service_run.sh index 0a3784a..d860cfd 100755 --- a/bin/service_run.sh +++ b/bin/service_run.sh @@ -10,6 +10,15 @@ set -eu +RUNTIME_MODE="${RUNTIME_MODE:-docker}" + +if [ "$RUNTIME_MODE" != "docker" ]; then + echo "[service] ❌ ERROR: service_run.sh 是 Docker 入口，只在 RUNTIME_MODE=docker 时可用。" >&2 + echo "[service] 当前 RUNTIME_MODE=$RUNTIME_MODE" >&2 + echo "[service] Linux 宿主机部署请使用 systemd timer/service，不要运行此脚本。" >&2 + exit 1 +fi + # 等 PG 就绪（避免 race：market_sync 比 postgres 早启） if [ -n "${PG_HOST:-}" ]; then echo "[service] wait for PG ${PG_HOST}:${PG_PORT:-5432}..." diff --git a/bin/systemd/market-sync-mairui-indicators.service b/bin/systemd/market-sync-mairui-indicators.service new file mode 100644 index 0000000..8762c0a --- /dev/null +++ b/bin/systemd/market-sync-mairui-indicators.service @@ -0,0 +1,32 @@ +[Unit] +Description=Market data MACD/KDJ/BOLL indicators sync (mairui, daily 16:40) +Documentation=file:///home/gao/Development/quant_home/market_sync/bin/market_sync_mairui_indicators_run.sh +After=network-online.target market-sync.service +Wants=network-online.target + +[Service] +Type=oneshot +WorkingDirectory=/home/gao/Development/quant_home/market_sync +User=gao +Group=gao + +# 入口脚本：单独跑 mairui_indicators 一个 task +# mairui 16:30 后发布指标数据 → 16:40 触发留 buffer +ExecStart=/home/gao/Development/quant_home/market_sync/bin/market_sync_mairui_indicators_run.sh + +# 硬上限 3h（5204 只 × 3 指标，正常 ~20min，全量时可能更久） +TimeoutStartSec=10800 + +# 不要 Restart=（oneshot 失败就让 OnFailure= 发通知，别自动重跑） +# Restart=no 是 oneshot 默认值 + +# 日志走 journald +StandardOutput=journal +StandardError=journal +SyslogIdentifier=market-sync-mairui-indicators + +# 环境 +Environment=PYTHONUNBUFFERED=1 + +[Install] +WantedBy=multi-user.target diff --git a/bin/systemd/market-sync-mairui-indicators.timer b/bin/systemd/market-sync-mairui-indicators.timer new file mode 100644 index 0000000..6210284 --- /dev/null +++ b/bin/systemd/market-sync-mairui-indicators.timer @@ -0,0 +1,14 @@ +[Unit] +Description=Schedule MACD/KDJ/BOLL indicators sync — Mon..Fri 16:40 Asia/Shanghai +# market-sync-mairui-indicators.service 是这个 timer 的执行单元 + +[Timer] +# mairui 16:30 后发布指标数据 → 16:40 触发留 buffer +OnCalendar=Mon..Fri 16:40:00 Asia/Shanghai +# 系统关机 / 错过执行时，下次开机补跑一次（避免漏数据） +Persistent=true +# 单位（service）的精确名称 +Unit=market-sync-mairui-indicators.service + +[Install] +WantedBy=timers.target diff --git a/bin/systemd/market-sync-mairui-ma-daily.service b/bin/systemd/market-sync-mairui-ma-daily.service new file mode 100644 index 0000000..4deef25 --- /dev/null +++ b/bin/systemd/market-sync-mairui-ma-daily.service @@ -0,0 +1,32 @@ +[Unit] +Description=Market data MA daily sync (local derivative, daily 16:30) +Documentation=file:///home/gao/Development/quant_home/market_sync/bin/market_sync_mairui_ma_daily_run.sh +After=network-online.target market-sync.service +Wants=network-online.target + +[Service] +Type=oneshot +WorkingDirectory=/home/gao/Development/quant_home/market_sync +User=gao +Group=gao + +# 入口脚本：单独跑 mairui_ma_daily 一个 task +# 本地派生任务，依赖 kline_stock 已更新 +ExecStart=/home/gao/Development/quant_home/market_sync/bin/market_sync_mairui_ma_daily_run.sh + +# 硬上限 2h（正常 ~30min，留 buffer） +TimeoutStartSec=7200 + +# 不要 Restart=（oneshot 失败就让 OnFailure= 发通知，别自动重跑） +# Restart=no 是 oneshot 默认值 + +# 日志走 journald +StandardOutput=journal +StandardError=journal +SyslogIdentifier=market-sync-mairui-ma-daily + +# 环境 +Environment=PYTHONUNBUFFERED=1 + +[Install] +WantedBy=multi-user.target diff --git a/bin/systemd/market-sync-mairui-ma-daily.timer b/bin/systemd/market-sync-mairui-ma-daily.timer new file mode 100644 index 0000000..7ba02d3 --- /dev/null +++ b/bin/systemd/market-sync-mairui-ma-daily.timer @@ -0,0 +1,14 @@ +[Unit] +Description=Schedule MA daily sync — Mon..Fri 16:30 Asia/Shanghai +# market-sync-mairui-ma-daily.service 是这个 timer 的执行单元 + +[Timer] +# kline_daily 15:40 / kline_index 15:30 跑完后，16:30 计算 MA +OnCalendar=Mon..Fri 16:30:00 Asia/Shanghai +# 系统关机 / 错过执行时，下次开机补跑一次（避免漏数据） +Persistent=true +# 单位（service）的精确名称 +Unit=market-sync-mairui-ma-daily.service + +[Install] +WantedBy=timers.target diff --git a/bin/systemd/market-sync-morning.service b/bin/systemd/market-sync-morning.service index 5c92fa2..eea8cc8 100644 --- a/bin/systemd/market-sync-morning.service +++ b/bin/systemd/market-sync-morning.service @@ -1,5 +1,5 @@ [Unit] -Description=Market sync morning pre-market — stock_basic + industry_sector +Description=Market sync morning pre-market — stock_basic Documentation=file:///home/gao/Development/quant_home/market_sync/bin/market_sync_morning_run.sh After=network-online.target Wants=network-online.target @@ -10,10 +10,10 @@ WorkingDirectory=/home/gao/Development/quant_home/market_sync User=gao Group=gao -# 早盘前只跑 stock_basic + industry_sector（避免和 15:30 runall 重复） +# 早盘前只跑 stock_basic（确保最新股票列表+股本快照） ExecStart=/home/gao/Development/quant_home/market_sync/bin/market_sync_morning_run.sh -# stock_basic ~9min + industry_sector ~16min + sleep 5s + buffer = 1h 够用 +# stock_basic ~9min + buffer = 1h 够用 TimeoutStartSec=3600 # 不要 Restart=（oneshot 失败就让 OnFailure= 发通知，别自动重跑） diff --git a/bin/systemd/market-sync-morning.timer b/bin/systemd/market-sync-morning.timer index 9a877a4..5449710 100644 --- a/bin/systemd/market-sync-morning.timer +++ b/bin/systemd/market-sync-morning.timer @@ -1,10 +1,10 @@ [Unit] Description=Schedule market sync morning — Mon..Fri 09:00 Asia/Shanghai # market-sync-morning.service 是这个 timer 的执行单元 -# 早盘前 09:00 触发：跑 stock_basic + industry_sector，给 09:30 开盘留 30min buffer +# 早盘前 09:00 触发：跑 stock_basic，给 09:30 开盘留 30min buffer [Timer] -# A 股开盘 09:30，09:00 跑 stock_basic (~9min) + industry_sector (~16min) = ~25min +# A 股开盘 09:30，09:00 跑 stock_basic (~9min) 即可 OnCalendar=Mon..Fri 09:00:00 Asia/Shanghai # 关机/错过时下次开机补跑 Persistent=true diff --git a/bin/systemd/market-sync-share.service b/bin/systemd/market-sync-share.service new file mode 100644 index 0000000..00defa8 --- /dev/null +++ b/bin/systemd/market-sync-share.service @@ -0,0 +1,31 @@ +[Unit] +Description=Market data 股本快照 sync (雪球源, weekly Sat 11:30) +Documentation=file:///home/gao/Development/quant_home/market_sync/bin/market_sync_share_run.sh +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +WorkingDirectory=/home/gao/Development/quant_home/market_sync +User=gao +Group=gao + +# 入口脚本：单独跑 share_snapshot 一个 task +ExecStart=/home/gao/Development/quant_home/market_sync/bin/market_sync_share_run.sh + +# 硬上限 30min（全市场 5200 只雪球 quote_detail，10 worker 约 3-8min，留 buffer） +TimeoutStartSec=1800 + +# 不要 Restart=（oneshot 失败就让 OnFailure= 发通知，别自动重跑） +# Restart=no 是 oneshot 默认值 + +# 日志走 journald（journalctl -u market-sync-share.service -f） +StandardOutput=journal +StandardError=journal +SyslogIdentifier=market-sync-share + +# 环境（不读 /etc/environment，只带这几个；.env 由 cli 内部 load_dotenv） +Environment=PYTHONUNBUFFERED=1 + +[Install] +WantedBy=multi-user.target diff --git a/bin/systemd/market-sync-share.timer b/bin/systemd/market-sync-share.timer new file mode 100644 index 0000000..a47034f --- /dev/null +++ b/bin/systemd/market-sync-share.timer @@ -0,0 +1,16 @@ +[Unit] +Description=Schedule share_snapshot sync — Sat 11:30 Asia/Shanghai +# market-sync-share.service 是这个 timer 的执行单元 +# 股本快照每周跑一次即可（雪球 quote_detail，周末收盘后数据稳定） + +[Timer] +# 周六 11:30（和 stock_node 同时段） +OnCalendar=Sat *-*-* 11:30:00 Asia/Shanghai +# 系统关机 / 错过执行时,下次开机补跑一次(避免漏数据) +Persistent=true +# 单位(service)的精确名称 +Unit=market-sync-share.service +# 不要 AccuracySec(默认 1min 漂移够用) + +[Install] +WantedBy=timers.target diff --git a/bin/systemd/market-sync-stock-node.service b/bin/systemd/market-sync-stock-node.service index 29789c5..bc0010d 100644 --- a/bin/systemd/market-sync-stock-node.service +++ b/bin/systemd/market-sync-stock-node.service @@ -1,7 +1,7 @@ [Unit] Description=Market data 股票-节点映射 sync (mairui /hszg, weekly Sat 11:30) Documentation=file:///home/gao/Development/quant_home/market_sync/bin/market_sync_stock_node_run.sh -After=network-online.target +After=network-online.target market-sync-morning.service Wants=network-online.target [Service] diff --git a/bin/systemd/market-sync-worker.service b/bin/systemd/market-sync-worker.service new file mode 100644 index 0000000..4192233 --- /dev/null +++ b/bin/systemd/market-sync-worker.service @@ -0,0 +1,25 @@ +[Unit] +Description=Market Data Sync Worker (PostgreSQL, scheduler + health monitor) +Documentation=file:///home/gao/Development/quant_home/market_sync/app/entrypoints/worker.py +# 依赖 network 和 PG +After=network-online.target postgresql.service +Wants=network-online.target + +[Service] +Type=simple +WorkingDirectory=/home/gao/Development/quant_home/market_sync +User=gao +Group=gao + +# 入口：启动进程内 scheduler + health monitor + recover 卡死任务 +# 注意：实际入口是 app/entrypoints/worker.py，不是 app/worker.py +# 2026-07-12 修复: 把 python -m app.worker 改为 python -m app.entrypoints.worker +ExecStart=/home/gao/Development/quant_home/market_sync/.venv/bin/python -m app.entrypoints.worker + +# 明确标记运行模式，让 worker.py 知道不要启动 scheduler +# （systemd 模式下同步由 timer 触发，避免双调度器重叠） +Environment=PYTHONUNBUFFERED=1 +Environment="RUNTIME_MODE=systemd" + +[Install] +WantedBy=multi-user.target diff --git a/bin/systemd/market-sync.service b/bin/systemd/market-sync.service index 718a400..097845c 100644 --- a/bin/systemd/market-sync.service +++ b/bin/systemd/market-sync.service @@ -13,10 +13,11 @@ Group=gao # 入口脚本（runall_once.py + structured_watch.sh 二合一） ExecStart=/home/gao/Development/quant_home/market_sync/bin/market_sync_run.sh -# 硬上限 4h（实际 ~80min + kline_5min 增量 70min, 总 ~2.5h, 留 buffer 给 retry / 慢任务） -# 历史 bug（2026-07-02）：2h 超时截断，share_snapshot / market_regime 没跑成 -# 详见 docs/works/2026-07-03-01-market-sync-timeout.md -TimeoutStartSec=14400 +# 硬上限 3h（实际 ~80min + kline_5min 增量 70min, 总 ~2.5h, 留 30min buffer） +# 2026-07-08 教训: 之前 4h 让 kline_daily 单点卡死拖 4h,runall 链全部受影响。 +# runall_once.py 已加 per-task multiprocessing timeout(单 task 最长 75min), +# 这里 3h 是 systemd 层兜底。 +TimeoutStartSec=10800 # 不要 Restart=（oneshot 失败就让 OnFailure= 发通知，别自动重跑—— # 重跑会撞外部 API 限流窗口，浪费 1.5h） diff --git a/bin/systemd_deploy.sh b/bin/systemd_deploy.sh deleted file mode 100755 index 9b3bc48..0000000 --- a/bin/systemd_deploy.sh +++ /dev/null @@ -1,185 +0,0 @@ -#!/bin/bash -# market_sync systemd 一键部署脚本 -# -# 用途：把 bin/systemd/*.service 和 *.timer 同步到 /etc/systemd/system/， -# 解决 work #04 检测到的 5 处 drift。 -# -# 运行：sudo bash bin/systemd_deploy.sh -# -# 设计原则： -# - 必须 sudo（写 /etc/systemd/system/ + daemon-reload） -# - 幂等：重复运行无副作用 -# - 显示 plan → 等待确认 → 执行 -# - 失败立即中止，不留半套状态 - -set -e - -PROJECT_ROOT="/home/gao/Development/quant_home/market_sync" -REPO_UNITS_DIR="$PROJECT_ROOT/bin/systemd" -ETC_UNITS_DIR="/etc/systemd/system" - -# ── 0. 前置检查 ───────────────────────────────────────────────────────────── -if [ "$(id -u)" -ne 0 ]; then - echo "❌ 必须 sudo 运行：sudo bash $0" - exit 1 -fi - -if [ ! -d "$REPO_UNITS_DIR" ]; then - echo "❌ repo unit 目录不存在: $REPO_UNITS_DIR" - exit 1 -fi - -# ── 1. 计算 plan ──────────────────────────────────────────────────────────── -declare -a TO_COPY=() # repo→etc 拷贝 -declare -a TO_REMOVE=() # etc 中孤儿（repo 已删） -declare -a TO_RESTART_SVC=() # service 文件变了需要 daemon-reload + restart - -# repo 中所有 market-sync* unit -mapfile -t REPO_UNITS < <(find "$REPO_UNITS_DIR" -maxdepth 1 -type f \( -name "*.service" -o -name "*.timer" \) -printf '%f\n' | sort) -# /etc 中所有 market-sync* unit -mapfile -t ETC_UNITS < <(find "$ETC_UNITS_DIR" -maxdepth 1 -type f \( -name "market-sync*.service" -o -name "market-sync*.timer" \) -printf '%f\n' | sort 2>/dev/null || true) - -# drift 1: repo 有 /etc 没装 -for u in "${REPO_UNITS[@]}"; do - if [ ! -f "$ETC_UNITS_DIR/$u" ]; then - TO_COPY+=("$u") - fi -done - -# drift 2: 两边都有但内容不一致 -for u in "${REPO_UNITS[@]}"; do - if [ -f "$ETC_UNITS_DIR/$u" ]; then - if ! diff -q "$REPO_UNITS_DIR/$u" "$ETC_UNITS_DIR/$u" > /dev/null 2>&1; then - TO_COPY+=("$u") - if [[ "$u" == *.service ]]; then - TO_RESTART_SVC+=("${u%.service}") - fi - fi - fi -done - -# drift 3: /etc 有 repo 没维护（孤儿） -for u in "${ETC_UNITS[@]}"; do - found=0 - for r in "${REPO_UNITS[@]}"; do - if [ "$r" = "$u" ]; then found=1; break; fi - done - if [ $found -eq 0 ]; then - TO_REMOVE+=("$u") - fi -done - -# ── 2. 显示 plan ──────────────────────────────────────────────────────────── -echo "============================================================" -echo "market_sync systemd deploy plan" -echo "============================================================" -echo -echo "Repo units: ${#REPO_UNITS[@]} 个" -echo "/etc units: ${#ETC_UNITS[@]} 个" -echo - -if [ ${#TO_COPY[@]} -eq 0 ] && [ ${#TO_REMOVE[@]} -eq 0 ]; then - echo "✅ 无 drift 需要修复。无需执行。" - exit 0 -fi - -if [ ${#TO_COPY[@]} -gt 0 ]; then - echo "📋 将要拷贝 ($(echo "${TO_COPY[@]}" | tr ' ' '\n' | wc -l) 个):" - for u in "${TO_COPY[@]}"; do - echo " cp bin/systemd/$u → /etc/systemd/system/$u" - done - echo -fi - -if [ ${#TO_RESTART_SVC[@]} -gt 0 ]; then - echo "🔄 将要重启 service ($(echo "${TO_RESTART_SVC[@]}" | tr ' ' '\n' | wc -l) 个):" - for s in "${TO_RESTART_SVC[@]}"; do - echo " systemctl reset-failed $s.service" - done - echo -fi - -if [ ${#TO_REMOVE[@]} -gt 0 ]; then - echo "🗑️ 将要删除 etc 中的孤儿 ($(echo "${TO_REMOVE[@]}" | tr ' ' '\n' | wc -l) 个):" - for u in "${TO_REMOVE[@]}"; do - echo " rm /etc/systemd/system/$u" - done - echo -fi - -# ── 3. 等待确认 ──────────────────────────────────────────────────────────── -read -p "确认执行？[y/N] " -n 1 -r -echo -if [[ ! $REPLY =~ ^[Yy]$ ]]; then - echo "已取消。" - exit 0 -fi - -# ── 4. 执行 ───────────────────────────────────────────────────────────────── -echo -echo "=== 执行 ===" - -# 4a. cp 漂移的 unit -for u in "${TO_COPY[@]}"; do - cp -v "$REPO_UNITS_DIR/$u" "$ETC_UNITS_DIR/$u" -done - -# 4b. 删孤儿 -for u in "${TO_REMOVE[@]}"; do - rm -v "$ETC_UNITS_DIR/$u" -done - -# 4c. reload systemd -echo -echo "=== systemctl daemon-reload ===" -systemctl daemon-reload - -# 4d. reset-failed + restart 修改过的 service -for s in "${TO_RESTART_SVC[@]}"; do - if systemctl is-enabled "${s}.service" 2>/dev/null | grep -q enabled; then - systemctl reset-failed "${s}.service" || true - echo " reset-failed ${s}.service (没自动 restart — 由 timer 自然触发)" - fi -done - -# 4e. 启用新 timer（如果有） -for u in "${TO_COPY[@]}"; do - if [[ "$u" == *.timer ]]; then - timer="${u%.timer}" - if systemctl list-unit-files "${timer}.timer" 2>/dev/null | grep -q "${timer}.timer"; then - if ! systemctl is-enabled "${timer}.timer" 2>/dev/null | grep -q enabled; then - systemctl enable "${timer}.timer" - echo " enable ${timer}.timer" - fi - fi - fi -done - -# ── 5. 验证 ──────────────────────────────────────────────────────────────── -echo -echo "=== 验证 ===" -echo "Repo units: ${#REPO_UNITS[@]}" -echo "/etc units: $(find "$ETC_UNITS_DIR" -maxdepth 1 -type f -name 'market-sync*.service' -o -name 'market-sync*.timer' | wc -l)" - -drift_count=0 -for u in "${REPO_UNITS[@]}"; do - if [ -f "$ETC_UNITS_DIR/$u" ]; then - if ! diff -q "$REPO_UNITS_DIR/$u" "$ETC_UNITS_DIR/$u" > /dev/null 2>&1; then - echo " ❌ $u 仍有 drift" - drift_count=$((drift_count + 1)) - fi - else - echo " ❌ $u 仍未部署" - drift_count=$((drift_count + 1)) - fi -done - -echo -if [ $drift_count -eq 0 ]; then - echo "✅ 所有 unit 已对齐，0 drift" - echo - echo "下一步：跑 .venv/bin/python bin/daily_sync_check.py --report-only 验证" -else - echo "❌ 还有 $drift_count 处 drift，请检查" - exit 1 -fi \ No newline at end of file diff --git a/bin/systemd_deploy.sh b/bin/systemd_deploy.sh new file mode 120000 index 0000000..0b14086 --- /dev/null +++ b/bin/systemd_deploy.sh @@ -0,0 +1 @@ +deploy/systemd/deploy.sh \ No newline at end of file diff --git a/bin/task_watch.py b/bin/task_watch.py new file mode 100644 index 0000000..6dcaee2 --- /dev/null +++ b/bin/task_watch.py @@ -0,0 +1,218 @@ +"""自适应任务跟踪监视器。 + +用法: + .venv/bin/python bin/task_watch.py # 单次检查 + .venv/bin/python bin/task_watch.py --watch # 持续监视（自动调整检查间隔） + +设计： + 1. 查询 dataset_registry 中 status='running' 的任务 + 2. 从 config 表中解析历史运行耗时（解析 lastMessage 中的 elapsed_sec） + 3. 估算剩余时间 = 历史平均耗时 - 已用时间 + 4. 自适应下次检查间隔 = clamp(剩余 * 0.15, 15s, 300s) + 无历史数据时默认 60s + 5. 持续监视模式下，每次检查后动态调整下一次的 sleep 时间 +""" +from __future__ import annotations + +import argparse +import json +import re +import sys +import time +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +_PROJECT_ROOT = Path(__file__).resolve().parent.parent +if str(_PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(_PROJECT_ROOT)) + +from sqlalchemy import create_engine, text + +from app.core.config import settings +from app.core.utils.logging import setup_logging, get_logger + +setup_logging() +logger = get_logger("task_watch") + +# ── 历史耗时解析 ───────────────────────────────────────────────────── + +_ELAPSED_RE = re.compile(r"elapsed[=_ ]sec[= ]?(\d+\.?\d*)", re.IGNORECASE) +_ELAPSED_SUFFIX_RE = re.compile(r"(\d+\.?\d*)s") + + +def _parse_elapsed_sec(message: str) -> float | None: + for pattern in (_ELAPSED_RE,): + m = pattern.search(message) + if m: + return float(m.group(1)) + return None + + +# ── 历史耗时数据库（从 config 表 lastMessage 解析）─────────────────── + + +def _load_history(conn: Any) -> dict[str, list[float]]: + history: dict[str, list[float]] = {} + rows = conn.execute( + text(""" + SELECT key, value + FROM market_data.config + WHERE category = 'schedule' + """) + ).fetchall() + for key, value_json in rows: + try: + val = json.loads(value_json) + except (json.JSONDecodeError, TypeError): + continue + job = val.get("job") or key.removeprefix("schedule_") + msg = val.get("lastMessage") or "" + elapsed = _parse_elapsed_sec(msg) + if elapsed and elapsed > 0: + history.setdefault(job, []).append(elapsed) + return history + + +# ── 活跃任务查询 ────────────────────────────────────────────────────── + + +def _fetch_running_tasks(conn: Any) -> list[dict[str, Any]]: + rows = conn.execute( + text(""" + SELECT dataset_id, status, started_at, message + FROM market_data.dataset_registry + WHERE status = 'running' + ORDER BY started_at NULLS LAST + """) + ).fetchall() + tasks = [] + for row in rows: + tasks.append({ + "dataset_id": row[0], + "status": row[1], + "started_at": row[2], + "message": row[3] or "", + }) + return tasks + + +# ── 估算 ────────────────────────────────────────────────────────────── + + +def _estimate( + task: dict[str, Any], + history: dict[str, list[float]], +) -> tuple[float | None, float | None, float | None]: + """返回 (历史平均耗时秒, 已用秒, 预估剩余秒)。""" + tid = task["dataset_id"] + starts = task["started_at"] + if starts is None: + return None, None, None + + started_wall = starts.replace(tzinfo=None) + elapsed = (datetime.now() - started_wall).total_seconds() + elapsed = max(round(elapsed, 1), 0) + + durations = history.get(tid, []) + if not durations: + return None, elapsed, None + + avg_duration = sum(durations) / len(durations) + remaining = max(round(avg_duration - elapsed, 1), 0) + return round(avg_duration, 1), elapsed, remaining + + +# ── 自适应检查间隔 ──────────────────────────────────────────────────── + + +def _next_interval(remaining: float | None) -> float: + if remaining is None: + return 60.0 + interval = remaining * 0.15 + return max(15.0, min(interval, 300.0)) + + +# ── 输出 ────────────────────────────────────────────────────────────── + + +def _fmt_sec(sec: float | None) -> str: + if sec is None: + return "N/A" + if sec < 60: + return f"{sec:.0f}s" + if sec < 3600: + return f"{sec/60:.1f}min" + return f"{sec/3600:.1f}h" + + +def _print_report( + tasks: list[dict[str, Any]], + history: dict[str, list[float]], + next_interval: float, +) -> None: + now = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + header = f"── Task Watch [{now}] ──" + print(header) + if not tasks: + print(" 无运行中的任务") + print(f" 下次检查: {_fmt_sec(next_interval)}") + return + + lines = [] + for t in tasks: + avg, elapsed, remaining = _estimate(t, history) + durations = history.get(t["dataset_id"], []) + progress = "" + if avg and elapsed: + pct = min(elapsed / avg * 100, 99.9) + progress = f" {pct:.0f}%" + line = ( + f" {t['dataset_id']:20s}" + f" elapsed={_fmt_sec(elapsed):>8s}" + f" avg={_fmt_sec(avg):>8s}" + f" remain={_fmt_sec(remaining):>8s}" + f"{progress}" + f" (历史 {len(durations)} 次)" + ) + lines.append(line) + + for l in lines: + print(l) + print(f" 下次检查: {_fmt_sec(next_interval)}") + print("─" * len(header)) + + +# ── 主循环 ──────────────────────────────────────────────────────────── + + +def main() -> None: + parser = argparse.ArgumentParser(description="自适应任务跟踪监视器") + parser.add_argument("--watch", action="store_true", help="持续监视模式（自动调整检查间隔）") + args = parser.parse_args() + + engine = create_engine(settings.pg_sqlalchemy_url()) + + while True: + with engine.connect() as conn: + history = _load_history(conn) + tasks = _fetch_running_tasks(conn) + + interval = 60.0 + if tasks: + intervals = [] + for t in tasks: + _, _, remaining = _estimate(t, history) + intervals.append(_next_interval(remaining)) + interval = min(intervals) if intervals else 60.0 + + _print_report(tasks, history, interval) + + if not args.watch: + break + + time.sleep(interval) + + +if __name__ == "__main__": + main() diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile new file mode 100644 index 0000000..781acc3 --- /dev/null +++ b/deploy/docker/Dockerfile @@ -0,0 +1,65 @@ +# syntax=docker/dockerfile:1.6 +# ───────────────────────────────────────────────────────────── +# market_sync 数据同步服务 +# 单镜像同时跑：FastAPI dashboard + 进程内 scheduler（worker） +# 部署：docker compose up -d +# ───────────────────────────────────────────────────────────── + +# --- builder stage: 装依赖到 venv --- +FROM python:3.11-slim AS builder + +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PIP_NO_CACHE_DIR=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 + +# 仅装编译期需要的系统包（很多 wheel 已预编译，但 psycopg2 / cryptography 可能要） +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential gcc libpq-dev \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +# 先 copy requirements 单独一层（利用 Docker 缓存：依赖不变就不重装） +COPY requirements.txt . +RUN python -m venv /app/.venv \ + && /app/.venv/bin/pip install --upgrade pip \ + && /app/.venv/bin/pip install -r requirements.txt + + +# --- runtime stage: 极简基础镜像 + 仅复制 venv 与代码 --- +FROM python:3.11-slim AS runtime + +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PATH="/app/.venv/bin:${PATH}" \ + PYTHONPATH=/app \ + # service 模式默认监听 + API_HOST=0.0.0.0 \ + API_PORT=8100 \ + # Docker 模式：单容器内使用进程内 scheduler + RUNTIME_MODE=docker + +# runtime 只需要 psycopg2 的运行时库 + tzdata（pandas tz aware 需要） +RUN apt-get update && apt-get install -y --no-install-recommends \ + libpq5 tzdata curl \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +# 从 builder 复制 venv（已编译好的依赖） +COPY --from=builder /app/.venv /app/.venv + +# 复制应用代码 +COPY app/ ./app/ +COPY bin/ ./bin/ + +# 默认启动 service 入口（FastAPI + 进程内 scheduler） +# 也可覆盖为 cli / worker 单跑某个 task： +# docker compose run --rm market_sync python -m app.entrypoints.cli sync kline_daily +EXPOSE 8100 + +HEALTHCHECK --interval=60s --timeout=5s --start-period=30s --retries=3 \ + CMD curl -fsS http://localhost:8100/api/health || exit 1 + +ENTRYPOINT ["/app/bin/service_run.sh"] \ No newline at end of file diff --git a/deploy/docker/docker-compose.yml b/deploy/docker/docker-compose.yml new file mode 100644 index 0000000..1bca35e --- /dev/null +++ b/deploy/docker/docker-compose.yml @@ -0,0 +1,86 @@ +# docker-compose.yml — 一键起 market_sync 数据同步服务 +# +# 包含： +# - postgres : PostgreSQL 16（market_data schema 的主库） +# - market_sync : 数据同步 + dashboard + scheduler（单进程） +# +# 用法： +# docker compose up -d # 后台启动 +# docker compose logs -f market_sync # 看同步日志 +# docker compose exec market_sync \ +# python -m app.entrypoints.cli sync stock_node --type2 2,3 +# +version: "3.9" + +services: + postgres: + image: postgres:16-alpine + restart: unless-stopped + environment: + POSTGRES_USER: market_sync + POSTGRES_PASSWORD: market_sync + POSTGRES_DB: market_data + ports: + - "5432:5432" + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U market_sync -d market_data"] + interval: 10s + timeout: 3s + retries: 5 + + market_sync: + build: + context: . + dockerfile: Dockerfile + image: market_sync:latest + restart: unless-stopped + depends_on: + postgres: + condition: service_healthy + environment: + # 运行模式：Docker 模式使用进程内 scheduler + RUNTIME_MODE: docker + # PG 连接（指向同 compose 的 postgres 服务） + PG_HOST: postgres + PG_PORT: "5432" + PG_USER: market_sync + PG_PASSWORD: market_sync + PG_DB_NAME: market_data + DB_BACKEND: pg + # 调度 + SCHEDULER_TIMEZONE: Asia/Shanghai + SCHEDULER_TICK_SECONDS: "5" + SCHEDULER_AUTO_SEED: "true" + # API + API_HOST: 0.0.0.0 + API_PORT: "8100" + # 日志 + LOG_DIR: /app/logs + LOG_LEVEL: INFO + # 数据源开关 + DS_BAOSTOCK_ENABLED: "true" + DS_SINA_ENABLED: "true" + # 凭证（可放进 .env / secrets 文件，避免明文） + MAIRUI_LICENCE: ${MAIRUI_LICENCE:-} + XUEQIU_TOKEN: ${XUEQIU_TOKEN:-} + # 麦蕊限速（钻石 100 RPS 留 buffer；服务化后保守点） + MAIRUI_RPS_LIMIT: "10" + # 节假日 + TRADING_HOLIDAYS: ${TRADING_HOLIDAYS:-} + ports: + - "8100:8100" + volumes: + # 持久化日志（任务失败时方便排查） + - sync_logs:/app/logs + healthcheck: + test: ["CMD", "curl", "-fsS", "http://localhost:8100/api/health"] + interval: 60s + timeout: 5s + retries: 3 + start_period: 30s + +volumes: + pgdata: + sync_logs: \ No newline at end of file diff --git a/deploy/systemd/deploy.sh b/deploy/systemd/deploy.sh new file mode 100755 index 0000000..6bc222b --- /dev/null +++ b/deploy/systemd/deploy.sh @@ -0,0 +1,211 @@ +#!/bin/bash +# market_sync systemd 一键部署脚本 +# +# 用途：把 deploy/systemd/units/*.service 和 *.timer 同步到 /etc/systemd/system/， +# 并自动启用 timer、处理 worker 服务状态。 +# +# 运行：sudo bash deploy/systemd/deploy.sh +# sudo bash deploy/systemd/deploy.sh --yes # 跳过确认 +# +# 设计原则： +# - 必须 sudo（写 /etc/systemd/system/ + daemon-reload） +# - 幂等：重复运行无副作用 +# - 显示 plan → 等待确认 → 执行 +# - 失败立即中止，不留半套状态 + +set -e + +PROJECT_ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +REPO_UNITS_DIR="$PROJECT_ROOT/deploy/systemd/units" +ETC_UNITS_DIR="/etc/systemd/system" +SKIP_CONFIRM=false + +# ── 0. 前置检查 ───────────────────────────────────────────────────────────── +if [ "$(id -u)" -ne 0 ]; then + echo "❌ 必须 sudo 运行：sudo bash $0" + exit 1 +fi + +if [ "${1:-}" = "--yes" ]; then + SKIP_CONFIRM=true +fi + +if [ ! -d "$REPO_UNITS_DIR" ]; then + echo "❌ repo unit 目录不存在: $REPO_UNITS_DIR" + exit 1 +fi + +# ── 1. 计算 plan ──────────────────────────────────────────────────────────── +declare -a TO_COPY=() # repo→etc 拷贝 +declare -a TO_REMOVE=() # etc 中孤儿（repo 已删） +declare -a TO_RESTART_SVC=() # service 文件变了需要 daemon-reload + restart + +# repo 中所有 market-sync* unit +mapfile -t REPO_UNITS < <(find "$REPO_UNITS_DIR" -maxdepth 1 -type f \( -name "*.service" -o -name "*.timer" \) -printf '%f\n' | sort) +# /etc 中所有 market-sync* unit +mapfile -t ETC_UNITS < <(find "$ETC_UNITS_DIR" -maxdepth 1 -type f \( -name "market-sync*.service" -o -name "market-sync*.timer" \) -printf '%f\n' | sort 2>/dev/null || true) + +# drift 1: repo 有 /etc 没装 +for u in "${REPO_UNITS[@]}"; do + if [ ! -f "$ETC_UNITS_DIR/$u" ]; then + TO_COPY+=("$u") + fi +done + +# drift 2: 两边都有但内容不一致 +for u in "${REPO_UNITS[@]}"; do + if [ -f "$ETC_UNITS_DIR/$u" ]; then + if ! diff -q "$REPO_UNITS_DIR/$u" "$ETC_UNITS_DIR/$u" > /dev/null 2>&1; then + TO_COPY+=("$u") + if [[ "$u" == *.service ]]; then + TO_RESTART_SVC+=("${u%.service}") + fi + fi + fi +done + +# drift 3: /etc 有 repo 没维护（孤儿） +for u in "${ETC_UNITS[@]}"; do + found=0 + for r in "${REPO_UNITS[@]}"; do + if [ "$r" = "$u" ]; then found=1; break; fi + done + if [ $found -eq 0 ]; then + TO_REMOVE+=("$u") + fi +done + +# ── 2. 显示 plan ──────────────────────────────────────────────────────────── +echo "============================================================" +echo "market_sync systemd deploy plan" +echo "============================================================" +echo "Repo units dir: $REPO_UNITS_DIR" +echo +echo "Repo units: ${#REPO_UNITS[@]} 个" +echo "/etc units: ${#ETC_UNITS[@]} 个" +echo + +if [ ${#TO_COPY[@]} -eq 0 ] && [ ${#TO_REMOVE[@]} -eq 0 ]; then + echo "✅ 无 drift 需要修复。" +else + if [ ${#TO_COPY[@]} -gt 0 ]; then + echo "📋 将要拷贝 (${#TO_COPY[@]} 个):" + for u in "${TO_COPY[@]}"; do + echo " cp deploy/systemd/units/$u → /etc/systemd/system/$u" + done + echo + fi + + if [ ${#TO_RESTART_SVC[@]} -gt 0 ]; then + echo "🔄 将要处理 service (${#TO_RESTART_SVC[@]} 个):" + for s in "${TO_RESTART_SVC[@]}"; do + echo " systemctl reset-failed $s.service" + done + echo + fi + + if [ ${#TO_REMOVE[@]} -gt 0 ]; then + echo "🗑️ 将要删除 etc 中的孤儿 (${#TO_REMOVE[@]} 个):" + for u in "${TO_REMOVE[@]}"; do + echo " rm /etc/systemd/system/$u" + done + echo + fi +fi + +# ── 3. 等待确认 ──────────────────────────────────────────────────────────── +if [ "$SKIP_CONFIRM" = false ]; then + read -p "确认执行？[y/N] " -n 1 -r + echo + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + echo "已取消。" + exit 0 + fi +fi + +# ── 4. 执行 ───────────────────────────────────────────────────────────────── +echo +echo "=== 执行 ===" + +# 4a. cp 漂移的 unit +for u in "${TO_COPY[@]}"; do + cp -v "$REPO_UNITS_DIR/$u" "$ETC_UNITS_DIR/$u" +done + +# 4b. 删孤儿 +for u in "${TO_REMOVE[@]}"; do + rm -v "$ETC_UNITS_DIR/$u" +done + +# 4c. reload systemd +echo +echo "=== systemctl daemon-reload ===" +systemctl daemon-reload + +# 4d. 停用并 mask worker 服务（systemd 模式下不需要进程内 scheduler） +if systemctl is-enabled market-sync-worker.service >/dev/null 2>&1 || \ + systemctl is-active market-sync-worker.service >/dev/null 2>&1; then + echo "=== 关闭 market-sync-worker.service（systemd 模式下由 timer 触发任务）===" + systemctl stop market-sync-worker.service || true + systemctl disable market-sync-worker.service || true + systemctl mask market-sync-worker.service || true +fi + +# 4e. reset-failed + restart 修改过的 service +for s in "${TO_RESTART_SVC[@]}"; do + if systemctl is-enabled "${s}.service" 2>/dev/null | grep -q enabled; then + systemctl reset-failed "${s}.service" || true + echo " reset-failed ${s}.service (由 timer 自然触发)" + fi +done + +# 4f. 启用所有 timer（幂等） +echo +echo "=== 启用 timers ===" +for u in "${REPO_UNITS[@]}"; do + if [[ "$u" == *.timer ]]; then + timer="${u%.timer}" + if ! systemctl is-enabled "${timer}.timer" >/dev/null 2>&1; then + systemctl enable "${timer}.timer" + echo " enable ${timer}.timer" + else + echo " ${timer}.timer 已启用" + fi + fi +done + +# ── 5. 验证 ──────────────────────────────────────────────────────────────── +echo +echo "=== 验证 ===" +echo "Repo units: ${#REPO_UNITS[@]}" +echo "/etc units: $(find "$ETC_UNITS_DIR" -maxdepth 1 -type f \( -name 'market-sync*.service' -o -name 'market-sync*.timer' \) | wc -l)" + +drift_count=0 +for u in "${REPO_UNITS[@]}"; do + if [ -f "$ETC_UNITS_DIR/$u" ]; then + if ! diff -q "$REPO_UNITS_DIR/$u" "$ETC_UNITS_DIR/$u" > /dev/null 2>&1; then + echo " ❌ $u 仍有 drift" + drift_count=$((drift_count + 1)) + fi + else + echo " ❌ $u 仍未部署" + drift_count=$((drift_count + 1)) + fi +done + +# 验证 worker 已被 mask +if systemctl is-enabled market-sync-worker.service 2>/dev/null | grep -q masked; then + echo " ✅ market-sync-worker.service 已 mask（systemd 模式下不启用进程内 scheduler）" +else + echo " ⚠️ market-sync-worker.service 未被 mask，systemd 模式下可能双调度器重叠" +fi + +echo +if [ $drift_count -eq 0 ]; then + echo "✅ 所有 unit 已对齐，0 drift" + echo + echo "下一步：跑 .venv/bin/python bin/daily_sync_check.py --report-only 验证" +else + echo "❌ 还有 $drift_count 处 drift，请检查" + exit 1 +fi diff --git a/deploy/systemd/deploy_split.sh b/deploy/systemd/deploy_split.sh new file mode 100644 index 0000000..38025e8 --- /dev/null +++ b/deploy/systemd/deploy_split.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# ── 停止旧 market-sync ── +sudo systemctl stop market-sync.timer +sudo systemctl disable market-sync.timer +sudo systemctl stop market-sync.service +sudo systemctl disable market-sync.service + +# ── 安装新独立 timer+service (4组) ── +cd /home/gao/Development/quant_home/market_sync/deploy/systemd/units + +for f in market-sync-kline-index market-sync-kline-daily market-sync-kline-5min market-sync-market-regime; do + sudo cp ${f}.service /etc/systemd/system/ + sudo cp ${f}.timer /etc/systemd/system/ +done + +# ── 更新 morning service：去掉已废弃的 industry_sector ── +# （只用 morning 跑 stock_basic，不再跑 industry_sector） +# 修改 morning run 脚本只跑 stock_basic +sudo systemctl daemon-reload + +# ── 启动新 timer ── +for f in market-sync-kline-index market-sync-kline-daily market-sync-kline-5min market-sync-market-regime; do + sudo systemctl enable ${f}.timer + sudo systemctl start ${f}.timer +done + +echo "=== 完成 ===" +systemctl list-timers --all 2>/dev/null | grep market-sync diff --git a/deploy/systemd/scripts/deploy_systemd.sh b/deploy/systemd/scripts/deploy_systemd.sh new file mode 100755 index 0000000..251e9c7 --- /dev/null +++ b/deploy/systemd/scripts/deploy_systemd.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# 部署 market-sync systemd units 到 /etc/systemd/system/ +# 2026-07-09 新增: market-sync-worker.service(长驻进程, 调度 schedule_* 任务) +# +# 用法: sudo bash bin/deploy_systemd.sh +# +# 跑这个脚本会: +# 1. 复制 bin/systemd/market-sync*.{service,timer} 到 /etc/systemd/system/ +# 2. systemctl daemon-reload +# 3. enable + start market-sync-worker.service +# 4. 不动已存在的 .timer (它们都已 enable, 重复 enable 无副作用) +set -euo pipefail + +REPO_DIR="$(cd "$(dirname "$0")/.." && pwd)" +SYSTEMD_SRC="$REPO_DIR/bin/systemd" +SYSTEMD_DST="/etc/systemd/system" + +if [ "$(id -u)" -ne 0 ]; then + echo "❌ 必须用 sudo 跑: sudo bash $0" >&2 + exit 1 +fi + +echo "=== 1. 复制 unit files ===" +for f in "$SYSTEMD_SRC"/market-sync*.{service,timer}; do + [ -f "$f" ] || continue + name="$(basename "$f")" + cp -v "$f" "$SYSTEMD_DST/$name" + chmod 644 "$SYSTEMD_DST/$name" +done + +echo "" +echo "=== 2. daemon-reload ===" +systemctl daemon-reload + +echo "" +echo "=== 3. 启用 + 启动 worker 长驻进程(关键!) ===" +if ! systemctl is-enabled --quiet market-sync-worker.service; then + systemctl enable market-sync-worker.service +fi +# 如果已通过 nohup 启动了, 先 kill 掉避免双跑 +if pgrep -f "app\.entrypoints\.worker" >/dev/null; then + echo " ⚠️ 检测到手动启动的 worker 进程, 先 kill" + pkill -TERM -f "app\.entrypoints\.worker" || true + sleep 2 + pkill -KILL -f "app\.entrypoints\.worker" 2>/dev/null || true +fi +systemctl restart market-sync-worker.service +sleep 2 +systemctl status market-sync-worker.service --no-pager | head -10 + +echo "" +echo "=== 4. 列出所有 market-sync timer (验证) ===" +systemctl list-timers market-sync* --no-pager 2>&1 | head -20 + +echo "" +echo "✅ 部署完成" +echo " - 长驻 worker: systemctl status market-sync-worker.service" +echo " - 看任务执行: tail -f logs/worker_manual_*.log (本进程) 或 journalctl -u market-sync-worker -f (systemd 接管后)" +echo " - 巡检: .venv/bin/python bin/daily_sync_check.py" diff --git a/deploy/systemd/scripts/systemd_deploy.sh b/deploy/systemd/scripts/systemd_deploy.sh new file mode 100755 index 0000000..9b3bc48 --- /dev/null +++ b/deploy/systemd/scripts/systemd_deploy.sh @@ -0,0 +1,185 @@ +#!/bin/bash +# market_sync systemd 一键部署脚本 +# +# 用途：把 bin/systemd/*.service 和 *.timer 同步到 /etc/systemd/system/， +# 解决 work #04 检测到的 5 处 drift。 +# +# 运行：sudo bash bin/systemd_deploy.sh +# +# 设计原则： +# - 必须 sudo（写 /etc/systemd/system/ + daemon-reload） +# - 幂等：重复运行无副作用 +# - 显示 plan → 等待确认 → 执行 +# - 失败立即中止，不留半套状态 + +set -e + +PROJECT_ROOT="/home/gao/Development/quant_home/market_sync" +REPO_UNITS_DIR="$PROJECT_ROOT/bin/systemd" +ETC_UNITS_DIR="/etc/systemd/system" + +# ── 0. 前置检查 ───────────────────────────────────────────────────────────── +if [ "$(id -u)" -ne 0 ]; then + echo "❌ 必须 sudo 运行：sudo bash $0" + exit 1 +fi + +if [ ! -d "$REPO_UNITS_DIR" ]; then + echo "❌ repo unit 目录不存在: $REPO_UNITS_DIR" + exit 1 +fi + +# ── 1. 计算 plan ──────────────────────────────────────────────────────────── +declare -a TO_COPY=() # repo→etc 拷贝 +declare -a TO_REMOVE=() # etc 中孤儿（repo 已删） +declare -a TO_RESTART_SVC=() # service 文件变了需要 daemon-reload + restart + +# repo 中所有 market-sync* unit +mapfile -t REPO_UNITS < <(find "$REPO_UNITS_DIR" -maxdepth 1 -type f \( -name "*.service" -o -name "*.timer" \) -printf '%f\n' | sort) +# /etc 中所有 market-sync* unit +mapfile -t ETC_UNITS < <(find "$ETC_UNITS_DIR" -maxdepth 1 -type f \( -name "market-sync*.service" -o -name "market-sync*.timer" \) -printf '%f\n' | sort 2>/dev/null || true) + +# drift 1: repo 有 /etc 没装 +for u in "${REPO_UNITS[@]}"; do + if [ ! -f "$ETC_UNITS_DIR/$u" ]; then + TO_COPY+=("$u") + fi +done + +# drift 2: 两边都有但内容不一致 +for u in "${REPO_UNITS[@]}"; do + if [ -f "$ETC_UNITS_DIR/$u" ]; then + if ! diff -q "$REPO_UNITS_DIR/$u" "$ETC_UNITS_DIR/$u" > /dev/null 2>&1; then + TO_COPY+=("$u") + if [[ "$u" == *.service ]]; then + TO_RESTART_SVC+=("${u%.service}") + fi + fi + fi +done + +# drift 3: /etc 有 repo 没维护（孤儿） +for u in "${ETC_UNITS[@]}"; do + found=0 + for r in "${REPO_UNITS[@]}"; do + if [ "$r" = "$u" ]; then found=1; break; fi + done + if [ $found -eq 0 ]; then + TO_REMOVE+=("$u") + fi +done + +# ── 2. 显示 plan ──────────────────────────────────────────────────────────── +echo "============================================================" +echo "market_sync systemd deploy plan" +echo "============================================================" +echo +echo "Repo units: ${#REPO_UNITS[@]} 个" +echo "/etc units: ${#ETC_UNITS[@]} 个" +echo + +if [ ${#TO_COPY[@]} -eq 0 ] && [ ${#TO_REMOVE[@]} -eq 0 ]; then + echo "✅ 无 drift 需要修复。无需执行。" + exit 0 +fi + +if [ ${#TO_COPY[@]} -gt 0 ]; then + echo "📋 将要拷贝 ($(echo "${TO_COPY[@]}" | tr ' ' '\n' | wc -l) 个):" + for u in "${TO_COPY[@]}"; do + echo " cp bin/systemd/$u → /etc/systemd/system/$u" + done + echo +fi + +if [ ${#TO_RESTART_SVC[@]} -gt 0 ]; then + echo "🔄 将要重启 service ($(echo "${TO_RESTART_SVC[@]}" | tr ' ' '\n' | wc -l) 个):" + for s in "${TO_RESTART_SVC[@]}"; do + echo " systemctl reset-failed $s.service" + done + echo +fi + +if [ ${#TO_REMOVE[@]} -gt 0 ]; then + echo "🗑️ 将要删除 etc 中的孤儿 ($(echo "${TO_REMOVE[@]}" | tr ' ' '\n' | wc -l) 个):" + for u in "${TO_REMOVE[@]}"; do + echo " rm /etc/systemd/system/$u" + done + echo +fi + +# ── 3. 等待确认 ──────────────────────────────────────────────────────────── +read -p "确认执行？[y/N] " -n 1 -r +echo +if [[ ! $REPLY =~ ^[Yy]$ ]]; then + echo "已取消。" + exit 0 +fi + +# ── 4. 执行 ───────────────────────────────────────────────────────────────── +echo +echo "=== 执行 ===" + +# 4a. cp 漂移的 unit +for u in "${TO_COPY[@]}"; do + cp -v "$REPO_UNITS_DIR/$u" "$ETC_UNITS_DIR/$u" +done + +# 4b. 删孤儿 +for u in "${TO_REMOVE[@]}"; do + rm -v "$ETC_UNITS_DIR/$u" +done + +# 4c. reload systemd +echo +echo "=== systemctl daemon-reload ===" +systemctl daemon-reload + +# 4d. reset-failed + restart 修改过的 service +for s in "${TO_RESTART_SVC[@]}"; do + if systemctl is-enabled "${s}.service" 2>/dev/null | grep -q enabled; then + systemctl reset-failed "${s}.service" || true + echo " reset-failed ${s}.service (没自动 restart — 由 timer 自然触发)" + fi +done + +# 4e. 启用新 timer（如果有） +for u in "${TO_COPY[@]}"; do + if [[ "$u" == *.timer ]]; then + timer="${u%.timer}" + if systemctl list-unit-files "${timer}.timer" 2>/dev/null | grep -q "${timer}.timer"; then + if ! systemctl is-enabled "${timer}.timer" 2>/dev/null | grep -q enabled; then + systemctl enable "${timer}.timer" + echo " enable ${timer}.timer" + fi + fi + fi +done + +# ── 5. 验证 ──────────────────────────────────────────────────────────────── +echo +echo "=== 验证 ===" +echo "Repo units: ${#REPO_UNITS[@]}" +echo "/etc units: $(find "$ETC_UNITS_DIR" -maxdepth 1 -type f -name 'market-sync*.service' -o -name 'market-sync*.timer' | wc -l)" + +drift_count=0 +for u in "${REPO_UNITS[@]}"; do + if [ -f "$ETC_UNITS_DIR/$u" ]; then + if ! diff -q "$REPO_UNITS_DIR/$u" "$ETC_UNITS_DIR/$u" > /dev/null 2>&1; then + echo " ❌ $u 仍有 drift" + drift_count=$((drift_count + 1)) + fi + else + echo " ❌ $u 仍未部署" + drift_count=$((drift_count + 1)) + fi +done + +echo +if [ $drift_count -eq 0 ]; then + echo "✅ 所有 unit 已对齐，0 drift" + echo + echo "下一步：跑 .venv/bin/python bin/daily_sync_check.py --report-only 验证" +else + echo "❌ 还有 $drift_count 处 drift，请检查" + exit 1 +fi \ No newline at end of file diff --git a/deploy/systemd/units/market-sync-daily-check.service b/deploy/systemd/units/market-sync-daily-check.service new file mode 100644 index 0000000..acea43c --- /dev/null +++ b/deploy/systemd/units/market-sync-daily-check.service @@ -0,0 +1,31 @@ +[Unit] +Description=Market sync daily check — reads dataset_registry, writes report, alerts on failures +Documentation=file:///home/gao/Development/quant_home/market_sync/bin/daily_sync_check.py +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +WorkingDirectory=/home/gao/Development/quant_home/market_sync +User=gao +Group=gao + +# 巡检：读 PG dataset_registry，对比今日预期，写 logs/daily_check_*.json +# 失败时 POST webhook 到 FAILURE_WEBHOOK_URL（HMAC-SHA256 签名） +ExecStart=/home/gao/Development/quant_home/market_sync/.venv/bin/python /home/gao/Development/quant_home/market_sync/bin/daily_sync_check.py + +# 巡检应 < 30s 完成；超时 = PG 不通或代码 bug，不让卡死 +TimeoutStartSec=120 + +# 不要 Restart=（oneshot 失败就让 OnFailure= 发通知，别自动重跑） +# 失败时 exit code = 1，可被 OnFailure= 钩住发通知 + +# 日志走 journald（journalctl -u market-sync-daily-check -f） +StandardOutput=journal +StandardError=journal +SyslogIdentifier=market-sync-daily-check + +Environment=PYTHONUNBUFFERED=1 + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/deploy/systemd/units/market-sync-daily-check.timer b/deploy/systemd/units/market-sync-daily-check.timer new file mode 100644 index 0000000..2ee10b8 --- /dev/null +++ b/deploy/systemd/units/market-sync-daily-check.timer @@ -0,0 +1,15 @@ +[Unit] +Description=Schedule market sync daily check — Daily 23:00 Asia/Shanghai +# market-sync-daily-check.service 是这个 timer 的执行单元 +# 23:00 触发：周一~五最后一个任务是 longhubang @ 22:00，留 1h buffer； +# 周六 23:00 给 stock_node @ 11:30 留 11h+ buffer + +[Timer] +# 每天 23:00 触发；Persistent=true 确保关机/错过时下次开机补跑 +OnCalendar=*-*-* 23:00:00 Asia/Shanghai +Persistent=true +Unit=market-sync-daily-check.service +# 不要 AccuracySec（默认 1min 漂移够用） + +[Install] +WantedBy=timers.target \ No newline at end of file diff --git a/deploy/systemd/units/market-sync-kline-5min.service b/deploy/systemd/units/market-sync-kline-5min.service new file mode 100644 index 0000000..fc25376 --- /dev/null +++ b/deploy/systemd/units/market-sync-kline-5min.service @@ -0,0 +1,20 @@ +[Unit] +Description=Market sync kline_5min — 5分钟K (评分核心) +Documentation=file:///home/gao/Development/quant_home/market_sync/bin/market_sync_kline_5min_run.sh +After=network-online.target market-sync-kline-daily.service +Wants=network-online.target + +[Service] +Type=oneshot +WorkingDirectory=/home/gao/Development/quant_home/market_sync +User=gao +Group=gao +ExecStart=/home/gao/Development/quant_home/market_sync/bin/market_sync_kline_5min_run.sh +TimeoutStartSec=7200 +StandardOutput=journal +StandardError=journal +SyslogIdentifier=market-sync-kline-5min +Environment=PYTHONUNBUFFERED=1 + +[Install] +WantedBy=multi-user.target diff --git a/deploy/systemd/units/market-sync-kline-5min.timer b/deploy/systemd/units/market-sync-kline-5min.timer new file mode 100644 index 0000000..20b8f0f --- /dev/null +++ b/deploy/systemd/units/market-sync-kline-5min.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Schedule kline_5min sync — Mon..Fri 16:00 Asia/Shanghai + +[Timer] +OnCalendar=Mon..Fri 16:00:00 Asia/Shanghai +Persistent=true +Unit=market-sync-kline-5min.service + +[Install] +WantedBy=timers.target diff --git a/deploy/systemd/units/market-sync-kline-daily-early.timer b/deploy/systemd/units/market-sync-kline-daily-early.timer new file mode 100644 index 0000000..3b51b6d --- /dev/null +++ b/deploy/systemd/units/market-sync-kline-daily-early.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Market sync kline_daily — 15:10 麦蕊优先抢跑 +Documentation=file:///home/gao/Development/quant_home/market_sync/bin/market_sync_kline_daily_run.sh + +[Timer] +OnCalendar=Mon..Fri 15:10:00 +Persistent=false +RandomizedDelaySec=0 + +[Install] +WantedBy=timers.target diff --git a/deploy/systemd/units/market-sync-kline-daily.service b/deploy/systemd/units/market-sync-kline-daily.service new file mode 100644 index 0000000..68fc701 --- /dev/null +++ b/deploy/systemd/units/market-sync-kline-daily.service @@ -0,0 +1,20 @@ +[Unit] +Description=Market sync kline_daily — 个股日K (评分核心) +Documentation=file:///home/gao/Development/quant_home/market_sync/bin/market_sync_kline_daily_run.sh +After=network-online.target market-sync-kline-index.service +Wants=network-online.target + +[Service] +Type=oneshot +WorkingDirectory=/home/gao/Development/quant_home/market_sync +User=gao +Group=gao +ExecStart=/home/gao/Development/quant_home/market_sync/bin/market_sync_kline_daily_run.sh +TimeoutStartSec=3600 +StandardOutput=journal +StandardError=journal +SyslogIdentifier=market-sync-kline-daily +Environment=PYTHONUNBUFFERED=1 + +[Install] +WantedBy=multi-user.target diff --git a/deploy/systemd/units/market-sync-kline-daily.timer b/deploy/systemd/units/market-sync-kline-daily.timer new file mode 100644 index 0000000..0b38254 --- /dev/null +++ b/deploy/systemd/units/market-sync-kline-daily.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Schedule kline_daily sync — Mon..Fri 15:40 Asia/Shanghai + +[Timer] +OnCalendar=Mon..Fri 15:40:00 Asia/Shanghai +Persistent=true +Unit=market-sync-kline-daily.service + +[Install] +WantedBy=timers.target diff --git a/deploy/systemd/units/market-sync-kline-index.service b/deploy/systemd/units/market-sync-kline-index.service new file mode 100644 index 0000000..ce198c3 --- /dev/null +++ b/deploy/systemd/units/market-sync-kline-index.service @@ -0,0 +1,20 @@ +[Unit] +Description=Market sync kline_index — 指数日K (独立, 非评分依赖) +Documentation=file:///home/gao/Development/quant_home/market_sync/bin/market_sync_kline_index_run.sh +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +WorkingDirectory=/home/gao/Development/quant_home/market_sync +User=gao +Group=gao +ExecStart=/home/gao/Development/quant_home/market_sync/bin/market_sync_kline_index_run.sh +TimeoutStartSec=600 +StandardOutput=journal +StandardError=journal +SyslogIdentifier=market-sync-kline-index +Environment=PYTHONUNBUFFERED=1 + +[Install] +WantedBy=multi-user.target diff --git a/deploy/systemd/units/market-sync-kline-index.timer b/deploy/systemd/units/market-sync-kline-index.timer new file mode 100644 index 0000000..e2e036f --- /dev/null +++ b/deploy/systemd/units/market-sync-kline-index.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Schedule kline_index sync — Mon..Fri 15:30 Asia/Shanghai + +[Timer] +OnCalendar=Mon..Fri 15:30:00 Asia/Shanghai +Persistent=true +Unit=market-sync-kline-index.service + +[Install] +WantedBy=timers.target diff --git a/deploy/systemd/units/market-sync-lhb.service b/deploy/systemd/units/market-sync-lhb.service new file mode 100644 index 0000000..8a3ad36 --- /dev/null +++ b/deploy/systemd/units/market-sync-lhb.service @@ -0,0 +1,33 @@ +[Unit] +Description=Market data 龙虎榜 sync (akshare 源, daily 22:00) +Documentation=file:///home/gao/Development/quant_home/market_sync/bin/market_sync_lhb_run.sh +After=network-online.target market-sync.service +Wants=network-online.target + +[Service] +Type=oneshot +WorkingDirectory=/home/gao/Development/quant_home/market_sync +User=gao +Group=gao + +# 入口脚本：单独跑 longhubang 一个 task +# 龙虎榜数据 19:00~21:00 陆续出齐 → 22:00 触发（与 moneyflow 21:35 错开避免并发） +ExecStart=/home/gao/Development/quant_home/market_sync/bin/market_sync_lhb_run.sh + +# 硬上限 1h（每日 ~200 次 akshare 调用 + DB upsert，3-5 分钟内完成，留 buffer） +TimeoutStartSec=3600 + +# 不要 Restart=（oneshot 失败就让 OnFailure= 发通知，别自动重跑—— +# 重跑会撞外部 API 限流窗口，浪费资源） +# Restart=no 是 oneshot 默认值 + +# 日志走 journald（journalctl -u market-sync-lhb.service -f） +StandardOutput=journal +StandardError=journal +SyslogIdentifier=market-sync-lhb + +# 环境（不读 /etc/environment，只带这几个；.env 由 cli 内部 load_dotenv） +Environment=PYTHONUNBUFFERED=1 + +[Install] +WantedBy=multi-user.target diff --git a/deploy/systemd/units/market-sync-lhb.timer b/deploy/systemd/units/market-sync-lhb.timer new file mode 100644 index 0000000..98de850 --- /dev/null +++ b/deploy/systemd/units/market-sync-lhb.timer @@ -0,0 +1,16 @@ +[Unit] +Description=Schedule 龙虎榜 sync — Mon..Fri 22:00 Asia/Shanghai +# market-sync-lhb.service 是这个 timer 的执行单元 +# 龙虎榜数据 19:00~21:00 陆续出齐 → 22:00 触发（与 moneyflow 21:35 错开） + +[Timer] +# 龙虎榜出齐 → 22:00 触发 +OnCalendar=Mon..Fri 22:00:00 Asia/Shanghai +# 系统关机 / 错过执行时，下次开机补跑一次（避免漏数据） +Persistent=true +# 单位（service）的精确名称 +Unit=market-sync-lhb.service +# 不要 AccuracySec（默认 1min 漂移够用，避免 22:00:00 整点打堆） + +[Install] +WantedBy=timers.target diff --git a/deploy/systemd/units/market-sync-mairui-indicators.service b/deploy/systemd/units/market-sync-mairui-indicators.service new file mode 100644 index 0000000..8762c0a --- /dev/null +++ b/deploy/systemd/units/market-sync-mairui-indicators.service @@ -0,0 +1,32 @@ +[Unit] +Description=Market data MACD/KDJ/BOLL indicators sync (mairui, daily 16:40) +Documentation=file:///home/gao/Development/quant_home/market_sync/bin/market_sync_mairui_indicators_run.sh +After=network-online.target market-sync.service +Wants=network-online.target + +[Service] +Type=oneshot +WorkingDirectory=/home/gao/Development/quant_home/market_sync +User=gao +Group=gao + +# 入口脚本：单独跑 mairui_indicators 一个 task +# mairui 16:30 后发布指标数据 → 16:40 触发留 buffer +ExecStart=/home/gao/Development/quant_home/market_sync/bin/market_sync_mairui_indicators_run.sh + +# 硬上限 3h（5204 只 × 3 指标，正常 ~20min，全量时可能更久） +TimeoutStartSec=10800 + +# 不要 Restart=（oneshot 失败就让 OnFailure= 发通知，别自动重跑） +# Restart=no 是 oneshot 默认值 + +# 日志走 journald +StandardOutput=journal +StandardError=journal +SyslogIdentifier=market-sync-mairui-indicators + +# 环境 +Environment=PYTHONUNBUFFERED=1 + +[Install] +WantedBy=multi-user.target diff --git a/deploy/systemd/units/market-sync-mairui-indicators.timer b/deploy/systemd/units/market-sync-mairui-indicators.timer new file mode 100644 index 0000000..6210284 --- /dev/null +++ b/deploy/systemd/units/market-sync-mairui-indicators.timer @@ -0,0 +1,14 @@ +[Unit] +Description=Schedule MACD/KDJ/BOLL indicators sync — Mon..Fri 16:40 Asia/Shanghai +# market-sync-mairui-indicators.service 是这个 timer 的执行单元 + +[Timer] +# mairui 16:30 后发布指标数据 → 16:40 触发留 buffer +OnCalendar=Mon..Fri 16:40:00 Asia/Shanghai +# 系统关机 / 错过执行时，下次开机补跑一次（避免漏数据） +Persistent=true +# 单位（service）的精确名称 +Unit=market-sync-mairui-indicators.service + +[Install] +WantedBy=timers.target diff --git a/deploy/systemd/units/market-sync-mairui-ma-daily.service b/deploy/systemd/units/market-sync-mairui-ma-daily.service new file mode 100644 index 0000000..4deef25 --- /dev/null +++ b/deploy/systemd/units/market-sync-mairui-ma-daily.service @@ -0,0 +1,32 @@ +[Unit] +Description=Market data MA daily sync (local derivative, daily 16:30) +Documentation=file:///home/gao/Development/quant_home/market_sync/bin/market_sync_mairui_ma_daily_run.sh +After=network-online.target market-sync.service +Wants=network-online.target + +[Service] +Type=oneshot +WorkingDirectory=/home/gao/Development/quant_home/market_sync +User=gao +Group=gao + +# 入口脚本：单独跑 mairui_ma_daily 一个 task +# 本地派生任务，依赖 kline_stock 已更新 +ExecStart=/home/gao/Development/quant_home/market_sync/bin/market_sync_mairui_ma_daily_run.sh + +# 硬上限 2h（正常 ~30min，留 buffer） +TimeoutStartSec=7200 + +# 不要 Restart=（oneshot 失败就让 OnFailure= 发通知，别自动重跑） +# Restart=no 是 oneshot 默认值 + +# 日志走 journald +StandardOutput=journal +StandardError=journal +SyslogIdentifier=market-sync-mairui-ma-daily + +# 环境 +Environment=PYTHONUNBUFFERED=1 + +[Install] +WantedBy=multi-user.target diff --git a/deploy/systemd/units/market-sync-mairui-ma-daily.timer b/deploy/systemd/units/market-sync-mairui-ma-daily.timer new file mode 100644 index 0000000..7ba02d3 --- /dev/null +++ b/deploy/systemd/units/market-sync-mairui-ma-daily.timer @@ -0,0 +1,14 @@ +[Unit] +Description=Schedule MA daily sync — Mon..Fri 16:30 Asia/Shanghai +# market-sync-mairui-ma-daily.service 是这个 timer 的执行单元 + +[Timer] +# kline_daily 15:40 / kline_index 15:30 跑完后，16:30 计算 MA +OnCalendar=Mon..Fri 16:30:00 Asia/Shanghai +# 系统关机 / 错过执行时，下次开机补跑一次（避免漏数据） +Persistent=true +# 单位（service）的精确名称 +Unit=market-sync-mairui-ma-daily.service + +[Install] +WantedBy=timers.target diff --git a/deploy/systemd/units/market-sync-market-regime.service b/deploy/systemd/units/market-sync-market-regime.service new file mode 100644 index 0000000..4d93992 --- /dev/null +++ b/deploy/systemd/units/market-sync-market-regime.service @@ -0,0 +1,20 @@ +[Unit] +Description=Market sync market_regime — 市场情绪 (本地计算, 非评分依赖) +Documentation=file:///home/gao/Development/quant_home/market_sync/bin/market_sync_market_regime_run.sh +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +WorkingDirectory=/home/gao/Development/quant_home/market_sync +User=gao +Group=gao +ExecStart=/home/gao/Development/quant_home/market_sync/bin/market_sync_market_regime_run.sh +TimeoutStartSec=600 +StandardOutput=journal +StandardError=journal +SyslogIdentifier=market-sync-market-regime +Environment=PYTHONUNBUFFERED=1 + +[Install] +WantedBy=multi-user.target diff --git a/deploy/systemd/units/market-sync-market-regime.timer b/deploy/systemd/units/market-sync-market-regime.timer new file mode 100644 index 0000000..7c9ca9e --- /dev/null +++ b/deploy/systemd/units/market-sync-market-regime.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Schedule market_regime sync — Mon..Fri 16:30 Asia/Shanghai + +[Timer] +OnCalendar=Mon..Fri 16:30:00 Asia/Shanghai +Persistent=true +Unit=market-sync-market-regime.service + +[Install] +WantedBy=timers.target diff --git a/deploy/systemd/units/market-sync-mcp.service b/deploy/systemd/units/market-sync-mcp.service new file mode 100644 index 0000000..bad1bbd --- /dev/null +++ b/deploy/systemd/units/market-sync-mcp.service @@ -0,0 +1,22 @@ +[Unit] +Description=market_sync MCP Server — Model Context Protocol for sync data +Documentation=file:///home/gao/Development/quant_home/market_sync/app/mcp_server.py +After=network-online.target postgres-container.service +Wants=network-online.target + +[Service] +Type=simple +WorkingDirectory=/home/gao/Development/quant_home/market_sync +User=gao +Group=gao +ExecStart=/home/gao/Development/quant_home/market_sync/.venv/bin/python -m app.mcp_server --sse --host 127.0.0.1 --port 8101 +Restart=on-failure +RestartSec=5 +TimeoutStartSec=30 +StandardOutput=journal +StandardError=journal +SyslogIdentifier=market-sync-mcp +Environment=PYTHONUNBUFFERED=1 + +[Install] +WantedBy=multi-user.target diff --git a/deploy/systemd/units/market-sync-moneyflow.service b/deploy/systemd/units/market-sync-moneyflow.service new file mode 100644 index 0000000..92ea37b --- /dev/null +++ b/deploy/systemd/units/market-sync-moneyflow.service @@ -0,0 +1,33 @@ +[Unit] +Description=Market data moneyflow sync (mairui transaction, daily 21:35) +Documentation=file:///home/gao/Development/quant_home/market_sync/bin/market_sync_moneyflow_run.sh +After=network-online.target market-sync.service +Wants=network-online.target + +[Service] +Type=oneshot +WorkingDirectory=/home/gao/Development/quant_home/market_sync +User=gao +Group=gao + +# 入口脚本：单独跑 moneyflow 一个 task +# mairui 文档：「更新：每日 21:30」—— 21:35 触发留 5min buffer +ExecStart=/home/gao/Development/quant_home/market_sync/bin/market_sync_moneyflow_run.sh + +# 硬上限 2h（5206 只 × 50 RPS = 104s 理论上限，留 buffer 给重试 / 慢响应） +TimeoutStartSec=7200 + +# 不要 Restart=（oneshot 失败就让 OnFailure= 发通知，别自动重跑—— +# 重跑会撞外部 API 限流窗口，浪费 1.5h） +# Restart=no 是 oneshot 默认值 + +# 日志走 journald（journalctl -u market-sync-moneyflow.service -f） +StandardOutput=journal +StandardError=journal +SyslogIdentifier=market-sync-moneyflow + +# 环境（不读 /etc/environment，只带这几个；.env 由 cli 内部 load_dotenv） +Environment=PYTHONUNBUFFERED=1 + +[Install] +WantedBy=multi-user.target diff --git a/deploy/systemd/units/market-sync-moneyflow.timer b/deploy/systemd/units/market-sync-moneyflow.timer new file mode 100644 index 0000000..5f20a5e --- /dev/null +++ b/deploy/systemd/units/market-sync-moneyflow.timer @@ -0,0 +1,16 @@ +[Unit] +Description=Schedule moneyflow sync — Mon..Fri 21:35 Asia/Shanghai +# market-sync-moneyflow.service 是这个 timer 的执行单元 +# mairui 文档：「更新：每日 21:30」—— 21:35 触发留 5 分钟 buffer + +[Timer] +# mairui 21:30 发布数据 → 21:35 触发 +OnCalendar=Mon..Fri 21:35:00 Asia/Shanghai +# 系统关机 / 错过执行时，下次开机补跑一次（避免漏数据） +Persistent=true +# 单位（service）的精确名称 +Unit=market-sync-moneyflow.service +# 不要 AccuracySec（默认 1min 漂移够用，避免 21:35:00 整点打堆） + +[Install] +WantedBy=timers.target diff --git a/deploy/systemd/units/market-sync-morning.service b/deploy/systemd/units/market-sync-morning.service new file mode 100644 index 0000000..eea8cc8 --- /dev/null +++ b/deploy/systemd/units/market-sync-morning.service @@ -0,0 +1,29 @@ +[Unit] +Description=Market sync morning pre-market — stock_basic +Documentation=file:///home/gao/Development/quant_home/market_sync/bin/market_sync_morning_run.sh +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +WorkingDirectory=/home/gao/Development/quant_home/market_sync +User=gao +Group=gao + +# 早盘前只跑 stock_basic（确保最新股票列表+股本快照） +ExecStart=/home/gao/Development/quant_home/market_sync/bin/market_sync_morning_run.sh + +# stock_basic ~9min + buffer = 1h 够用 +TimeoutStartSec=3600 + +# 不要 Restart=（oneshot 失败就让 OnFailure= 发通知，别自动重跑） + +# 日志走 journald（journalctl -u market-sync-morning -f） +StandardOutput=journal +StandardError=journal +SyslogIdentifier=market-sync-morning + +Environment=PYTHONUNBUFFERED=1 + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/deploy/systemd/units/market-sync-morning.timer b/deploy/systemd/units/market-sync-morning.timer new file mode 100644 index 0000000..5449710 --- /dev/null +++ b/deploy/systemd/units/market-sync-morning.timer @@ -0,0 +1,14 @@ +[Unit] +Description=Schedule market sync morning — Mon..Fri 09:00 Asia/Shanghai +# market-sync-morning.service 是这个 timer 的执行单元 +# 早盘前 09:00 触发：跑 stock_basic，给 09:30 开盘留 30min buffer + +[Timer] +# A 股开盘 09:30，09:00 跑 stock_basic (~9min) 即可 +OnCalendar=Mon..Fri 09:00:00 Asia/Shanghai +# 关机/错过时下次开机补跑 +Persistent=true +Unit=market-sync-morning.service + +[Install] +WantedBy=timers.target \ No newline at end of file diff --git a/deploy/systemd/units/market-sync-share.service b/deploy/systemd/units/market-sync-share.service new file mode 100644 index 0000000..00defa8 --- /dev/null +++ b/deploy/systemd/units/market-sync-share.service @@ -0,0 +1,31 @@ +[Unit] +Description=Market data 股本快照 sync (雪球源, weekly Sat 11:30) +Documentation=file:///home/gao/Development/quant_home/market_sync/bin/market_sync_share_run.sh +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +WorkingDirectory=/home/gao/Development/quant_home/market_sync +User=gao +Group=gao + +# 入口脚本：单独跑 share_snapshot 一个 task +ExecStart=/home/gao/Development/quant_home/market_sync/bin/market_sync_share_run.sh + +# 硬上限 30min（全市场 5200 只雪球 quote_detail，10 worker 约 3-8min，留 buffer） +TimeoutStartSec=1800 + +# 不要 Restart=（oneshot 失败就让 OnFailure= 发通知，别自动重跑） +# Restart=no 是 oneshot 默认值 + +# 日志走 journald（journalctl -u market-sync-share.service -f） +StandardOutput=journal +StandardError=journal +SyslogIdentifier=market-sync-share + +# 环境（不读 /etc/environment，只带这几个；.env 由 cli 内部 load_dotenv） +Environment=PYTHONUNBUFFERED=1 + +[Install] +WantedBy=multi-user.target diff --git a/deploy/systemd/units/market-sync-share.timer b/deploy/systemd/units/market-sync-share.timer new file mode 100644 index 0000000..a47034f --- /dev/null +++ b/deploy/systemd/units/market-sync-share.timer @@ -0,0 +1,16 @@ +[Unit] +Description=Schedule share_snapshot sync — Sat 11:30 Asia/Shanghai +# market-sync-share.service 是这个 timer 的执行单元 +# 股本快照每周跑一次即可（雪球 quote_detail，周末收盘后数据稳定） + +[Timer] +# 周六 11:30（和 stock_node 同时段） +OnCalendar=Sat *-*-* 11:30:00 Asia/Shanghai +# 系统关机 / 错过执行时,下次开机补跑一次(避免漏数据) +Persistent=true +# 单位(service)的精确名称 +Unit=market-sync-share.service +# 不要 AccuracySec(默认 1min 漂移够用) + +[Install] +WantedBy=timers.target diff --git a/deploy/systemd/units/market-sync-stock-node.service b/deploy/systemd/units/market-sync-stock-node.service new file mode 100644 index 0000000..bc0010d --- /dev/null +++ b/deploy/systemd/units/market-sync-stock-node.service @@ -0,0 +1,33 @@ +[Unit] +Description=Market data 股票-节点映射 sync (mairui /hszg, weekly Sat 11:30) +Documentation=file:///home/gao/Development/quant_home/market_sync/bin/market_sync_stock_node_run.sh +After=network-online.target market-sync-morning.service +Wants=network-online.target + +[Service] +Type=oneshot +WorkingDirectory=/home/gao/Development/quant_home/market_sync +User=gao +Group=gao + +# 入口脚本：单独跑 stock_node 一个 task +# mairui /hszg 数据每周六 11:00 更新 → 11:30 触发,留 30 min buffer +ExecStart=/home/gao/Development/quant_home/market_sync/bin/market_sync_stock_node_run.sh + +# 硬上限 2h(1100+ 调用 + 写库 + retry buffer) +TimeoutStartSec=7200 + +# 不要 Restart=(oneshot 失败就让 OnFailure= 发通知,别自动重跑—— +# 重跑会撞 mairui 配额窗口) +# Restart=no 是 oneshot 默认值 + +# 日志走 journald(journalctl -u market-sync-stock-node.service -f) +StandardOutput=journal +StandardError=journal +SyslogIdentifier=market-sync-stock-node + +# 环境(不读 /etc/environment,只带这几个;.env 由 cli 内部 load_dotenv) +Environment=PYTHONUNBUFFERED=1 + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/deploy/systemd/units/market-sync-stock-node.timer b/deploy/systemd/units/market-sync-stock-node.timer new file mode 100644 index 0000000..acf48ba --- /dev/null +++ b/deploy/systemd/units/market-sync-stock-node.timer @@ -0,0 +1,16 @@ +[Unit] +Description=Schedule stock_node sync — Sat 11:30 Asia/Shanghai +# market-sync-stock-node.service 是这个 timer 的执行单元 +# mairui /hszg 数据每周六 11:00 更新 → 11:30 触发(30 min buffer) + +[Timer] +# 周六 11:30(mairui 11:00 更新后) +OnCalendar=Sat *-*-* 11:30:00 Asia/Shanghai +# 系统关机 / 错过执行时,下次开机补跑一次(避免漏数据) +Persistent=true +# 单位(service)的精确名称 +Unit=market-sync-stock-node.service +# 不要 AccuracySec(默认 1min 漂移够用,避免 11:30:00 整点打堆) + +[Install] +WantedBy=timers.target \ No newline at end of file diff --git a/deploy/systemd/units/market-sync-tick.service b/deploy/systemd/units/market-sync-tick.service new file mode 100644 index 0000000..4a3742d --- /dev/null +++ b/deploy/systemd/units/market-sync-tick.service @@ -0,0 +1,33 @@ +[Unit] +Description=Market data tick-by-tick trade sync (mairui hsrl/zbjy, daily 21:05) +Documentation=file:///home/gao/Development/quant_home/market_sync/bin/market_sync_tick_run.sh +After=network-online.target market-sync.service +Wants=network-online.target + +[Service] +Type=oneshot +WorkingDirectory=/home/gao/Development/quant_home/market_sync +User=gao +Group=gao + +# 入口脚本：单独跑 tick_trade 一个 task（mairui 21:00 发布数据， +# 这个 service 在 21:05 触发，留 5min buffer；force=True 绕过 task 内部 21:00 门控） +ExecStart=/home/gao/Development/quant_home/market_sync/bin/market_sync_tick_run.sh + +# 硬上限 2h（5000 只 × 1 RPS ≈ 17min，留 buffer 给 retry / 慢任务） +TimeoutStartSec=7200 + +# 不要 Restart=（oneshot 失败就让 OnFailure= 发通知，别自动重跑—— +# 重跑会撞外部 API 限流窗口，浪费 1.5h） +# Restart=no 是 oneshot 默认值 + +# 日志走 journald（journalctl -u market-sync-tick.service -f） +StandardOutput=journal +StandardError=journal +SyslogIdentifier=market-sync-tick + +# 环境（不读 /etc/environment，只带这几个；.env 由 cli 内部 load_dotenv） +Environment=PYTHONUNBUFFERED=1 + +[Install] +WantedBy=multi-user.target diff --git a/deploy/systemd/units/market-sync-tick.timer b/deploy/systemd/units/market-sync-tick.timer new file mode 100644 index 0000000..05ace68 --- /dev/null +++ b/deploy/systemd/units/market-sync-tick.timer @@ -0,0 +1,17 @@ +[Unit] +Description=Schedule tick_trade sync — Mon..Fri 21:05 Asia/Shanghai +# market-sync-tick.service 是这个 timer 的执行单元 +# 触发顺序：market-sync.service (15:30) 之后 market-sync-tick.service (21:05) +# 但 systemd timer 独立触发，After= 只是声明依赖关系（非强制） + +[Timer] +# mairui 文档：「更新：每日 21:00」—— 21:05 触发留 5 分钟 buffer +OnCalendar=Mon..Fri 21:05:00 Asia/Shanghai +# 系统关机 / 错过执行时，下次开机补跑一次（避免漏数据） +Persistent=true +# 单位（service）的精确名称 +Unit=market-sync-tick.service +# 不要 AccuracySec（默认 1min 漂移够用，避免 21:05:00 整点打堆） + +[Install] +WantedBy=timers.target diff --git a/deploy/systemd/units/market-sync-worker.service b/deploy/systemd/units/market-sync-worker.service new file mode 100644 index 0000000..4192233 --- /dev/null +++ b/deploy/systemd/units/market-sync-worker.service @@ -0,0 +1,25 @@ +[Unit] +Description=Market Data Sync Worker (PostgreSQL, scheduler + health monitor) +Documentation=file:///home/gao/Development/quant_home/market_sync/app/entrypoints/worker.py +# 依赖 network 和 PG +After=network-online.target postgresql.service +Wants=network-online.target + +[Service] +Type=simple +WorkingDirectory=/home/gao/Development/quant_home/market_sync +User=gao +Group=gao + +# 入口：启动进程内 scheduler + health monitor + recover 卡死任务 +# 注意：实际入口是 app/entrypoints/worker.py，不是 app/worker.py +# 2026-07-12 修复: 把 python -m app.worker 改为 python -m app.entrypoints.worker +ExecStart=/home/gao/Development/quant_home/market_sync/.venv/bin/python -m app.entrypoints.worker + +# 明确标记运行模式，让 worker.py 知道不要启动 scheduler +# （systemd 模式下同步由 timer 触发，避免双调度器重叠） +Environment=PYTHONUNBUFFERED=1 +Environment="RUNTIME_MODE=systemd" + +[Install] +WantedBy=multi-user.target diff --git a/deploy/systemd/units/market-sync.service b/deploy/systemd/units/market-sync.service new file mode 100644 index 0000000..097845c --- /dev/null +++ b/deploy/systemd/units/market-sync.service @@ -0,0 +1,35 @@ +[Unit] +Description=Market Data Full Sync (PostgreSQL, 9 sync tasks end-to-end) +Documentation=file:///home/gao/Development/quant_home/market_sync/bin/market_sync_run.sh +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +WorkingDirectory=/home/gao/Development/quant_home/market_sync +User=gao +Group=gao + +# 入口脚本（runall_once.py + structured_watch.sh 二合一） +ExecStart=/home/gao/Development/quant_home/market_sync/bin/market_sync_run.sh + +# 硬上限 3h（实际 ~80min + kline_5min 增量 70min, 总 ~2.5h, 留 30min buffer） +# 2026-07-08 教训: 之前 4h 让 kline_daily 单点卡死拖 4h,runall 链全部受影响。 +# runall_once.py 已加 per-task multiprocessing timeout(单 task 最长 75min), +# 这里 3h 是 systemd 层兜底。 +TimeoutStartSec=10800 + +# 不要 Restart=（oneshot 失败就让 OnFailure= 发通知，别自动重跑—— +# 重跑会撞外部 API 限流窗口，浪费 1.5h） +# Restart=no 是 oneshot 默认值 + +# 日志走 journald（journalctl -u market-sync.service -f） +StandardOutput=journal +StandardError=journal +SyslogIdentifier=market-sync + +# 环境（不读 /etc/environment，只带这几个；.env 由 runall_once.py 内部 load_dotenv） +Environment=PYTHONUNBUFFERED=1 + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/deploy/systemd/units/market-sync.timer b/deploy/systemd/units/market-sync.timer new file mode 100644 index 0000000..013560e --- /dev/null +++ b/deploy/systemd/units/market-sync.timer @@ -0,0 +1,15 @@ +[Unit] +Description=Schedule market data full sync — Mon..Fri 15:30 Asia/Shanghai +# market-sync.service 是这个 timer 的执行单元 + +[Timer] +# A 股收盘时间 15:00，30 分钟 buffer 让最后一笔 tick 落地 +OnCalendar=Mon..Fri 15:30:00 Asia/Shanghai +# 系统关机 / 错过执行时，下次开机补跑一次（避免漏数据） +Persistent=true +# 单位（service）的精确名称 +Unit=market-sync.service +# 不要 AccuracySec（默认 1min 漂移够用，避免 15:30:00 整点打堆） + +[Install] +WantedBy=timers.target \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 709eee8..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,84 +0,0 @@ -# docker-compose.yml — 一键起 market_sync 数据同步服务 -# -# 包含： -# - postgres : PostgreSQL 16（market_data schema 的主库） -# - market_sync : 数据同步 + dashboard + scheduler（单进程） -# -# 用法： -# docker compose up -d # 后台启动 -# docker compose logs -f market_sync # 看同步日志 -# docker compose exec market_sync \ -# python -m app.entrypoints.cli sync stock_node --type2 2,3 -# -version: "3.9" - -services: - postgres: - image: postgres:16-alpine - restart: unless-stopped - environment: - POSTGRES_USER: market_sync - POSTGRES_PASSWORD: market_sync - POSTGRES_DB: market_data - ports: - - "5432:5432" - volumes: - - pgdata:/var/lib/postgresql/data - healthcheck: - test: ["CMD-SHELL", "pg_isready -U market_sync -d market_data"] - interval: 10s - timeout: 3s - retries: 5 - - market_sync: - build: - context: . - dockerfile: Dockerfile - image: market_sync:latest - restart: unless-stopped - depends_on: - postgres: - condition: service_healthy - environment: - # PG 连接（指向同 compose 的 postgres 服务） - PG_HOST: postgres - PG_PORT: "5432" - PG_USER: market_sync - PG_PASSWORD: market_sync - PG_DB_NAME: market_data - DB_BACKEND: pg - # 调度 - SCHEDULER_TIMEZONE: Asia/Shanghai - SCHEDULER_TICK_SECONDS: "5" - SCHEDULER_AUTO_SEED: "true" - # API - API_HOST: 0.0.0.0 - API_PORT: "8100" - # 日志 - LOG_DIR: /app/logs - LOG_LEVEL: INFO - # 数据源开关 - DS_BAOSTOCK_ENABLED: "true" - DS_SINA_ENABLED: "true" - # 凭证（可放进 .env / secrets 文件，避免明文） - MAIRUI_LICENCE: ${MAIRUI_LICENCE:-} - XUEQIU_TOKEN: ${XUEQIU_TOKEN:-} - # 麦蕊限速（钻石 100 RPS 留 buffer；服务化后保守点） - MAIRUI_RPS_LIMIT: "10" - # 节假日 - TRADING_HOLIDAYS: ${TRADING_HOLIDAYS:-} - ports: - - "8100:8100" - volumes: - # 持久化日志（任务失败时方便排查） - - sync_logs:/app/logs - healthcheck: - test: ["CMD", "curl", "-fsS", "http://localhost:8100/api/health"] - interval: 60s - timeout: 5s - retries: 3 - start_period: 30s - -volumes: - pgdata: - sync_logs: \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 120000 index 0000000..5275a4c --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1 @@ +deploy/docker/docker-compose.yml \ No newline at end of file diff --git a/opencode.json b/opencode.json new file mode 100644 index 0000000..ac1c561 --- /dev/null +++ b/opencode.json @@ -0,0 +1,10 @@ +{ + "$schema": "https://opencode.ai/config.json", + "mcp": { + "market_sync": { + "type": "remote", + "url": "http://127.0.0.1:8101/sse", + "enabled": true + } + } +} diff --git a/scripts/run_all_sync.sh b/scripts/run_all_sync.sh index 2d62e7f..e13a27f 100755 --- a/scripts/run_all_sync.sh +++ b/scripts/run_all_sync.sh @@ -1,81 +1,52 @@ #!/bin/bash -# 串行跑所有 6 个同步任务，后台模式，写到 logs/ -# 用法：./run_all_sync.sh [STEP] -# STEP 为空 = 跑 1-6 -# STEP=1 只跑 stock_basic，等等 +set -u -set -e -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" -cd "$PROJECT_DIR" +PROJECT_ROOT="/home/gao/Development/quant_home/market_sync" +LOG_DIR="$PROJECT_ROOT/logs" +RUN_LOG="$LOG_DIR/run_all_sync_$(date +%Y%m%d_%H%M%S).log" -mkdir -p logs -LOG_DIR="$PROJECT_DIR/logs" -TIMESTAMP=$(date +%Y%m%d_%H%M%S) -RUN_LOG="$LOG_DIR/run_all_$TIMESTAMP.log" +cd "$PROJECT_ROOT" -# 读 token -if [ -f "$PROJECT_DIR/.env" ]; then - set -a - source "$PROJECT_DIR/.env" - set +a -fi - -# 把 .env 里的 TRADING_HOLIDAYS 转成 -- 跳过 -# XUEQIU_TOKEN 在 .env 里 - -source .venv/bin/activate -export PYTHONPATH="$PROJECT_DIR" - -LOG_PREFIX="[$TIMESTAMP]" +LOG_PREFIX="[run_all_sync]" run_step() { - local step_name="$1" - local task_id="$2" - local extra_args="$3" - local step_log="$LOG_DIR/step_${step_name}_$TIMESTAMP.log" - - echo "$LOG_PREFIX === Step $step_name: $task_id ===" | tee -a "$RUN_LOG" - echo "$LOG_PREFIX log: $step_log" | tee -a "$RUN_LOG" - - # 跑任务（前台模式，方便顺序执行 + 立即看到结果） - python -m app.entrypoints.cli sync "$task_id" $extra_args 2>&1 | tee "$step_log" | tail -10 - local exit_code=${PIPESTATUS[0]} - - if [ $exit_code -eq 0 ]; then - echo "$LOG_PREFIX ✓ Step $step_name OK" | tee -a "$RUN_LOG" + local step=$1 task=$2 extra_args=${3:-} + echo "$LOG_PREFIX === 步骤 $step: $task ===" | tee -a "$RUN_LOG" + if [ -n "$extra_args" ]; then + .venv/bin/python -m app.entrypoints.cli sync "$task" $extra_args 2>&1 | tee -a "$RUN_LOG" else - echo "$LOG_PREFIX ✗ Step $step_name FAIL (exit=$exit_code)" | tee -a "$RUN_LOG" - return $exit_code + .venv/bin/python -m app.entrypoints.cli sync "$task" 2>&1 | tee -a "$RUN_LOG" fi + local rc=${PIPESTATUS[0]} + if [ $rc -ne 0 ]; then + echo "$LOG_PREFIX ✗ 步骤 $step ($task) 失败 (exit=$rc)" | tee -a "$RUN_LOG" + return $rc + fi + echo "$LOG_PREFIX ✓ 步骤 $step ($task) 完成" | tee -a "$RUN_LOG" } case "${1:-all}" in 1|stock_basic) run_step "1_stock_basic" "stock_basic" ;; - 2|industry_sector) run_step "2_industry_sector" "industry_sector" ;; - 3|kline_index) run_step "3_kline_index" "kline_index" ;; - 4|kline_daily) run_step "4_kline_daily" "kline_daily" "--workers 20" ;; - 5|share_snapshot) run_step "5_share_snapshot" "share_snapshot" "--workers 10" ;; - 6|market_regime) run_step "6_market_regime" "market_regime" ;; - 7|kline_5min) run_step "7_kline_5min" "kline_5min" "--workers 10" ;; - 8|moneyflow) run_step "8_moneyflow" "moneyflow" "--workers 10" ;; - 9|sector_features) run_step "9_sector_features" "sector_features" ;; - 10|tick_trade) run_step "10_tick_trade" "tick_trade" "--workers 5 --force" ;; + 2|kline_index) run_step "2_kline_index" "kline_index" ;; + 3|kline_daily) run_step "3_kline_daily" "kline_daily" "--workers 20" ;; + 4|share_snapshot) run_step "4_share_snapshot" "share_snapshot" "--workers 10" ;; + 5|market_regime) run_step "5_market_regime" "market_regime" ;; + 6|kline_5min) run_step "6_kline_5min" "kline_5min" "--workers 10" ;; + 7|moneyflow) run_step "7_moneyflow" "moneyflow" "--workers 10" ;; + 8|tick_trade) run_step "8_tick_trade" "tick_trade" "--workers 5 --force" ;; all) run_step "1_stock_basic" "stock_basic" || exit 1 - run_step "2_industry_sector" "industry_sector" || exit 1 - run_step "3_kline_index" "kline_index" || exit 1 - run_step "4_kline_daily" "kline_daily" "--workers 20" || exit 1 - run_step "5_share_snapshot" "share_snapshot" "--workers 10" || exit 1 - run_step "6_market_regime" "market_regime" || exit 1 - run_step "7_kline_5min" "kline_5min" "--workers 10" || exit 1 - run_step "8_moneyflow" "moneyflow" "--workers 10" || exit 1 - run_step "9_sector_features" "sector_features" || exit 1 - run_step "10_tick_trade" "tick_trade" "--workers 5 --force" || exit 1 - echo "$LOG_PREFIX ✓ 全部 10 步完成" | tee -a "$RUN_LOG" + run_step "2_kline_index" "kline_index" || exit 1 + run_step "3_kline_daily" "kline_daily" "--workers 20" || exit 1 + run_step "4_share_snapshot" "share_snapshot" "--workers 10" || exit 1 + run_step "5_market_regime" "market_regime" || exit 1 + run_step "6_kline_5min" "kline_5min" "--workers 10" || exit 1 + run_step "7_moneyflow" "moneyflow" "--workers 10" || exit 1 + run_step "8_tick_trade" "tick_trade" "--workers 5 --force" || exit 1 + echo "$LOG_PREFIX ✓ 全部 8 步完成" | tee -a "$RUN_LOG" ;; *) - echo "usage: $0 [1|2|3|4|5|6|7|8|9|10|all]" + echo "usage: $0 [1|2|3|4|5|6|7|8|all]" exit 1 ;; esac diff --git a/tests/test_schema_models.py b/tests/test_schema_models.py index 394941b..8602119 100644 --- a/tests/test_schema_models.py +++ b/tests/test_schema_models.py @@ -7,24 +7,25 @@ from __future__ import annotations def test_orm_metadata_registers_all_tables(): - """ORMBase.metadata 应该注册 23 张 PG 表（项目主业务表）。""" + """ORMBase.metadata 应该注册 25 张 PG 表（项目主业务表）。""" from app.core.db import models # 触发全部模型 import from app.core.db.orm import ORMBase # 注意：SA 2.x 的 MetaData.tables 既能按 bare name 也能按 schema-qualified name 索引 # （keyed 容器），所以用 values() 拿 Table 对象再用 .name 拿 bare 名 bare_names = {t.name for t in ORMBase.metadata.tables.values()} - assert len(bare_names) == 23, f"期望 23 张 ORM 表，实际 {len(bare_names)}: {bare_names}" + assert len(bare_names) == 21, f"期望 21 张 ORM 表，实际 {len(bare_names)}: {bare_names}" expected = { "config", "dataset_registry", "stocks", "indices", "kline_stock", "kline_index", "kline_5min", "moneyflow", "share", - "sectors", "stock_sector_map", "industry", - "sector_indices", "sector_features_daily", "market_regime_daily", + "market_regime_daily", "tick_trade", "longhubang_daily", "longhubang_seat", # 2026-07-01 龙虎榜 "kline_stock_ma_daily", # 2026-07-04 mairui_ma_daily "node_categories", "nodes", "stock_node_map", # 2026-07-02 股票-节点映射 "sync_history", # 2026-07-07 同步历史表 + # 2026-07-08 mairui 技术指标 MACD/KDJ/BOLL + "kline_stock_macd_daily", "kline_stock_kdj_daily", "kline_stock_boll_daily", } assert bare_names == expected, f"ORM 表名集合与期望不符: {bare_names ^ expected}" diff --git a/tests/test_smoke.py b/tests/test_smoke.py index cb335fd..ffbbc0c 100644 --- a/tests/test_smoke.py +++ b/tests/test_smoke.py @@ -35,10 +35,10 @@ def test_tasks_registry(): from app.tasks import TASKS, get_task assert isinstance(TASKS, dict) # 注：每次新增 task 都要更新这里的数字 - # 当前 13 个：stock_basic, kline_daily, kline_index, kline_5min, tick_trade, - # moneyflow, industry_sector, sector_features, share_snapshot, market_regime, - # longhubang, mairui_ma_daily, stock_node - assert len(TASKS) == 13 + # 当前 12 个：stock_basic, kline_daily, kline_index, kline_5min, tick_trade, + # moneyflow, share_snapshot, market_regime, + # longhubang, mairui_ma_daily, stock_node, mairui_indicators + assert len(TASKS) == 12 # get_task 应该返回实例 task = get_task("kline_daily") assert task.dataset_id == "kline_daily" @@ -46,10 +46,10 @@ def test_tasks_registry(): def test_sync_definitions(): from app.core.sync.registry import SYNC_DEFINITIONS - assert len(SYNC_DEFINITIONS) == 13 + assert len(SYNC_DEFINITIONS) == 12 # 所有 dataset_id 应唯一 ids = [d["dataset_id"] for d in SYNC_DEFINITIONS] - assert len(set(ids)) == 13 + assert len(set(ids)) == 12 # tick_trade 应在其中且 sort_order=45 tt = next(d for d in SYNC_DEFINITIONS if d["dataset_id"] == "tick_trade") assert tt["sort_order"] == 45