chore: baseline — ORM migration + systemd deploy + MCP server + daily check
- ORM migration: models/ops 重构 - deploy: 标准化 systemd timer/service 部署体系 - MCP server: 5 个只读工具(任务/状态/数据集) - daily check: 数据一致性巡检 - watch: task_watch 后台监控 - kline_daily: 麦蕊优先,时间门禁15:00 - 删除: task_industry_sector, task_sector_features
This commit is contained in:
@@ -0,0 +1,65 @@
|
||||
# syntax=docker/dockerfile:1.6
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# market_sync 数据同步服务
|
||||
# 单镜像同时跑:FastAPI dashboard + 进程内 scheduler(worker)
|
||||
# 部署:docker compose up -d
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
|
||||
# --- builder stage: 装依赖到 venv ---
|
||||
FROM python:3.11-slim AS builder
|
||||
|
||||
ENV PYTHONUNBUFFERED=1 \
|
||||
PYTHONDONTWRITEBYTECODE=1 \
|
||||
PIP_NO_CACHE_DIR=1 \
|
||||
PIP_DISABLE_PIP_VERSION_CHECK=1
|
||||
|
||||
# 仅装编译期需要的系统包(很多 wheel 已预编译,但 psycopg2 / cryptography 可能要)
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
build-essential gcc libpq-dev \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# 先 copy requirements 单独一层(利用 Docker 缓存:依赖不变就不重装)
|
||||
COPY requirements.txt .
|
||||
RUN python -m venv /app/.venv \
|
||||
&& /app/.venv/bin/pip install --upgrade pip \
|
||||
&& /app/.venv/bin/pip install -r requirements.txt
|
||||
|
||||
|
||||
# --- runtime stage: 极简基础镜像 + 仅复制 venv 与代码 ---
|
||||
FROM python:3.11-slim AS runtime
|
||||
|
||||
ENV PYTHONUNBUFFERED=1 \
|
||||
PYTHONDONTWRITEBYTECODE=1 \
|
||||
PATH="/app/.venv/bin:${PATH}" \
|
||||
PYTHONPATH=/app \
|
||||
# service 模式默认监听
|
||||
API_HOST=0.0.0.0 \
|
||||
API_PORT=8100 \
|
||||
# Docker 模式:单容器内使用进程内 scheduler
|
||||
RUNTIME_MODE=docker
|
||||
|
||||
# runtime 只需要 psycopg2 的运行时库 + tzdata(pandas tz aware 需要)
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
libpq5 tzdata curl \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# 从 builder 复制 venv(已编译好的依赖)
|
||||
COPY --from=builder /app/.venv /app/.venv
|
||||
|
||||
# 复制应用代码
|
||||
COPY app/ ./app/
|
||||
COPY bin/ ./bin/
|
||||
|
||||
# 默认启动 service 入口(FastAPI + 进程内 scheduler)
|
||||
# 也可覆盖为 cli / worker 单跑某个 task:
|
||||
# docker compose run --rm market_sync python -m app.entrypoints.cli sync kline_daily
|
||||
EXPOSE 8100
|
||||
|
||||
HEALTHCHECK --interval=60s --timeout=5s --start-period=30s --retries=3 \
|
||||
CMD curl -fsS http://localhost:8100/api/health || exit 1
|
||||
|
||||
ENTRYPOINT ["/app/bin/service_run.sh"]
|
||||
@@ -0,0 +1,86 @@
|
||||
# docker-compose.yml — 一键起 market_sync 数据同步服务
|
||||
#
|
||||
# 包含:
|
||||
# - postgres : PostgreSQL 16(market_data schema 的主库)
|
||||
# - market_sync : 数据同步 + dashboard + scheduler(单进程)
|
||||
#
|
||||
# 用法:
|
||||
# docker compose up -d # 后台启动
|
||||
# docker compose logs -f market_sync # 看同步日志
|
||||
# docker compose exec market_sync \
|
||||
# python -m app.entrypoints.cli sync stock_node --type2 2,3
|
||||
#
|
||||
version: "3.9"
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16-alpine
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
POSTGRES_USER: market_sync
|
||||
POSTGRES_PASSWORD: market_sync
|
||||
POSTGRES_DB: market_data
|
||||
ports:
|
||||
- "5432:5432"
|
||||
volumes:
|
||||
- pgdata:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U market_sync -d market_data"]
|
||||
interval: 10s
|
||||
timeout: 3s
|
||||
retries: 5
|
||||
|
||||
market_sync:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
image: market_sync:latest
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
# 运行模式:Docker 模式使用进程内 scheduler
|
||||
RUNTIME_MODE: docker
|
||||
# PG 连接(指向同 compose 的 postgres 服务)
|
||||
PG_HOST: postgres
|
||||
PG_PORT: "5432"
|
||||
PG_USER: market_sync
|
||||
PG_PASSWORD: market_sync
|
||||
PG_DB_NAME: market_data
|
||||
DB_BACKEND: pg
|
||||
# 调度
|
||||
SCHEDULER_TIMEZONE: Asia/Shanghai
|
||||
SCHEDULER_TICK_SECONDS: "5"
|
||||
SCHEDULER_AUTO_SEED: "true"
|
||||
# API
|
||||
API_HOST: 0.0.0.0
|
||||
API_PORT: "8100"
|
||||
# 日志
|
||||
LOG_DIR: /app/logs
|
||||
LOG_LEVEL: INFO
|
||||
# 数据源开关
|
||||
DS_BAOSTOCK_ENABLED: "true"
|
||||
DS_SINA_ENABLED: "true"
|
||||
# 凭证(可放进 .env / secrets 文件,避免明文)
|
||||
MAIRUI_LICENCE: ${MAIRUI_LICENCE:-}
|
||||
XUEQIU_TOKEN: ${XUEQIU_TOKEN:-}
|
||||
# 麦蕊限速(钻石 100 RPS 留 buffer;服务化后保守点)
|
||||
MAIRUI_RPS_LIMIT: "10"
|
||||
# 节假日
|
||||
TRADING_HOLIDAYS: ${TRADING_HOLIDAYS:-}
|
||||
ports:
|
||||
- "8100:8100"
|
||||
volumes:
|
||||
# 持久化日志(任务失败时方便排查)
|
||||
- sync_logs:/app/logs
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-fsS", "http://localhost:8100/api/health"]
|
||||
interval: 60s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
start_period: 30s
|
||||
|
||||
volumes:
|
||||
pgdata:
|
||||
sync_logs:
|
||||
Executable
+211
@@ -0,0 +1,211 @@
|
||||
#!/bin/bash
|
||||
# market_sync systemd 一键部署脚本
|
||||
#
|
||||
# 用途:把 deploy/systemd/units/*.service 和 *.timer 同步到 /etc/systemd/system/,
|
||||
# 并自动启用 timer、处理 worker 服务状态。
|
||||
#
|
||||
# 运行:sudo bash deploy/systemd/deploy.sh
|
||||
# sudo bash deploy/systemd/deploy.sh --yes # 跳过确认
|
||||
#
|
||||
# 设计原则:
|
||||
# - 必须 sudo(写 /etc/systemd/system/ + daemon-reload)
|
||||
# - 幂等:重复运行无副作用
|
||||
# - 显示 plan → 等待确认 → 执行
|
||||
# - 失败立即中止,不留半套状态
|
||||
|
||||
set -e
|
||||
|
||||
PROJECT_ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||
REPO_UNITS_DIR="$PROJECT_ROOT/deploy/systemd/units"
|
||||
ETC_UNITS_DIR="/etc/systemd/system"
|
||||
SKIP_CONFIRM=false
|
||||
|
||||
# ── 0. 前置检查 ─────────────────────────────────────────────────────────────
|
||||
if [ "$(id -u)" -ne 0 ]; then
|
||||
echo "❌ 必须 sudo 运行:sudo bash $0"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "${1:-}" = "--yes" ]; then
|
||||
SKIP_CONFIRM=true
|
||||
fi
|
||||
|
||||
if [ ! -d "$REPO_UNITS_DIR" ]; then
|
||||
echo "❌ repo unit 目录不存在: $REPO_UNITS_DIR"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ── 1. 计算 plan ────────────────────────────────────────────────────────────
|
||||
declare -a TO_COPY=() # repo→etc 拷贝
|
||||
declare -a TO_REMOVE=() # etc 中孤儿(repo 已删)
|
||||
declare -a TO_RESTART_SVC=() # service 文件变了需要 daemon-reload + restart
|
||||
|
||||
# repo 中所有 market-sync* unit
|
||||
mapfile -t REPO_UNITS < <(find "$REPO_UNITS_DIR" -maxdepth 1 -type f \( -name "*.service" -o -name "*.timer" \) -printf '%f\n' | sort)
|
||||
# /etc 中所有 market-sync* unit
|
||||
mapfile -t ETC_UNITS < <(find "$ETC_UNITS_DIR" -maxdepth 1 -type f \( -name "market-sync*.service" -o -name "market-sync*.timer" \) -printf '%f\n' | sort 2>/dev/null || true)
|
||||
|
||||
# drift 1: repo 有 /etc 没装
|
||||
for u in "${REPO_UNITS[@]}"; do
|
||||
if [ ! -f "$ETC_UNITS_DIR/$u" ]; then
|
||||
TO_COPY+=("$u")
|
||||
fi
|
||||
done
|
||||
|
||||
# drift 2: 两边都有但内容不一致
|
||||
for u in "${REPO_UNITS[@]}"; do
|
||||
if [ -f "$ETC_UNITS_DIR/$u" ]; then
|
||||
if ! diff -q "$REPO_UNITS_DIR/$u" "$ETC_UNITS_DIR/$u" > /dev/null 2>&1; then
|
||||
TO_COPY+=("$u")
|
||||
if [[ "$u" == *.service ]]; then
|
||||
TO_RESTART_SVC+=("${u%.service}")
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
# drift 3: /etc 有 repo 没维护(孤儿)
|
||||
for u in "${ETC_UNITS[@]}"; do
|
||||
found=0
|
||||
for r in "${REPO_UNITS[@]}"; do
|
||||
if [ "$r" = "$u" ]; then found=1; break; fi
|
||||
done
|
||||
if [ $found -eq 0 ]; then
|
||||
TO_REMOVE+=("$u")
|
||||
fi
|
||||
done
|
||||
|
||||
# ── 2. 显示 plan ────────────────────────────────────────────────────────────
|
||||
echo "============================================================"
|
||||
echo "market_sync systemd deploy plan"
|
||||
echo "============================================================"
|
||||
echo "Repo units dir: $REPO_UNITS_DIR"
|
||||
echo
|
||||
echo "Repo units: ${#REPO_UNITS[@]} 个"
|
||||
echo "/etc units: ${#ETC_UNITS[@]} 个"
|
||||
echo
|
||||
|
||||
if [ ${#TO_COPY[@]} -eq 0 ] && [ ${#TO_REMOVE[@]} -eq 0 ]; then
|
||||
echo "✅ 无 drift 需要修复。"
|
||||
else
|
||||
if [ ${#TO_COPY[@]} -gt 0 ]; then
|
||||
echo "📋 将要拷贝 (${#TO_COPY[@]} 个):"
|
||||
for u in "${TO_COPY[@]}"; do
|
||||
echo " cp deploy/systemd/units/$u → /etc/systemd/system/$u"
|
||||
done
|
||||
echo
|
||||
fi
|
||||
|
||||
if [ ${#TO_RESTART_SVC[@]} -gt 0 ]; then
|
||||
echo "🔄 将要处理 service (${#TO_RESTART_SVC[@]} 个):"
|
||||
for s in "${TO_RESTART_SVC[@]}"; do
|
||||
echo " systemctl reset-failed $s.service"
|
||||
done
|
||||
echo
|
||||
fi
|
||||
|
||||
if [ ${#TO_REMOVE[@]} -gt 0 ]; then
|
||||
echo "🗑️ 将要删除 etc 中的孤儿 (${#TO_REMOVE[@]} 个):"
|
||||
for u in "${TO_REMOVE[@]}"; do
|
||||
echo " rm /etc/systemd/system/$u"
|
||||
done
|
||||
echo
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── 3. 等待确认 ────────────────────────────────────────────────────────────
|
||||
if [ "$SKIP_CONFIRM" = false ]; then
|
||||
read -p "确认执行?[y/N] " -n 1 -r
|
||||
echo
|
||||
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
|
||||
echo "已取消。"
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── 4. 执行 ─────────────────────────────────────────────────────────────────
|
||||
echo
|
||||
echo "=== 执行 ==="
|
||||
|
||||
# 4a. cp 漂移的 unit
|
||||
for u in "${TO_COPY[@]}"; do
|
||||
cp -v "$REPO_UNITS_DIR/$u" "$ETC_UNITS_DIR/$u"
|
||||
done
|
||||
|
||||
# 4b. 删孤儿
|
||||
for u in "${TO_REMOVE[@]}"; do
|
||||
rm -v "$ETC_UNITS_DIR/$u"
|
||||
done
|
||||
|
||||
# 4c. reload systemd
|
||||
echo
|
||||
echo "=== systemctl daemon-reload ==="
|
||||
systemctl daemon-reload
|
||||
|
||||
# 4d. 停用并 mask worker 服务(systemd 模式下不需要进程内 scheduler)
|
||||
if systemctl is-enabled market-sync-worker.service >/dev/null 2>&1 || \
|
||||
systemctl is-active market-sync-worker.service >/dev/null 2>&1; then
|
||||
echo "=== 关闭 market-sync-worker.service(systemd 模式下由 timer 触发任务)==="
|
||||
systemctl stop market-sync-worker.service || true
|
||||
systemctl disable market-sync-worker.service || true
|
||||
systemctl mask market-sync-worker.service || true
|
||||
fi
|
||||
|
||||
# 4e. reset-failed + restart 修改过的 service
|
||||
for s in "${TO_RESTART_SVC[@]}"; do
|
||||
if systemctl is-enabled "${s}.service" 2>/dev/null | grep -q enabled; then
|
||||
systemctl reset-failed "${s}.service" || true
|
||||
echo " reset-failed ${s}.service (由 timer 自然触发)"
|
||||
fi
|
||||
done
|
||||
|
||||
# 4f. 启用所有 timer(幂等)
|
||||
echo
|
||||
echo "=== 启用 timers ==="
|
||||
for u in "${REPO_UNITS[@]}"; do
|
||||
if [[ "$u" == *.timer ]]; then
|
||||
timer="${u%.timer}"
|
||||
if ! systemctl is-enabled "${timer}.timer" >/dev/null 2>&1; then
|
||||
systemctl enable "${timer}.timer"
|
||||
echo " enable ${timer}.timer"
|
||||
else
|
||||
echo " ${timer}.timer 已启用"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
# ── 5. 验证 ────────────────────────────────────────────────────────────────
|
||||
echo
|
||||
echo "=== 验证 ==="
|
||||
echo "Repo units: ${#REPO_UNITS[@]}"
|
||||
echo "/etc units: $(find "$ETC_UNITS_DIR" -maxdepth 1 -type f \( -name 'market-sync*.service' -o -name 'market-sync*.timer' \) | wc -l)"
|
||||
|
||||
drift_count=0
|
||||
for u in "${REPO_UNITS[@]}"; do
|
||||
if [ -f "$ETC_UNITS_DIR/$u" ]; then
|
||||
if ! diff -q "$REPO_UNITS_DIR/$u" "$ETC_UNITS_DIR/$u" > /dev/null 2>&1; then
|
||||
echo " ❌ $u 仍有 drift"
|
||||
drift_count=$((drift_count + 1))
|
||||
fi
|
||||
else
|
||||
echo " ❌ $u 仍未部署"
|
||||
drift_count=$((drift_count + 1))
|
||||
fi
|
||||
done
|
||||
|
||||
# 验证 worker 已被 mask
|
||||
if systemctl is-enabled market-sync-worker.service 2>/dev/null | grep -q masked; then
|
||||
echo " ✅ market-sync-worker.service 已 mask(systemd 模式下不启用进程内 scheduler)"
|
||||
else
|
||||
echo " ⚠️ market-sync-worker.service 未被 mask,systemd 模式下可能双调度器重叠"
|
||||
fi
|
||||
|
||||
echo
|
||||
if [ $drift_count -eq 0 ]; then
|
||||
echo "✅ 所有 unit 已对齐,0 drift"
|
||||
echo
|
||||
echo "下一步:跑 .venv/bin/python bin/daily_sync_check.py --report-only 验证"
|
||||
else
|
||||
echo "❌ 还有 $drift_count 处 drift,请检查"
|
||||
exit 1
|
||||
fi
|
||||
@@ -0,0 +1,28 @@
|
||||
#!/bin/bash
|
||||
# ── 停止旧 market-sync ──
|
||||
sudo systemctl stop market-sync.timer
|
||||
sudo systemctl disable market-sync.timer
|
||||
sudo systemctl stop market-sync.service
|
||||
sudo systemctl disable market-sync.service
|
||||
|
||||
# ── 安装新独立 timer+service (4组) ──
|
||||
cd /home/gao/Development/quant_home/market_sync/deploy/systemd/units
|
||||
|
||||
for f in market-sync-kline-index market-sync-kline-daily market-sync-kline-5min market-sync-market-regime; do
|
||||
sudo cp ${f}.service /etc/systemd/system/
|
||||
sudo cp ${f}.timer /etc/systemd/system/
|
||||
done
|
||||
|
||||
# ── 更新 morning service:去掉已废弃的 industry_sector ──
|
||||
# (只用 morning 跑 stock_basic,不再跑 industry_sector)
|
||||
# 修改 morning run 脚本只跑 stock_basic
|
||||
sudo systemctl daemon-reload
|
||||
|
||||
# ── 启动新 timer ──
|
||||
for f in market-sync-kline-index market-sync-kline-daily market-sync-kline-5min market-sync-market-regime; do
|
||||
sudo systemctl enable ${f}.timer
|
||||
sudo systemctl start ${f}.timer
|
||||
done
|
||||
|
||||
echo "=== 完成 ==="
|
||||
systemctl list-timers --all 2>/dev/null | grep market-sync
|
||||
Executable
+59
@@ -0,0 +1,59 @@
|
||||
#!/bin/bash
|
||||
# 部署 market-sync systemd units 到 /etc/systemd/system/
|
||||
# 2026-07-09 新增: market-sync-worker.service(长驻进程, 调度 schedule_* 任务)
|
||||
#
|
||||
# 用法: sudo bash bin/deploy_systemd.sh
|
||||
#
|
||||
# 跑这个脚本会:
|
||||
# 1. 复制 bin/systemd/market-sync*.{service,timer} 到 /etc/systemd/system/
|
||||
# 2. systemctl daemon-reload
|
||||
# 3. enable + start market-sync-worker.service
|
||||
# 4. 不动已存在的 .timer (它们都已 enable, 重复 enable 无副作用)
|
||||
set -euo pipefail
|
||||
|
||||
REPO_DIR="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
SYSTEMD_SRC="$REPO_DIR/bin/systemd"
|
||||
SYSTEMD_DST="/etc/systemd/system"
|
||||
|
||||
if [ "$(id -u)" -ne 0 ]; then
|
||||
echo "❌ 必须用 sudo 跑: sudo bash $0" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "=== 1. 复制 unit files ==="
|
||||
for f in "$SYSTEMD_SRC"/market-sync*.{service,timer}; do
|
||||
[ -f "$f" ] || continue
|
||||
name="$(basename "$f")"
|
||||
cp -v "$f" "$SYSTEMD_DST/$name"
|
||||
chmod 644 "$SYSTEMD_DST/$name"
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "=== 2. daemon-reload ==="
|
||||
systemctl daemon-reload
|
||||
|
||||
echo ""
|
||||
echo "=== 3. 启用 + 启动 worker 长驻进程(关键!) ==="
|
||||
if ! systemctl is-enabled --quiet market-sync-worker.service; then
|
||||
systemctl enable market-sync-worker.service
|
||||
fi
|
||||
# 如果已通过 nohup 启动了, 先 kill 掉避免双跑
|
||||
if pgrep -f "app\.entrypoints\.worker" >/dev/null; then
|
||||
echo " ⚠️ 检测到手动启动的 worker 进程, 先 kill"
|
||||
pkill -TERM -f "app\.entrypoints\.worker" || true
|
||||
sleep 2
|
||||
pkill -KILL -f "app\.entrypoints\.worker" 2>/dev/null || true
|
||||
fi
|
||||
systemctl restart market-sync-worker.service
|
||||
sleep 2
|
||||
systemctl status market-sync-worker.service --no-pager | head -10
|
||||
|
||||
echo ""
|
||||
echo "=== 4. 列出所有 market-sync timer (验证) ==="
|
||||
systemctl list-timers market-sync* --no-pager 2>&1 | head -20
|
||||
|
||||
echo ""
|
||||
echo "✅ 部署完成"
|
||||
echo " - 长驻 worker: systemctl status market-sync-worker.service"
|
||||
echo " - 看任务执行: tail -f logs/worker_manual_*.log (本进程) 或 journalctl -u market-sync-worker -f (systemd 接管后)"
|
||||
echo " - 巡检: .venv/bin/python bin/daily_sync_check.py"
|
||||
Executable
+185
@@ -0,0 +1,185 @@
|
||||
#!/bin/bash
|
||||
# market_sync systemd 一键部署脚本
|
||||
#
|
||||
# 用途:把 bin/systemd/*.service 和 *.timer 同步到 /etc/systemd/system/,
|
||||
# 解决 work #04 检测到的 5 处 drift。
|
||||
#
|
||||
# 运行:sudo bash bin/systemd_deploy.sh
|
||||
#
|
||||
# 设计原则:
|
||||
# - 必须 sudo(写 /etc/systemd/system/ + daemon-reload)
|
||||
# - 幂等:重复运行无副作用
|
||||
# - 显示 plan → 等待确认 → 执行
|
||||
# - 失败立即中止,不留半套状态
|
||||
|
||||
set -e
|
||||
|
||||
PROJECT_ROOT="/home/gao/Development/quant_home/market_sync"
|
||||
REPO_UNITS_DIR="$PROJECT_ROOT/bin/systemd"
|
||||
ETC_UNITS_DIR="/etc/systemd/system"
|
||||
|
||||
# ── 0. 前置检查 ─────────────────────────────────────────────────────────────
|
||||
if [ "$(id -u)" -ne 0 ]; then
|
||||
echo "❌ 必须 sudo 运行:sudo bash $0"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -d "$REPO_UNITS_DIR" ]; then
|
||||
echo "❌ repo unit 目录不存在: $REPO_UNITS_DIR"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ── 1. 计算 plan ────────────────────────────────────────────────────────────
|
||||
declare -a TO_COPY=() # repo→etc 拷贝
|
||||
declare -a TO_REMOVE=() # etc 中孤儿(repo 已删)
|
||||
declare -a TO_RESTART_SVC=() # service 文件变了需要 daemon-reload + restart
|
||||
|
||||
# repo 中所有 market-sync* unit
|
||||
mapfile -t REPO_UNITS < <(find "$REPO_UNITS_DIR" -maxdepth 1 -type f \( -name "*.service" -o -name "*.timer" \) -printf '%f\n' | sort)
|
||||
# /etc 中所有 market-sync* unit
|
||||
mapfile -t ETC_UNITS < <(find "$ETC_UNITS_DIR" -maxdepth 1 -type f \( -name "market-sync*.service" -o -name "market-sync*.timer" \) -printf '%f\n' | sort 2>/dev/null || true)
|
||||
|
||||
# drift 1: repo 有 /etc 没装
|
||||
for u in "${REPO_UNITS[@]}"; do
|
||||
if [ ! -f "$ETC_UNITS_DIR/$u" ]; then
|
||||
TO_COPY+=("$u")
|
||||
fi
|
||||
done
|
||||
|
||||
# drift 2: 两边都有但内容不一致
|
||||
for u in "${REPO_UNITS[@]}"; do
|
||||
if [ -f "$ETC_UNITS_DIR/$u" ]; then
|
||||
if ! diff -q "$REPO_UNITS_DIR/$u" "$ETC_UNITS_DIR/$u" > /dev/null 2>&1; then
|
||||
TO_COPY+=("$u")
|
||||
if [[ "$u" == *.service ]]; then
|
||||
TO_RESTART_SVC+=("${u%.service}")
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
# drift 3: /etc 有 repo 没维护(孤儿)
|
||||
for u in "${ETC_UNITS[@]}"; do
|
||||
found=0
|
||||
for r in "${REPO_UNITS[@]}"; do
|
||||
if [ "$r" = "$u" ]; then found=1; break; fi
|
||||
done
|
||||
if [ $found -eq 0 ]; then
|
||||
TO_REMOVE+=("$u")
|
||||
fi
|
||||
done
|
||||
|
||||
# ── 2. 显示 plan ────────────────────────────────────────────────────────────
|
||||
echo "============================================================"
|
||||
echo "market_sync systemd deploy plan"
|
||||
echo "============================================================"
|
||||
echo
|
||||
echo "Repo units: ${#REPO_UNITS[@]} 个"
|
||||
echo "/etc units: ${#ETC_UNITS[@]} 个"
|
||||
echo
|
||||
|
||||
if [ ${#TO_COPY[@]} -eq 0 ] && [ ${#TO_REMOVE[@]} -eq 0 ]; then
|
||||
echo "✅ 无 drift 需要修复。无需执行。"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ ${#TO_COPY[@]} -gt 0 ]; then
|
||||
echo "📋 将要拷贝 ($(echo "${TO_COPY[@]}" | tr ' ' '\n' | wc -l) 个):"
|
||||
for u in "${TO_COPY[@]}"; do
|
||||
echo " cp bin/systemd/$u → /etc/systemd/system/$u"
|
||||
done
|
||||
echo
|
||||
fi
|
||||
|
||||
if [ ${#TO_RESTART_SVC[@]} -gt 0 ]; then
|
||||
echo "🔄 将要重启 service ($(echo "${TO_RESTART_SVC[@]}" | tr ' ' '\n' | wc -l) 个):"
|
||||
for s in "${TO_RESTART_SVC[@]}"; do
|
||||
echo " systemctl reset-failed $s.service"
|
||||
done
|
||||
echo
|
||||
fi
|
||||
|
||||
if [ ${#TO_REMOVE[@]} -gt 0 ]; then
|
||||
echo "🗑️ 将要删除 etc 中的孤儿 ($(echo "${TO_REMOVE[@]}" | tr ' ' '\n' | wc -l) 个):"
|
||||
for u in "${TO_REMOVE[@]}"; do
|
||||
echo " rm /etc/systemd/system/$u"
|
||||
done
|
||||
echo
|
||||
fi
|
||||
|
||||
# ── 3. 等待确认 ────────────────────────────────────────────────────────────
|
||||
read -p "确认执行?[y/N] " -n 1 -r
|
||||
echo
|
||||
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
|
||||
echo "已取消。"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ── 4. 执行 ─────────────────────────────────────────────────────────────────
|
||||
echo
|
||||
echo "=== 执行 ==="
|
||||
|
||||
# 4a. cp 漂移的 unit
|
||||
for u in "${TO_COPY[@]}"; do
|
||||
cp -v "$REPO_UNITS_DIR/$u" "$ETC_UNITS_DIR/$u"
|
||||
done
|
||||
|
||||
# 4b. 删孤儿
|
||||
for u in "${TO_REMOVE[@]}"; do
|
||||
rm -v "$ETC_UNITS_DIR/$u"
|
||||
done
|
||||
|
||||
# 4c. reload systemd
|
||||
echo
|
||||
echo "=== systemctl daemon-reload ==="
|
||||
systemctl daemon-reload
|
||||
|
||||
# 4d. reset-failed + restart 修改过的 service
|
||||
for s in "${TO_RESTART_SVC[@]}"; do
|
||||
if systemctl is-enabled "${s}.service" 2>/dev/null | grep -q enabled; then
|
||||
systemctl reset-failed "${s}.service" || true
|
||||
echo " reset-failed ${s}.service (没自动 restart — 由 timer 自然触发)"
|
||||
fi
|
||||
done
|
||||
|
||||
# 4e. 启用新 timer(如果有)
|
||||
for u in "${TO_COPY[@]}"; do
|
||||
if [[ "$u" == *.timer ]]; then
|
||||
timer="${u%.timer}"
|
||||
if systemctl list-unit-files "${timer}.timer" 2>/dev/null | grep -q "${timer}.timer"; then
|
||||
if ! systemctl is-enabled "${timer}.timer" 2>/dev/null | grep -q enabled; then
|
||||
systemctl enable "${timer}.timer"
|
||||
echo " enable ${timer}.timer"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
# ── 5. 验证 ────────────────────────────────────────────────────────────────
|
||||
echo
|
||||
echo "=== 验证 ==="
|
||||
echo "Repo units: ${#REPO_UNITS[@]}"
|
||||
echo "/etc units: $(find "$ETC_UNITS_DIR" -maxdepth 1 -type f -name 'market-sync*.service' -o -name 'market-sync*.timer' | wc -l)"
|
||||
|
||||
drift_count=0
|
||||
for u in "${REPO_UNITS[@]}"; do
|
||||
if [ -f "$ETC_UNITS_DIR/$u" ]; then
|
||||
if ! diff -q "$REPO_UNITS_DIR/$u" "$ETC_UNITS_DIR/$u" > /dev/null 2>&1; then
|
||||
echo " ❌ $u 仍有 drift"
|
||||
drift_count=$((drift_count + 1))
|
||||
fi
|
||||
else
|
||||
echo " ❌ $u 仍未部署"
|
||||
drift_count=$((drift_count + 1))
|
||||
fi
|
||||
done
|
||||
|
||||
echo
|
||||
if [ $drift_count -eq 0 ]; then
|
||||
echo "✅ 所有 unit 已对齐,0 drift"
|
||||
echo
|
||||
echo "下一步:跑 .venv/bin/python bin/daily_sync_check.py --report-only 验证"
|
||||
else
|
||||
echo "❌ 还有 $drift_count 处 drift,请检查"
|
||||
exit 1
|
||||
fi
|
||||
@@ -0,0 +1,31 @@
|
||||
[Unit]
|
||||
Description=Market sync daily check — reads dataset_registry, writes report, alerts on failures
|
||||
Documentation=file:///home/gao/Development/quant_home/market_sync/bin/daily_sync_check.py
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
WorkingDirectory=/home/gao/Development/quant_home/market_sync
|
||||
User=gao
|
||||
Group=gao
|
||||
|
||||
# 巡检:读 PG dataset_registry,对比今日预期,写 logs/daily_check_*.json
|
||||
# 失败时 POST webhook 到 FAILURE_WEBHOOK_URL(HMAC-SHA256 签名)
|
||||
ExecStart=/home/gao/Development/quant_home/market_sync/.venv/bin/python /home/gao/Development/quant_home/market_sync/bin/daily_sync_check.py
|
||||
|
||||
# 巡检应 < 30s 完成;超时 = PG 不通或代码 bug,不让卡死
|
||||
TimeoutStartSec=120
|
||||
|
||||
# 不要 Restart=(oneshot 失败就让 OnFailure= 发通知,别自动重跑)
|
||||
# 失败时 exit code = 1,可被 OnFailure= 钩住发通知
|
||||
|
||||
# 日志走 journald(journalctl -u market-sync-daily-check -f)
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=market-sync-daily-check
|
||||
|
||||
Environment=PYTHONUNBUFFERED=1
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,15 @@
|
||||
[Unit]
|
||||
Description=Schedule market sync daily check — Daily 23:00 Asia/Shanghai
|
||||
# market-sync-daily-check.service 是这个 timer 的执行单元
|
||||
# 23:00 触发:周一~五最后一个任务是 longhubang @ 22:00,留 1h buffer;
|
||||
# 周六 23:00 给 stock_node @ 11:30 留 11h+ buffer
|
||||
|
||||
[Timer]
|
||||
# 每天 23:00 触发;Persistent=true 确保关机/错过时下次开机补跑
|
||||
OnCalendar=*-*-* 23:00:00 Asia/Shanghai
|
||||
Persistent=true
|
||||
Unit=market-sync-daily-check.service
|
||||
# 不要 AccuracySec(默认 1min 漂移够用)
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -0,0 +1,20 @@
|
||||
[Unit]
|
||||
Description=Market sync kline_5min — 5分钟K (评分核心)
|
||||
Documentation=file:///home/gao/Development/quant_home/market_sync/bin/market_sync_kline_5min_run.sh
|
||||
After=network-online.target market-sync-kline-daily.service
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
WorkingDirectory=/home/gao/Development/quant_home/market_sync
|
||||
User=gao
|
||||
Group=gao
|
||||
ExecStart=/home/gao/Development/quant_home/market_sync/bin/market_sync_kline_5min_run.sh
|
||||
TimeoutStartSec=7200
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=market-sync-kline-5min
|
||||
Environment=PYTHONUNBUFFERED=1
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,10 @@
|
||||
[Unit]
|
||||
Description=Schedule kline_5min sync — Mon..Fri 16:00 Asia/Shanghai
|
||||
|
||||
[Timer]
|
||||
OnCalendar=Mon..Fri 16:00:00 Asia/Shanghai
|
||||
Persistent=true
|
||||
Unit=market-sync-kline-5min.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -0,0 +1,11 @@
|
||||
[Unit]
|
||||
Description=Market sync kline_daily — 15:10 麦蕊优先抢跑
|
||||
Documentation=file:///home/gao/Development/quant_home/market_sync/bin/market_sync_kline_daily_run.sh
|
||||
|
||||
[Timer]
|
||||
OnCalendar=Mon..Fri 15:10:00
|
||||
Persistent=false
|
||||
RandomizedDelaySec=0
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -0,0 +1,20 @@
|
||||
[Unit]
|
||||
Description=Market sync kline_daily — 个股日K (评分核心)
|
||||
Documentation=file:///home/gao/Development/quant_home/market_sync/bin/market_sync_kline_daily_run.sh
|
||||
After=network-online.target market-sync-kline-index.service
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
WorkingDirectory=/home/gao/Development/quant_home/market_sync
|
||||
User=gao
|
||||
Group=gao
|
||||
ExecStart=/home/gao/Development/quant_home/market_sync/bin/market_sync_kline_daily_run.sh
|
||||
TimeoutStartSec=3600
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=market-sync-kline-daily
|
||||
Environment=PYTHONUNBUFFERED=1
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,10 @@
|
||||
[Unit]
|
||||
Description=Schedule kline_daily sync — Mon..Fri 15:40 Asia/Shanghai
|
||||
|
||||
[Timer]
|
||||
OnCalendar=Mon..Fri 15:40:00 Asia/Shanghai
|
||||
Persistent=true
|
||||
Unit=market-sync-kline-daily.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -0,0 +1,20 @@
|
||||
[Unit]
|
||||
Description=Market sync kline_index — 指数日K (独立, 非评分依赖)
|
||||
Documentation=file:///home/gao/Development/quant_home/market_sync/bin/market_sync_kline_index_run.sh
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
WorkingDirectory=/home/gao/Development/quant_home/market_sync
|
||||
User=gao
|
||||
Group=gao
|
||||
ExecStart=/home/gao/Development/quant_home/market_sync/bin/market_sync_kline_index_run.sh
|
||||
TimeoutStartSec=600
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=market-sync-kline-index
|
||||
Environment=PYTHONUNBUFFERED=1
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,10 @@
|
||||
[Unit]
|
||||
Description=Schedule kline_index sync — Mon..Fri 15:30 Asia/Shanghai
|
||||
|
||||
[Timer]
|
||||
OnCalendar=Mon..Fri 15:30:00 Asia/Shanghai
|
||||
Persistent=true
|
||||
Unit=market-sync-kline-index.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -0,0 +1,33 @@
|
||||
[Unit]
|
||||
Description=Market data 龙虎榜 sync (akshare 源, daily 22:00)
|
||||
Documentation=file:///home/gao/Development/quant_home/market_sync/bin/market_sync_lhb_run.sh
|
||||
After=network-online.target market-sync.service
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
WorkingDirectory=/home/gao/Development/quant_home/market_sync
|
||||
User=gao
|
||||
Group=gao
|
||||
|
||||
# 入口脚本:单独跑 longhubang 一个 task
|
||||
# 龙虎榜数据 19:00~21:00 陆续出齐 → 22:00 触发(与 moneyflow 21:35 错开避免并发)
|
||||
ExecStart=/home/gao/Development/quant_home/market_sync/bin/market_sync_lhb_run.sh
|
||||
|
||||
# 硬上限 1h(每日 ~200 次 akshare 调用 + DB upsert,3-5 分钟内完成,留 buffer)
|
||||
TimeoutStartSec=3600
|
||||
|
||||
# 不要 Restart=(oneshot 失败就让 OnFailure= 发通知,别自动重跑——
|
||||
# 重跑会撞外部 API 限流窗口,浪费资源)
|
||||
# Restart=no 是 oneshot 默认值
|
||||
|
||||
# 日志走 journald(journalctl -u market-sync-lhb.service -f)
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=market-sync-lhb
|
||||
|
||||
# 环境(不读 /etc/environment,只带这几个;.env 由 cli 内部 load_dotenv)
|
||||
Environment=PYTHONUNBUFFERED=1
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,16 @@
|
||||
[Unit]
|
||||
Description=Schedule 龙虎榜 sync — Mon..Fri 22:00 Asia/Shanghai
|
||||
# market-sync-lhb.service 是这个 timer 的执行单元
|
||||
# 龙虎榜数据 19:00~21:00 陆续出齐 → 22:00 触发(与 moneyflow 21:35 错开)
|
||||
|
||||
[Timer]
|
||||
# 龙虎榜出齐 → 22:00 触发
|
||||
OnCalendar=Mon..Fri 22:00:00 Asia/Shanghai
|
||||
# 系统关机 / 错过执行时,下次开机补跑一次(避免漏数据)
|
||||
Persistent=true
|
||||
# 单位(service)的精确名称
|
||||
Unit=market-sync-lhb.service
|
||||
# 不要 AccuracySec(默认 1min 漂移够用,避免 22:00:00 整点打堆)
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -0,0 +1,32 @@
|
||||
[Unit]
|
||||
Description=Market data MACD/KDJ/BOLL indicators sync (mairui, daily 16:40)
|
||||
Documentation=file:///home/gao/Development/quant_home/market_sync/bin/market_sync_mairui_indicators_run.sh
|
||||
After=network-online.target market-sync.service
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
WorkingDirectory=/home/gao/Development/quant_home/market_sync
|
||||
User=gao
|
||||
Group=gao
|
||||
|
||||
# 入口脚本:单独跑 mairui_indicators 一个 task
|
||||
# mairui 16:30 后发布指标数据 → 16:40 触发留 buffer
|
||||
ExecStart=/home/gao/Development/quant_home/market_sync/bin/market_sync_mairui_indicators_run.sh
|
||||
|
||||
# 硬上限 3h(5204 只 × 3 指标,正常 ~20min,全量时可能更久)
|
||||
TimeoutStartSec=10800
|
||||
|
||||
# 不要 Restart=(oneshot 失败就让 OnFailure= 发通知,别自动重跑)
|
||||
# Restart=no 是 oneshot 默认值
|
||||
|
||||
# 日志走 journald
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=market-sync-mairui-indicators
|
||||
|
||||
# 环境
|
||||
Environment=PYTHONUNBUFFERED=1
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,14 @@
|
||||
[Unit]
|
||||
Description=Schedule MACD/KDJ/BOLL indicators sync — Mon..Fri 16:40 Asia/Shanghai
|
||||
# market-sync-mairui-indicators.service 是这个 timer 的执行单元
|
||||
|
||||
[Timer]
|
||||
# mairui 16:30 后发布指标数据 → 16:40 触发留 buffer
|
||||
OnCalendar=Mon..Fri 16:40:00 Asia/Shanghai
|
||||
# 系统关机 / 错过执行时,下次开机补跑一次(避免漏数据)
|
||||
Persistent=true
|
||||
# 单位(service)的精确名称
|
||||
Unit=market-sync-mairui-indicators.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -0,0 +1,32 @@
|
||||
[Unit]
|
||||
Description=Market data MA daily sync (local derivative, daily 16:30)
|
||||
Documentation=file:///home/gao/Development/quant_home/market_sync/bin/market_sync_mairui_ma_daily_run.sh
|
||||
After=network-online.target market-sync.service
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
WorkingDirectory=/home/gao/Development/quant_home/market_sync
|
||||
User=gao
|
||||
Group=gao
|
||||
|
||||
# 入口脚本:单独跑 mairui_ma_daily 一个 task
|
||||
# 本地派生任务,依赖 kline_stock 已更新
|
||||
ExecStart=/home/gao/Development/quant_home/market_sync/bin/market_sync_mairui_ma_daily_run.sh
|
||||
|
||||
# 硬上限 2h(正常 ~30min,留 buffer)
|
||||
TimeoutStartSec=7200
|
||||
|
||||
# 不要 Restart=(oneshot 失败就让 OnFailure= 发通知,别自动重跑)
|
||||
# Restart=no 是 oneshot 默认值
|
||||
|
||||
# 日志走 journald
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=market-sync-mairui-ma-daily
|
||||
|
||||
# 环境
|
||||
Environment=PYTHONUNBUFFERED=1
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,14 @@
|
||||
[Unit]
|
||||
Description=Schedule MA daily sync — Mon..Fri 16:30 Asia/Shanghai
|
||||
# market-sync-mairui-ma-daily.service 是这个 timer 的执行单元
|
||||
|
||||
[Timer]
|
||||
# kline_daily 15:40 / kline_index 15:30 跑完后,16:30 计算 MA
|
||||
OnCalendar=Mon..Fri 16:30:00 Asia/Shanghai
|
||||
# 系统关机 / 错过执行时,下次开机补跑一次(避免漏数据)
|
||||
Persistent=true
|
||||
# 单位(service)的精确名称
|
||||
Unit=market-sync-mairui-ma-daily.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -0,0 +1,20 @@
|
||||
[Unit]
|
||||
Description=Market sync market_regime — 市场情绪 (本地计算, 非评分依赖)
|
||||
Documentation=file:///home/gao/Development/quant_home/market_sync/bin/market_sync_market_regime_run.sh
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
WorkingDirectory=/home/gao/Development/quant_home/market_sync
|
||||
User=gao
|
||||
Group=gao
|
||||
ExecStart=/home/gao/Development/quant_home/market_sync/bin/market_sync_market_regime_run.sh
|
||||
TimeoutStartSec=600
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=market-sync-market-regime
|
||||
Environment=PYTHONUNBUFFERED=1
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,10 @@
|
||||
[Unit]
|
||||
Description=Schedule market_regime sync — Mon..Fri 16:30 Asia/Shanghai
|
||||
|
||||
[Timer]
|
||||
OnCalendar=Mon..Fri 16:30:00 Asia/Shanghai
|
||||
Persistent=true
|
||||
Unit=market-sync-market-regime.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -0,0 +1,22 @@
|
||||
[Unit]
|
||||
Description=market_sync MCP Server — Model Context Protocol for sync data
|
||||
Documentation=file:///home/gao/Development/quant_home/market_sync/app/mcp_server.py
|
||||
After=network-online.target postgres-container.service
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
WorkingDirectory=/home/gao/Development/quant_home/market_sync
|
||||
User=gao
|
||||
Group=gao
|
||||
ExecStart=/home/gao/Development/quant_home/market_sync/.venv/bin/python -m app.mcp_server --sse --host 127.0.0.1 --port 8101
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
TimeoutStartSec=30
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=market-sync-mcp
|
||||
Environment=PYTHONUNBUFFERED=1
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,33 @@
|
||||
[Unit]
|
||||
Description=Market data moneyflow sync (mairui transaction, daily 21:35)
|
||||
Documentation=file:///home/gao/Development/quant_home/market_sync/bin/market_sync_moneyflow_run.sh
|
||||
After=network-online.target market-sync.service
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
WorkingDirectory=/home/gao/Development/quant_home/market_sync
|
||||
User=gao
|
||||
Group=gao
|
||||
|
||||
# 入口脚本:单独跑 moneyflow 一个 task
|
||||
# mairui 文档:「更新:每日 21:30」—— 21:35 触发留 5min buffer
|
||||
ExecStart=/home/gao/Development/quant_home/market_sync/bin/market_sync_moneyflow_run.sh
|
||||
|
||||
# 硬上限 2h(5206 只 × 50 RPS = 104s 理论上限,留 buffer 给重试 / 慢响应)
|
||||
TimeoutStartSec=7200
|
||||
|
||||
# 不要 Restart=(oneshot 失败就让 OnFailure= 发通知,别自动重跑——
|
||||
# 重跑会撞外部 API 限流窗口,浪费 1.5h)
|
||||
# Restart=no 是 oneshot 默认值
|
||||
|
||||
# 日志走 journald(journalctl -u market-sync-moneyflow.service -f)
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=market-sync-moneyflow
|
||||
|
||||
# 环境(不读 /etc/environment,只带这几个;.env 由 cli 内部 load_dotenv)
|
||||
Environment=PYTHONUNBUFFERED=1
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,16 @@
|
||||
[Unit]
|
||||
Description=Schedule moneyflow sync — Mon..Fri 21:35 Asia/Shanghai
|
||||
# market-sync-moneyflow.service 是这个 timer 的执行单元
|
||||
# mairui 文档:「更新:每日 21:30」—— 21:35 触发留 5 分钟 buffer
|
||||
|
||||
[Timer]
|
||||
# mairui 21:30 发布数据 → 21:35 触发
|
||||
OnCalendar=Mon..Fri 21:35:00 Asia/Shanghai
|
||||
# 系统关机 / 错过执行时,下次开机补跑一次(避免漏数据)
|
||||
Persistent=true
|
||||
# 单位(service)的精确名称
|
||||
Unit=market-sync-moneyflow.service
|
||||
# 不要 AccuracySec(默认 1min 漂移够用,避免 21:35:00 整点打堆)
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -0,0 +1,29 @@
|
||||
[Unit]
|
||||
Description=Market sync morning pre-market — stock_basic
|
||||
Documentation=file:///home/gao/Development/quant_home/market_sync/bin/market_sync_morning_run.sh
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
WorkingDirectory=/home/gao/Development/quant_home/market_sync
|
||||
User=gao
|
||||
Group=gao
|
||||
|
||||
# 早盘前只跑 stock_basic(确保最新股票列表+股本快照)
|
||||
ExecStart=/home/gao/Development/quant_home/market_sync/bin/market_sync_morning_run.sh
|
||||
|
||||
# stock_basic ~9min + buffer = 1h 够用
|
||||
TimeoutStartSec=3600
|
||||
|
||||
# 不要 Restart=(oneshot 失败就让 OnFailure= 发通知,别自动重跑)
|
||||
|
||||
# 日志走 journald(journalctl -u market-sync-morning -f)
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=market-sync-morning
|
||||
|
||||
Environment=PYTHONUNBUFFERED=1
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,14 @@
|
||||
[Unit]
|
||||
Description=Schedule market sync morning — Mon..Fri 09:00 Asia/Shanghai
|
||||
# market-sync-morning.service 是这个 timer 的执行单元
|
||||
# 早盘前 09:00 触发:跑 stock_basic,给 09:30 开盘留 30min buffer
|
||||
|
||||
[Timer]
|
||||
# A 股开盘 09:30,09:00 跑 stock_basic (~9min) 即可
|
||||
OnCalendar=Mon..Fri 09:00:00 Asia/Shanghai
|
||||
# 关机/错过时下次开机补跑
|
||||
Persistent=true
|
||||
Unit=market-sync-morning.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -0,0 +1,31 @@
|
||||
[Unit]
|
||||
Description=Market data 股本快照 sync (雪球源, weekly Sat 11:30)
|
||||
Documentation=file:///home/gao/Development/quant_home/market_sync/bin/market_sync_share_run.sh
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
WorkingDirectory=/home/gao/Development/quant_home/market_sync
|
||||
User=gao
|
||||
Group=gao
|
||||
|
||||
# 入口脚本:单独跑 share_snapshot 一个 task
|
||||
ExecStart=/home/gao/Development/quant_home/market_sync/bin/market_sync_share_run.sh
|
||||
|
||||
# 硬上限 30min(全市场 5200 只雪球 quote_detail,10 worker 约 3-8min,留 buffer)
|
||||
TimeoutStartSec=1800
|
||||
|
||||
# 不要 Restart=(oneshot 失败就让 OnFailure= 发通知,别自动重跑)
|
||||
# Restart=no 是 oneshot 默认值
|
||||
|
||||
# 日志走 journald(journalctl -u market-sync-share.service -f)
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=market-sync-share
|
||||
|
||||
# 环境(不读 /etc/environment,只带这几个;.env 由 cli 内部 load_dotenv)
|
||||
Environment=PYTHONUNBUFFERED=1
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,16 @@
|
||||
[Unit]
|
||||
Description=Schedule share_snapshot sync — Sat 11:30 Asia/Shanghai
|
||||
# market-sync-share.service 是这个 timer 的执行单元
|
||||
# 股本快照每周跑一次即可(雪球 quote_detail,周末收盘后数据稳定)
|
||||
|
||||
[Timer]
|
||||
# 周六 11:30(和 stock_node 同时段)
|
||||
OnCalendar=Sat *-*-* 11:30:00 Asia/Shanghai
|
||||
# 系统关机 / 错过执行时,下次开机补跑一次(避免漏数据)
|
||||
Persistent=true
|
||||
# 单位(service)的精确名称
|
||||
Unit=market-sync-share.service
|
||||
# 不要 AccuracySec(默认 1min 漂移够用)
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -0,0 +1,33 @@
|
||||
[Unit]
|
||||
Description=Market data 股票-节点映射 sync (mairui /hszg, weekly Sat 11:30)
|
||||
Documentation=file:///home/gao/Development/quant_home/market_sync/bin/market_sync_stock_node_run.sh
|
||||
After=network-online.target market-sync-morning.service
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
WorkingDirectory=/home/gao/Development/quant_home/market_sync
|
||||
User=gao
|
||||
Group=gao
|
||||
|
||||
# 入口脚本:单独跑 stock_node 一个 task
|
||||
# mairui /hszg 数据每周六 11:00 更新 → 11:30 触发,留 30 min buffer
|
||||
ExecStart=/home/gao/Development/quant_home/market_sync/bin/market_sync_stock_node_run.sh
|
||||
|
||||
# 硬上限 2h(1100+ 调用 + 写库 + retry buffer)
|
||||
TimeoutStartSec=7200
|
||||
|
||||
# 不要 Restart=(oneshot 失败就让 OnFailure= 发通知,别自动重跑——
|
||||
# 重跑会撞 mairui 配额窗口)
|
||||
# Restart=no 是 oneshot 默认值
|
||||
|
||||
# 日志走 journald(journalctl -u market-sync-stock-node.service -f)
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=market-sync-stock-node
|
||||
|
||||
# 环境(不读 /etc/environment,只带这几个;.env 由 cli 内部 load_dotenv)
|
||||
Environment=PYTHONUNBUFFERED=1
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,16 @@
|
||||
[Unit]
|
||||
Description=Schedule stock_node sync — Sat 11:30 Asia/Shanghai
|
||||
# market-sync-stock-node.service 是这个 timer 的执行单元
|
||||
# mairui /hszg 数据每周六 11:00 更新 → 11:30 触发(30 min buffer)
|
||||
|
||||
[Timer]
|
||||
# 周六 11:30(mairui 11:00 更新后)
|
||||
OnCalendar=Sat *-*-* 11:30:00 Asia/Shanghai
|
||||
# 系统关机 / 错过执行时,下次开机补跑一次(避免漏数据)
|
||||
Persistent=true
|
||||
# 单位(service)的精确名称
|
||||
Unit=market-sync-stock-node.service
|
||||
# 不要 AccuracySec(默认 1min 漂移够用,避免 11:30:00 整点打堆)
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -0,0 +1,33 @@
|
||||
[Unit]
|
||||
Description=Market data tick-by-tick trade sync (mairui hsrl/zbjy, daily 21:05)
|
||||
Documentation=file:///home/gao/Development/quant_home/market_sync/bin/market_sync_tick_run.sh
|
||||
After=network-online.target market-sync.service
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
WorkingDirectory=/home/gao/Development/quant_home/market_sync
|
||||
User=gao
|
||||
Group=gao
|
||||
|
||||
# 入口脚本:单独跑 tick_trade 一个 task(mairui 21:00 发布数据,
|
||||
# 这个 service 在 21:05 触发,留 5min buffer;force=True 绕过 task 内部 21:00 门控)
|
||||
ExecStart=/home/gao/Development/quant_home/market_sync/bin/market_sync_tick_run.sh
|
||||
|
||||
# 硬上限 2h(5000 只 × 1 RPS ≈ 17min,留 buffer 给 retry / 慢任务)
|
||||
TimeoutStartSec=7200
|
||||
|
||||
# 不要 Restart=(oneshot 失败就让 OnFailure= 发通知,别自动重跑——
|
||||
# 重跑会撞外部 API 限流窗口,浪费 1.5h)
|
||||
# Restart=no 是 oneshot 默认值
|
||||
|
||||
# 日志走 journald(journalctl -u market-sync-tick.service -f)
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=market-sync-tick
|
||||
|
||||
# 环境(不读 /etc/environment,只带这几个;.env 由 cli 内部 load_dotenv)
|
||||
Environment=PYTHONUNBUFFERED=1
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,17 @@
|
||||
[Unit]
|
||||
Description=Schedule tick_trade sync — Mon..Fri 21:05 Asia/Shanghai
|
||||
# market-sync-tick.service 是这个 timer 的执行单元
|
||||
# 触发顺序:market-sync.service (15:30) 之后 market-sync-tick.service (21:05)
|
||||
# 但 systemd timer 独立触发,After= 只是声明依赖关系(非强制)
|
||||
|
||||
[Timer]
|
||||
# mairui 文档:「更新:每日 21:00」—— 21:05 触发留 5 分钟 buffer
|
||||
OnCalendar=Mon..Fri 21:05:00 Asia/Shanghai
|
||||
# 系统关机 / 错过执行时,下次开机补跑一次(避免漏数据)
|
||||
Persistent=true
|
||||
# 单位(service)的精确名称
|
||||
Unit=market-sync-tick.service
|
||||
# 不要 AccuracySec(默认 1min 漂移够用,避免 21:05:00 整点打堆)
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -0,0 +1,25 @@
|
||||
[Unit]
|
||||
Description=Market Data Sync Worker (PostgreSQL, scheduler + health monitor)
|
||||
Documentation=file:///home/gao/Development/quant_home/market_sync/app/entrypoints/worker.py
|
||||
# 依赖 network 和 PG
|
||||
After=network-online.target postgresql.service
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
WorkingDirectory=/home/gao/Development/quant_home/market_sync
|
||||
User=gao
|
||||
Group=gao
|
||||
|
||||
# 入口:启动进程内 scheduler + health monitor + recover 卡死任务
|
||||
# 注意:实际入口是 app/entrypoints/worker.py,不是 app/worker.py
|
||||
# 2026-07-12 修复: 把 python -m app.worker 改为 python -m app.entrypoints.worker
|
||||
ExecStart=/home/gao/Development/quant_home/market_sync/.venv/bin/python -m app.entrypoints.worker
|
||||
|
||||
# 明确标记运行模式,让 worker.py 知道不要启动 scheduler
|
||||
# (systemd 模式下同步由 timer 触发,避免双调度器重叠)
|
||||
Environment=PYTHONUNBUFFERED=1
|
||||
Environment="RUNTIME_MODE=systemd"
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,35 @@
|
||||
[Unit]
|
||||
Description=Market Data Full Sync (PostgreSQL, 9 sync tasks end-to-end)
|
||||
Documentation=file:///home/gao/Development/quant_home/market_sync/bin/market_sync_run.sh
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
WorkingDirectory=/home/gao/Development/quant_home/market_sync
|
||||
User=gao
|
||||
Group=gao
|
||||
|
||||
# 入口脚本(runall_once.py + structured_watch.sh 二合一)
|
||||
ExecStart=/home/gao/Development/quant_home/market_sync/bin/market_sync_run.sh
|
||||
|
||||
# 硬上限 3h(实际 ~80min + kline_5min 增量 70min, 总 ~2.5h, 留 30min buffer)
|
||||
# 2026-07-08 教训: 之前 4h 让 kline_daily 单点卡死拖 4h,runall 链全部受影响。
|
||||
# runall_once.py 已加 per-task multiprocessing timeout(单 task 最长 75min),
|
||||
# 这里 3h 是 systemd 层兜底。
|
||||
TimeoutStartSec=10800
|
||||
|
||||
# 不要 Restart=(oneshot 失败就让 OnFailure= 发通知,别自动重跑——
|
||||
# 重跑会撞外部 API 限流窗口,浪费 1.5h)
|
||||
# Restart=no 是 oneshot 默认值
|
||||
|
||||
# 日志走 journald(journalctl -u market-sync.service -f)
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=market-sync
|
||||
|
||||
# 环境(不读 /etc/environment,只带这几个;.env 由 runall_once.py 内部 load_dotenv)
|
||||
Environment=PYTHONUNBUFFERED=1
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,15 @@
|
||||
[Unit]
|
||||
Description=Schedule market data full sync — Mon..Fri 15:30 Asia/Shanghai
|
||||
# market-sync.service 是这个 timer 的执行单元
|
||||
|
||||
[Timer]
|
||||
# A 股收盘时间 15:00,30 分钟 buffer 让最后一笔 tick 落地
|
||||
OnCalendar=Mon..Fri 15:30:00 Asia/Shanghai
|
||||
# 系统关机 / 错过执行时,下次开机补跑一次(避免漏数据)
|
||||
Persistent=true
|
||||
# 单位(service)的精确名称
|
||||
Unit=market-sync.service
|
||||
# 不要 AccuracySec(默认 1min 漂移够用,避免 15:30:00 整点打堆)
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
Reference in New Issue
Block a user