diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..a745e26 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,31 @@ +# .dockerignore — 减小 build context +.git +.github +.vscode +.idea +.claude +.venv +venv +__pycache__ +*.pyc +*.pyo +*.pyd +.pytest_cache +.mypy_cache +.ruff_cache +.coverage +htmlcov/ +*.log +logs/ +.DS_Store +*.swp +*.swo +node_modules +frontend/node_modules +tests/ +docs/ +*.md +!README.md +# 已通过 requirements.txt 装好,不重复打包 +**/*.tar.gz +**/*.whl \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..c63e383 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,63 @@ +# syntax=docker/dockerfile:1.6 +# ───────────────────────────────────────────────────────────── +# market_sync 数据同步服务 +# 单镜像同时跑:FastAPI dashboard + 进程内 scheduler(worker) +# 部署:docker compose up -d +# ───────────────────────────────────────────────────────────── + +# --- builder stage: 装依赖到 venv --- +FROM python:3.11-slim AS builder + +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PIP_NO_CACHE_DIR=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 + +# 仅装编译期需要的系统包(很多 wheel 已预编译,但 psycopg2 / cryptography 可能要) +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential gcc libpq-dev \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +# 先 copy requirements 单独一层(利用 Docker 缓存:依赖不变就不重装) +COPY requirements.txt . +RUN python -m venv /app/.venv \ + && /app/.venv/bin/pip install --upgrade pip \ + && /app/.venv/bin/pip install -r requirements.txt + + +# --- runtime stage: 极简基础镜像 + 仅复制 venv 与代码 --- +FROM python:3.11-slim AS runtime + +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PATH="/app/.venv/bin:${PATH}" \ + PYTHONPATH=/app \ + # service 模式默认监听 + API_HOST=0.0.0.0 \ + API_PORT=8100 + +# runtime 只需要 psycopg2 的运行时库 + tzdata(pandas tz aware 需要) +RUN apt-get update && apt-get install -y --no-install-recommends \ + libpq5 tzdata curl \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +# 从 builder 复制 venv(已编译好的依赖) +COPY --from=builder /app/.venv /app/.venv + +# 复制应用代码 +COPY app/ ./app/ +COPY bin/ ./bin/ + +# 默认启动 service 入口(FastAPI + 进程内 scheduler) +# 也可覆盖为 cli / worker 单跑某个 task: +# docker compose run --rm market_sync python -m app.entrypoints.cli sync kline_daily +EXPOSE 8100 + +HEALTHCHECK --interval=60s --timeout=5s --start-period=30s --retries=3 \ + CMD curl -fsS http://localhost:8100/api/health || exit 1 + +ENTRYPOINT ["/app/bin/service_run.sh"] \ No newline at end of file diff --git a/app/entrypoints/worker.py b/app/entrypoints/worker.py index 004a855..df18f22 100644 --- a/app/entrypoints/worker.py +++ b/app/entrypoints/worker.py @@ -1,11 +1,14 @@ """纯 worker 模式(无 web)— 启动调度器 + 健康监控。 -用法:python -m app.worker +用法: + python -m app.worker # 独立 worker 进程 + python -c "from app.entrypoints.worker import start_scheduler_thread; start_scheduler_thread()" # 在 web 进程内 inline 跑 """ from __future__ import annotations import json import sys +import threading import time from pathlib import Path @@ -21,6 +24,54 @@ setup_logging() logger = get_logger("worker") +def start_scheduler_thread() -> None: + """在当前进程内启动 scheduler + 健康监控(daemon thread)。 + + 给 service_run.sh 用:在 uvicorn 同进程跑后台 scheduler,避免 + docker 一个容器跑两个进程(supervisord / s6 那种方案)。 + 幂等:多次调用只启动一次。 + """ + # 1. 注册数据源 + seed config + from app.core.datasource.registry import ( + build_default_registry, + seed_datasource_configs, + start_health_monitor, + ) + build_default_registry() + seed_datasource_configs() + + # 2. seed sync registry + 恢复卡死任务 + from app.core.sync.registry import recover_interrupted_syncs, seed_sync_registry + seed_sync_registry() + n = recover_interrupted_syncs() + if n > 0: + logger.warning(f"[worker] 恢复了 {n} 个中断的同步任务") + + # 3. seed 节假日 + if settings.holidays_list: + db_ops.upsert_config( + "trading_calendar_holidays", + json.dumps(settings.holidays_list, ensure_ascii=False), + category="general", + description="A 股休市日", + ) + + # 4. seed 默认计划任务 + if settings.scheduler_auto_seed: + from app.core.scheduler.scheduler import seed_schedule_configs + seed_schedule_configs() + + # 5. 注册 sync job + 启动调度器 + 健康监控 + from app.core.scheduler.scheduler import ( + register_sync_jobs, + start_scheduler, + ) + register_sync_jobs() + start_scheduler() + start_health_monitor() + logger.info("[worker] scheduler + health monitor 已 inline 启动") + + def main(): logger.info("[worker] market_data_sync worker 启动") diff --git a/bin/service_run.sh b/bin/service_run.sh new file mode 100755 index 0000000..0a3784a --- /dev/null +++ b/bin/service_run.sh @@ -0,0 +1,69 @@ +#!/bin/sh +# service_run.sh — Docker 入口 +# +# 单进程跑:FastAPI dashboard(uvicorn)+ 进程内 scheduler(worker) +# 用 uvicorn 主进程 + 一个 daemon thread 跑 worker 的 scheduler 循环。 +# +# 设计原因:docker-compose 一容器一服务最简单,systemd timer 的事交给 +# 进程内 scheduler(避免 docker 里跑 systemd)。原来 .env 里的 schedule +# 表就是为这个准备的。 + +set -eu + +# 等 PG 就绪(避免 race:market_sync 比 postgres 早启) +if [ -n "${PG_HOST:-}" ]; then + echo "[service] wait for PG ${PG_HOST}:${PG_PORT:-5432}..." + for i in $(seq 1 30); do + if python -c " +import socket, sys +s = socket.socket() +s.settimeout(2) +try: + s.connect(('${PG_HOST}', ${PG_PORT:-5432})) + print('pg ready') +except Exception as e: + sys.exit(1) +" >/dev/null 2>&1; then + echo "[service] PG up after ${i}s" + break + fi + sleep 1 + done +fi + +# seed datasource + sync registry + schedule(幂等) +python -c " +from app.core.datasource.registry import build_default_registry, seed_datasource_configs +from app.core.sync.registry import seed_sync_registry, recover_interrupted_syncs +from app.core.scheduler.scheduler import seed_schedule_configs +build_default_registry() +seed_datasource_configs() +seed_sync_registry() +recover_interrupted_syncs() +seed_schedule_configs() +print('[service] seed 完成') +" + +# 后台启动 scheduler(独立线程) +python -c " +from app.entrypoints.worker import start_scheduler_thread +start_scheduler_thread() +print('[service] scheduler thread started') + +import time +while True: + time.sleep(60) +" & +SCHEDULER_PID=$! + +# trap SIGTERM 优雅退出 +trap "echo '[service] stopping scheduler'; kill ${SCHEDULER_PID} 2>/dev/null || true; wait" TERM INT + +# 前台跑 uvicorn(主进程) +echo "[service] starting uvicorn on ${API_HOST:-0.0.0.0}:${API_PORT:-8100}" +exec uvicorn app.api.main:app \ + --host "${API_HOST:-0.0.0.0}" \ + --port "${API_PORT:-8100}" \ + --workers 1 \ + --log-level "${LOG_LEVEL:-info}" \ + --no-access-log \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..709eee8 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,84 @@ +# docker-compose.yml — 一键起 market_sync 数据同步服务 +# +# 包含: +# - postgres : PostgreSQL 16(market_data schema 的主库) +# - market_sync : 数据同步 + dashboard + scheduler(单进程) +# +# 用法: +# docker compose up -d # 后台启动 +# docker compose logs -f market_sync # 看同步日志 +# docker compose exec market_sync \ +# python -m app.entrypoints.cli sync stock_node --type2 2,3 +# +version: "3.9" + +services: + postgres: + image: postgres:16-alpine + restart: unless-stopped + environment: + POSTGRES_USER: market_sync + POSTGRES_PASSWORD: market_sync + POSTGRES_DB: market_data + ports: + - "5432:5432" + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U market_sync -d market_data"] + interval: 10s + timeout: 3s + retries: 5 + + market_sync: + build: + context: . + dockerfile: Dockerfile + image: market_sync:latest + restart: unless-stopped + depends_on: + postgres: + condition: service_healthy + environment: + # PG 连接(指向同 compose 的 postgres 服务) + PG_HOST: postgres + PG_PORT: "5432" + PG_USER: market_sync + PG_PASSWORD: market_sync + PG_DB_NAME: market_data + DB_BACKEND: pg + # 调度 + SCHEDULER_TIMEZONE: Asia/Shanghai + SCHEDULER_TICK_SECONDS: "5" + SCHEDULER_AUTO_SEED: "true" + # API + API_HOST: 0.0.0.0 + API_PORT: "8100" + # 日志 + LOG_DIR: /app/logs + LOG_LEVEL: INFO + # 数据源开关 + DS_BAOSTOCK_ENABLED: "true" + DS_SINA_ENABLED: "true" + # 凭证(可放进 .env / secrets 文件,避免明文) + MAIRUI_LICENCE: ${MAIRUI_LICENCE:-} + XUEQIU_TOKEN: ${XUEQIU_TOKEN:-} + # 麦蕊限速(钻石 100 RPS 留 buffer;服务化后保守点) + MAIRUI_RPS_LIMIT: "10" + # 节假日 + TRADING_HOLIDAYS: ${TRADING_HOLIDAYS:-} + ports: + - "8100:8100" + volumes: + # 持久化日志(任务失败时方便排查) + - sync_logs:/app/logs + healthcheck: + test: ["CMD", "curl", "-fsS", "http://localhost:8100/api/health"] + interval: 60s + timeout: 5s + retries: 3 + start_period: 30s + +volumes: + pgdata: + sync_logs: \ No newline at end of file