kyugao/market_sync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(service): dockerize — Dockerfile + docker-compose + 统一入口

Browse Source 
把项目打包成可部署的数据同步服务：

- Dockerfile (multi-stage builder/runtime, slim base, 含 healthcheck)
- docker-compose.yml (postgres:16-alpine + market_sync, 持久化 pgdata + logs)
- .dockerignore (减 build context)
- bin/service_run.sh (单进程跑 uvicorn + scheduler 线程)
- app/entrypoints/worker.py 抽 start_scheduler_thread() (幂等)

部署:
  docker compose up -d
  # 然后访问 http://localhost:8100/docs 看 FastAPI

设计选择:
  - 单容器一服务（uvicorn 主 + scheduler daemon thread），不用 supervisord
  - 原 systemd timer 由进程内 scheduler 取代（配置已存在于 config 表）
  - PG 用 docker volume 持久化；同步日志单独 volume
This commit is contained in:
gao
2026-07-07 16:30:56 +08:00
parent a8ea132924
commit fbc3cc84f0
5 changed files with 299 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.dockerignore
+31
View File
@@ -0,0 +1,31 @@
# .dockerignore — 减小 build context
.git
.github
.vscode
.idea
.claude
.venv
venv
__pycache__
*.pyc
*.pyo
*.pyd
.pytest_cache
.mypy_cache
.ruff_cache
.coverage
htmlcov/
*.log
logs/
.DS_Store
*.swp
*.swo
node_modules
frontend/node_modules
tests/
docs/
*.md
!README.md
# 已通过 requirements.txt 装好，不重复打包
**/*.tar.gz
**/*.whl
Dockerfile
+63
View File
@@ -0,0 +1,63 @@
# syntax=docker/dockerfile:1.6
# ─────────────────────────────────────────────────────────────
# market_sync 数据同步服务
# 单镜像同时跑：FastAPI dashboard + 进程内 scheduler（worker）
# 部署：docker compose up -d
# ─────────────────────────────────────────────────────────────
# --- builder stage: 装依赖到 venv ---
FROM python:3.11-slim AS builder
ENV PYTHONUNBUFFERED=1 \
PYTHONDONTWRITEBYTECODE=1 \
PIP_NO_CACHE_DIR=1 \
PIP_DISABLE_PIP_VERSION_CHECK=1
# 仅装编译期需要的系统包（很多 wheel 已预编译，但 psycopg2 / cryptography 可能要）
RUN apt-get update && apt-get install -y --no-install-recommends \
build-essential gcc libpq-dev \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /app
# 先 copy requirements 单独一层（利用 Docker 缓存：依赖不变就不重装）
COPY requirements.txt .
RUN python -m venv /app/.venv \
&& /app/.venv/bin/pip install --upgrade pip \
&& /app/.venv/bin/pip install -r requirements.txt
# --- runtime stage: 极简基础镜像 + 仅复制 venv 与代码 ---
FROM python:3.11-slim AS runtime
ENV PYTHONUNBUFFERED=1 \
PYTHONDONTWRITEBYTECODE=1 \
PATH="/app/.venv/bin:${PATH}" \
PYTHONPATH=/app \
# service 模式默认监听
API_HOST=0.0.0.0 \
API_PORT=8100
# runtime 只需要 psycopg2 的运行时库 + tzdata（pandas tz aware 需要）
RUN apt-get update && apt-get install -y --no-install-recommends \
libpq5 tzdata curl \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /app
# 从 builder 复制 venv（已编译好的依赖）
COPY --from=builder /app/.venv /app/.venv
# 复制应用代码
COPY app/ ./app/
COPY bin/ ./bin/
# 默认启动 service 入口（FastAPI + 进程内 scheduler）
# 也可覆盖为 cli / worker 单跑某个 task：
# docker compose run --rm market_sync python -m app.entrypoints.cli sync kline_daily
EXPOSE 8100
HEALTHCHECK --interval=60s --timeout=5s --start-period=30s --retries=3 \
CMD curl -fsS http://localhost:8100/api/health || exit 1
ENTRYPOINT ["/app/bin/service_run.sh"]
app/entrypoints/worker.py
+52 -1
View File
@@ -1,11 +1,14 @@
"""纯 worker 模式（无 web）— 启动调度器 + 健康监控。
用法：python -m app.worker
用法：
python -m app.worker # 独立 worker 进程
python -c "from app.entrypoints.worker import start_scheduler_thread; start_scheduler_thread()" # 在 web 进程内 inline 跑
"""
from __future__ import annotations
import json
import sys
import threading
import time
from pathlib import Path
@@ -21,6 +24,54 @@ setup_logging()
logger = get_logger("worker")
def start_scheduler_thread() -> None:
"""在当前进程内启动 scheduler + 健康监控（daemon thread）。
给 service_run.sh 用：在 uvicorn 同进程跑后台 scheduler，避免
docker 一个容器跑两个进程（supervisord / s6 那种方案）。
幂等：多次调用只启动一次。
"""
# 1. 注册数据源 + seed config
from app.core.datasource.registry import (
build_default_registry,
seed_datasource_configs,
start_health_monitor,
)
build_default_registry()
seed_datasource_configs()
# 2. seed sync registry + 恢复卡死任务
from app.core.sync.registry import recover_interrupted_syncs, seed_sync_registry
seed_sync_registry()
n = recover_interrupted_syncs()
if n > 0:
logger.warning(f"[worker] 恢复了 {n} 个中断的同步任务")
# 3. seed 节假日
if settings.holidays_list:
db_ops.upsert_config(
"trading_calendar_holidays",
json.dumps(settings.holidays_list, ensure_ascii=False),
category="general",
description="A 股休市日",
)
# 4. seed 默认计划任务
if settings.scheduler_auto_seed:
from app.core.scheduler.scheduler import seed_schedule_configs
seed_schedule_configs()
# 5. 注册 sync job + 启动调度器 + 健康监控
from app.core.scheduler.scheduler import (
register_sync_jobs,
start_scheduler,
)
register_sync_jobs()
start_scheduler()
start_health_monitor()
logger.info("[worker] scheduler + health monitor 已 inline 启动")
def main():
logger.info("[worker] market_data_sync worker 启动")
bin/service_run.sh
Executable
+69
View File
@@ -0,0 +1,69 @@
#!/bin/sh
# service_run.sh — Docker 入口
#
# 单进程跑：FastAPI dashboard（uvicorn）+ 进程内 scheduler（worker）
# 用 uvicorn 主进程 + 一个 daemon thread 跑 worker 的 scheduler 循环。
#
# 设计原因：docker-compose 一容器一服务最简单，systemd timer 的事交给
# 进程内 scheduler（避免 docker 里跑 systemd）。原来 .env 里的 schedule
# 表就是为这个准备的。
set -eu
# 等 PG 就绪（避免 race：market_sync 比 postgres 早启）
if [ -n "${PG_HOST:-}" ]; then
echo "[service] wait for PG ${PG_HOST}:${PG_PORT:-5432}..."
for i in $(seq 1 30); do
if python -c "
import socket, sys
s = socket.socket()
s.settimeout(2)
try:
s.connect(('${PG_HOST}', ${PG_PORT:-5432}))
print('pg ready')
except Exception as e:
sys.exit(1)
" >/dev/null 2>&1; then
echo "[service] PG up after ${i}s"
break
fi
sleep 1
done
fi
# seed datasource + sync registry + schedule（幂等）
python -c "
from app.core.datasource.registry import build_default_registry, seed_datasource_configs
from app.core.sync.registry import seed_sync_registry, recover_interrupted_syncs
from app.core.scheduler.scheduler import seed_schedule_configs
build_default_registry()
seed_datasource_configs()
seed_sync_registry()
recover_interrupted_syncs()
seed_schedule_configs()
print('[service] seed 完成')
"
# 后台启动 scheduler（独立线程）
python -c "
from app.entrypoints.worker import start_scheduler_thread
start_scheduler_thread()
print('[service] scheduler thread started')
import time
while True:
time.sleep(60)
" &
SCHEDULER_PID=$!
# trap SIGTERM 优雅退出
trap "echo '[service] stopping scheduler'; kill ${SCHEDULER_PID} 2>/dev/null || true; wait" TERM INT
# 前台跑 uvicorn（主进程）
echo "[service] starting uvicorn on ${API_HOST:-0.0.0.0}:${API_PORT:-8100}"
exec uvicorn app.api.main:app \
--host "${API_HOST:-0.0.0.0}" \
--port "${API_PORT:-8100}" \
--workers 1 \
--log-level "${LOG_LEVEL:-info}" \
--no-access-log
docker-compose.yml
+84
View File
@@ -0,0 +1,84 @@
# docker-compose.yml — 一键起 market_sync 数据同步服务
#
# 包含：
# - postgres : PostgreSQL 16（market_data schema 的主库）
# - market_sync : 数据同步 + dashboard + scheduler（单进程）
#
# 用法：
# docker compose up -d # 后台启动
# docker compose logs -f market_sync # 看同步日志
# docker compose exec market_sync \
# python -m app.entrypoints.cli sync stock_node --type2 2,3
#
version: "3.9"
services:
postgres:
image: postgres:16-alpine
restart: unless-stopped
environment:
POSTGRES_USER: market_sync
POSTGRES_PASSWORD: market_sync
POSTGRES_DB: market_data
ports:
- "5432:5432"
volumes:
- pgdata:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U market_sync -d market_data"]
interval: 10s
timeout: 3s
retries: 5
market_sync:
build:
context: .
dockerfile: Dockerfile
image: market_sync:latest
restart: unless-stopped
depends_on:
postgres:
condition: service_healthy
environment:
# PG 连接（指向同 compose 的 postgres 服务）
PG_HOST: postgres
PG_PORT: "5432"
PG_USER: market_sync
PG_PASSWORD: market_sync
PG_DB_NAME: market_data
DB_BACKEND: pg
# 调度
SCHEDULER_TIMEZONE: Asia/Shanghai
SCHEDULER_TICK_SECONDS: "5"
SCHEDULER_AUTO_SEED: "true"
# API
API_HOST: 0.0.0.0
API_PORT: "8100"
# 日志
LOG_DIR: /app/logs
LOG_LEVEL: INFO
# 数据源开关
DS_BAOSTOCK_ENABLED: "true"
DS_SINA_ENABLED: "true"
# 凭证（可放进 .env / secrets 文件，避免明文）
MAIRUI_LICENCE: ${MAIRUI_LICENCE:-}
XUEQIU_TOKEN: ${XUEQIU_TOKEN:-}
# 麦蕊限速（钻石 100 RPS 留 buffer；服务化后保守点）
MAIRUI_RPS_LIMIT: "10"
# 节假日
TRADING_HOLIDAYS: ${TRADING_HOLIDAYS:-}
ports:
- "8100:8100"
volumes:
# 持久化日志（任务失败时方便排查）
- sync_logs:/app/logs
healthcheck:
test: ["CMD", "curl", "-fsS", "http://localhost:8100/api/health"]
interval: 60s
timeout: 5s
retries: 3
start_period: 30s
volumes:
pgdata:
sync_logs: