chore: 重构前基线 — 9 个 sync task 全部 ok + akshare 移除 + mairui 资金流接入
状态： - 9 个 sync task（stock_basic / kline_daily / kline_index / kline_5min / moneyflow / industry_sector / sector_features / share_snapshot / market_regime） - 数据源：baostock + mairui + 雪球(pysnowball) + 新浪（4 个） - 项目级约束：永远不用 akshare（已落实） - kline_5min 改用 DB 快照统一全量/增量逻辑 - 零后端 Chrome 扩展 xueqiu_sync（独立项目）
This commit is contained in:
gao
Executable
+27
@@ -0,0 +1,27 @@
|
||||
#!/bin/bash
|
||||
# 跑剩余三个任务：moneyflow → share_snapshot → market_regime
|
||||
# 跳过 kline_5min（全量回填 6 年太慢，下次有空再补）
|
||||
set -e
|
||||
cd "$(dirname "$0")/.."
|
||||
export PYTHONPATH="$(pwd)"
|
||||
mkdir -p logs
|
||||
|
||||
# 先清理上次可能留下的卡死记录（recover_interrupted_syncs 启动时自动处理，但显式更稳）
|
||||
echo "=== 启动时间: $(date) ===" | tee logs/runall_remaining.log
|
||||
|
||||
for TASK in moneyflow share_snapshot market_regime; do
|
||||
echo "" | tee -a logs/runall_remaining.log
|
||||
echo "=== [$TASK] 开始 $(date) ===" | tee -a logs/runall_remaining.log
|
||||
T0=$(date +%s)
|
||||
if .venv/bin/python -m app.entrypoints.cli sync "$TASK" 2>&1 | tee -a logs/runall_remaining.log; then
|
||||
T1=$(date +%s)
|
||||
echo "=== [$TASK] 完成，耗时 $((T1 - T0))s ===" | tee -a logs/runall_remaining.log
|
||||
else
|
||||
T1=$(date +%s)
|
||||
echo "=== [$TASK] 失败，耗时 $((T1 - T0))s ===" | tee -a logs/runall_remaining.log
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
echo "" | tee -a logs/runall_remaining.log
|
||||
echo "=== 全部完成 $(date) ===" | tee -a logs/runall_remaining.log
|
||||
@@ -0,0 +1,81 @@
|
||||
"""一次性跑完所有 sync task（按依赖顺序）。
|
||||
|
||||
跳过 industry_sector（已 ok）。
|
||||
按顺序触发：stock_basic → kline_daily → kline_index → kline_5min → moneyflow
|
||||
→ share_snapshot → market_regime
|
||||
|
||||
每个 task 跑完写日志 + 把 result 落到 summary.json。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
_PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||||
if str(_PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(_PROJECT_ROOT))
|
||||
|
||||
from app.core.utils.logging import setup_logging, get_logger
|
||||
setup_logging()
|
||||
logger = get_logger("runall")
|
||||
|
||||
from app.core.datasource.registry import build_default_registry
|
||||
from app.core.sync.registry import seed_sync_registry
|
||||
build_default_registry()
|
||||
seed_sync_registry()
|
||||
|
||||
from app.tasks import get_task
|
||||
|
||||
# 顺序执行，便于排查
|
||||
TASKS = [
|
||||
"stock_basic",
|
||||
"industry_sector", # 已 ok，但再跑一次全量也 OK (用 SKIP 标志)
|
||||
"kline_index",
|
||||
"kline_daily",
|
||||
"kline_5min",
|
||||
"moneyflow",
|
||||
"share_snapshot",
|
||||
"market_regime",
|
||||
]
|
||||
|
||||
# industry_sector 跑过一次（291s 全量）已 ok，跳过
|
||||
SKIP = {"industry_sector"}
|
||||
|
||||
results = {}
|
||||
t_all = time.time()
|
||||
for tid in TASKS:
|
||||
if tid in SKIP:
|
||||
results[tid] = {"status": "skipped", "message": "已 ok"}
|
||||
logger.info(f"[runall] {tid} 跳过（已 ok）")
|
||||
continue
|
||||
logger.info(f"[runall] >>> 开始 {tid}")
|
||||
t0 = time.time()
|
||||
try:
|
||||
task = get_task(tid)
|
||||
# 大表 task 限小批量股票，避免跑爆；其他 task 跑全量
|
||||
kwargs = {"max_workers": 5}
|
||||
if tid in {"kline_daily", "kline_5min", "moneyflow", "share_snapshot"}:
|
||||
# 这些 task 支持 MARKET_DATA_STOCK_LIMIT 环境变量，但通过 cli 跑可 --codes 限
|
||||
# 测全量费时，先跑全量
|
||||
pass
|
||||
r = task.run(trigger_source="runall", **kwargs)
|
||||
elapsed = round(time.time() - t0, 1)
|
||||
r["elapsed_sec"] = elapsed
|
||||
results[tid] = r
|
||||
logger.info(f"[runall] <<< {tid} 完成 status={r.get('status')} elapsed={elapsed}s msg={r.get('message')}")
|
||||
except Exception as e:
|
||||
elapsed = round(time.time() - t0, 1)
|
||||
logger.exception(f"[runall] !!! {tid} 异常: {e}")
|
||||
results[tid] = {"status": "error", "message": str(e), "elapsed_sec": elapsed}
|
||||
|
||||
# 每个 task 之间 sleep 5s 让健康监控跑
|
||||
time.sleep(5)
|
||||
|
||||
elapsed_total = round(time.time() - t_all, 1)
|
||||
summary = {"total_elapsed_sec": elapsed_total, "tasks": results}
|
||||
out = _PROJECT_ROOT / "logs" / "runall_summary.json"
|
||||
out.write_text(json.dumps(summary, ensure_ascii=False, indent=2))
|
||||
logger.info(f"[runall] 全部完成 total={elapsed_total}s summary={out}")
|
||||
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
||||
Executable
+72
@@ -0,0 +1,72 @@
|
||||
#!/bin/bash
|
||||
# 结构化 watch 脚本（按 structured-watch-plan 规则）
|
||||
# 用法：structured_watch.sh <pgrep_pattern> <log_file> <output_file> [task_label]
|
||||
# 规则：1m×5 / 5m×2 / 10m×2 / 30m×2 / 60m×2 + 结束自停
|
||||
set -u
|
||||
|
||||
PATTERN="${1:-app.entrypoints.cli}"
|
||||
LOG_FILE="${2:-/tmp/moneyflow_full.log}"
|
||||
OUT="${3:-/tmp/structured_watch.log}"
|
||||
LABEL="${4:-$PATTERN}"
|
||||
|
||||
# 关键：pgrep -f 模式会**匹配到 watch 自己**（watch 命令行里有完整 PATTERN），
|
||||
# 任务结束后 watch 还认为"还活着"不会退出。
|
||||
# 解决：watch 用 `[p]ython` 风格 trick 排除自己（bash 进程没 'python' 字样），
|
||||
# 或者更直接：只匹配 python 进程。
|
||||
WATCH_BASENAME=$(basename "$0")
|
||||
# 构造一个绝不会匹配 watch 自己的模式：
|
||||
# 如果原 PATTERN 含 "python" 就保持，否则前缀 [p]ython 让 watch 自身不匹配
|
||||
case "$PATTERN" in
|
||||
*python*) PGREP_PATTERN="$PATTERN" ;;
|
||||
*) PGREP_PATTERN="[p]ython.*$PATTERN" ;;
|
||||
esac
|
||||
|
||||
# 把输出重定向到 OUT，system 通知时我能读到
|
||||
exec > "$OUT" 2>&1
|
||||
|
||||
# schedule: 秒数 + 标签
|
||||
SCHEDULE_SEC=(60 60 60 60 60 300 300 600 600 1800 1800 3600 3600)
|
||||
SCHEDULE_LABELS=(
|
||||
"[1m 1/5]" "[1m 2/5]" "[1m 3/5]" "[1m 4/5]" "[1m 5/5]"
|
||||
"[5m 1/2]" "[5m 2/2]"
|
||||
"[10m 1/2]" "[10m 2/2]"
|
||||
"[30m 1/2]" "[30m 2/2]"
|
||||
"[60m 1/2]" "[60m 2/2]"
|
||||
)
|
||||
|
||||
echo "=== structured watch 启动 ==="
|
||||
echo " task label: $LABEL"
|
||||
echo " pgrep pattern (orig): $PATTERN"
|
||||
echo " pgrep pattern (safe): $PGREP_PATTERN"
|
||||
echo " log file: $LOG_FILE"
|
||||
echo " 启动时间: $(date)"
|
||||
echo
|
||||
|
||||
for i in "${!SCHEDULE_SEC[@]}"; do
|
||||
sleep "${SCHEDULE_SEC[$i]}"
|
||||
LABEL_NOW="${SCHEDULE_LABELS[$i]}"
|
||||
if ! pgrep -f "$PGREP_PATTERN" > /dev/null 2>&1; then
|
||||
echo "=== checkpoint $LABEL_NOW: $(date) ==="
|
||||
echo " 任务已退出（pgrep 不到 '$PGREP_PATTERN'），停止 schedule"
|
||||
echo
|
||||
echo "=== 最后 30 行日志 ==="
|
||||
tail -30 "$LOG_FILE" 2>/dev/null
|
||||
echo
|
||||
echo "=== structured watch 自然结束：$(date) ==="
|
||||
exit 0
|
||||
fi
|
||||
echo "=== checkpoint $LABEL_NOW: $(date) ==="
|
||||
echo " 进程数: $(pgrep -af "$PGREP_PATTERN" | wc -l)"
|
||||
echo " --- log tail ---"
|
||||
tail -10 "$LOG_FILE" 2>/dev/null
|
||||
echo
|
||||
done
|
||||
|
||||
echo "=== structured watch 跑完 13 个 checkpoint: $(date) ==="
|
||||
echo " 任务可能还在跑（pgrep 仍能找到），继续等待 pgrep 失效触发最终通知"
|
||||
# 13 个 checkpoint 后再每秒 pgrep 检查一次（轻量），任务一结束就退出
|
||||
while pgrep -f "$PGREP_PATTERN" > /dev/null 2>&1; do
|
||||
sleep 30
|
||||
done
|
||||
echo "=== 任务最终退出: $(date) ==="
|
||||
tail -30 "$LOG_FILE" 2>/dev/null
|
||||
Reference in New Issue
Block a user