feat(sync): sync_history 表 + base.py 自动记录每次同步
新增 23 张表 sync_history:append-only,每次 task.run() 完成追加一行。 字段: dataset_id / run_date / status / trigger_source / started_at / finished_at / elapsed_sec / rows_written / message / error / stats (JSONB) / triggered_by 索引: (dataset_id, started_at) / (started_at) / (status) vs dataset_registry 的区别: - dataset_registry 只保留「最后一次状态」便于快查 - sync_history 保留完整历史,支持看板按天/按 task 维度聚合 + 失败回溯 自动落库: - SyncTask.run() 成功后调 _record_history() - 异常路径也写(避免排查时漏掉崩溃的 run) - 落库失败仅 warning 不阻塞主流程 新增 db_ops helpers: - insert_sync_history() - list_sync_history(dataset_id?, days, limit) 看板/API 用 - daily_sync_summary(days) 按日期聚合 ok/warning/error/blocked _ensure_schema 改造: 从 no-op 改为用 superuser 连接跑 create_all(),SA 2.x IF NOT EXISTS 幂等。 这样新增 model 不用手动跑 pg_bootstrap。生产首次部署仍建议跑 pg_bootstrap (它还要建 role / schema)。 测试: tests/test_schema_models.py 22 → 23 张表
This commit is contained in:
+68
-1
@@ -4,6 +4,7 @@
|
||||
- 状态自动更新(running → success / failed)
|
||||
- 进度回调
|
||||
- 异常捕获 → 写 dataset_registry
|
||||
- 每次 run 完成追加一行 sync_history(看板/MCP/失败排查)
|
||||
|
||||
子类只需实现 _run() 即可。
|
||||
"""
|
||||
@@ -14,6 +15,7 @@ import traceback
|
||||
from datetime import datetime, time as dtime
|
||||
from typing import Any, Optional
|
||||
|
||||
from app.core.db import ops as db_ops
|
||||
from app.core.sync.registry import (
|
||||
mark_sync_failed,
|
||||
mark_sync_progress,
|
||||
@@ -55,7 +57,10 @@ class SyncTask:
|
||||
# ── 公开入口 ─────────────────────────────────────────
|
||||
|
||||
def run(self, *, trigger_source: str = "manual", **kwargs) -> dict[str, Any]:
|
||||
"""统一入口:自动包 mark_running / mark_success / mark_failed。"""
|
||||
"""统一入口:自动包 mark_running / mark_success / mark_failed。
|
||||
|
||||
完成后追加一行 sync_history(看板 / MCP / 失败排查共用)。
|
||||
"""
|
||||
# 1. 确保 health check 跑过(CLI / 手动触发场景下 schedule 后台不会先跑)
|
||||
self._ensure_health_checked()
|
||||
|
||||
@@ -66,6 +71,7 @@ class SyncTask:
|
||||
message=f"开始同步 {self.dataset_id}...",
|
||||
)
|
||||
t0 = time.time()
|
||||
started_at = datetime.now()
|
||||
try:
|
||||
result = self._run(trigger_source=trigger_source, **kwargs)
|
||||
elapsed = round(time.time() - t0, 1)
|
||||
@@ -94,6 +100,17 @@ class SyncTask:
|
||||
logger.info(f"[{self.dataset_id}] 完成 {status}: {message} ({elapsed}s)")
|
||||
# 同步 schedule config 的 lastRun(fix Bug 2)
|
||||
self._maybe_mirror_schedule_status(trigger_source, status, message)
|
||||
# 追加 sync_history(每次 run 留痕,看板/MCP/失败排查共用)
|
||||
self._record_history(
|
||||
trigger_source=trigger_source,
|
||||
started_at=started_at,
|
||||
finished_at=datetime.now(),
|
||||
elapsed_sec=elapsed,
|
||||
status=status,
|
||||
message=message,
|
||||
error=result.get("error", ""),
|
||||
result=result,
|
||||
)
|
||||
return result
|
||||
except Exception as e:
|
||||
elapsed = round(time.time() - t0, 1)
|
||||
@@ -101,6 +118,17 @@ class SyncTask:
|
||||
err_msg = f"{type(e).__name__}: {e}"
|
||||
mark_sync_failed(self.dataset_id, message=err_msg, error=tb)
|
||||
logger.error(f"[{self.dataset_id}] 失败: {err_msg}\n{tb}")
|
||||
# 异常路径也要写 sync_history
|
||||
self._record_history(
|
||||
trigger_source=trigger_source,
|
||||
started_at=started_at,
|
||||
finished_at=datetime.now(),
|
||||
elapsed_sec=elapsed,
|
||||
status="error",
|
||||
message=err_msg,
|
||||
error=tb,
|
||||
result={"status": "error"},
|
||||
)
|
||||
return {
|
||||
"status": "error",
|
||||
"message": err_msg,
|
||||
@@ -108,6 +136,45 @@ class SyncTask:
|
||||
"elapsed_sec": elapsed,
|
||||
}
|
||||
|
||||
def _record_history(
|
||||
self, *,
|
||||
trigger_source: str,
|
||||
started_at: datetime,
|
||||
finished_at: datetime,
|
||||
elapsed_sec: float,
|
||||
status: str,
|
||||
message: str,
|
||||
error: str,
|
||||
result: dict[str, Any],
|
||||
) -> None:
|
||||
"""把这次 run 的核心指标落 sync_history(看板/MCP 共用)。
|
||||
|
||||
stats 字段收集 _run() 返回 dict 里的统计字段(ok/fail/skip/total/rows 等)。
|
||||
"""
|
||||
try:
|
||||
stats_keys = {"ok", "fail", "skip", "total", "rows", "rows_written",
|
||||
"categories", "nodes", "mappings", "days", "sectors"}
|
||||
stats = {k: result[k] for k in stats_keys if k in result}
|
||||
rows_written = int(stats.get("rows", 0) or stats.get("rows_written", 0)
|
||||
or stats.get("mappings", 0) or 0)
|
||||
db_ops.insert_sync_history(
|
||||
dataset_id=self.dataset_id,
|
||||
run_date=started_at.date(),
|
||||
status=status,
|
||||
trigger_source=trigger_source,
|
||||
started_at=started_at,
|
||||
finished_at=finished_at,
|
||||
elapsed_sec=elapsed_sec,
|
||||
rows_written=rows_written,
|
||||
message=message[:1000] if message else "",
|
||||
error=error[:2000] if error else "",
|
||||
stats=stats,
|
||||
triggered_by=trigger_source,
|
||||
)
|
||||
except Exception as e:
|
||||
# 落库失败不应阻塞主流程
|
||||
logger.warning(f"[{self.dataset_id}] sync_history 落库失败(不影响主流程): {e}")
|
||||
|
||||
def _ensure_health_checked(self) -> None:
|
||||
"""若内存里 health_results 为空,跑一次。"""
|
||||
try:
|
||||
|
||||
Reference in New Issue
Block a user