From 5cb64f181a60e47a87ec5dd089f9950005bd80fa Mon Sep 17 00:00:00 2001 From: gao Date: Tue, 7 Jul 2026 17:54:05 +0800 Subject: [PATCH] =?UTF-8?q?feat(api):=20dashboard=20=E7=9C=8B=E6=9D=BF=20+?= =?UTF-8?q?=20config=20=E9=85=8D=E7=BD=AE=E9=A1=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Dashboard (M3): - 路由 /api/dashboard/{daily,history,today,datasets,stats}/ - daily_summary: 按日期聚合 ok/warning/error/blocked - history: 同步历史详情（按 task / 日期过滤） - today: 今日 KPI + 任务列表 - datasets: dataset_registry + 最近 7d run 合并 - stats: 各表行数 + 日期范围（17 张表） HTML SPA: app/api/static/dashboard.html - 5 张状态卡片 + 7 天趋势表 + 任务状态表 + 数据规模表 + 历史详情 - 自动 30s 刷新, 纯 vanilla JS (无框架依赖) Config (M5): - 路由 /api/config/{GET list, GET key, PUT upsert, DELETE}/ - 凭证类自动脱敏 (MAIRUI_LICENCE / XUEQIU_TOKEN / MAIRUI_RPS_LIMIT) - POST /api/config/reload: 把 .env / 环境变量已知项回填到 config 表 - 分类过滤 + key 搜索 + 凭证提示 HTML SPA: app/api/static/config.html - 列表 + 编辑弹窗 + 删除确认 - 凭证类编辑时强制重新输入完整值（避免显示脱敏误以为是新值） - 一键从环境变量回填 FastAPI 主入口挂载: - include_router(dashboard, config) - mount('/static', StaticFiles) 提供 HTML --- app/api/main.py | 11 +- app/api/routes/config.py | 120 ++++++++++++++++ app/api/routes/dashboard.py | 133 +++++++++++++++++ app/api/static/config.html | 264 ++++++++++++++++++++++++++++++++++ app/api/static/dashboard.html | 231 +++++++++++++++++++++++++++++ 5 files changed, 758 insertions(+), 1 deletion(-) create mode 100644 app/api/routes/config.py create mode 100644 app/api/routes/dashboard.py create mode 100644 app/api/static/config.html create mode 100644 app/api/static/dashboard.html diff --git a/app/api/main.py b/app/api/main.py index 0d16bb5..54f7155 100644 --- a/app/api/main.py +++ b/app/api/main.py @@ -13,10 +13,11 @@ if str(_PROJECT_ROOT) not in sys.path: from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware +from fastapi.staticfiles import StaticFiles from app.core.config import settings from app.core.utils.logging import get_logger, setup_logging -from app.api.routes import health, sync, datasources, schedule +from app.api.routes import health, sync, datasources, schedule, dashboard, config setup_logging() logger = get_logger("api") @@ -38,6 +39,14 @@ app.include_router(health.router) app.include_router(sync.router) app.include_router(datasources.router) app.include_router(schedule.router) +app.include_router(dashboard.router) +app.include_router(config.router) + +# 看板 HTML 静态文件 +import pathlib +_static_dir = pathlib.Path(__file__).resolve().parent / "static" +if _static_dir.exists(): + app.mount("/static", StaticFiles(directory=str(_static_dir)), name="static") # ── 启动钩子 ────────────────────────────────────────────────────────── diff --git a/app/api/routes/config.py b/app/api/routes/config.py new file mode 100644 index 0000000..47d81da --- /dev/null +++ b/app/api/routes/config.py @@ -0,0 +1,120 @@ +"""配置管理 API：读写 config 表（MAIRUI_LICENCE / XUEQIU_TOKEN 等可视化配置）。 + +凭证类配置（MAIRUI_LICENCE / XUEQIU_TOKEN）写到 config 表后,可在下次 +任务运行时通过 settings 重新读取(注: 当前代码是 os.environ.get 一启动就 +固化,改 config 表不会立即生效 — 见 README 说明)。 +""" +from __future__ import annotations + +from typing import Optional + +from fastapi import APIRouter, Body, HTTPException, Query + +from app.core.db import ops as db_ops +from app.core.db.models import Config + +router = APIRouter(prefix="/api/config", tags=["config"]) + + +# 已知凭证类配置: 写入时建议显示为占位符,读出时也建议脱敏 +SENSITIVE_KEYS = {"MAIRUI_LICENCE", "XUEQIU_TOKEN", "MAIRUI_RPS_LIMIT"} + + +def _mask(key: str, value: str) -> str: + """凭证类配置显示前 8 位 + ***。""" + if key in SENSITIVE_KEYS and value: + if len(value) <= 8: + return "*" * len(value) + return value[:8] + "***" + return value + + +@router.get("/") +def list_configs( + category: Optional[str] = Query(None), + search: Optional[str] = Query(None, description="按 key 模糊搜索"), +): + """列出所有 config 项。凭证类自动脱敏。""" + rows = db_ops.fetch_all_config() + if category: + rows = [r for r in rows if r.get("category") == category] + if search: + s = search.lower() + rows = [r for r in rows if s in r["key"].lower()] + # 脱敏 + for r in rows: + r["value"] = _mask(r["key"], r.get("value", "")) + return {"count": len(rows), "configs": rows} + + +@router.get("/{key}") +def get_config(key: str): + r = db_ops.fetch_config_by_key(key) + if not r: + raise HTTPException(status_code=404, detail=f"config {key!r} not found") + r["value"] = _mask(key, r.get("value", "")) + return r + + +@router.put("/{key}") +def upsert_config( + key: str, + body: dict = Body(...), +): + """新建或更新一条 config。""" + value = body.get("value", "") + category = body.get("category", "general") + description = body.get("description", "") + if not isinstance(value, str): + raise HTTPException(status_code=400, detail="value must be a string") + db_ops.upsert_config(key=key, value=value, category=category, description=description) + return {"ok": True, "key": key} + + +@router.delete("/{key}") +def delete_config(key: str): + db_ops.delete_config(key) + return {"ok": True, "key": key} + + +@router.get("/categories/list") +def list_categories(): + """返回所有 category（用于前端下拉）。""" + from sqlalchemy import distinct, select + with db_ops.get_session() as s: + cats = s.execute(select(distinct(Config.category))).scalars().all() + return {"categories": sorted([c for c in cats if c])} + + +@router.post("/reload") +def reload_from_env(): + """把 .env / 环境变量里的已知配置回填到 config 表（覆盖式）。""" + import os + from app.core.config import settings + + # 已知 env-driven 配置项 + env_keys = [ + ("MAIRUI_LICENCE", os.environ.get("MAIRUI_LICENCE", ""), "general", "麦蕊智数 licence"), + ("XUEQIU_TOKEN", os.environ.get("XUEQIU_TOKEN", ""), "credential", "雪球 token (含 xq_a_token=...;u=...)"), + ("MAIRUI_RPS_LIMIT", os.environ.get("MAIRUI_RPS_LIMIT", ""), "tuning", "麦蕊智数 RPS 限速"), + ("DS_BAOSTOCK_ENABLED", str(settings.ds_baostock_enabled).lower(), "datasource", "baostock 开关"), + ("DS_SINA_ENABLED", str(settings.ds_sina_enabled).lower(), "datasource", "新浪 开关"), + ("TRADING_HOLIDAYS", settings.trading_holidays, "trading_calendar", "A 股休市日 (YYYY-MM-DD,逗号分隔)"), + ("API_PORT", str(settings.api_port), "api", "API 监听端口"), + ("SCHEDULER_TICK_SECONDS", str(settings.scheduler_tick_seconds), "scheduler", "调度器扫描间隔"), + ] + + n = 0 + for key, value, cat, desc in env_keys: + if not value: + continue + db_ops.upsert_config(key=key, value=value, category=cat, description=desc) + n += 1 + return {"ok": True, "synced": n} + + +@router.get("/page") +def config_page(): + """配置页 HTML 入口（前端单文件 SPA）。""" + from fastapi.responses import RedirectResponse + return RedirectResponse(url="/static/config.html") \ No newline at end of file diff --git a/app/api/routes/dashboard.py b/app/api/routes/dashboard.py new file mode 100644 index 0000000..c5485c2 --- /dev/null +++ b/app/api/routes/dashboard.py @@ -0,0 +1,133 @@ +"""看板 API：每天同步情况 / 任务历史 / 数据集统计。 + +复用 db_ops.list_sync_history() + daily_sync_summary()。 +""" +from __future__ import annotations + +from datetime import date + +from fastapi import APIRouter, Query + +from app.core.db import ops as db_ops + +router = APIRouter(prefix="/api/dashboard", tags=["dashboard"]) + + +@router.get("/daily") +def daily_summary(days: int = Query(7, ge=1, le=90)): + """按日期聚合的同步状态计数（看板首页用）。""" + return {"days": days, "summary": db_ops.daily_sync_summary(days=days)} + + +@router.get("/history") +def sync_history( + dataset_id: str | None = Query(None), + days: int = Query(7, ge=1, le=90), + limit: int = Query(200, ge=1, le=1000), + run_date: str | None = Query(None, description="YYYY-MM-DD 过滤某天"), +): + """同步历史记录（每次 run 一行，append-only）。""" + rd = None + if run_date: + try: + rd = date.fromisoformat(run_date) + except ValueError: + return {"error": f"run_date must be YYYY-MM-DD, got {run_date!r}"} + rows = db_ops.list_sync_history( + dataset_id=dataset_id, run_date=rd, days=days, limit=limit, + ) + return {"dataset_id": dataset_id, "days": days, "count": len(rows), "history": rows} + + +@router.get("/today") +def today_summary(): + """今天的同步情况（看板头部卡片用）。""" + rows = db_ops.daily_sync_summary(days=1) + today_str = date.today().isoformat() + today = next((r for r in rows if r["date"] == today_str), None) or { + "date": today_str, "ok": 0, "warning": 0, "error": 0, "blocked": 0, + "total": 0, "tasks_run": 0, + } + # 列出今天每个 task 的最新状态 + history = db_ops.list_sync_history(days=1, limit=500) + return {"summary": today, "tasks": history} + + +@router.get("/datasets") +def dataset_overview(): + """所有 dataset 的当前状态 + 最近一次 run。 + + 合并 dataset_registry（当前状态）+ sync_history（最近一次历史）。 + """ + from app.core.sync import get_registry_status + registry = get_registry_status() + history = db_ops.list_sync_history(days=7, limit=500) + last_by_task: dict[str, dict] = {} + for h in history: + if h["dataset_id"] not in last_by_task: + last_by_task[h["dataset_id"]] = h + enriched = [] + for r in registry: + ds_id = r["dataset_id"] + last = last_by_task.get(ds_id) + enriched.append({ + **r, + "last_run": last, + "history_count_7d": sum(1 for h in history if h["dataset_id"] == ds_id), + }) + return {"count": len(enriched), "datasets": enriched} + + +@router.get("/stats") +def data_stats(): + """数据集的数据范围/数据量（用于看板"数据规模"卡片）。""" + from sqlalchemy import func, select + from app.core.db.models import ( + KlineStock, KlineIndex, Kline5Min, Moneyflow, Share, TickTrade, + SectorIndices, SectorFeaturesDaily, MarketRegimeDaily, + LonghubangDaily, LonghubangSeat, Stocks, Industry, + NodeCategory, Node, StockNodeMap, KlineStockMADaily, + ) + from app.core.db.orm import SessionLocal + + targets = [ + ("stocks", Stocks), + ("kline_stock", KlineStock), + ("kline_index", KlineIndex), + ("kline_5min", Kline5Min), + ("kline_stock_ma_daily", KlineStockMADaily), + ("moneyflow", Moneyflow), + ("share", Share), + ("tick_trade", TickTrade), + ("sector_indices", SectorIndices), + ("sector_features_daily", SectorFeaturesDaily), + ("market_regime_daily", MarketRegimeDaily), + ("longhubang_daily", LonghubangDaily), + ("longhubang_seat", LonghubangSeat), + ("industry", Industry), + ("node_categories", NodeCategory), + ("nodes", Node), + ("stock_node_map", StockNodeMap), + ] + out = [] + with SessionLocal() as s: + for name, model in targets: + try: + count = s.execute(select(func.count()).select_from(model)).scalar() or 0 + # 尝试拿最早/最新 trade_date + date_range = None + if hasattr(model, "trade_date"): + mn = s.execute(select(func.min(model.trade_date))).scalar() + mx = s.execute(select(func.max(model.trade_date))).scalar() + date_range = {"min": str(mn), "max": str(mx)} if mn else None + out.append({"table": name, "rows": int(count), "date_range": date_range}) + except Exception as e: + out.append({"table": name, "rows": -1, "error": str(e)[:100]}) + return {"tables": out} + + +@router.get("/") +def dashboard_index(): + """看板 HTML 入口（前端单文件 SPA）。""" + from fastapi.responses import FileResponse, RedirectResponse + return RedirectResponse(url="/static/dashboard.html") \ No newline at end of file diff --git a/app/api/static/config.html b/app/api/static/config.html new file mode 100644 index 0000000..ca75847 --- /dev/null +++ b/app/api/static/config.html @@ -0,0 +1,264 @@ + + + + + +market_sync 参数配置 + + + + +

⚙️ 参数配置

+
+ 读写 market_data.config 表。 + · 📊 数据看板 + · FastAPI Docs +
+ +
+ +
+ + + + + + + +
+ + + + + + + +
KeyValue分类说明更新时间操作
加载中...
+ + + + + + + + \ No newline at end of file diff --git a/app/api/static/dashboard.html b/app/api/static/dashboard.html new file mode 100644 index 0000000..e070f08 --- /dev/null +++ b/app/api/static/dashboard.html @@ -0,0 +1,231 @@ + + + + + +market_sync 数据看板 + + + + +

📊 market_sync 数据看板

+
+ 加载中... + · 自动每 30s 刷新 + · 数据源: /api/dashboard/* +
+ + +

今日 (今日运行情况)

+
+
总任务
-
+
✓ OK
-
+
⚠ Warning
-
+
✗ Error
-
+
⊘ Blocked
-
+
+ + +

最近 7 天每日同步情况

+ + + + + + + +
日期OKWarningErrorBlockedTotalTasks Run
加载中...
+ + +

所有同步任务 (实时状态 + 最近 7 天)

+ + + + + + +
TaskNameSourceStatusLast Run7d RunsRowsElapsed
加载中...
+ + +

数据规模

+ + + +
TableRowsDate Range
加载中...
+ + +

同步历史 (最近 200 条)

+
+ + + +
+ + + + + + + +
StartedTaskStatusTriggerElapsed (s)RowsMessage
加载中...
+ + + + + \ No newline at end of file