性能优化: 任务列表瘦身+详情分页+auto状态独立存储+磁盘统计缓存+db异步同步; 统计数值未读取时显示-

- /api/tasks 响应 1.57MB -> 12KB (latest_run 只带摘要, 一次读 runs.json) - /api/stats 磁盘占用加 30s 缓存, 去除重复全量读 - 详情接口分页返回 results (默认100条/页), 前端表格分页+页码跳转 - auto 任务 pending/visited 队列迁移到 data/auto_state/ 独立文件 (tasks.json 1.6MB -> 14KB) - MySQL 同步改后台线程异步执行 (db.sync_run_async/upsert_task_async), 不再阻塞爬虫和 API - 启动时自动迁移存量 auto 状态 - 统计/回收站/运行中数值在未读到真实数据前显示 '-'
2026-08-12 09:28:16 +08:00
parent 66ba1a7cfc
commit 796e667533
8 changed files with 47204 additions and 79 deletions
@@ -45,26 +45,32 @@ async function loadTasks() {
    pill.textContent = `运行中任务: ${running}`;
    pill.classList.toggle("active", running > 0);
    const st = await api("/api/status");
-    $("version").textContent = `v${st.version}`;
+    $("version").textContent = st && st.version ? `v${st.version}` : "v-";
  } catch (e) {
    toast("加载任务失败: " + e.message, true);
  }
  loadStats();
 }

+const STAT_IDS = ["stTasks", "stRunning", "stRuns", "stOk", "stFail", "stImgs", "stDisk"];
+
 async function loadStats() {
+  // 未读到真实数据前, 一律显示 "-"
+  STAT_IDS.forEach((id) => { $(id).textContent = "-"; });
+  $("trashCount").textContent = "-";
  try {
    const s = await api("/api/stats");
-    $("stTasks").textContent = s.tasks;
-    $("stRunning").textContent = s.running;
-    $("stRuns").textContent = s.runs;
-    $("stOk").textContent = s.ok;
-    $("stFail").textContent = s.fail;
-    $("stImgs").textContent = s.images;
-    $("stDisk").textContent = s.disk_mb >= 1024
-      ? (s.disk_mb / 1024).toFixed(1) + " GB" : s.disk_mb + " MB";
-    $("trashCount").textContent = s.trash || 0;
-  } catch (e) { /* 统计失败忽略 */ }
+    $("stTasks").textContent = s.tasks ?? "-";
+    $("stRunning").textContent = s.running ?? "-";
+    $("stRuns").textContent = s.runs ?? "-";
+    $("stOk").textContent = s.ok ?? "-";
+    $("stFail").textContent = s.fail ?? "-";
+    $("stImgs").textContent = s.images ?? "-";
+    const mb = Number(s.disk_mb);
+    $("stDisk").textContent = Number.isFinite(mb) && s.disk_mb != null
+      ? (mb >= 1024 ? (mb / 1024).toFixed(1) + " GB" : mb + " MB") : "-";
+    $("trashCount").textContent = s.trash ?? "-";
+  } catch (e) { /* 统计失败保持 "-" */ }
 }

 function taskStatusBadge(t) {
@@ -516,13 +522,15 @@ async function submitForm(e) {
 }

 /* ---------------- 详情 ---------------- */
+const DETAIL_PAGE_SIZE = 100;   // 详情结果每页条数
+
 async function openDetail(tid) {
  try {
-    const t = await api(`/api/tasks/${tid}`);
+    const t = await api(`/api/tasks/${tid}?page_size=${DETAIL_PAGE_SIZE}`);
    state.detail.task = t;
    const runs = t.runs || [];
-    state.detail.runId = runs.find((r) => r.status === "running")
-      ? runs[0].id : (runs[0] ? runs[0].id : null);
+    const cur = t.run;
+    state.detail.runId = cur ? cur.id : (runs[0] ? runs[0].id : null);
    state.detail.logOffset = 0;
    $("detailTitle").textContent = `任务详情 · ${t.name}`;
    renderDetail();
@@ -531,11 +539,38 @@ async function openDetail(tid) {
  } catch (e) { toast(e.message, true); }
 }

+async function loadRunPage(tid, rid, page) {
+  return api(`/api/tasks/${tid}?run=${rid}&page=${page}&page_size=${DETAIL_PAGE_SIZE}`);
+}
+
+async function selectRun(rid) {
+  const t = state.detail.task;
+  if (!t) return;
+  try {
+    const d = await loadRunPage(t.id, rid, 1);
+    state.detail.task = d;
+    state.detail.runId = d.run ? d.run.id : null;
+    state.detail.logOffset = 0;
+    renderDetail();
+    startLogPoll();
+  } catch (e) { toast(e.message, true); }
+}
+
+async function goRunPage(p) {
+  const t = state.detail.task;
+  if (!t || !state.detail.runId) return;
+  try {
+    const d = await loadRunPage(t.id, state.detail.runId, p);
+    state.detail.task = d;
+    renderDetail();
+  } catch (e) { toast(e.message, true); }
+}
+
 function renderDetail() {
  const t = state.detail.task;
  const runs = t.runs || [];
-  const run = runs.find((r) => r.id === state.detail.runId) || runs[0] || null;
-  state.detail.runId = run ? run.id : null;
+  const run = t.run || null;
+  state.detail.runId = run ? run.id : (runs[0] ? runs[0].id : null);
  const cfg = t.config || {};

  const meta = `
@@ -546,8 +581,8 @@ function renderDetail() {
      <span>输出: <b>${esc(cfg.out_dir || "out/" + t.id)}</b></span>
      ${t.mode === "auto" ? `
        <span>起始: <b>${esc((t.auto && t.auto.seed_url) || "")}</b></span>
-        <span>待爬缓存: <b>${(t.auto && (t.auto.pending || []).length) || 0}</b> 条（停止后可继续爬取）</span>
-        <span>已爬: <b>${(t.auto && (t.auto.visited || []).length) || 0}</b> 页</span>` : ""}
+        <span>待爬缓存: <b>${t.auto_pending_count ?? "-"}</b> 条（停止后可继续爬取）</span>
+        <span>已爬: <b>${t.auto_visited_count ?? "-"}</b> 页</span>` : ""}
    </div>`;

  let sched = "";
@@ -604,7 +639,7 @@ function renderRunPanel(t, run) {
    const meta = r.meta_file
      ? `<span class="file-link" onclick="showMeta('${t.id}','${esc(r.meta_file)}')">📋 元数据</span>` : "—";
    return `<tr>
-      <td>${i + 1}</td>
+      <td>${(run.results_total ? (run.run_page - 1) * DETAIL_PAGE_SIZE : 0) + i + 1}</td>
      <td class="${statusCls}">${r.status}</td>
      <td class="title-cell" title="${esc(r.title)}">${esc(r.title)}</td>
      <td class="url-cell" title="${esc(r.url)}">${esc(r.url)}</td>
@@ -621,6 +656,18 @@ function renderRunPanel(t, run) {
        onclick="previewFile('${t.id}','${esc(im.file)}','图片预览')">`).join("")}
    </div>` : "";

+  const total = run.results_total || results.length;
+  const pages = run.run_pages || 1;
+  const page = run.run_page || 1;
+  const pager = total > 0 ? `
+    <div class="pager">
+      ${total > results.length ? `
+        <button class="btn sm" ${page <= 1 ? "disabled" : ""} onclick="goRunPage(${page - 1})">◀ 上一页</button>
+        <span class="pager-info">第 <b>${page}</b> / ${pages} 页 · 共 ${total} 条</span>
+        <button class="btn sm" ${page >= pages ? "disabled" : ""} onclick="goRunPage(${page + 1})">下一页 ▶</button>`
+      : `<span class="pager-info">共 ${total} 条</span>`}
+    </div>` : "";
+
  return `
  <div class="run-panel">
    <div class="run-stats">
@@ -640,7 +687,8 @@ function renderRunPanel(t, run) {
          <thead><tr><th>#</th><th>状态</th><th>标题</th><th>网址</th><th>爬取时间</th><th>HTML</th><th>TXT</th><th>图片</th><th>元数据</th><th>错误</th></tr></thead>
          <tbody>${rows}</tbody>
        </table>
-      </div>` : '<div class="card-line">暂无结果</div>'}
+      </div>
+      ${pager}` : '<div class="card-line">暂无结果</div>'}
    ${thumbHtml}
    <div class="logs" id="logBox"></div>
  </div>`;
@@ -651,13 +699,6 @@ function scrollThumbs() {
  if (box) box.scrollIntoView({ behavior: "smooth", block: "center" });
 }

-async function selectRun(rid) {
-  state.detail.runId = rid;
-  state.detail.logOffset = 0;
-  renderDetail();
-  startLogPoll();
-}
-
 async function startLogPoll() {
  stopLogPoll();
  const t = state.detail.task;
@@ -14,9 +14,9 @@
      <input id="searchInput" placeholder="搜索网址 / 标题 / 任务名..." maxlength="100">
      <button id="btnSearch" class="btn sm primary" title="搜索">🔍</button>
    </div>
-    <span class="status-pill" id="statusPill">运行中任务: 0</span>
+    <span class="status-pill" id="statusPill">运行中任务: -</span>
    <button id="btnTheme" class="btn ghost" title="切换日间/夜间主题">☀️</button>
-    <button id="btnTrash" class="btn ghost" title="回收站">🗑️ 回收站 <span id="trashCount" class="trash-count">0</span></button>
+    <button id="btnTrash" class="btn ghost" title="回收站">🗑️ 回收站 <span id="trashCount" class="trash-count">-</span></button>
    <button id="btnRefresh" class="btn ghost">⟳ 刷新</button>
    <button id="btnNew" class="btn primary">＋ 新建任务</button>
  </div>
@@ -24,13 +24,13 @@

 <main>
  <div class="stats-bar" id="statsBar">
-    <div class="stat-card"><div class="stat-num" id="stTasks">0</div><div class="stat-label">📋 任务总数</div></div>
-    <div class="stat-card"><div class="stat-num num-run" id="stRunning">0</div><div class="stat-label">🔄 正在运行</div></div>
-    <div class="stat-card"><div class="stat-num" id="stRuns">0</div><div class="stat-label">📊 累计运行次数</div></div>
-    <div class="stat-card"><div class="stat-num num-ok" id="stOk">0</div><div class="stat-label">✅ 成功页面</div></div>
-    <div class="stat-card"><div class="stat-num num-fail" id="stFail">0</div><div class="stat-label">❌ 失败页面</div></div>
-    <div class="stat-card"><div class="stat-num" id="stImgs">0</div><div class="stat-label">🖼️ 已爬图片</div></div>
-    <div class="stat-card"><div class="stat-num" id="stDisk">0</div><div class="stat-label">💾 磁盘占用</div></div>
+    <div class="stat-card"><div class="stat-num" id="stTasks">-</div><div class="stat-label">📋 任务总数</div></div>
+    <div class="stat-card"><div class="stat-num num-run" id="stRunning">-</div><div class="stat-label">🔄 正在运行</div></div>
+    <div class="stat-card"><div class="stat-num" id="stRuns">-</div><div class="stat-label">📊 累计运行次数</div></div>
+    <div class="stat-card"><div class="stat-num num-ok" id="stOk">-</div><div class="stat-label">✅ 成功页面</div></div>
+    <div class="stat-card"><div class="stat-num num-fail" id="stFail">-</div><div class="stat-label">❌ 失败页面</div></div>
+    <div class="stat-card"><div class="stat-num" id="stImgs">-</div><div class="stat-label">🖼️ 已爬图片</div></div>
+    <div class="stat-card"><div class="stat-num" id="stDisk">-</div><div class="stat-label">💾 磁盘占用</div></div>
  </div>
  <div id="taskList" class="task-grid"></div>
  <div id="emptyState" class="empty hidden">
@@ -275,3 +275,15 @@ td.title-cell { max-width: 220px; overflow: hidden; text-overflow: ellipsis; }
@keyframes fadein { from { opacity: 0; transform: translate(-50%, -8px); } }
 ::-webkit-scrollbar { width: 8px; height: 8px; }
 ::-webkit-scrollbar-thumb { background: var(--thumb); border-radius: 4px; }
+
+/* 结果分页控件 */
+.pager {
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  gap: 12px;
+  padding: 10px 0;
+  flex-wrap: wrap;
+}
+.pager-info { color: var(--text-dim, #999); font-size: 13px; }
+.pager .btn:disabled { opacity: 0.4; cursor: not-allowed; }