diff --git a/app.py b/app.py index 25da80a..b591147 100644 --- a/app.py +++ b/app.py @@ -15,7 +15,7 @@ from flask import Flask, jsonify, request, send_file, send_from_directory import store import db import notify -from engine import CrawlJob, probe_links +from engine import CrawlJob, probe_links, normalize_url from scheduler import Scheduler, cron_next, interval_delta HERE = os.path.dirname(os.path.abspath(__file__)) @@ -435,6 +435,54 @@ def api_trash_clear(): # ---------------- API: 运行控制 ---------------- +@app.route("/api/tasks//retry-failed", methods=["POST"]) +def api_retry_failed(tid): + """重爬失败页: 提取指定 run (默认最新) 中失败的 URL, 从已爬集合解除标记并注入 + 待爬队列头部; 返回注入数量, 调用方可随后点「继续爬取」重爬这些页面""" + task = store.get_task(tid) + if not task: + return jsonify({"error": "任务不存在"}), 404 + if task.get("mode") != "auto": + return jsonify({"error": "仅自动模式任务支持重爬失败页"}), 400 + try: + body = request.get_json(force=True) or {} + except Exception: + body = {} + rid = request.args.get("run", "") or body.get("run", "") + runs = store.get_runs(tid) + cur = None + if rid: + cur = next((r for r in runs if r["id"] == rid), None) + else: + cur = runs[-1] if runs else None + if not cur: + return jsonify({"error": "运行记录不存在"}), 404 + failed = [res.get("url") for res in cur.get("results", []) + if res.get("status") == "FAIL" and res.get("url")] + if not failed: + return jsonify({"error": "该运行记录没有失败页面", "injected": 0}) + st = store.load_auto_state(tid, task) + visited = set(st.get("visited", [])) + pending = st.get("pending", []) + pending_urls = {normalize_url(p.get("url", "")) for p in pending} + # 待重爬: 已爬过且不在待爬队列中的失败 URL (去重) + to_inject, seen = [], set() + for u in failed: + key = normalize_url(u) + if key in visited and key not in pending_urls and key not in seen: + seen.add(key) + to_inject.append({"url": u, "depth": 0, "source": "retry-failed"}) + if not to_inject: + return jsonify({"error": "失败页面均已爬或已在待爬队列中", "injected": 0}) + # 解除已爬标记 + remove_keys = {normalize_url(u) for u in failed} + visited = {v for v in visited if normalize_url(v) not in remove_keys} + # 注入队列头部, 优先重爬 + pending = to_inject + pending + store.save_auto_state(tid, {"pending": pending, "visited": sorted(visited)}) + return jsonify({"injected": len(to_inject), "pending_total": len(pending)}) + + @app.route("/api/tasks//continue", methods=["POST"]) def api_continue(tid): """继续爬取: auto 任务从待爬缓存队列接着爬 (跳过起始网址, 保留已爬集合)""" diff --git a/static/app.js b/static/app.js index d457d9b..db21759 100644 --- a/static/app.js +++ b/static/app.js @@ -618,6 +618,26 @@ function pageWindow(cur, pages, width) { return win; } +/* 重爬失败页: 注入待爬队列并自动继续爬取 */ +async function retryFailed() { + const t = state.detail.task; + if (!t || !state.detail.runId) return; + if (!confirm("将把本次运行失败的页面重新加入待爬队列并立即继续爬取,确定?")) return; + try { + const d = await api(`/api/tasks/${t.id}/retry-failed`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ run: state.detail.runId }), + }); + toast(`已注入 ${d.injected} 个失败页,开始继续爬取…`); + if ((d.injected || 0) > 0) { + await api(`/api/tasks/${t.id}/continue`, { method: "POST" }); + } + loadTasks(); + openDetail(t.id); + } catch (e) { toast(e.message, true); } +} + function renderDetail() { const t = state.detail.task; const runs = t.runs || []; @@ -752,6 +772,8 @@ function renderRunPanel(t, run) { ✅ ${s.ok} ❌ ${s.fail} 🖼️ ${s.images} + ${t.mode === "auto" && (s.fail || 0) > 0 && run.status !== "running" && run.status !== "paused" ? ` + ` : ""} ${prog}
当前: ${esc(run.progress.current_url || "")}