v1.0.14: 自动爬取支持深度/页数无限制(默认0/1000), 待爬链接持久化缓存队列(停止后续爬), 起始网址不去重, 清空缓存API

2026-08-11 21:07:31 +08:00
parent 3ad3d00ca8
commit 66ba1a7cfc
5 changed files with 91 additions and 25 deletions
@@ -51,7 +51,9 @@
 ### 4. 自动爬取模式
 给定一个起始网址，系统**持续递归**爬取：爬取页面 → 自动发现符合规则的链接 → 继续爬取 → 继续发现……直到**无新链接可爬**时自动结束。
 - **🧪 试爬取**：先用起始网址试跑一次，展示规则筛选后的链接清单（将爬取哪些、排除哪些及原因），确认规则符合预期后再正式开爬
- **最大页数（安全上限）**：无深度限制，但达到页数上限自动停止，防止动态无限链接的站点失控
+- **最大爬取深度**：0=无限制（默认），N=只爬 N 层
+- **最大页数**：0=无限制，默认 1000 作安全上限（防止动态无限链接的站点失控）
+- **🔄 缓存续爬**：提取但未爬取的链接自动存入缓存队列（连同已爬集合一并持久化），任务停止/中断后再次运行，从缓存队列**继续爬取**，已爬过的不会重复；起始网址每次运行都重新爬取（不去重）；规则变更后可点「🧹 清空缓存」重新开始
 - **包含规则**：只爬包含指定子串（或正则）的链接
 - **排除规则**：跳过匹配的链接（如 login、/tag/）
 - **仅同域名**：限制在起始网站内
@@ -448,6 +448,27 @@ def api_probe():
    return jsonify(result)


+@app.route("/api/tasks/<tid>/clear_cache", methods=["POST"])
+def api_clear_cache(tid):
+    """清空自动任务的待爬缓存队列与已爬集合 (规则变更后重新开始用)"""
+    task = store.get_task(tid)
+    if not task:
+        return jsonify({"error": "任务不存在"}), 404
+    if task.get("mode") != "auto":
+        return jsonify({"error": "仅自动爬取任务支持清空缓存"}), 400
+    with JOBS_LOCK:
+        job = JOBS.get(tid)
+        if job and job.is_running():
+            return jsonify({"error": "任务正在运行，无法清空缓存"}), 409
+    auto = task.setdefault("auto", {})
+    auto["pending"] = []
+    auto["visited"] = []
+    task["updated_at"] = now_str()
+    store.upsert_task(task)
+    db.upsert_task(task)
+    return jsonify({"ok": True, "msg": "缓存队列已清空"})
+
+
@app.route("/api/tasks/<tid>/probe", methods=["POST"])
 def api_task_probe(tid):
    """对已保存的自动任务执行试爬取 (使用保存的规则)"""
@@ -21,6 +21,7 @@ from playwright_stealth import Stealth

 import notify
 import store
+import db

 HERE = os.path.dirname(os.path.abspath(__file__))
 DATA_DIR = os.path.join(HERE, "data")
@@ -567,34 +568,48 @@ class CrawlJob:
        return included

    def _crawl_auto(self):
-        """自动爬取: 持续递归 爬取->发现链接->爬取... 直到无新链接可爬或达到安全上限"""
+        """自动爬取: 持续递归 爬取->发现链接->爬取... 直到无新链接可爬或达到上限
+        - max_depth: 0=无限制, N=只爬 N 层
+        - max_pages: 0=无限制, N=安全上限
+        - 提取但未爬取的链接持久化到 auto.pending, 已爬集合持久化到 auto.visited
+        - 停止后再次运行从缓存队列继续爬; 起始网址每次运行都重新爬(不去重)
+        """
        run = self.run
        auto = self.task.get("auto", {})
        seed = auto.get("seed_url", "")
-        max_pages = int(auto.get("max_pages", 200) or 200)   # 安全上限, 防失控
-        run["progress"]["total"] = max_pages
+        max_pages = int(auto.get("max_pages", 1000) or 0)     # 0 = 无限制
+        max_depth = int(auto.get("max_depth", 0) or 0)        # 0 = 无限制
        out_dir = self._resolve_out_dir()
        run["out_dir"] = out_dir
        os.makedirs(out_dir, exist_ok=True)
+
+        # 恢复持久化状态: 已爬集合 + 上次未爬完的缓存队列
+        visited = set(auto.get("visited", []) or [])
+        pending = auto.get("pending", []) or []
+        # 起始网址每次运行都爬(不做去重), 缓存队列继续消费
+        queue = [(seed, 0, "")]
+        if pending:
+            queue.extend((p["url"], p.get("depth", 0), p.get("source", "")) for p in pending)
+        queued = set(visited)
+        for u, _d, _s in queue:
+            queued.add(normalize_url(u))
+
+        run["progress"]["total"] = len(queue)
        self._persist()

        p, browser, ctx, page, cookie_file = self._open_browser()
-        queue = [(seed, 0, "")]               # (url, depth, 来源链接)
-        visited = set()                        # 规范化 URL 去重
-        queued = set([normalize_url(seed)])
-        idx = 0
        try:
            while queue and not self._stop.is_set():
                self._wait_if_paused()
-                url, depth, src = queue.pop(0)
-                key = normalize_url(url)
-                if key in visited:
-                    continue
-                if len(visited) >= max_pages:
+                if max_pages > 0 and len(visited) >= max_pages:
                    self._log("info", f"达到最大页数上限 {max_pages}, 停止")
                    break
+                url, depth, src = queue.pop(0)
+                key = normalize_url(url)
+                if key in visited and url != seed:     # 起始网址不去重, 其余已爬跳过
+                    continue
                visited.add(key)
-                idx += 1
+                idx = len(visited)
                run["progress"]["current_url"] = url
                run["progress"]["done"] = len(visited)
                self._persist()
@@ -603,8 +618,8 @@ class CrawlJob:
                run["results"].append(entry)
                self._bump_stats(entry)
                self._persist()
-                # 无深度限制: 只要页面爬取成功就继续发现链接, 递归直到队列为空
-                if entry["status"] == "OK":
+                # 无深度限制或未达深度限制时持续发现链接
+                if entry["status"] == "OK" and (max_depth == 0 or depth < max_depth):
                    for link in self._discover_links(page):
                        lk = normalize_url(link)
                        if lk not in visited and lk not in queued:
@@ -612,9 +627,18 @@ class CrawlJob:
                            queue.append((link, depth + 1, url))
                if entry["status"] == "OK":
                    self._delay()
+            run["progress"]["total"] = len(visited)
+            if not self._stop.is_set() and len(queue) == 0:
+                self._log("info", f"无新链接可爬, 任务结束 (共 {len(visited)} 页)")
        finally:
            self._close_browser(p, browser, ctx, cookie_file)
-        run["progress"]["total"] = len(visited)
-        if not self._stop.is_set() and len(visited) < max_pages:
-            self._log("info", f"无新链接可爬, 任务结束 (共 {len(visited)} 页)")
+            # 无论完成/停止/异常, 都保存缓存队列与已爬集合, 便于下次继续
+            try:
+                auto["pending"] = [
+                    {"url": u, "depth": d, "source": s} for u, d, s in queue]
+                auto["visited"] = list(visited)
+                store.upsert_task(self.task)
+                db.upsert_task(self.task)
+            except Exception:
+                pass
        self._persist()
@@ -434,7 +434,8 @@ async function openEdit(tid) {
    f.elements["exclude"].value = (t.auto.exclude || []).join("\n");
    f.elements["same_domain"].checked = t.auto.same_domain !== false;
    f.elements["use_regex"].checked = !!t.auto.use_regex;
-    f.elements["max_pages"].value = t.auto.max_pages ?? 200;
+    f.elements["max_pages"].value = t.auto.max_pages ?? 1000;
+    f.elements["max_depth"].value = t.auto.max_depth ?? 0;
  }
  syncScheduleUI();
  $("formHint").textContent = t.running
@@ -480,7 +481,8 @@ async function submitForm(e) {
      exclude: splitLines(f.elements["exclude"].value),
      same_domain: f.elements["same_domain"].checked,
      use_regex: f.elements["use_regex"].checked,
-      max_pages: parseInt(f.elements["max_pages"].value) || 200,
+      max_pages: parseInt(f.elements["max_pages"].value) || 0,
+      max_depth: parseInt(f.elements["max_depth"].value) || 0,
    };
  }
  if (mode === "scheduled") {
@@ -542,7 +544,10 @@ function renderDetail() {
      <span>任务ID: <b>${t.id}</b></span>
      <span>创建: <b>${fmtTime(t.created_at)}</b></span>
      <span>输出: <b>${esc(cfg.out_dir || "out/" + t.id)}</b></span>
-      ${t.mode === "auto" ? `<span>起始: <b>${esc((t.auto && t.auto.seed_url) || "")}</b></span>` : ""}
+      ${t.mode === "auto" ? `
+        <span>起始: <b>${esc((t.auto && t.auto.seed_url) || "")}</b></span>
+        <span>待爬缓存: <b>${(t.auto && (t.auto.pending || []).length) || 0}</b> 条（停止后可继续爬取）</span>
+        <span>已爬: <b>${(t.auto && (t.auto.visited || []).length) || 0}</b> 页</span>` : ""}
    </div>`;

  let sched = "";
@@ -563,7 +568,8 @@ function renderDetail() {
        <button class="btn" onclick="actTask('${t.id}','pause')">⏸ 暂停</button>
        <button class="btn danger" onclick="actTask('${t.id}','stop')">⏹ 终止</button>`
      : `<button class="btn primary" onclick="actTask('${t.id}','start')">▶ 立即执行</button>`}
-      ${t.mode === "auto" ? `<button class="btn" onclick="probeTask('${t.id}')">🧪 试爬取</button>` : ""}
+      ${t.mode === "auto" ? `<button class="btn" onclick="probeTask('${t.id}')">🧪 试爬取</button>
+        <button class="btn" ${t.running ? "disabled" : ""} onclick="clearCache('${t.id}')">🧹 清空缓存</button>` : ""}
      <button class="btn" onclick="openEdit('${t.id}')">✏️ 编辑配置</button>
    </div>`;

@@ -719,6 +725,15 @@ async function probeTask(tid) {
  } catch (e) { toast(e.message, true); }
 }

+async function clearCache(tid) {
+  if (!confirm("清空待爬缓存与已爬记录？\n下次运行将从起始网址重新开始爬取。")) return;
+  try {
+    const r = await api(`/api/tasks/${tid}/clear_cache`, { method: "POST" });
+    toast(r.msg || "已清空");
+    openDetail(tid);
+  } catch (e) { toast(e.message, true); }
+}
+
 function renderProbe(r) {
  const body = $("probeBody");
  if (!r.ok) {
@@ -132,8 +132,12 @@
            <div class="field check"><label><input name="same_domain" type="checkbox" checked> 仅爬同域名</label></div>
            <div class="field check"><label><input name="use_regex" type="checkbox"> 规则按正则匹配</label></div>
          </div>
-          <div class="field"><label>最大页数（安全上限，达到后自动停止；不设深度限制，会一直递归爬取到无新链接可爬为止）</label>
-            <input name="max_pages" type="number" value="200"></div>
+          <div class="row2">
+            <div class="field"><label>最大爬取深度（0=无限制，N=只爬 N 层）</label>
+              <input name="max_depth" type="number" min="0" value="0"></div>
+            <div class="field"><label>最大页数（0=无限制，默认 1000 作安全上限）</label>
+              <input name="max_pages" type="number" min="0" value="1000"></div>
+          </div>
        </div>
      </div>
      <div class="modal-foot">