v1.0.14: 自动爬取支持深度/页数无限制(默认0/1000), 待爬链接持久化缓存队列(停止后续爬), 起始网址不去重, 清空缓存API

README.md

@@ -51,7 +51,9 @@
 ### 4. 自动爬取模式
 给定一个起始网址，系统**持续递归**爬取：爬取页面 → 自动发现符合规则的链接 → 继续爬取 → 继续发现……直到**无新链接可爬**时自动结束。
 - **🧪 试爬取**：先用起始网址试跑一次，展示规则筛选后的链接清单（将爬取哪些、排除哪些及原因），确认规则符合预期后再正式开爬
-- **最大页数（安全上限）**：无深度限制，但达到页数上限自动停止，防止动态无限链接的站点失控
+- **最大爬取深度**：0=无限制（默认），N=只爬 N 层
+- **最大页数**：0=无限制，默认 1000 作安全上限（防止动态无限链接的站点失控）
+- **🔄 缓存续爬**：提取但未爬取的链接自动存入缓存队列（连同已爬集合一并持久化），任务停止/中断后再次运行，从缓存队列**继续爬取**，已爬过的不会重复；起始网址每次运行都重新爬取（不去重）；规则变更后可点「🧹 清空缓存」重新开始
 - **包含规则**：只爬包含指定子串（或正则）的链接
 - **排除规则**：跳过匹配的链接（如 login、/tag/）
 - **仅同域名**：限制在起始网站内

app.py

@@ -448,6 +448,27 @@ def api_probe():
     return jsonify(result)
 
 
+@app.route("/api/tasks/<tid>/clear_cache", methods=["POST"])
+def api_clear_cache(tid):
+    """清空自动任务的待爬缓存队列与已爬集合 (规则变更后重新开始用)"""
+    task = store.get_task(tid)
+    if not task:
+        return jsonify({"error": "任务不存在"}), 404
+    if task.get("mode") != "auto":
+        return jsonify({"error": "仅自动爬取任务支持清空缓存"}), 400
+    with JOBS_LOCK:
+        job = JOBS.get(tid)
+        if job and job.is_running():
+            return jsonify({"error": "任务正在运行，无法清空缓存"}), 409
+    auto = task.setdefault("auto", {})
+    auto["pending"] = []
+    auto["visited"] = []
+    task["updated_at"] = now_str()
+    store.upsert_task(task)
+    db.upsert_task(task)
+    return jsonify({"ok": True, "msg": "缓存队列已清空"})
+
+
 @app.route("/api/tasks/<tid>/probe", methods=["POST"])
 def api_task_probe(tid):
     """对已保存的自动任务执行试爬取 (使用保存的规则)"""

engine.py

@@ -21,6 +21,7 @@ from playwright_stealth import Stealth
 
 import notify
 import store
+import db
 
 HERE = os.path.dirname(os.path.abspath(__file__))
 DATA_DIR = os.path.join(HERE, "data")
@@ -567,34 +568,48 @@ class CrawlJob:
         return included
 
     def _crawl_auto(self):
-        """自动爬取: 持续递归 爬取->发现链接->爬取... 直到无新链接可爬或达到安全上限"""
+        """自动爬取: 持续递归 爬取->发现链接->爬取... 直到无新链接可爬或达到上限
+        - max_depth: 0=无限制, N=只爬 N 层
+        - max_pages: 0=无限制, N=安全上限
+        - 提取但未爬取的链接持久化到 auto.pending, 已爬集合持久化到 auto.visited
+        - 停止后再次运行从缓存队列继续爬; 起始网址每次运行都重新爬(不去重)
+        """
         run = self.run
         auto = self.task.get("auto", {})
         seed = auto.get("seed_url", "")
-        max_pages = int(auto.get("max_pages", 200) or 200)   # 安全上限, 防失控
+        max_pages = int(auto.get("max_pages", 1000) or 0)     # 0 = 无限制
-        run["progress"]["total"] = max_pages
+        max_depth = int(auto.get("max_depth", 0) or 0)        # 0 = 无限制
         out_dir = self._resolve_out_dir()
         run["out_dir"] = out_dir
         os.makedirs(out_dir, exist_ok=True)
+
+        # 恢复持久化状态: 已爬集合 + 上次未爬完的缓存队列
+        visited = set(auto.get("visited", []) or [])
+        pending = auto.get("pending", []) or []
+        # 起始网址每次运行都爬(不做去重), 缓存队列继续消费
+        queue = [(seed, 0, "")]
+        if pending:
+            queue.extend((p["url"], p.get("depth", 0), p.get("source", "")) for p in pending)
+        queued = set(visited)
+        for u, _d, _s in queue:
+            queued.add(normalize_url(u))
+
+        run["progress"]["total"] = len(queue)
         self._persist()
 
         p, browser, ctx, page, cookie_file = self._open_browser()
-        queue = [(seed, 0, "")]               # (url, depth, 来源链接)
-        visited = set()                        # 规范化 URL 去重
-        queued = set([normalize_url(seed)])
-        idx = 0
         try:
             while queue and not self._stop.is_set():
                 self._wait_if_paused()
-                url, depth, src = queue.pop(0)
+                if max_pages > 0 and len(visited) >= max_pages:
-                key = normalize_url(url)
-                if key in visited:
-                    continue
-                if len(visited) >= max_pages:
                     self._log("info", f"达到最大页数上限 {max_pages}, 停止")
                     break
+                url, depth, src = queue.pop(0)
+                key = normalize_url(url)
+                if key in visited and url != seed:     # 起始网址不去重, 其余已爬跳过
+                    continue
                 visited.add(key)
-                idx += 1
+                idx = len(visited)
                 run["progress"]["current_url"] = url
                 run["progress"]["done"] = len(visited)
                 self._persist()
@@ -603,8 +618,8 @@ class CrawlJob:
                 run["results"].append(entry)
                 self._bump_stats(entry)
                 self._persist()
-                # 无深度限制: 只要页面爬取成功就继续发现链接, 递归直到队列为空
+                # 无深度限制或未达深度限制时持续发现链接
-                if entry["status"] == "OK":
+                if entry["status"] == "OK" and (max_depth == 0 or depth < max_depth):
                     for link in self._discover_links(page):
                         lk = normalize_url(link)
                         if lk not in visited and lk not in queued:
@@ -612,9 +627,18 @@ class CrawlJob:
                             queue.append((link, depth + 1, url))
                 if entry["status"] == "OK":
                     self._delay()
+            run["progress"]["total"] = len(visited)
+            if not self._stop.is_set() and len(queue) == 0:
+                self._log("info", f"无新链接可爬, 任务结束 (共 {len(visited)} 页)")
         finally:
             self._close_browser(p, browser, ctx, cookie_file)
-        run["progress"]["total"] = len(visited)
+            # 无论完成/停止/异常, 都保存缓存队列与已爬集合, 便于下次继续
-        if not self._stop.is_set() and len(visited) < max_pages:
+            try:
-            self._log("info", f"无新链接可爬, 任务结束 (共 {len(visited)} 页)")
+                auto["pending"] = [
+                    {"url": u, "depth": d, "source": s} for u, d, s in queue]
+                auto["visited"] = list(visited)
+                store.upsert_task(self.task)
+                db.upsert_task(self.task)
+            except Exception:
+                pass
         self._persist()

static/app.js

@@ -434,7 +434,8 @@ async function openEdit(tid) {
     f.elements["exclude"].value = (t.auto.exclude || []).join("\n");
     f.elements["same_domain"].checked = t.auto.same_domain !== false;
     f.elements["use_regex"].checked = !!t.auto.use_regex;
-    f.elements["max_pages"].value = t.auto.max_pages ?? 200;
+    f.elements["max_pages"].value = t.auto.max_pages ?? 1000;
+    f.elements["max_depth"].value = t.auto.max_depth ?? 0;
   }
   syncScheduleUI();
   $("formHint").textContent = t.running
@@ -480,7 +481,8 @@ async function submitForm(e) {
       exclude: splitLines(f.elements["exclude"].value),
       same_domain: f.elements["same_domain"].checked,
       use_regex: f.elements["use_regex"].checked,
-      max_pages: parseInt(f.elements["max_pages"].value) || 200,
+      max_pages: parseInt(f.elements["max_pages"].value) || 0,
+      max_depth: parseInt(f.elements["max_depth"].value) || 0,
     };
   }
   if (mode === "scheduled") {
@@ -542,7 +544,10 @@ function renderDetail() {
       <span>任务ID: <b>${t.id}</b></span>
       <span>创建: <b>${fmtTime(t.created_at)}</b></span>
       <span>输出: <b>${esc(cfg.out_dir || "out/" + t.id)}</b></span>
-      ${t.mode === "auto" ? `<span>起始: <b>${esc((t.auto && t.auto.seed_url) || "")}</b></span>` : ""}
+      ${t.mode === "auto" ? `
+        <span>起始: <b>${esc((t.auto && t.auto.seed_url) || "")}</b></span>
+        <span>待爬缓存: <b>${(t.auto && (t.auto.pending || []).length) || 0}</b> 条（停止后可继续爬取）</span>
+        <span>已爬: <b>${(t.auto && (t.auto.visited || []).length) || 0}</b> 页</span>` : ""}
     </div>`;
 
   let sched = "";
@@ -563,7 +568,8 @@ function renderDetail() {
         <button class="btn" onclick="actTask('${t.id}','pause')">⏸ 暂停</button>
         <button class="btn danger" onclick="actTask('${t.id}','stop')">⏹ 终止</button>`
       : `<button class="btn primary" onclick="actTask('${t.id}','start')">▶ 立即执行</button>`}
-      ${t.mode === "auto" ? `<button class="btn" onclick="probeTask('${t.id}')">🧪 试爬取</button>` : ""}
+      ${t.mode === "auto" ? `<button class="btn" onclick="probeTask('${t.id}')">🧪 试爬取</button>
+        <button class="btn" ${t.running ? "disabled" : ""} onclick="clearCache('${t.id}')">🧹 清空缓存</button>` : ""}
       <button class="btn" onclick="openEdit('${t.id}')">✏️ 编辑配置</button>
     </div>`;
 
@@ -719,6 +725,15 @@ async function probeTask(tid) {
   } catch (e) { toast(e.message, true); }
 }
 
+async function clearCache(tid) {
+  if (!confirm("清空待爬缓存与已爬记录？\n下次运行将从起始网址重新开始爬取。")) return;
+  try {
+    const r = await api(`/api/tasks/${tid}/clear_cache`, { method: "POST" });
+    toast(r.msg || "已清空");
+    openDetail(tid);
+  } catch (e) { toast(e.message, true); }
+}
+
 function renderProbe(r) {
   const body = $("probeBody");
   if (!r.ok) {

static/index.html

@@ -132,8 +132,12 @@
             <div class="field check"><label><input name="same_domain" type="checkbox" checked> 仅爬同域名</label></div>
             <div class="field check"><label><input name="use_regex" type="checkbox"> 规则按正则匹配</label></div>
           </div>
-          <div class="field"><label>最大页数（安全上限，达到后自动停止；不设深度限制，会一直递归爬取到无新链接可爬为止）</label>
+          <div class="row2">
-            <input name="max_pages" type="number" value="200"></div>
+            <div class="field"><label>最大爬取深度（0=无限制，N=只爬 N 层）</label>
+              <input name="max_depth" type="number" min="0" value="0"></div>
+            <div class="field"><label>最大页数（0=无限制，默认 1000 作安全上限）</label>
+              <input name="max_pages" type="number" min="0" value="1000"></div>
+          </div>
         </div>
       </div>
       <div class="modal-foot">