v1.0.13: 自动爬取改为持续递归(无深度限制, 爬到无新链接为止), max_pages作安全上限(默认200)

engine.py

@@ -567,11 +567,11 @@ class CrawlJob:
         return included
 
     def _crawl_auto(self):
+        """自动爬取: 持续递归 爬取->发现链接->爬取... 直到无新链接可爬或达到安全上限"""
         run = self.run
         auto = self.task.get("auto", {})
         seed = auto.get("seed_url", "")
-        max_pages = int(auto.get("max_pages", 50) or 50)
-        max_depth = int(auto.get("max_depth", 2) or 2)
+        max_pages = int(auto.get("max_pages", 200) or 200)   # 安全上限, 防失控
         run["progress"]["total"] = max_pages
         out_dir = self._resolve_out_dir()
         run["out_dir"] = out_dir
@@ -591,6 +591,7 @@ class CrawlJob:
                 if key in visited:
                     continue
                 if len(visited) >= max_pages:
+                    self._log("info", f"达到最大页数上限 {max_pages}, 停止")
                     break
                 visited.add(key)
                 idx += 1
@@ -602,7 +603,8 @@ class CrawlJob:
                 run["results"].append(entry)
                 self._bump_stats(entry)
                 self._persist()
-                if entry["status"] == "OK" and depth < max_depth:
+                # 无深度限制: 只要页面爬取成功就继续发现链接, 递归直到队列为空
+                if entry["status"] == "OK":
                     for link in self._discover_links(page):
                         lk = normalize_url(link)
                         if lk not in visited and lk not in queued:
@@ -613,4 +615,6 @@ class CrawlJob:
         finally:
             self._close_browser(p, browser, ctx, cookie_file)
         run["progress"]["total"] = len(visited)
+        if not self._stop.is_set() and len(visited) < max_pages:
+            self._log("info", f"无新链接可爬, 任务结束 (共 {len(visited)} 页)")
         self._persist()