diff --git a/README.md b/README.md index af32556..6bdaf48 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,9 @@ ### 4. 自动爬取模式 给定一个起始网址，系统**持续递归**爬取：爬取页面 → 自动发现符合规则的链接 → 继续爬取 → 继续发现……直到**无新链接可爬**时自动结束。 - **🧪 试爬取**：先用起始网址试跑一次，展示规则筛选后的链接清单（将爬取哪些、排除哪些及原因），确认规则符合预期后再正式开爬 -- **最大页数（安全上限）**：无深度限制，但达到页数上限自动停止，防止动态无限链接的站点失控 +- **最大爬取深度**：0=无限制（默认），N=只爬 N 层 +- **最大页数**：0=无限制，默认 1000 作安全上限（防止动态无限链接的站点失控） +- **🔄 缓存续爬**：提取但未爬取的链接自动存入缓存队列（连同已爬集合一并持久化），任务停止/中断后再次运行，从缓存队列**继续爬取**，已爬过的不会重复；起始网址每次运行都重新爬取（不去重）；规则变更后可点「🧹 清空缓存」重新开始 - **包含规则**：只爬包含指定子串（或正则）的链接 - **排除规则**：跳过匹配的链接（如 login、/tag/） - **仅同域名**：限制在起始网站内 diff --git a/app.py b/app.py index d0a4721..6a88999 100644 --- a/app.py +++ b/app.py @@ -448,6 +448,27 @@ def api_probe(): return jsonify(result) +@app.route("/api/tasks//clear_cache", methods=["POST"]) +def api_clear_cache(tid): + """清空自动任务的待爬缓存队列与已爬集合 (规则变更后重新开始用)""" + task = store.get_task(tid) + if not task: + return jsonify({"error": "任务不存在"}), 404 + if task.get("mode") != "auto": + return jsonify({"error": "仅自动爬取任务支持清空缓存"}), 400 + with JOBS_LOCK: + job = JOBS.get(tid) + if job and job.is_running(): + return jsonify({"error": "任务正在运行，无法清空缓存"}), 409 + auto = task.setdefault("auto", {}) + auto["pending"] = [] + auto["visited"] = [] + task["updated_at"] = now_str() + store.upsert_task(task) + db.upsert_task(task) + return jsonify({"ok": True, "msg": "缓存队列已清空"}) + + @app.route("/api/tasks//probe", methods=["POST"]) def api_task_probe(tid): """对已保存的自动任务执行试爬取 (使用保存的规则)""" diff --git a/engine.py b/engine.py index c3be35b..c7a8c4d 100644 --- a/engine.py +++ b/engine.py @@ -21,6 +21,7 @@ from playwright_stealth import Stealth import notify import store +import db HERE = os.path.dirname(os.path.abspath(__file__)) DATA_DIR = os.path.join(HERE, "data") @@ -567,34 +568,48 @@ class CrawlJob: return included def _crawl_auto(self): - """自动爬取: 持续递归 爬取->发现链接->爬取... 直到无新链接可爬或达到安全上限""" + """自动爬取: 持续递归 爬取->发现链接->爬取... 直到无新链接可爬或达到上限 + - max_depth: 0=无限制, N=只爬 N 层 + - max_pages: 0=无限制, N=安全上限 + - 提取但未爬取的链接持久化到 auto.pending, 已爬集合持久化到 auto.visited + - 停止后再次运行从缓存队列继续爬; 起始网址每次运行都重新爬(不去重) + """ run = self.run auto = self.task.get("auto", {}) seed = auto.get("seed_url", "") - max_pages = int(auto.get("max_pages", 200) or 200) # 安全上限, 防失控 - run["progress"]["total"] = max_pages + max_pages = int(auto.get("max_pages", 1000) or 0) # 0 = 无限制 + max_depth = int(auto.get("max_depth", 0) or 0) # 0 = 无限制 out_dir = self._resolve_out_dir() run["out_dir"] = out_dir os.makedirs(out_dir, exist_ok=True) + + # 恢复持久化状态: 已爬集合 + 上次未爬完的缓存队列 + visited = set(auto.get("visited", []) or []) + pending = auto.get("pending", []) or [] + # 起始网址每次运行都爬(不做去重), 缓存队列继续消费 + queue = [(seed, 0, "")] + if pending: + queue.extend((p["url"], p.get("depth", 0), p.get("source", "")) for p in pending) + queued = set(visited) + for u, _d, _s in queue: + queued.add(normalize_url(u)) + + run["progress"]["total"] = len(queue) self._persist() p, browser, ctx, page, cookie_file = self._open_browser() - queue = [(seed, 0, "")] # (url, depth, 来源链接) - visited = set() # 规范化 URL 去重 - queued = set([normalize_url(seed)]) - idx = 0 try: while queue and not self._stop.is_set(): self._wait_if_paused() - url, depth, src = queue.pop(0) - key = normalize_url(url) - if key in visited: - continue - if len(visited) >= max_pages: + if max_pages > 0 and len(visited) >= max_pages: self._log("info", f"达到最大页数上限 {max_pages}, 停止") break + url, depth, src = queue.pop(0) + key = normalize_url(url) + if key in visited and url != seed: # 起始网址不去重, 其余已爬跳过 + continue visited.add(key) - idx += 1 + idx = len(visited) run["progress"]["current_url"] = url run["progress"]["done"] = len(visited) self._persist() @@ -603,8 +618,8 @@ class CrawlJob: run["results"].append(entry) self._bump_stats(entry) self._persist() - # 无深度限制: 只要页面爬取成功就继续发现链接, 递归直到队列为空 - if entry["status"] == "OK": + # 无深度限制或未达深度限制时持续发现链接 + if entry["status"] == "OK" and (max_depth == 0 or depth < max_depth): for link in self._discover_links(page): lk = normalize_url(link) if lk not in visited and lk not in queued: @@ -612,9 +627,18 @@ class CrawlJob: queue.append((link, depth + 1, url)) if entry["status"] == "OK": self._delay() + run["progress"]["total"] = len(visited) + if not self._stop.is_set() and len(queue) == 0: + self._log("info", f"无新链接可爬, 任务结束 (共 {len(visited)} 页)") finally: self._close_browser(p, browser, ctx, cookie_file) - run["progress"]["total"] = len(visited) - if not self._stop.is_set() and len(visited) < max_pages: - self._log("info", f"无新链接可爬, 任务结束 (共 {len(visited)} 页)") + # 无论完成/停止/异常, 都保存缓存队列与已爬集合, 便于下次继续 + try: + auto["pending"] = [ + {"url": u, "depth": d, "source": s} for u, d, s in queue] + auto["visited"] = list(visited) + store.upsert_task(self.task) + db.upsert_task(self.task) + except Exception: + pass self._persist() diff --git a/static/app.js b/static/app.js index 250de28..5377158 100644 --- a/static/app.js +++ b/static/app.js @@ -434,7 +434,8 @@ async function openEdit(tid) { f.elements["exclude"].value = (t.auto.exclude || []).join("\n"); f.elements["same_domain"].checked = t.auto.same_domain !== false; f.elements["use_regex"].checked = !!t.auto.use_regex; - f.elements["max_pages"].value = t.auto.max_pages ?? 200; + f.elements["max_pages"].value = t.auto.max_pages ?? 1000; + f.elements["max_depth"].value = t.auto.max_depth ?? 0; } syncScheduleUI(); $("formHint").textContent = t.running @@ -480,7 +481,8 @@ async function submitForm(e) { exclude: splitLines(f.elements["exclude"].value), same_domain: f.elements["same_domain"].checked, use_regex: f.elements["use_regex"].checked, - max_pages: parseInt(f.elements["max_pages"].value) || 200, + max_pages: parseInt(f.elements["max_pages"].value) || 0, + max_depth: parseInt(f.elements["max_depth"].value) || 0, }; } if (mode === "scheduled") { @@ -542,7 +544,10 @@ function renderDetail() { 任务ID: ${t.id} 创建: ${fmtTime(t.created_at)} 输出: ${esc(cfg.out_dir || "out/" + t.id)} - ${t.mode === "auto" ? `起始: ${esc((t.auto && t.auto.seed_url) || "")}` : ""} + ${t.mode === "auto" ? ` + 起始: ${esc((t.auto && t.auto.seed_url) || "")} + 待爬缓存: ${(t.auto && (t.auto.pending || []).length) || 0} 条（停止后可继续爬取） + 已爬: ${(t.auto && (t.auto.visited || []).length) || 0} 页` : ""} `; let sched = ""; @@ -563,7 +568,8 @@ function renderDetail() { ` : ``} - ${t.mode === "auto" ? `` : ""} + ${t.mode === "auto" ? ` + ` : ""} `; @@ -719,6 +725,15 @@ async function probeTask(tid) { } catch (e) { toast(e.message, true); } } +async function clearCache(tid) { + if (!confirm("清空待爬缓存与已爬记录？\n下次运行将从起始网址重新开始爬取。")) return; + try { + const r = await api(`/api/tasks/${tid}/clear_cache`, { method: "POST" }); + toast(r.msg || "已清空"); + openDetail(tid); + } catch (e) { toast(e.message, true); } +} + function renderProbe(r) { const body = $("probeBody"); if (!r.ok) { diff --git a/static/index.html b/static/index.html index e0f23a3..f38a767 100644 --- a/static/index.html +++ b/static/index.html @@ -132,8 +132,12 @@
-
-
+
+
+
+
+
+