From ec053266a7f04be54ca75599bedc5a7883266c9e Mon Sep 17 00:00:00 2001 From: hz4th_coder Date: Fri, 14 Aug 2026 10:28:18 +0800 Subject: [PATCH] =?UTF-8?q?v1.4.0=20=E4=BF=AE=E5=A4=8D=E5=8F=8D=E7=88=AC?= =?UTF-8?q?=E8=AF=AF=E5=88=A4+=E6=B5=8F=E8=A7=88=E5=99=A8=E5=B4=A9?= =?UTF-8?q?=E6=BA=83=E7=A9=BA=E8=BD=AC,=20=E8=AF=A6=E6=83=85=E9=A1=B5?= =?UTF-8?q?=E5=A4=9A=E5=85=83=E7=BF=BB=E9=A1=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 修复反爬误判: is_challenge_page 仅匹配 title 关键词 + html 前 20KB 强特征标记, 不再全文匹配 captcha/challenge 等词 (博客园正文提 captcha 的页面被误杀) - 修复浏览器崩溃后空转: 检测 Target page/context/browser closed, 自动重启浏览器并重试当前 URL (不消耗重试配额, 单页最多重建3次); 页面读取连续异常3次快速失败; 新增 browser_max_pages 配置定期重启浏览器防内存膨胀 - 详情页多元翻页: 首页/末页/页码列表(带省略号)/页码跳转输入框(回车/GO)/每页条数选择(50/100/200/500) --- engine.py | 113 ++++++++++++++++++++++++++++++++++++++++------- static/app.js | 85 +++++++++++++++++++++++++++++++---- static/style.css | 17 +++++++ 3 files changed, 191 insertions(+), 24 deletions(-) diff --git a/engine.py b/engine.py index 3214d5a..1ba1973 100644 --- a/engine.py +++ b/engine.py @@ -30,12 +30,21 @@ IMG_EXTS = (".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp") DEFAULT_UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " "(KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36") -_CHALLENGE_MARKS = [ +# title 中出现任一标记即判定反爬 (title 短小可靠, 不会误伤正文) +_CHALLENGE_TITLE_MARKS = [ "access denied", "403 forbidden", "just a moment", "attention required", "captcha", "bot check", "cf-challenge", "verify you are human", "checking your browser", "enable javascript and cookies", ] +# html 中仅匹配强特征标记, 且只在 head 区域(前 20KB)搜索。 +# 不能全文匹配 "captcha"/"challenge" 等词——正常文章正文提到这些词会被误判为反爬。 +_CHALLENGE_HTML_MARKS = [ + "cf-challenge", "challenge-platform", "cf-browser-verification", + "verify you are human", "checking your browser", + "enable javascript and cookies", "just a moment", +] + # 反爬拦截类错误信号: 命中后不再重试 (重试无意义且拖慢任务) _ANTI_CRAWL_MARKS = [ "403", "429", "503", "forbidden", "access denied", "too many requests", @@ -50,14 +59,31 @@ def is_anti_crawl_error(err): return any(m in s for m in _ANTI_CRAWL_MARKS) +_BROWSER_DEAD_MARKS = [ + "browser has been closed", "target page, context or browser has been closed", + "has been disposed", "execution context was destroyed", "browser closed", + "page closed", "target closed", "crash", +] + + +def is_browser_dead_error(err): + """判断错误是否属于浏览器/页面失效 (需重启浏览器后重试, 重试同一 URL 才有意义)""" + s = str(err or "").lower() + return any(m in s for m in _BROWSER_DEAD_MARKS) + + def is_challenge_page(title, html): - """判断是否仍在反爬验证页 (仅关键词启发, 避免误伤正常小页面)""" - low = html.lower() + """判断是否仍在反爬验证页 + - title 出现反爬关键词: 判定 (可靠, 反爬页 title 基本都会变) + - html 仅在前 20KB(head 区域)出现强特征标记时判定, + 避免正文含 captcha/challenge 等词的正常页面被误杀 + """ t = (title or "").lower() - for mark in _CHALLENGE_MARKS: - if mark in t or mark in low: + for mark in _CHALLENGE_TITLE_MARKS: + if mark in t: return True - return False + low = (html or "").lower()[:20000] + return any(m in low for m in _CHALLENGE_HTML_MARKS) def safe_name(url, idx): @@ -231,6 +257,14 @@ class CrawlJob: self._auto_pending = None self._cfg_lock = threading.RLock() self.thread = None + # 浏览器句柄 (p, browser, ctx, page, cookie_file); 崩溃后重建 + self._browser = None + + def _page(self): + return self._browser[3] if self._browser else None + + def _ctx(self): + return self._browser[2] if self._browser else None # ---------------- 控制接口 ---------------- def start(self): @@ -309,9 +343,13 @@ class CrawlJob: pass Stealth().apply_stealth_sync(ctx) page = ctx.new_page() - return p, browser, ctx, page, cookie_file + self._browser = (p, browser, ctx, page, cookie_file) + return self._browser - def _close_browser(self, p, browser, ctx, cookie_file): + def _close_browser(self): + if not self._browser: + return + p, browser, ctx, _page, cookie_file = self._browser try: json.dump(ctx.cookies(), open(cookie_file, "w")) except Exception: @@ -324,10 +362,24 @@ class CrawlJob: p.stop() except Exception: pass + self._browser = None + + def _reopen_browser(self): + """浏览器失效后重建: 保存 cookie -> 关闭旧实例 -> 启动新实例""" + self._close_browser() + time.sleep(1) + self._open_browser() + self._log("info", "浏览器已重启") + + def _need_browser_restart(self, done_count): + """每爬 N 页主动重启一次浏览器, 防止长时间运行内存膨胀导致崩溃 (browser_max_pages=0 关闭)""" + limit = int(self._cfg("browser_max_pages", 0) or 0) + return limit > 0 and done_count > 1 and (done_count - 1) % limit == 0 def _wait_page_settle(self, page, timeout_s): last_title, stable = "", 0 challenge_hits = 0 + err_streak = 0 # 连续读取失败计数 start = time.time() while time.time() - start < timeout_s: if self._stop.is_set(): @@ -338,7 +390,11 @@ class CrawlJob: title = page.title() html = page.content() except Exception: + err_streak += 1 + if err_streak >= 3: # 页面/浏览器已失效, 提前失败而非干等到超时 + raise RuntimeError("Target page, context or browser has been closed") continue # 正在跳转 + err_streak = 0 if is_challenge_page(title, html): challenge_hits += 1 if challenge_hits >= 8: # 持续 8 秒仍是验证页, 判定反爬, 尽早放弃 @@ -461,7 +517,9 @@ class CrawlJob: "source_url": source_url, "depth": depth, "images": [], "attempts": 0, } - for attempt in range(retries + 1): + dead_strikes = 0 # 浏览器失效重建次数 (防止无限重建) + attempt = 0 + while attempt <= retries: if self._stop.is_set(): entry["error"] = "任务已终止" break @@ -485,6 +543,22 @@ class CrawlJob: except Exception as e: entry["error"] = str(e) entry["crawl_time"] = store.now_str() + if is_browser_dead_error(e): + # 浏览器/页面失效: 重启浏览器后重试同一 URL, 不消耗重试次数 + dead_strikes += 1 + if dead_strikes > 3: + entry["error"] = f"浏览器多次重启仍失效, 放弃: {e}" + self._log("error", f"浏览器多次重启仍失效, 放弃 {url}") + break + self._log("warn", f"浏览器已失效({e}), 重启后重试: {url}") + try: + self._reopen_browser() + page, ctx = self._page(), self._ctx() + except Exception as re_err: + entry["error"] = f"浏览器重启失败: {re_err}" + self._log("error", f"浏览器重启失败, 放弃 {url}: {re_err}") + break + continue self._log("warn", f"第{attempt + 1}次失败 {url}: {e}") if is_anti_crawl_error(e): # 反爬拦截: 重试也过不去, 直接放弃, 不再消耗重试次数 @@ -499,6 +573,7 @@ class CrawlJob: break self._wait_if_paused() time.sleep(0.3) + attempt += 1 self._write_page_meta(out_dir, base, entry) return entry @@ -562,17 +637,20 @@ class CrawlJob: os.makedirs(out_dir, exist_ok=True) self._persist() - p, browser, ctx, page, cookie_file = self._open_browser() + self._open_browser() try: for i, url in enumerate(urls, 1): if self._stop.is_set(): self._log("info", "收到终止信号, 停止爬取") break self._wait_if_paused() + if self._need_browser_restart(i): + self._log("info", f"已爬 {i - 1} 页, 主动重启浏览器") + self._reopen_browser() run["progress"]["current_url"] = url run["progress"]["done"] = i - 1 self._persist() - entry = self._retry_crawl(page, ctx, url, i, out_dir) + entry = self._retry_crawl(self._page(), self._ctx(), url, i, out_dir) run["results"].append(entry) self._bump_stats(entry) run["progress"]["done"] = i @@ -580,7 +658,7 @@ class CrawlJob: if entry["status"] == "OK": self._delay() finally: - self._close_browser(p, browser, ctx, cookie_file) + self._close_browser() def _discover_links(self, page): """从当前页面提取符合规则的链接""" @@ -637,7 +715,7 @@ class CrawlJob: self._auto_pending = len(queue) self._persist() - p, browser, ctx, page, cookie_file = self._open_browser() + self._open_browser() try: while queue and not self._stop.is_set(): self._wait_if_paused() @@ -650,6 +728,9 @@ class CrawlJob: elif len(visited) >= max_pages: self._log("info", f"达到最大页数上限 {max_pages}, 停止") break + if self._need_browser_restart(len(visited)): + self._log("info", f"已爬 {len(visited) - 1} 页, 主动重启浏览器") + self._reopen_browser() url, depth, src = queue.pop(0) key = normalize_url(url) if key in visited and url != seed: # 起始网址不去重, 其余已爬跳过 @@ -661,14 +742,14 @@ class CrawlJob: run["progress"]["current_url"] = url run["progress"]["done"] = len(visited) self._persist() - entry = self._retry_crawl(page, ctx, url, idx, out_dir, + entry = self._retry_crawl(self._page(), self._ctx(), url, idx, out_dir, source_url=src, depth=depth) run["results"].append(entry) self._bump_stats(entry) self._persist() # 无深度限制或未达深度限制时持续发现链接 if entry["status"] == "OK" and (max_depth == 0 or depth < max_depth): - for link in self._discover_links(page): + for link in self._discover_links(self._page()): lk = normalize_url(link) if lk not in visited and lk not in queued: queued.add(lk) @@ -680,7 +761,7 @@ class CrawlJob: if not self._stop.is_set() and len(queue) == 0: self._log("info", f"无新链接可爬, 任务结束 (共 {len(visited)} 页)") finally: - self._close_browser(p, browser, ctx, cookie_file) + self._close_browser() # 无论完成/停止/异常, 都保存缓存队列与已爬集合, 便于下次继续 try: store.save_auto_state(self.task["id"], { diff --git a/static/app.js b/static/app.js index f0ce1da..d457d9b 100644 --- a/static/app.js +++ b/static/app.js @@ -3,12 +3,13 @@ const $ = (id) => document.getElementById(id); const MODE_LABEL = { batch: "批量", scheduled: "定时", auto: "自动" }; +const DETAIL_PAGE_SIZE = 100; // 详情结果每页条数 (默认) const state = { tasks: [], editTask: null, // 正在编辑的任务 mode: "batch", // 当前表单模式 - detail: { task: null, runId: null, logOffset: 0, timer: null }, + detail: { task: null, runId: null, logOffset: 0, timer: null, pageSize: DETAIL_PAGE_SIZE }, logTimer: null, }; let formDirty = false; // 新建/编辑表单是否有未保存修改 @@ -522,7 +523,6 @@ async function submitForm(e) { } /* ---------------- 详情 ---------------- */ -const DETAIL_PAGE_SIZE = 100; // 详情结果每页条数 async function openDetail(tid) { try { @@ -531,6 +531,7 @@ async function openDetail(tid) { const runs = t.runs || []; const cur = t.run; state.detail.runId = cur ? cur.id : (runs[0] ? runs[0].id : null); + state.detail.pageSize = (t.run && t.run.run_page_size) || DETAIL_PAGE_SIZE; state.detail.logOffset = 0; $("detailTitle").textContent = `任务详情 · ${t.name}`; renderDetail(); @@ -539,8 +540,9 @@ async function openDetail(tid) { } catch (e) { toast(e.message, true); } } -async function loadRunPage(tid, rid, page) { - return api(`/api/tasks/${tid}?run=${rid}&page=${page}&page_size=${DETAIL_PAGE_SIZE}`); +async function loadRunPage(tid, rid, page, size) { + const ps = size || state.detail.pageSize || DETAIL_PAGE_SIZE; + return api(`/api/tasks/${tid}?run=${rid}&page=${page}&page_size=${ps}`); } async function selectRun(rid) { @@ -550,6 +552,7 @@ async function selectRun(rid) { const d = await loadRunPage(t.id, rid, 1); state.detail.task = d; state.detail.runId = d.run ? d.run.id : null; + state.detail.pageSize = (d.run && d.run.run_page_size) || DETAIL_PAGE_SIZE; state.detail.logOffset = 0; renderDetail(); startLogPoll(); @@ -559,13 +562,62 @@ async function selectRun(rid) { async function goRunPage(p) { const t = state.detail.task; if (!t || !state.detail.runId) return; + const ps = (t.run && t.run.run_page_size) || state.detail.pageSize || DETAIL_PAGE_SIZE; try { - const d = await loadRunPage(t.id, state.detail.runId, p); + const d = await loadRunPage(t.id, state.detail.runId, p, ps); state.detail.task = d; + state.detail.pageSize = (d.run && d.run.run_page_size) || ps; renderDetail(); } catch (e) { toast(e.message, true); } } +/* 跳转到指定页码 (输入框回车/点击 GO) */ +function jumpRunPage() { + const t = state.detail.task; + if (!t || !state.detail.runId || !t.run) return; + const inp = $("pagerJump"); + const pages = t.run.run_pages || 1; + let p = parseInt(inp.value, 10); + if (!p || isNaN(p)) p = 1; + p = Math.min(Math.max(p, 1), pages); + inp.value = p; + goRunPage(p); +} + +/* 切换每页条数 */ +async function changePageSize(size) { + const t = state.detail.task; + if (!t || !state.detail.runId) return; + try { + const d = await loadRunPage(t.id, state.detail.runId, 1, parseInt(size, 10) || 100); + state.detail.task = d; + state.detail.pageSize = (d.run && d.run.run_page_size) || 100; + renderDetail(); + } catch (e) { toast(e.message, true); } +} + +/* 生成页码窗口列表, 0 表示省略号: 1 2 3 … 97 98 99 100 */ +function pageWindow(cur, pages, width) { + width = width || 5; + const win = []; + if (pages <= width + 2) { + for (let i = 1; i <= pages; i++) win.push(i); + return win; + } + win.push(1); + let lo = Math.max(2, cur - Math.floor(width / 2)); + let hi = Math.min(pages - 1, cur + Math.floor(width / 2)); + if (hi - lo < width - 1) { + if (lo <= 2) hi = lo + width - 1; + else lo = hi - width + 1; + } + if (lo > 2) win.push(0); + for (let i = lo; i <= hi; i++) win.push(i); + if (hi < pages - 1) win.push(0); + win.push(pages); + return win; +} + function renderDetail() { const t = state.detail.task; const runs = t.runs || []; @@ -664,13 +716,30 @@ function renderRunPanel(t, run) { const total = run.results_total || results.length; const pages = run.run_pages || 1; const page = run.run_page || 1; + const pageSize = run.run_page_size || DETAIL_PAGE_SIZE; + const multi = pages > 1; const pager = total > 0 ? `
- ${total > results.length ? ` - + ${multi ? ` + + + ${pageWindow(page, pages).map((p) => p === 0 + ? '…' + : ``).join("")} + + 第 ${page} / ${pages} 页 · 共 ${total} 条 - ` + 跳至 + 页 + + ` : `共 ${total} 条`} + 每页 + 条 +
` : ""; return ` diff --git a/static/style.css b/static/style.css index 8bc5000..f703aa6 100644 --- a/static/style.css +++ b/static/style.css @@ -287,3 +287,20 @@ td.title-cell { max-width: 220px; overflow: hidden; text-overflow: ellipsis; } } .pager-info { color: var(--text-dim, #999); font-size: 13px; } .pager .btn:disabled { opacity: 0.4; cursor: not-allowed; } +.pager-pages { display: flex; align-items: center; gap: 4px; flex-wrap: wrap; } +.pager-pages .page-btn { min-width: 30px; padding: 4px 6px; } +.pager-pages .page-btn.active { + background: var(--accent, #2d6cdf); + color: #fff; + border-color: var(--accent, #2d6cdf); + font-weight: 600; +} +.pager-ellipsis { color: var(--text-dim, #999); padding: 0 2px; user-select: none; } +.pager-jump { display: inline-flex; align-items: center; gap: 4px; color: var(--text-dim, #999); font-size: 13px; } +.pager-jump input { + width: 56px; + padding: 3px 6px; + text-align: center; +} +.pager-size { display: inline-flex; align-items: center; gap: 4px; color: var(--text-dim, #999); font-size: 13px; } +.pager-size select { padding: 3px 4px; width: auto; }