v1.4.0 修复反爬误判+浏览器崩溃空转, 详情页多元翻页

- 修复反爬误判: is_challenge_page 仅匹配 title 关键词 + html 前 20KB 强特征标记, 不再全文匹配 captcha/challenge 等词 (博客园正文提 captcha 的页面被误杀) - 修复浏览器崩溃后空转: 检测 Target page/context/browser closed, 自动重启浏览器并重试当前 URL (不消耗重试配额, 单页最多重建3次); 页面读取连续异常3次快速失败; 新增 browser_max_pages 配置定期重启浏览器防内存膨胀 - 详情页多元翻页: 首页/末页/页码列表(带省略号)/页码跳转输入框(回车/GO)/每页条数选择(50/100/200/500)
2026-08-14 10:28:25 +08:00
parent 32eebf3dd1
commit ec053266a7
3 changed files with 191 additions and 24 deletions
@@ -30,12 +30,21 @@ IMG_EXTS = (".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp")
 DEFAULT_UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
              "(KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36")

-_CHALLENGE_MARKS = [
+# title 中出现任一标记即判定反爬 (title 短小可靠, 不会误伤正文)
+_CHALLENGE_TITLE_MARKS = [
    "access denied", "403 forbidden", "just a moment", "attention required",
    "captcha", "bot check", "cf-challenge", "verify you are human",
    "checking your browser", "enable javascript and cookies",
 ]

+# html 中仅匹配强特征标记, 且只在 head 区域(前 20KB)搜索。
+# 不能全文匹配 "captcha"/"challenge" 等词——正常文章正文提到这些词会被误判为反爬。
+_CHALLENGE_HTML_MARKS = [
+    "cf-challenge", "challenge-platform", "cf-browser-verification",
+    "verify you are human", "checking your browser",
+    "enable javascript and cookies", "just a moment",
+]
+
 # 反爬拦截类错误信号: 命中后不再重试 (重试无意义且拖慢任务)
 _ANTI_CRAWL_MARKS = [
    "403", "429", "503", "forbidden", "access denied", "too many requests",
@@ -50,14 +59,31 @@ def is_anti_crawl_error(err):
    return any(m in s for m in _ANTI_CRAWL_MARKS)


+_BROWSER_DEAD_MARKS = [
+    "browser has been closed", "target page, context or browser has been closed",
+    "has been disposed", "execution context was destroyed", "browser closed",
+    "page closed", "target closed", "crash",
+]
+
+
+def is_browser_dead_error(err):
+    """判断错误是否属于浏览器/页面失效 (需重启浏览器后重试, 重试同一 URL 才有意义)"""
+    s = str(err or "").lower()
+    return any(m in s for m in _BROWSER_DEAD_MARKS)
+
+
 def is_challenge_page(title, html):
-    """判断是否仍在反爬验证页 (仅关键词启发, 避免误伤正常小页面)"""
-    low = html.lower()
+    """判断是否仍在反爬验证页
+    - title 出现反爬关键词: 判定 (可靠, 反爬页 title 基本都会变)
+    - html 仅在前 20KB(head 区域)出现强特征标记时判定,
+      避免正文含 captcha/challenge 等词的正常页面被误杀
+    """
    t = (title or "").lower()
-    for mark in _CHALLENGE_MARKS:
-        if mark in t or mark in low:
+    for mark in _CHALLENGE_TITLE_MARKS:
+        if mark in t:
            return True
-    return False
+    low = (html or "").lower()[:20000]
+    return any(m in low for m in _CHALLENGE_HTML_MARKS)


 def safe_name(url, idx):
@@ -231,6 +257,14 @@ class CrawlJob:
        self._auto_pending = None
        self._cfg_lock = threading.RLock()
        self.thread = None
+        # 浏览器句柄 (p, browser, ctx, page, cookie_file); 崩溃后重建
+        self._browser = None
+
+    def _page(self):
+        return self._browser[3] if self._browser else None
+
+    def _ctx(self):
+        return self._browser[2] if self._browser else None

    # ---------------- 控制接口 ----------------
    def start(self):
@@ -309,9 +343,13 @@ class CrawlJob:
                pass
        Stealth().apply_stealth_sync(ctx)
        page = ctx.new_page()
-        return p, browser, ctx, page, cookie_file
+        self._browser = (p, browser, ctx, page, cookie_file)
+        return self._browser

-    def _close_browser(self, p, browser, ctx, cookie_file):
+    def _close_browser(self):
+        if not self._browser:
+            return
+        p, browser, ctx, _page, cookie_file = self._browser
        try:
            json.dump(ctx.cookies(), open(cookie_file, "w"))
        except Exception:
@@ -324,10 +362,24 @@ class CrawlJob:
            p.stop()
        except Exception:
            pass
+        self._browser = None
+
+    def _reopen_browser(self):
+        """浏览器失效后重建: 保存 cookie -> 关闭旧实例 -> 启动新实例"""
+        self._close_browser()
+        time.sleep(1)
+        self._open_browser()
+        self._log("info", "浏览器已重启")
+
+    def _need_browser_restart(self, done_count):
+        """每爬 N 页主动重启一次浏览器, 防止长时间运行内存膨胀导致崩溃 (browser_max_pages=0 关闭)"""
+        limit = int(self._cfg("browser_max_pages", 0) or 0)
+        return limit > 0 and done_count > 1 and (done_count - 1) % limit == 0

    def _wait_page_settle(self, page, timeout_s):
        last_title, stable = "", 0
        challenge_hits = 0
+        err_streak = 0                        # 连续读取失败计数
        start = time.time()
        while time.time() - start < timeout_s:
            if self._stop.is_set():
@@ -338,7 +390,11 @@ class CrawlJob:
                title = page.title()
                html = page.content()
            except Exception:
+                err_streak += 1
+                if err_streak >= 3:           # 页面/浏览器已失效, 提前失败而非干等到超时
+                    raise RuntimeError("Target page, context or browser has been closed")
                continue                      # 正在跳转
+            err_streak = 0
            if is_challenge_page(title, html):
                challenge_hits += 1
                if challenge_hits >= 8:       # 持续 8 秒仍是验证页, 判定反爬, 尽早放弃
@@ -461,7 +517,9 @@ class CrawlJob:
            "source_url": source_url, "depth": depth,
            "images": [], "attempts": 0,
        }
-        for attempt in range(retries + 1):
+        dead_strikes = 0                      # 浏览器失效重建次数 (防止无限重建)
+        attempt = 0
+        while attempt <= retries:
            if self._stop.is_set():
                entry["error"] = "任务已终止"
                break
@@ -485,6 +543,22 @@ class CrawlJob:
            except Exception as e:
                entry["error"] = str(e)
                entry["crawl_time"] = store.now_str()
+                if is_browser_dead_error(e):
+                    # 浏览器/页面失效: 重启浏览器后重试同一 URL, 不消耗重试次数
+                    dead_strikes += 1
+                    if dead_strikes > 3:
+                        entry["error"] = f"浏览器多次重启仍失效, 放弃: {e}"
+                        self._log("error", f"浏览器多次重启仍失效, 放弃 {url}")
+                        break
+                    self._log("warn", f"浏览器已失效({e}), 重启后重试: {url}")
+                    try:
+                        self._reopen_browser()
+                        page, ctx = self._page(), self._ctx()
+                    except Exception as re_err:
+                        entry["error"] = f"浏览器重启失败: {re_err}"
+                        self._log("error", f"浏览器重启失败, 放弃 {url}: {re_err}")
+                        break
+                    continue
                self._log("warn", f"第{attempt + 1}次失败 {url}: {e}")
                if is_anti_crawl_error(e):
                    # 反爬拦截: 重试也过不去, 直接放弃, 不再消耗重试次数
@@ -499,6 +573,7 @@ class CrawlJob:
                            break
                        self._wait_if_paused()
                        time.sleep(0.3)
+                attempt += 1
        self._write_page_meta(out_dir, base, entry)
        return entry

@@ -562,17 +637,20 @@ class CrawlJob:
        os.makedirs(out_dir, exist_ok=True)
        self._persist()

-        p, browser, ctx, page, cookie_file = self._open_browser()
+        self._open_browser()
        try:
            for i, url in enumerate(urls, 1):
                if self._stop.is_set():
                    self._log("info", "收到终止信号, 停止爬取")
                    break
                self._wait_if_paused()
+                if self._need_browser_restart(i):
+                    self._log("info", f"已爬 {i - 1} 页, 主动重启浏览器")
+                    self._reopen_browser()
                run["progress"]["current_url"] = url
                run["progress"]["done"] = i - 1
                self._persist()
-                entry = self._retry_crawl(page, ctx, url, i, out_dir)
+                entry = self._retry_crawl(self._page(), self._ctx(), url, i, out_dir)
                run["results"].append(entry)
                self._bump_stats(entry)
                run["progress"]["done"] = i
@@ -580,7 +658,7 @@ class CrawlJob:
                if entry["status"] == "OK":
                    self._delay()
        finally:
-            self._close_browser(p, browser, ctx, cookie_file)
+            self._close_browser()

    def _discover_links(self, page):
        """从当前页面提取符合规则的链接"""
@@ -637,7 +715,7 @@ class CrawlJob:
        self._auto_pending = len(queue)
        self._persist()

-        p, browser, ctx, page, cookie_file = self._open_browser()
+        self._open_browser()
        try:
            while queue and not self._stop.is_set():
                self._wait_if_paused()
@@ -650,6 +728,9 @@ class CrawlJob:
                    elif len(visited) >= max_pages:
                        self._log("info", f"达到最大页数上限 {max_pages}, 停止")
                        break
+                if self._need_browser_restart(len(visited)):
+                    self._log("info", f"已爬 {len(visited) - 1} 页, 主动重启浏览器")
+                    self._reopen_browser()
                url, depth, src = queue.pop(0)
                key = normalize_url(url)
                if key in visited and url != seed:     # 起始网址不去重, 其余已爬跳过
@@ -661,14 +742,14 @@ class CrawlJob:
                run["progress"]["current_url"] = url
                run["progress"]["done"] = len(visited)
                self._persist()
-                entry = self._retry_crawl(page, ctx, url, idx, out_dir,
+                entry = self._retry_crawl(self._page(), self._ctx(), url, idx, out_dir,
                                          source_url=src, depth=depth)
                run["results"].append(entry)
                self._bump_stats(entry)
                self._persist()
                # 无深度限制或未达深度限制时持续发现链接
                if entry["status"] == "OK" and (max_depth == 0 or depth < max_depth):
-                    for link in self._discover_links(page):
+                    for link in self._discover_links(self._page()):
                        lk = normalize_url(link)
                        if lk not in visited and lk not in queued:
                            queued.add(lk)
@@ -680,7 +761,7 @@ class CrawlJob:
            if not self._stop.is_set() and len(queue) == 0:
                self._log("info", f"无新链接可爬, 任务结束 (共 {len(visited)} 页)")
        finally:
-            self._close_browser(p, browser, ctx, cookie_file)
+            self._close_browser()
            # 无论完成/停止/异常, 都保存缓存队列与已爬集合, 便于下次继续
            try:
                store.save_auto_state(self.task["id"], {
@@ -3,12 +3,13 @@

 const $ = (id) => document.getElementById(id);
 const MODE_LABEL = { batch: "批量", scheduled: "定时", auto: "自动" };
+const DETAIL_PAGE_SIZE = 100;   // 详情结果每页条数 (默认)

 const state = {
  tasks: [],
  editTask: null,          // 正在编辑的任务
  mode: "batch",           // 当前表单模式
-  detail: { task: null, runId: null, logOffset: 0, timer: null },
+  detail: { task: null, runId: null, logOffset: 0, timer: null, pageSize: DETAIL_PAGE_SIZE },
  logTimer: null,
 };
 let formDirty = false;      // 新建/编辑表单是否有未保存修改
@@ -522,7 +523,6 @@ async function submitForm(e) {
 }

 /* ---------------- 详情 ---------------- */
-const DETAIL_PAGE_SIZE = 100;   // 详情结果每页条数

 async function openDetail(tid) {
  try {
@@ -531,6 +531,7 @@ async function openDetail(tid) {
    const runs = t.runs || [];
    const cur = t.run;
    state.detail.runId = cur ? cur.id : (runs[0] ? runs[0].id : null);
+    state.detail.pageSize = (t.run && t.run.run_page_size) || DETAIL_PAGE_SIZE;
    state.detail.logOffset = 0;
    $("detailTitle").textContent = `任务详情 · ${t.name}`;
    renderDetail();
@@ -539,8 +540,9 @@ async function openDetail(tid) {
  } catch (e) { toast(e.message, true); }
 }

-async function loadRunPage(tid, rid, page) {
-  return api(`/api/tasks/${tid}?run=${rid}&page=${page}&page_size=${DETAIL_PAGE_SIZE}`);
+async function loadRunPage(tid, rid, page, size) {
+  const ps = size || state.detail.pageSize || DETAIL_PAGE_SIZE;
+  return api(`/api/tasks/${tid}?run=${rid}&page=${page}&page_size=${ps}`);
 }

 async function selectRun(rid) {
@@ -550,6 +552,7 @@ async function selectRun(rid) {
    const d = await loadRunPage(t.id, rid, 1);
    state.detail.task = d;
    state.detail.runId = d.run ? d.run.id : null;
+    state.detail.pageSize = (d.run && d.run.run_page_size) || DETAIL_PAGE_SIZE;
    state.detail.logOffset = 0;
    renderDetail();
    startLogPoll();
@@ -559,13 +562,62 @@ async function selectRun(rid) {
 async function goRunPage(p) {
  const t = state.detail.task;
  if (!t || !state.detail.runId) return;
+  const ps = (t.run && t.run.run_page_size) || state.detail.pageSize || DETAIL_PAGE_SIZE;
  try {
-    const d = await loadRunPage(t.id, state.detail.runId, p);
+    const d = await loadRunPage(t.id, state.detail.runId, p, ps);
    state.detail.task = d;
+    state.detail.pageSize = (d.run && d.run.run_page_size) || ps;
    renderDetail();
  } catch (e) { toast(e.message, true); }
 }

+/* 跳转到指定页码 (输入框回车/点击 GO) */
+function jumpRunPage() {
+  const t = state.detail.task;
+  if (!t || !state.detail.runId || !t.run) return;
+  const inp = $("pagerJump");
+  const pages = t.run.run_pages || 1;
+  let p = parseInt(inp.value, 10);
+  if (!p || isNaN(p)) p = 1;
+  p = Math.min(Math.max(p, 1), pages);
+  inp.value = p;
+  goRunPage(p);
+}
+
+/* 切换每页条数 */
+async function changePageSize(size) {
+  const t = state.detail.task;
+  if (!t || !state.detail.runId) return;
+  try {
+    const d = await loadRunPage(t.id, state.detail.runId, 1, parseInt(size, 10) || 100);
+    state.detail.task = d;
+    state.detail.pageSize = (d.run && d.run.run_page_size) || 100;
+    renderDetail();
+  } catch (e) { toast(e.message, true); }
+}
+
+/* 生成页码窗口列表, 0 表示省略号: 1 2 3 … 97 98 99 100 */
+function pageWindow(cur, pages, width) {
+  width = width || 5;
+  const win = [];
+  if (pages <= width + 2) {
+    for (let i = 1; i <= pages; i++) win.push(i);
+    return win;
+  }
+  win.push(1);
+  let lo = Math.max(2, cur - Math.floor(width / 2));
+  let hi = Math.min(pages - 1, cur + Math.floor(width / 2));
+  if (hi - lo < width - 1) {
+    if (lo <= 2) hi = lo + width - 1;
+    else lo = hi - width + 1;
+  }
+  if (lo > 2) win.push(0);
+  for (let i = lo; i <= hi; i++) win.push(i);
+  if (hi < pages - 1) win.push(0);
+  win.push(pages);
+  return win;
+}
+
 function renderDetail() {
  const t = state.detail.task;
  const runs = t.runs || [];
@@ -664,13 +716,30 @@ function renderRunPanel(t, run) {
  const total = run.results_total || results.length;
  const pages = run.run_pages || 1;
  const page = run.run_page || 1;
+  const pageSize = run.run_page_size || DETAIL_PAGE_SIZE;
+  const multi = pages > 1;
  const pager = total > 0 ? `
    <div class="pager">
-      ${total > results.length ? `
-        <button class="btn sm" ${page <= 1 ? "disabled" : ""} onclick="goRunPage(${page - 1})">◀ 上一页</button>
+      ${multi ? `
+        <button class="btn sm" title="首页" ${page <= 1 ? "disabled" : ""} onclick="goRunPage(1)">⏮</button>
+        <button class="btn sm" title="上一页" ${page <= 1 ? "disabled" : ""} onclick="goRunPage(${page - 1})">◀</button>
+        <span class="pager-pages">${pageWindow(page, pages).map((p) => p === 0
+          ? '<span class="pager-ellipsis">…</span>'
+          : `<button class="btn sm page-btn ${p === page ? "active" : ""}" onclick="goRunPage(${p})">${p}</button>`).join("")}</span>
+        <button class="btn sm" title="下一页" ${page >= pages ? "disabled" : ""} onclick="goRunPage(${page + 1})">▶</button>
+        <button class="btn sm" title="末页" ${page >= pages ? "disabled" : ""} onclick="goRunPage(${pages})">⏭</button>
        <span class="pager-info">第 <b>${page}</b> / ${pages} 页 · 共 ${total} 条</span>
-        <button class="btn sm" ${page >= pages ? "disabled" : ""} onclick="goRunPage(${page + 1})">下一页 ▶</button>`
+        <span class="pager-jump">跳至
+          <input type="number" id="pagerJump" min="1" max="${pages}" value="${page}"
+                 onkeydown="if(event.key==='Enter')jumpRunPage()"> 页
+          <button class="btn sm" onclick="jumpRunPage()">GO</button>
+        </span>`
      : `<span class="pager-info">共 ${total} 条</span>`}
+      <span class="pager-size">每页
+        <select id="pagerSize" onchange="changePageSize(this.value)">
+          ${[50, 100, 200, 500].map((s) => `<option value="${s}" ${s === pageSize ? "selected" : ""}>${s}</option>`).join("")}
+        </select> 条
+      </span>
    </div>` : "";

  return `
@@ -287,3 +287,20 @@ td.title-cell { max-width: 220px; overflow: hidden; text-overflow: ellipsis; }
 }
 .pager-info { color: var(--text-dim, #999); font-size: 13px; }
 .pager .btn:disabled { opacity: 0.4; cursor: not-allowed; }
+.pager-pages { display: flex; align-items: center; gap: 4px; flex-wrap: wrap; }
+.pager-pages .page-btn { min-width: 30px; padding: 4px 6px; }
+.pager-pages .page-btn.active {
+  background: var(--accent, #2d6cdf);
+  color: #fff;
+  border-color: var(--accent, #2d6cdf);
+  font-weight: 600;
+}
+.pager-ellipsis { color: var(--text-dim, #999); padding: 0 2px; user-select: none; }
+.pager-jump { display: inline-flex; align-items: center; gap: 4px; color: var(--text-dim, #999); font-size: 13px; }
+.pager-jump input {
+  width: 56px;
+  padding: 3px 6px;
+  text-align: center;
+}
+.pager-size { display: inline-flex; align-items: center; gap: 4px; color: var(--text-dim, #999); font-size: 13px; }
+.pager-size select { padding: 3px 4px; width: auto; }