From 0482284cc0c377c40a11b7ec7aa8d1862efe8cb9 Mon Sep 17 00:00:00 2001 From: hz4th_coder Date: Wed, 12 Aug 2026 09:37:23 +0800 Subject: [PATCH] =?UTF-8?q?=E5=8F=8D=E7=88=AC=E6=8B=A6=E6=88=AA=E4=B8=8D?= =?UTF-8?q?=E5=86=8D=E9=87=8D=E8=AF=95:=20=E5=88=A4=E5=AE=9A=E4=B8=BA?= =?UTF-8?q?=E5=8F=8D=E7=88=AC=E7=B1=BB=E9=94=99=E8=AF=AF(403/429/503/Cloud?= =?UTF-8?q?flare/captcha/=E9=AA=8C=E8=AF=81=E9=A1=B5=E7=AD=89)=E7=9B=B4?= =?UTF-8?q?=E6=8E=A5=E6=94=BE=E5=BC=83=E8=AF=A5=E9=A1=B5,=20=E8=8A=82?= =?UTF-8?q?=E7=9C=81=E9=87=8D=E8=AF=95=E6=97=B6=E9=97=B4;=20=E9=AA=8C?= =?UTF-8?q?=E8=AF=81=E9=A1=B5=E6=8C=81=E7=BB=AD8=E7=A7=92=E5=8D=B3?= =?UTF-8?q?=E5=88=A4=E5=AE=9A=E5=8F=8D=E7=88=AC,=20=E4=B8=8D=E5=86=8D?= =?UTF-8?q?=E5=B9=B2=E7=AD=89=E6=BB=A1=E8=B6=85=E6=97=B6(=E9=BB=98?= =?UTF-8?q?=E8=AE=A460s)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - engine.is_anti_crawl_error: 反爬错误信号关键词判定 - _retry_crawl: 反爬错误直接 break, entry.error 标注'反爬拦截, 跳过重试' - _wait_page_settle/_settle_wait: 连续8秒检测到验证页即判定反爬尽早返回 - 集成测试: 反爬页 attempts=1 且 8.1s 放弃; 非反爬错误仍 attempts=retry_count+1 正常重试 --- engine.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/engine.py b/engine.py index f906610..5ce85f9 100644 --- a/engine.py +++ b/engine.py @@ -36,6 +36,19 @@ _CHALLENGE_MARKS = [ "checking your browser", "enable javascript and cookies", ] +# 反爬拦截类错误信号: 命中后不再重试 (重试无意义且拖慢任务) +_ANTI_CRAWL_MARKS = [ + "403", "429", "503", "forbidden", "access denied", "too many requests", + "captcha", "cloudflare", "challenge", "verify you are human", + "just a moment", "blocked", "被反爬拦截", +] + + +def is_anti_crawl_error(err): + """判断错误是否属于反爬拦截 (此类失败重试也无法通过, 直接放弃该页)""" + s = str(err or "").lower() + return any(m in s for m in _ANTI_CRAWL_MARKS) + def is_challenge_page(title, html): """判断是否仍在反爬验证页 (仅关键词启发, 避免误伤正常小页面)""" @@ -56,6 +69,7 @@ def safe_name(url, idx): def _settle_wait(page, timeout_s): """等待页面稳定(无 stop/pause 检查, 供试爬取等独立流程使用)""" last_title, stable = "", 0 + challenge_hits = 0 start = time.time() while time.time() - start < timeout_s: time.sleep(1) @@ -65,8 +79,12 @@ def _settle_wait(page, timeout_s): except Exception: continue if is_challenge_page(title, html): + challenge_hits += 1 + if challenge_hits >= 8: # 持续 8 秒仍是验证页, 判定反爬, 尽早放弃 + return True, title, html stable = 0 continue + challenge_hits = 0 if title == last_title: stable += 1 if stable >= 2 and len(html) > 1000: @@ -306,6 +324,7 @@ class CrawlJob: def _wait_page_settle(self, page, timeout_s): last_title, stable = "", 0 + challenge_hits = 0 start = time.time() while time.time() - start < timeout_s: if self._stop.is_set(): @@ -318,8 +337,12 @@ class CrawlJob: except Exception: continue # 正在跳转 if is_challenge_page(title, html): + challenge_hits += 1 + if challenge_hits >= 8: # 持续 8 秒仍是验证页, 判定反爬, 尽早放弃 + return True, title, html stable = 0 continue + challenge_hits = 0 if title == last_title: stable += 1 if stable >= 2 and len(html) > 1000: @@ -460,6 +483,11 @@ class CrawlJob: entry["error"] = str(e) entry["crawl_time"] = store.now_str() self._log("warn", f"第{attempt + 1}次失败 {url}: {e}") + if is_anti_crawl_error(e): + # 反爬拦截: 重试也过不去, 直接放弃, 不再消耗重试次数 + entry["error"] = f"反爬拦截, 跳过重试: {e}" + self._log("warn", f"判定为反爬拦截, 放弃重试: {url}") + break if attempt < retries: self._wait_if_paused() t0 = time.time()