diff --git a/engine.py b/engine.py index f906610..5ce85f9 100644 --- a/engine.py +++ b/engine.py @@ -36,6 +36,19 @@ _CHALLENGE_MARKS = [ "checking your browser", "enable javascript and cookies", ] +# 反爬拦截类错误信号: 命中后不再重试 (重试无意义且拖慢任务) +_ANTI_CRAWL_MARKS = [ + "403", "429", "503", "forbidden", "access denied", "too many requests", + "captcha", "cloudflare", "challenge", "verify you are human", + "just a moment", "blocked", "被反爬拦截", +] + + +def is_anti_crawl_error(err): + """判断错误是否属于反爬拦截 (此类失败重试也无法通过, 直接放弃该页)""" + s = str(err or "").lower() + return any(m in s for m in _ANTI_CRAWL_MARKS) + def is_challenge_page(title, html): """判断是否仍在反爬验证页 (仅关键词启发, 避免误伤正常小页面)""" @@ -56,6 +69,7 @@ def safe_name(url, idx): def _settle_wait(page, timeout_s): """等待页面稳定(无 stop/pause 检查, 供试爬取等独立流程使用)""" last_title, stable = "", 0 + challenge_hits = 0 start = time.time() while time.time() - start < timeout_s: time.sleep(1) @@ -65,8 +79,12 @@ def _settle_wait(page, timeout_s): except Exception: continue if is_challenge_page(title, html): + challenge_hits += 1 + if challenge_hits >= 8: # 持续 8 秒仍是验证页, 判定反爬, 尽早放弃 + return True, title, html stable = 0 continue + challenge_hits = 0 if title == last_title: stable += 1 if stable >= 2 and len(html) > 1000: @@ -306,6 +324,7 @@ class CrawlJob: def _wait_page_settle(self, page, timeout_s): last_title, stable = "", 0 + challenge_hits = 0 start = time.time() while time.time() - start < timeout_s: if self._stop.is_set(): @@ -318,8 +337,12 @@ class CrawlJob: except Exception: continue # 正在跳转 if is_challenge_page(title, html): + challenge_hits += 1 + if challenge_hits >= 8: # 持续 8 秒仍是验证页, 判定反爬, 尽早放弃 + return True, title, html stable = 0 continue + challenge_hits = 0 if title == last_title: stable += 1 if stable >= 2 and len(html) > 1000: @@ -460,6 +483,11 @@ class CrawlJob: entry["error"] = str(e) entry["crawl_time"] = store.now_str() self._log("warn", f"第{attempt + 1}次失败 {url}: {e}") + if is_anti_crawl_error(e): + # 反爬拦截: 重试也过不去, 直接放弃, 不再消耗重试次数 + entry["error"] = f"反爬拦截, 跳过重试: {e}" + self._log("warn", f"判定为反爬拦截, 放弃重试: {url}") + break if attempt < retries: self._wait_if_paused() t0 = time.time()