反爬拦截不再重试: 判定为反爬类错误(403/429/503/Cloudflare/captcha/验证页等)直接放弃该页, 节省重试时间; 验证页持续8秒即判定反爬, 不再干等满超时(默认60s)

- engine.is_anti_crawl_error: 反爬错误信号关键词判定
- _retry_crawl: 反爬错误直接 break, entry.error 标注'反爬拦截, 跳过重试'
- _wait_page_settle/_settle_wait: 连续8秒检测到验证页即判定反爬尽早返回
- 集成测试: 反爬页 attempts=1 且 8.1s 放弃; 非反爬错误仍 attempts=retry_count+1 正常重试

engine.py

@@ -36,6 +36,19 @@ _CHALLENGE_MARKS = [
     "checking your browser", "enable javascript and cookies",
 ]
 
+# 反爬拦截类错误信号: 命中后不再重试 (重试无意义且拖慢任务)
+_ANTI_CRAWL_MARKS = [
+    "403", "429", "503", "forbidden", "access denied", "too many requests",
+    "captcha", "cloudflare", "challenge", "verify you are human",
+    "just a moment", "blocked", "被反爬拦截",
+]
+
+
+def is_anti_crawl_error(err):
+    """判断错误是否属于反爬拦截 (此类失败重试也无法通过, 直接放弃该页)"""
+    s = str(err or "").lower()
+    return any(m in s for m in _ANTI_CRAWL_MARKS)
+
 
 def is_challenge_page(title, html):
     """判断是否仍在反爬验证页 (仅关键词启发, 避免误伤正常小页面)"""
@@ -56,6 +69,7 @@ def safe_name(url, idx):
 def _settle_wait(page, timeout_s):
     """等待页面稳定(无 stop/pause 检查, 供试爬取等独立流程使用)"""
     last_title, stable = "", 0
+    challenge_hits = 0
     start = time.time()
     while time.time() - start < timeout_s:
         time.sleep(1)
@@ -65,8 +79,12 @@ def _settle_wait(page, timeout_s):
         except Exception:
             continue
         if is_challenge_page(title, html):
+            challenge_hits += 1
+            if challenge_hits >= 8:        # 持续 8 秒仍是验证页, 判定反爬, 尽早放弃
+                return True, title, html
             stable = 0
             continue
+        challenge_hits = 0
         if title == last_title:
             stable += 1
             if stable >= 2 and len(html) > 1000:
@@ -306,6 +324,7 @@ class CrawlJob:
 
     def _wait_page_settle(self, page, timeout_s):
         last_title, stable = "", 0
+        challenge_hits = 0
         start = time.time()
         while time.time() - start < timeout_s:
             if self._stop.is_set():
@@ -318,8 +337,12 @@ class CrawlJob:
             except Exception:
                 continue                      # 正在跳转
             if is_challenge_page(title, html):
+                challenge_hits += 1
+                if challenge_hits >= 8:       # 持续 8 秒仍是验证页, 判定反爬, 尽早放弃
+                    return True, title, html
                 stable = 0
                 continue
+            challenge_hits = 0
             if title == last_title:
                 stable += 1
                 if stable >= 2 and len(html) > 1000:
@@ -460,6 +483,11 @@ class CrawlJob:
                 entry["error"] = str(e)
                 entry["crawl_time"] = store.now_str()
                 self._log("warn", f"第{attempt + 1}次失败 {url}: {e}")
+                if is_anti_crawl_error(e):
+                    # 反爬拦截: 重试也过不去, 直接放弃, 不再消耗重试次数
+                    entry["error"] = f"反爬拦截, 跳过重试: {e}"
+                    self._log("warn", f"判定为反爬拦截, 放弃重试: {url}")
+                    break
                 if attempt < retries:
                     self._wait_if_paused()
                     t0 = time.time()