From 5a137514e82391517e2822001265c7b69a15a424 Mon Sep 17 00:00:00 2001
From: hz4th_coder <hz4th_coder@tphai.com>
Date: Tue, 11 Aug 2026 17:57:59 +0800
Subject: [PATCH] =?UTF-8?q?v1.0.8:=20=E8=87=AA=E5=8A=A8=E7=88=AC=E5=8F=96U?=
 =?UTF-8?q?RL=E8=A7=84=E8=8C=83=E5=8C=96=E5=8E=BB=E9=87=8D(=E9=94=9A?=
 =?UTF-8?q?=E7=82=B9/=E8=B7=9F=E8=B8=AA=E5=8F=82=E6=95=B0/=E5=B0=BE?=
 =?UTF-8?q?=E9=83=A8=E6=96=9C=E6=9D=A0/=E9=BB=98=E8=AE=A4=E7=AB=AF?=
 =?UTF-8?q?=E5=8F=A3)=20+=20=E4=BF=AE=E5=A4=8D=E5=8F=8D=E7=88=AC=E5=90=AF?=
 =?UTF-8?q?=E5=8F=91=E5=BC=8F=E8=AF=AF=E4=BC=A4=E5=B0=8F=E9=A1=B5=E9=9D=A2?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 engine.py | 47 +++++++++++++++++++++++++++++++++++++++--------
 1 file changed, 39 insertions(+), 8 deletions(-)

diff --git a/engine.py b/engine.py
index 491ad39..2876168 100644
--- a/engine.py
+++ b/engine.py
@@ -37,13 +37,12 @@ _CHALLENGE_MARKS = [
 
 
 def is_challenge_page(title, html):
+    """判断是否仍在反爬验证页 (仅关键词启发, 避免误伤正常小页面)"""
     low = html.lower()
     t = (title or "").lower()
     for mark in _CHALLENGE_MARKS:
         if mark in t or mark in low:
             return True
-    if len(html) < 5000 and ("<article" not in low and "<main" not in low):
-        return True
     return False
 
 
@@ -77,6 +76,36 @@ def _settle_wait(page, timeout_s):
     return True, page.title(), page.content()
 
 
+_TRACKING_PARAMS = {
+    "utm_source", "utm_medium", "utm_campaign", "utm_term", "utm_content",
+    "fbclid", "gclid", "yclid", "mc_cid", "mc_eid", "ref", "ref_src",
+}
+
+
+def normalize_url(url):
+    """URL 规范化 (用于去重): 去锚点/跟踪参数/尾部斜杠/默认端口, host 小写"""
+    try:
+        p = urllib.parse.urlparse(str(url))
+        host = (p.hostname or "").lower()
+        if not host:
+            return str(url)
+        port = ""
+        if p.port and p.port not in (80, 443):
+            port = f":{p.port}"
+        path = p.path or "/"
+        if len(path) > 1 and path.endswith("/"):
+            path = path.rstrip("/")
+        query = ""
+        if p.query:
+            kept = [kv for kv in p.query.split("&")
+                    if kv.split("=", 1)[0].lower() not in _TRACKING_PARAMS]
+            if kept:
+                query = "?" + "&".join(kept)
+        return f"{p.scheme.lower()}://{host}{port}{path}{query}"
+    except Exception:
+        return str(url)
+
+
 def filter_links(hrefs, seed_url, include=None, exclude=None,
                  same_domain=True, use_regex=False):
     """按规则过滤链接, 返回 (included, excluded); excluded 含排除原因"""
@@ -551,18 +580,19 @@ class CrawlJob:
 
         p, browser, ctx, page, cookie_file = self._open_browser()
         queue = [(seed, 0, "")]               # (url, depth, 来源链接)
-        visited = set()
-        queued = set([seed])
+        visited = set()                        # 规范化 URL 去重
+        queued = set([normalize_url(seed)])
         idx = 0
         try:
             while queue and not self._stop.is_set():
                 self._wait_if_paused()
                 url, depth, src = queue.pop(0)
-                if url in visited:
+                key = normalize_url(url)
+                if key in visited:
                     continue
                 if len(visited) >= max_pages:
                     break
-                visited.add(url)
+                visited.add(key)
                 idx += 1
                 run["progress"]["current_url"] = url
                 run["progress"]["done"] = len(visited)
@@ -574,8 +604,9 @@ class CrawlJob:
                 self._persist()
                 if entry["status"] == "OK" and depth < max_depth:
                     for link in self._discover_links(page):
-                        if link not in visited and link not in queued:
-                            queued.add(link)
+                        lk = normalize_url(link)
+                        if lk not in visited and lk not in queued:
+                            queued.add(lk)
                             queue.append((link, depth + 1, url))
                 if entry["status"] == "OK":
                     self._delay()