通用爬虫系统 v1.0.0: 批量/定时/自动爬取 + Web管理界面 + 图片/通知/热更新

2026-08-11 12:44:18 +08:00
commit dede5003a4
15 changed files with 2251 additions and 0 deletions
@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+"""单页测试: Playwright + stealth + 系统Chrome 抓 TPU"""
+import sys, time
+from playwright.sync_api import sync_playwright
+from playwright_stealth import Stealth
+
+URL = sys.argv[1] if len(sys.argv) > 1 else "https://www.techpowerup.com/"
+
+with sync_playwright() as p:
+    browser = p.chromium.launch(
+        headless=True,
+        executable_path="/usr/bin/google-chrome",
+        args=["--disable-blink-features=AutomationControlled", "--no-sandbox", "--disable-gpu"],
+    )
+    ctx = browser.new_context(
+        user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
+        viewport={"width": 1920, "height": 1080},
+        locale="en-US",
+    )
+    Stealth().apply_stealth_sync(ctx)  # 隐藏 webdriver/headless 特征
+    page = ctx.new_page()
+    try:
+        page.goto(URL, wait_until="domcontentloaded", timeout=60000)
+        # 等 JS 挑战自动通过: 最多等 40s, 轮询检查标题/内容特征
+        for i in range(40):
+            time.sleep(1)
+            try:
+                title = page.title()
+                html = page.content()
+            except Exception:
+                continue  # 页面正在跳转, 稍后再试
+            if "Access Denied" in title or "403" in title:
+                print(f"[{i}s] 403 Access Denied"); break
+            if "bot check" in html.lower() and len(html) < 5000:
+                continue
+            if len(html) > 30000 or "news" in html.lower():
+                print(f"[{i}s] 通过! title={title!r} html_size={len(html)}")
+                break
+        else:
+            print("超时未通过, 当前 title:", page.title())
+        # 打印正文摘要作为证据
+        text = page.inner_text("body")[:800]
+        print("--- body 前800字符 ---")
+        print(text)
+    except Exception as e:
+        print("ERROR:", e)
+    finally:
+        browser.close()