v1.7.2 修复慢加载页面报 'Operation timed out':agent-browser 打开超时自动切 Playwright 重试
- 根因: agent-browser open 导航内部超时约30s(CDP command timed out: Page.navigate),慢加载/持续加载页面(如资源挂起导致 load 事件永不触发)会报 Failed to open URL: Operation timed out - agent-browser open 子进程超时 30s→45s,让其返回自己的超时报错而非被硬杀 - 超时判定放宽到 timed out/timeout 全变体,触发自动回退: capture_webpage 与 smart_capture 在 agent-browser 打开超时时自动切 Playwright 重试(domcontentloaded 更宽容,不等全部资源) - Playwright goto domcontentloaded 超时 30s→60s - 错误提示更友好(说明慢加载+已自动回退+建议) - 实测: 挂起资源测试页 agent-browser 超时→playwright 成功, 耗时~39s; 正常页面无回归
This commit is contained in:
@@ -315,7 +315,8 @@ class AgentBrowserSession:
|
||||
# 添加反爬虫检测的 User-Agent
|
||||
env = os.environ.copy()
|
||||
env['AGENT_BROWSER_USER_AGENT'] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
|
||||
return self.run(["open", url], timeout=30000)
|
||||
# agent-browser 内部导航等待约 30s 超时,这里留足 45s 让它返回自己的超时报错(避免被 subprocess 硬杀)
|
||||
return self.run(["open", url], timeout=45000)
|
||||
|
||||
def set_viewport(self, width, height):
|
||||
"""设置视口大小"""
|
||||
@@ -410,6 +411,8 @@ def capture_with_agent_browser(
|
||||
# 检查是否是反爬虫拦截
|
||||
if "403" in stdout or "Access Denied" in stdout:
|
||||
return {"success": False, "error": "网站反爬虫拦截 (403),建议使用 Playwright 后端或手动添加请求头"}
|
||||
if "timed out" in stderr.lower() or "timeout" in stderr.lower():
|
||||
return {"success": False, "error": "页面加载超时(可能是慢加载/持续加载页面),已自动切换 Playwright 后端重试;若仍失败请增大等待时间或换其他后端"}
|
||||
return {"success": False, "error": f"Failed to open URL: {stderr}"}
|
||||
|
||||
# 等待页面加载
|
||||
@@ -595,7 +598,7 @@ async def capture_with_playwright(
|
||||
await stealth_async(page)
|
||||
|
||||
try:
|
||||
await page.goto(url, wait_until="domcontentloaded", timeout=30000)
|
||||
await page.goto(url, wait_until="domcontentloaded", timeout=60000)
|
||||
except Exception as e:
|
||||
# 如果 domcontentloaded 超时,尝试 commit
|
||||
try:
|
||||
@@ -733,6 +736,8 @@ def smart_capture_agent_browser(url, cfg, wait_time, viewport):
|
||||
session.set_viewport(vw, vh)
|
||||
success, _, err = session.open(url)
|
||||
if not success:
|
||||
if "timed out" in err.lower() or "timeout" in err.lower():
|
||||
return {"success": False, "error": "页面加载超时(可能是慢加载/持续加载页面),已自动切换 Playwright 后端重试;若仍失败请增大等待时间或换其他后端"}
|
||||
return {"success": False, "error": f"打开网页失败: {err}"}
|
||||
session.wait(wait_time)
|
||||
|
||||
@@ -845,7 +850,7 @@ async def smart_capture_playwright(url, cfg, wait_time, viewport):
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
await page.goto(url, wait_until="domcontentloaded", timeout=30000)
|
||||
await page.goto(url, wait_until="domcontentloaded", timeout=60000)
|
||||
except Exception:
|
||||
try:
|
||||
await page.goto(url, wait_until="commit", timeout=30000)
|
||||
@@ -945,7 +950,19 @@ def smart_capture(url, cfg, wait_time, viewport, backend="auto"):
|
||||
backend = "agent-browser" if AGENT_BROWSER_AVAILABLE else "playwright"
|
||||
|
||||
if backend == "agent-browser":
|
||||
return smart_capture_agent_browser(url, cfg, wait_time, viewport)
|
||||
result = smart_capture_agent_browser(url, cfg, wait_time, viewport)
|
||||
# 打开页面超时 → 自动切 Playwright 重试(domcontentloaded 更宽容)
|
||||
if (not result.get("success")
|
||||
and "自动切换 Playwright" in (result.get("error") or "")
|
||||
and PLAYWRIGHT_AVAILABLE):
|
||||
try:
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
result = loop.run_until_complete(smart_capture_playwright(url, cfg, wait_time, viewport))
|
||||
loop.close()
|
||||
except Exception as e:
|
||||
return {"success": False, "error": str(e)}
|
||||
return result
|
||||
elif backend == "playwright":
|
||||
try:
|
||||
loop = asyncio.new_event_loop()
|
||||
@@ -991,6 +1008,22 @@ def capture_webpage(
|
||||
url, action, scroll_times, scroll_delay, full_page, viewport, wait_time
|
||||
)
|
||||
result["backend"] = "agent-browser"
|
||||
# agent-browser 打开页面超时(慢加载/持续加载页)→ 自动切 Playwright 重试(domcontentloaded 更宽容)
|
||||
if (not result.get("success")
|
||||
and "自动切换 Playwright" in (result.get("error") or "")
|
||||
and PLAYWRIGHT_AVAILABLE):
|
||||
try:
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
result = loop.run_until_complete(
|
||||
capture_with_playwright(
|
||||
url, action, scroll_times, scroll_delay, full_page, viewport, wait_time
|
||||
)
|
||||
)
|
||||
loop.close()
|
||||
result["backend"] = "playwright"
|
||||
except Exception as e:
|
||||
return {"success": False, "error": str(e)}
|
||||
return result
|
||||
|
||||
elif backend == "playwright":
|
||||
|
||||
Reference in New Issue
Block a user