增强 agent-browser 反爬虫支持
- 自动设置 AGENT_BROWSER_SOCKET_DIR 环境变量 - 添加真实 User-Agent 绕过反爬虫检测 - 增加 403 错误检测和提示 - 解决 techpowerup.com 等网站拦截问题
This commit is contained in:
13
app.py
13
app.py
@@ -59,12 +59,17 @@ class AgentBrowserSession:
|
||||
def run(self, cmd, timeout=60000):
|
||||
"""执行 agent-browser 命令"""
|
||||
full_cmd = self.base_cmd + cmd
|
||||
# 设置必要的环境变量
|
||||
env = os.environ.copy()
|
||||
env['AGENT_BROWSER_SOCKET_DIR'] = '/tmp/agent-browser-sockets'
|
||||
env['AGENT_BROWSER_USER_AGENT'] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
|
||||
try:
|
||||
result = subprocess.run(
|
||||
full_cmd,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout / 1000
|
||||
timeout=timeout / 1000,
|
||||
env=env
|
||||
)
|
||||
return result.returncode == 0, result.stdout, result.stderr
|
||||
except subprocess.TimeoutExpired:
|
||||
@@ -74,6 +79,9 @@ class AgentBrowserSession:
|
||||
|
||||
def open(self, url):
|
||||
"""打开网页"""
|
||||
# 添加反爬虫检测的 User-Agent
|
||||
env = os.environ.copy()
|
||||
env['AGENT_BROWSER_USER_AGENT'] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
|
||||
return self.run(["open", url], timeout=30000)
|
||||
|
||||
def set_viewport(self, width, height):
|
||||
@@ -134,6 +142,9 @@ def capture_with_agent_browser(
|
||||
# 打开网页
|
||||
success, stdout, stderr = session.open(url)
|
||||
if not success:
|
||||
# 检查是否是反爬虫拦截
|
||||
if "403" in stdout or "Access Denied" in stdout:
|
||||
return {"success": False, "error": "网站反爬虫拦截 (403),建议使用 Playwright 后端或手动添加请求头"}
|
||||
return {"success": False, "error": f"Failed to open URL: {stderr}"}
|
||||
|
||||
# 等待页面加载
|
||||
|
||||
Reference in New Issue
Block a user