diff --git a/PROJECT_SUMMARY.md b/PROJECT_SUMMARY.md new file mode 100644 index 0000000..75e325f --- /dev/null +++ b/PROJECT_SUMMARY.md @@ -0,0 +1,43 @@ +# Web Capture API 项目 + +## 待推送信息 + +- **仓库地址**: http://121.40.164.32:12007/hz4th_coder/web-capture-api.git +- **当前版本**: v1.0.2 +- **提交历史**: + - v1.0.0: 初始化项目 + - v1.0.1: 修改端口为 16025 + - v1.0.2: 修复 agent-browser socket 权限问题 + +## 操作步骤 + +### 1. 在 Git 平台创建仓库 + +请大爷登录 Git 平台 (http://121.40.164.32:12007) 并创建仓库: +- 仓库名称: `web-capture-api` +- 所属用户: `hz4th_coder` +- 可见性: 公开或私有(根据需要) + +### 2. 推送代码 + +仓库创建后,运行以下命令: + +```bash +cd /home/openclaw/.openclaw/workspace-hz4th_coder/works/web-capture-api +git push -u origin master --tags +``` + +## 项目功能 + +- 网页截图(全页或视口) +- HTML 代码提取 +- 自动滚动加载动态内容 +- 自定义视口大小 +- Web 界面和 RESTful API +- 支持 agent-browser 和 Playwright 双后端 +- Docker 部署支持 + +## 服务地址 + +- Web 界面: http://192.168.0.101:16025 +- API 接口: http://192.168.0.101:16025/api/capture \ No newline at end of file diff --git a/app.py b/app.py index 00bc584..c8f4915 100644 --- a/app.py +++ b/app.py @@ -59,12 +59,17 @@ class AgentBrowserSession: def run(self, cmd, timeout=60000): """执行 agent-browser 命令""" full_cmd = self.base_cmd + cmd + # 设置必要的环境变量 + env = os.environ.copy() + env['AGENT_BROWSER_SOCKET_DIR'] = '/tmp/agent-browser-sockets' + env['AGENT_BROWSER_USER_AGENT'] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36' try: result = subprocess.run( full_cmd, capture_output=True, text=True, - timeout=timeout / 1000 + timeout=timeout / 1000, + env=env ) return result.returncode == 0, result.stdout, result.stderr except subprocess.TimeoutExpired: @@ -74,6 +79,9 @@ class AgentBrowserSession: def open(self, url): """打开网页""" + # 添加反爬虫检测的 User-Agent + env = os.environ.copy() + env['AGENT_BROWSER_USER_AGENT'] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36' return self.run(["open", url], timeout=30000) def set_viewport(self, width, height): @@ -134,6 +142,9 @@ def capture_with_agent_browser( # 打开网页 success, stdout, stderr = session.open(url) if not success: + # 检查是否是反爬虫拦截 + if "403" in stdout or "Access Denied" in stdout: + return {"success": False, "error": "网站反爬虫拦截 (403),建议使用 Playwright 后端或手动添加请求头"} return {"success": False, "error": f"Failed to open URL: {stderr}"} # 等待页面加载