commit ed42f65c5675c7b7172adef240daa5c26697fa11 Author: hz4th_coder Date: Sat Jul 4 23:13:33 2026 +0800 初始化项目: Web Capture API v1.0.0 功能: - 网页截图（全页或视口） - HTML代码提取 - 自动滚动加载动态内容 - 自定义视口大小 - Web界面和RESTful API - 支持两种后端: agent-browser 和 Playwright - Docker部署支持 包含: - Flask REST API - 前端Web界面 - 测试脚本 - 安装脚本 - Dockerfile和docker-compose - systemd服务配置 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2ff8484 --- /dev/null +++ b/.gitignore @@ -0,0 +1,36 @@ +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +venv/ +env/ +ENV/ +.venv + +# IDE +.vscode/ +.idea/ +*.swp +*.swo + +# 测试 +.pytest_cache/ +.coverage +htmlcov/ + +# 临时文件 +*.log +*.tmp +.DS_Store +Thumbs.db + +# 截图输出（可选） +captures/ +*.png +*.html + +# 环境变量 +.env +.env.local \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..a89c77f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,58 @@ +FROM python:3.12-slim + +LABEL maintainer="hz4th_coder@tphai.com" +LABEL description="Web Capture API - 网页截图与代码提取服务" + +# 设置环境变量 +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 + +# 安装系统依赖 +RUN apt-get update && apt-get install -y --no-install-recommends \ + wget \ + gnupg \ + curl \ + ca-certificates \ + fonts-liberation \ + libasound2 \ + libatk-bridge2.0-0 \ + libatk1.0-0 \ + libatspi2.0-0 \ + libcups2 \ + libdbus-1-3 \ + libdrm2 \ + libgbm1 \ + libgtk-3-0 \ + libnspr4 \ + libnss3 \ + libwayland-client0 \ + libxcomposite1 \ + libxdamage1 \ + libxfixes3 \ + libxkbcommon0 \ + libxrandr2 \ + xdg-utils \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +# 安装Python依赖 +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +# 安装Playwright浏览器 +RUN playwright install chromium --with-deps + +# 复制应用代码 +COPY . . + +# 创建临时目录 +RUN mkdir -p /tmp/web_captures + +EXPOSE 5000 + +# 健康检查 +HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ + CMD curl -f http://localhost:5000/health || exit 1 + +CMD ["gunicorn", "-w", "4", "-b", "0.0.0.0:5000", "--timeout", "120", "app:app"] \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..291102e --- /dev/null +++ b/README.md @@ -0,0 +1,277 @@ +# Web Capture API + +网页截图与代码提取服务 + +## 功能特性 + +- 📸 **网页截图** - 全页或视口截图 +- 📄 **HTML提取** - 提取完整网页源码 +- 🔄 **动态内容** - 支持滚动加载动态内容 +- 🎯 **自定义视口** - 可调整浏览器窗口大小 +- ⚡ **RESTful API** - 简单易用的HTTP接口 +- 🌐 **Web界面** - 直观的前端页面 + +## 系统要求 + +- Python 3.10+ 或 Python 3.12 +- Node.js 18+ (用于 agent-browser) +- 现代浏览器（Chromium/Chrome） + +## 快速开始 + +### 方案一：使用 agent-browser（推荐） + +agent-browser 是基于 Rust 的高性能浏览器自动化工具。 + +```bash +# 1. 安装 Python 依赖 +pip install --user flask flask-cors gunicorn + +# 2. 安装 agent-browser +npm install -g agent-browser + +# 3. 安装浏览器依赖（需要 sudo） +sudo apt-get install -y \ + libxcb-shm0 libx11-xcb1 libx11-6 libxcb1 libxext6 \ + libxrandr2 libxcomposite1 libxcursor1 libxdamage1 \ + libxfixes3 libxi6 libgtk-3-0 libpangocairo-1.0-0 \ + libpango-1.0-0 libatk1.0-0 libcairo-gobject2 libcairo2 \ + libgdk-pixbuf-2.0-0 libxrender1 libasound2 libfreetype6 \ + libfontconfig1 libdbus-1-3 libnss3 libnspr4 libatk-bridge2.0-0 \ + libdrm2 libxkbcommon0 libatspi2.0-0 libcups2 libxshmfence1 \ + libgbm1 fonts-noto-color-emoji fonts-noto-cjk fonts-freefont-ttf + +# 4. 安装浏览器 +agent-browser install + +# 5. 启动服务 +python3.12 app.py +``` + +### 方案二：使用 Playwright + +Playwright 是微软开发的浏览器自动化库。 + +```bash +# 1. 安装依赖 +pip install --user playwright flask flask-cors gunicorn + +# 2. 安装浏览器 +python3.12 -m playwright install chromium + +# 3. 启动服务 +python3.12 app.py +``` + +## API 接口 + +### POST /api/capture + +**请求参数：** + +```json +{ + "url": "https://example.com", // 必填：目标网址 + "action": "screenshot", // 必填：screenshot 或 html + "scroll_times": 3, // 可选：滚动次数，默认0 + "scroll_delay": 1000, // 可选：滚动间隔(ms)，默认1000 + "full_page": true, // 可选：全页截图，默认false + "viewport": { // 可选：视口大小 + "width": 1920, + "height": 1080 + }, + "wait_time": 2000, // 可选：页面加载等待(ms)，默认2000 + "backend": "auto" // 可选：auto/agent-browser/playwright +} +``` + +**响应：** + +- `screenshot` 模式：返回 PNG 图片 +- `html` 模式：返回 JSON `{"success": true, "html": "..."}` + +### GET /health + +健康检查接口 + +### GET /api + +API 信息接口 + +## 使用示例 + +### 命令行调用 + +```bash +# 基础截图 +curl -X POST http://localhost:5000/api/capture \ + -H "Content-Type: application/json" \ + -d '{"url": "https://example.com", "action": "screenshot"}' \ + --output screenshot.png + +# 提取HTML +curl -X POST http://localhost:5000/api/capture \ + -H "Content-Type: application/json" \ + -d '{"url": "https://example.com", "action": "html"}' + +# 滚动加载后全页截图 +curl -X POST http://localhost:5000/api/capture \ + -H "Content-Type: application/json" \ + -d '{ + "url": "https://news.ycombinator.com", + "action": "screenshot", + "scroll_times": 5, + "scroll_delay": 1500, + "full_page": true + }' \ + --output full_page.png + +# 自定义视口 +curl -X POST http://localhost:5000/api/capture \ + -H "Content-Type: application/json" \ + -d '{ + "url": "https://example.com", + "action": "screenshot", + "viewport": {"width": 375, "height": 812} + }' \ + --output mobile.png +``` + +### Python 调用 + +```python +import requests + +# 截图 +response = requests.post( + 'http://localhost:5000/api/capture', + json={ + 'url': 'https://example.com', + 'action': 'screenshot' + } +) +with open('screenshot.png', 'wb') as f: + f.write(response.content) + +# 提取HTML +response = requests.post( + 'http://localhost:5000/api/capture', + json={ + 'url': 'https://example.com', + 'action': 'html' + } +) +html = response.json()['html'] +print(html) +``` + +### JavaScript 调用 + +```javascript +// 截图 +const response = await fetch('http://localhost:5000/api/capture', { + method: 'POST', + headers: {'Content-Type': 'application/json'}, + body: JSON.stringify({ + url: 'https://example.com', + action: 'screenshot', + scroll_times: 3, + full_page: true + }) +}); +const blob = await response.blob(); + +// 提取HTML +const response = await fetch('http://localhost:5000/api/capture', { + method: 'POST', + headers: {'Content-Type': 'application/json'}, + body: JSON.stringify({ + url: 'https://example.com', + action: 'html' + }) +}); +const data = await response.json(); +console.log(data.html); +``` + +## 生产部署 + +### 使用 Gunicorn + +```bash +gunicorn -w 4 -b 0.0.0.0:5000 app:app +``` + +### 使用 Systemd 服务 + +```bash +# 复制服务文件 +sudo cp web-capture-api.service /etc/systemd/system/ + +# 启动服务 +sudo systemctl start web-capture-api +sudo systemctl enable web-capture-api +``` + +### 使用 Docker + +```dockerfile +FROM python:3.12-slim + +# 安装依赖 +RUN apt-get update && apt-get install -y \ + wget gnupg curl \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt +RUN playwright install chromium --with-deps + +COPY . . +EXPOSE 5000 +CMD ["gunicorn", "-w", "4", "-b", "0.0.0.0:5000", "app:app"] +``` + +```bash +docker build -t web-capture-api . +docker run -p 5000:5000 web-capture-api +``` + +## 性能优化 + +- 使用 `--headless` 模式减少资源消耗 +- 合理设置 `wait_time` 避免不必要的等待 +- 滚动次数适度，避免过长处理时间 +- 生产环境使用 Gunicorn 多进程 + +## 故障排除 + +### 浏览器启动失败 + +检查是否安装了所有依赖： + +```bash +# agent-browser +agent-browser install --with-deps + +# playwright +python3.12 -m playwright install-deps chromium +``` + +### 权限错误 + +确保用户有权限访问浏览器缓存目录： + +```bash +mkdir -p ~/.cache/ms-playwright +chmod -R 755 ~/.cache/ms-playwright +``` + +### 内存不足 + +减少并发进程数或增加系统内存。 + +## 许可证 + +MIT License \ No newline at end of file diff --git a/app.py b/app.py new file mode 100644 index 0000000..c18246b --- /dev/null +++ b/app.py @@ -0,0 +1,401 @@ +#!/home/hz1/miniconda3/envs/openclaw/bin/python3.12 +""" +Web Capture API - 网页截图与代码提取服务 +使用 agent-browser (Rust + Playwright) 或 Playwright +""" + +import os +import json +import asyncio +import subprocess +import tempfile +import uuid +import time +from pathlib import Path +from flask import Flask, request, jsonify, send_file, render_template +from flask_cors import CORS +from datetime import datetime + +# 检查 Playwright 是否可用 +try: + from playwright.async_api import async_playwright + PLAYWRIGHT_AVAILABLE = True +except ImportError: + PLAYWRIGHT_AVAILABLE = False + +# 检查 agent-browser 是否可用 +def check_agent_browser(): + try: + result = subprocess.run( + ["which", "agent-browser"], + capture_output=True, + text=True + ) + return result.returncode == 0 + except: + return False + +AGENT_BROWSER_AVAILABLE = check_agent_browser() + +app = Flask(__name__) +CORS(app) + +# 配置 +CAPTURE_DIR = Path(tempfile.gettempdir()) / "web_captures" +CAPTURE_DIR.mkdir(exist_ok=True) + + +class AgentBrowserSession: + """agent-browser 会话管理器""" + + def __init__(self, session_id=None): + self.session_id = session_id or str(uuid.uuid4())[:8] + self.base_cmd = ["agent-browser", "--session", self.session_id] + + def run(self, cmd, timeout=60000): + """执行 agent-browser 命令""" + full_cmd = self.base_cmd + cmd + try: + result = subprocess.run( + full_cmd, + capture_output=True, + text=True, + timeout=timeout / 1000 + ) + return result.returncode == 0, result.stdout, result.stderr + except subprocess.TimeoutExpired: + return False, "", "Command timeout" + except Exception as e: + return False, "", str(e) + + def open(self, url): + """打开网页""" + return self.run(["open", url], timeout=30000) + + def set_viewport(self, width, height): + """设置视口大小""" + return self.run(["set", "viewport", str(width), str(height)]) + + def screenshot(self, output_path, full_page=False): + """截图""" + cmd = ["screenshot", str(output_path)] + if full_page: + cmd.append("--full") + return self.run(cmd, timeout=30000) + + def get_html(self): + """获取HTML""" + success, output, error = self.run( + ["eval", "document.documentElement.outerHTML"], + timeout=10000 + ) + return success, output, error + + def scroll_down(self, pixels=800): + """向下滚动""" + return self.run(["scroll", "down", str(pixels)]) + + def wait(self, ms): + """等待""" + return self.run(["wait", str(ms)]) + + def close(self): + """关闭浏览器""" + try: + self.run(["close"]) + except: + pass + + +def capture_with_agent_browser( + url: str, + action: str = "screenshot", + scroll_times: int = 0, + scroll_delay: int = 1000, + full_page: bool = False, + viewport: dict = None, + wait_time: int = 2000 +): + """ + 使用 agent-browser 捕获网页 + """ + session = AgentBrowserSession() + temp_file = None + + try: + # 设置视口 + if viewport: + session.set_viewport(viewport.get("width", 1920), viewport.get("height", 1080)) + + # 打开网页 + success, stdout, stderr = session.open(url) + if not success: + return {"success": False, "error": f"Failed to open URL: {stderr}"} + + # 等待页面加载 + session.wait(wait_time) + + # 滚动加载动态内容 + if scroll_times > 0: + for i in range(scroll_times): + session.scroll_down(800) + session.wait(scroll_delay) + + # 执行操作 + if action == "screenshot": + temp_file = CAPTURE_DIR / f"{session.session_id}.png" + success, _, error = session.screenshot(temp_file, full_page=full_page) + + if success and temp_file.exists(): + return {"success": True, "file_path": str(temp_file)} + else: + return {"success": False, "error": f"Screenshot failed: {error}"} + + elif action == "html": + success, html, error = session.get_html() + if success: + return {"success": True, "html": html} + else: + return {"success": False, "error": f"Get HTML failed: {error}"} + + else: + return {"success": False, "error": f"Unknown action: {action}"} + + except Exception as e: + return {"success": False, "error": str(e)} + + finally: + session.close() + + +async def capture_with_playwright( + url: str, + action: str = "screenshot", + scroll_times: int = 0, + scroll_delay: int = 1000, + full_page: bool = False, + viewport: dict = None, + wait_time: int = 2000 +): + """ + 使用 Playwright 捕获网页 + """ + if not PLAYWRIGHT_AVAILABLE: + return {"success": False, "error": "Playwright not installed"} + + session_id = str(uuid.uuid4())[:8] + + try: + async with async_playwright() as p: + browser = await p.chromium.launch(headless=True) + + viewport_settings = viewport or {"width": 1920, "height": 1080} + + context = await browser.new_context( + viewport={ + "width": viewport_settings.get("width", 1920), + "height": viewport_settings.get("height", 1080) + }, + user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36" + ) + + page = await context.new_page() + + try: + await page.goto(url, wait_until="networkidle", timeout=30000) + except: + await page.goto(url, wait_until="domcontentloaded", timeout=30000) + + if wait_time > 0: + await page.wait_for_timeout(wait_time) + + if scroll_times > 0: + for i in range(scroll_times): + await page.evaluate("window.scrollBy(0, 800)") + await page.wait_for_timeout(scroll_delay) + await page.wait_for_timeout(500) + + result = {} + + if action == "screenshot": + temp_file = CAPTURE_DIR / f"{session_id}.png" + await page.screenshot(path=str(temp_file), full_page=full_page) + result = {"success": True, "file_path": str(temp_file)} + + elif action == "html": + html = await page.content() + result = {"success": True, "html": html} + + else: + result = {"success": False, "error": f"Unknown action: {action}"} + + await browser.close() + return result + + except Exception as e: + return {"success": False, "error": str(e)} + + +def capture_webpage( + url: str, + action: str = "screenshot", + scroll_times: int = 0, + scroll_delay: int = 1000, + full_page: bool = False, + viewport: dict = None, + wait_time: int = 2000, + backend: str = "auto" +): + """ + 捕获网页 + + Args: + backend: "auto", "agent-browser", "playwright" + """ + # 选择后端 + if backend == "auto": + if AGENT_BROWSER_AVAILABLE: + backend = "agent-browser" + elif PLAYWRIGHT_AVAILABLE: + backend = "playwright" + else: + return {"success": False, "error": "No browser backend available. Install agent-browser or playwright."} + + # 使用对应后端 + if backend == "agent-browser": + if not AGENT_BROWSER_AVAILABLE: + return {"success": False, "error": "agent-browser not available"} + return capture_with_agent_browser( + url, action, scroll_times, scroll_delay, full_page, viewport, wait_time + ) + + elif backend == "playwright": + if not PLAYWRIGHT_AVAILABLE: + return {"success": False, "error": "Playwright not available"} + + try: + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + result = loop.run_until_complete( + capture_with_playwright( + url, action, scroll_times, scroll_delay, full_page, viewport, wait_time + ) + ) + loop.close() + return result + except Exception as e: + return {"success": False, "error": str(e)} + + else: + return {"success": False, "error": f"Unknown backend: {backend}"} + + +@app.route('/') +def index(): + """主页""" + return render_template('index.html') + + +@app.route('/api') +def api_info(): + """API信息""" + return jsonify({ + "service": "Web Capture API", + "version": "2.0.0", + "backends": { + "agent-browser": "available" if AGENT_BROWSER_AVAILABLE else "unavailable", + "playwright": "available" if PLAYWRIGHT_AVAILABLE else "unavailable" + }, + "endpoints": { + "/api/capture": "POST - Capture webpage screenshot or HTML", + "/health": "GET - Health check" + } + }) + + +@app.route('/health') +def health(): + """健康检查""" + status = "healthy" if (AGENT_BROWSER_AVAILABLE or PLAYWRIGHT_AVAILABLE) else "degraded" + return jsonify({ + "status": status, + "agent-browser": "installed" if AGENT_BROWSER_AVAILABLE else "not found", + "playwright": "installed" if PLAYWRIGHT_AVAILABLE else "not found", + "timestamp": datetime.now().isoformat() + }) + + +@app.route('/api/capture', methods=['POST']) +def capture(): + """ + 捕获网页接口 + + 请求体: + { + "url": "https://example.com", + "action": "screenshot" | "html", + "scroll_times": 0, + "scroll_delay": 1000, + "full_page": false, + "viewport": {"width": 1920, "height": 1080}, + "wait_time": 2000, + "backend": "auto" | "agent-browser" | "playwright" + } + """ + data = request.get_json() + + if not data: + return jsonify({"success": False, "error": "No JSON data provided"}), 400 + + url = data.get("url") + if not url: + return jsonify({"success": False, "error": "URL is required"}), 400 + + # 添加协议前缀 + if not url.startswith(("http://", "https://")): + url = "https://" + url + + action = data.get("action", "screenshot") + scroll_times = int(data.get("scroll_times", 0)) + scroll_delay = int(data.get("scroll_delay", 1000)) + full_page = bool(data.get("full_page", False)) + viewport = data.get("viewport") + wait_time = int(data.get("wait_time", 2000)) + backend = data.get("backend", "auto") + + result = capture_webpage( + url=url, + action=action, + scroll_times=scroll_times, + scroll_delay=scroll_delay, + full_page=full_page, + viewport=viewport, + wait_time=wait_time, + backend=backend + ) + + if not result["success"]: + return jsonify(result), 400 + + if action == "screenshot": + file_path = result["file_path"] + return send_file(file_path, mimetype='image/png') + + elif action == "html": + return jsonify(result) + + +if __name__ == '__main__': + print("🌐 Web Capture API Starting...") + print(f"📁 Capture directory: {CAPTURE_DIR}") + print(f"🔧 Backends:") + print(f" - agent-browser: {'✅' if AGENT_BROWSER_AVAILABLE else '❌'}") + print(f" - playwright: {'✅' if PLAYWRIGHT_AVAILABLE else '❌'}") + + if not AGENT_BROWSER_AVAILABLE and not PLAYWRIGHT_AVAILABLE: + print("\n⚠️ 没有可用的浏览器后端，请安装其中一个:") + print(" agent-browser: npm install -g agent-browser && agent-browser install") + print(" playwright: pip install playwright && playwright install chromium") + + print("\n🚀 Server running on http://0.0.0.0:5000") + app.run(host='0.0.0.0', port=5000, debug=True) \ No newline at end of file diff --git a/deploy.sh b/deploy.sh new file mode 100755 index 0000000..5bd69a9 --- /dev/null +++ b/deploy.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# 部署脚本 + +set -e + +echo "🚀 开始部署 Web Capture API..." + +# 进入项目目录 +cd /home/openclaw/.openclaw/workspace-hz4th_coder/works/web-capture-api + +# 安装 Python 依赖 +echo "📥 安装 Python 依赖..." +pip3.12 install --user playwright flask flask-cors gunicorn + +# 安装 Playwright 浏览器 +echo "🌐 安装 Playwright 浏览器..." +python3.12 -m playwright install chromium + +echo "" +echo "✅ 部署完成！" +echo "" +echo "运行方式：" +echo " 开发模式: python3.12 app.py" +echo " 生产模式: gunicorn -w 4 -b 0.0.0.0:5000 app:app" +echo "" +echo "访问地址: http://localhost:5000" \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..00fa23e --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,20 @@ +version: '3.8' + +services: + web-capture-api: + build: . + image: web-capture-api:latest + container_name: web-capture-api + ports: + - "5000:5000" + environment: + - TZ=Asia/Shanghai + volumes: + - ./captures:/tmp/web_captures + restart: unless-stopped + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:5000/health"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 10s \ No newline at end of file diff --git a/install.sh b/install.sh new file mode 100755 index 0000000..d1e96b0 --- /dev/null +++ b/install.sh @@ -0,0 +1,116 @@ +#!/bin/bash +# 一键安装脚本 + +set -e + +echo "🚀 Web Capture API 安装脚本" +echo "================================" +echo "" + +# 检测系统 +if [[ "$OSTYPE" == "linux-gnu"* ]]; then + OS="linux" +elif [[ "$OSTYPE" == "darwin"* ]]; then + OS="macos" +else + echo "❌ 不支持的操作系统: $OSTYPE" + exit 1 +fi + +echo "📋 系统检测: $OS" + +# 检查 Python +echo "" +echo "📦 检查 Python..." +if command -v python3.12 &> /dev/null; then + PYTHON=python3.12 +elif command -v python3 &> /dev/null; then + PYTHON=python3 +else + echo "❌ 未找到 Python 3，请先安装 Python 3.10+" + exit 1 +fi +echo "✅ 找到 Python: $($PYTHON --version)" + +# 检查 pip +if ! $PYTHON -m pip --version &> /dev/null; then + echo "❌ 未找到 pip，请先安装 pip" + exit 1 +fi + +# 检查 Node.js (用于 agent-browser) +echo "" +echo "📦 检查 Node.js..." +if command -v node &> /dev/null; then + NODE_VERSION=$(node --version) + echo "✅ 找到 Node.js: $NODE_VERSION" + HAS_NODE=true +else + echo "⚠️ 未找到 Node.js，将跳过 agent-browser 安装" + HAS_NODE=false +fi + +# 安装 Python 依赖 +echo "" +echo "📥 安装 Python 依赖..." +$PYTHON -m pip install --user playwright flask flask-cors gunicorn + +# 询问使用哪个后端 +echo "" +echo "🔧 选择浏览器后端:" +echo " 1) Playwright (推荐，自动安装浏览器)" +echo " 2) agent-browser (高性能，需要手动安装浏览器依赖)" +read -p "请选择 [1/2]: " choice + +case $choice in + 1) + echo "" + echo "🌐 安装 Playwright 浏览器..." + $PYTHON -m playwright install chromium + + if [ "$OS" == "linux" ]; then + echo "" + echo "⚠️ 如果浏览器启动失败，可能需要安装系统依赖：" + echo " $PYTHON -m playwright install-deps chromium" + fi + ;; + 2) + if [ "$HAS_NODE" = false ]; then + echo "❌ 安装 agent-browser 需要 Node.js" + exit 1 + fi + + echo "" + echo "🌐 安装 agent-browser..." + npm install -g agent-browser + + if [ "$OS" == "linux" ]; then + echo "" + echo "⚠️ agent-browser 需要浏览器依赖，请运行：" + echo " agent-browser install --with-deps" + echo "" + echo "如果需要 sudo 权限，请运行：" + echo " sudo agent-browser install --with-deps" + else + agent-browser install + fi + ;; + *) + echo "❌ 无效选择" + exit 1 + ;; +esac + +# 完成 +echo "" +echo "✅ 安装完成！" +echo "" +echo "🚀 启动服务:" +echo " cd $(pwd)" +echo " $PYTHON app.py" +echo "" +echo "📖 访问地址:" +echo " http://localhost:5000" +echo "" +echo "📚 API 文档:" +echo " http://localhost:5000/api" \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..f72b12b --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +Flask==3.0.0 +flask-cors==6.0.5 +gunicorn==26.0.0 +playwright==1.48.0 \ No newline at end of file diff --git a/templates/index.html b/templates/index.html new file mode 100644 index 0000000..de67b77 --- /dev/null +++ b/templates/index.html @@ -0,0 +1,480 @@ + + + + + + Web Capture - 网页截图与代码提取 + + + +
+
+

🌐 Web Capture

+

网页截图与代码提取服务

+
+ +
+
+
+ + +
+ +
+ +
+ + +
+
+ +
+ +
+ +
+
+ +
+ +
+
+ + +
+
+ + +
+
+ + +
+
+
+ +
+ +
+
+ + +
+
+ + +
+
+
+ + +
+ +
+ +

正在处理，请稍候...

+
+ +
+ +
+

捕获结果

+
+
+ + + +
+
+
+ +
+

📚 API 文档

+

POST /api/capture

+ 
{
+  "url": "https://example.com",
+  "action": "screenshot",  // 或 "html"
+  "scroll_times": 3,
+  "scroll_delay": 1000,
+  "full_page": true,
+  "viewport": {"width": 1920, "height": 1080},
+  "wait_time": 2000
+}
+
+
+ + + + \ No newline at end of file diff --git a/test_api.py b/test_api.py new file mode 100755 index 0000000..9f52d6a --- /dev/null +++ b/test_api.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +""" +测试脚本 - 测试 Web Capture API +""" + +import requests +import json +import sys + +API_URL = "http://localhost:5000" + + +def test_health(): + """测试健康检查""" + print("\n🔍 测试健康检查...") + resp = requests.get(f"{API_URL}/health") + print(f"状态: {resp.status_code}") + print(f"响应: {json.dumps(resp.json(), indent=2, ensure_ascii=False)}") + return resp.status_code == 200 + + +def test_screenshot(): + """测试截图功能""" + print("\n📸 测试截图功能...") + + data = { + "url": "https://example.com", + "action": "screenshot", + "wait_time": 2000 + } + + resp = requests.post( + f"{API_URL}/api/capture", + json=data + ) + + if resp.status_code == 200: + with open("test_screenshot.png", "wb") as f: + f.write(resp.content) + print("✅ 截图成功，已保存为 test_screenshot.png") + return True + else: + print(f"❌ 截图失败: {resp.text}") + return False + + +def test_html(): + """测试HTML提取""" + print("\n📄 测试HTML提取...") + + data = { + "url": "https://example.com", + "action": "html" + } + + resp = requests.post( + f"{API_URL}/api/capture", + json=data + ) + + if resp.status_code == 200: + result = resp.json() + print(f"✅ HTML提取成功，长度: {len(result.get('html', ''))}") + print(f"前200字符: {result.get('html', '')[:200]}...") + return True + else: + print(f"❌ HTML提取失败: {resp.text}") + return False + + +def test_scroll(): + """测试滚动加载""" + print("\n🔄 测试滚动加载...") + + data = { + "url": "https://news.ycombinator.com", + "action": "screenshot", + "scroll_times": 3, + "scroll_delay": 1000, + "full_page": True + } + + resp = requests.post( + f"{API_URL}/api/capture", + json=data + ) + + if resp.status_code == 200: + with open("test_scroll.png", "wb") as f: + f.write(resp.content) + print("✅ 滚动截图成功，已保存为 test_scroll.png") + return True + else: + print(f"❌ 滚动截图失败: {resp.text}") + return False + + +if __name__ == "__main__": + print("🚀 Web Capture API 测试") + print("=" * 50) + + tests = [ + ("健康检查", test_health), + ("截图功能", test_screenshot), + ("HTML提取", test_html), + ("滚动加载", test_scroll) + ] + + results = [] + for name, test_func in tests: + try: + result = test_func() + results.append((name, result)) + except Exception as e: + print(f"❌ {name} 异常: {e}") + results.append((name, False)) + + print("\n" + "=" * 50) + print("测试结果汇总:") + for name, result in results: + status = "✅ 通过" if result else "❌ 失败" + print(f" {name}: {status}") \ No newline at end of file diff --git a/web-capture-api.service b/web-capture-api.service new file mode 100644 index 0000000..3ae53d4 --- /dev/null +++ b/web-capture-api.service @@ -0,0 +1,14 @@ +[Unit] +Description=Web Capture API - 网页截图与代码提取服务 +After=network.target + +[Service] +Type=simple +User=openclaw +WorkingDirectory=/home/openclaw/.openclaw/workspace-hz4th_coder/works/web-capture-api +ExecStart=/home/openclaw/.openclaw/workspace-hz4th_coder/works/web-capture-api/venv/bin/gunicorn -w 4 -b 0.0.0.0:5000 app:app +Restart=always +RestartSec=10 + +[Install] +WantedBy=multi-user.target \ No newline at end of file