Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
17431e86cf | ||
|
|
82ff8a2610 | ||
|
|
1e69e7ca57 | ||
|
|
951f941eee |
@@ -315,7 +315,8 @@ class AgentBrowserSession:
|
|||||||
# 添加反爬虫检测的 User-Agent
|
# 添加反爬虫检测的 User-Agent
|
||||||
env = os.environ.copy()
|
env = os.environ.copy()
|
||||||
env['AGENT_BROWSER_USER_AGENT'] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
|
env['AGENT_BROWSER_USER_AGENT'] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
|
||||||
return self.run(["open", url], timeout=30000)
|
# agent-browser 内部导航等待约 30s 超时,这里留足 45s 让它返回自己的超时报错(避免被 subprocess 硬杀)
|
||||||
|
return self.run(["open", url], timeout=45000)
|
||||||
|
|
||||||
def set_viewport(self, width, height):
|
def set_viewport(self, width, height):
|
||||||
"""设置视口大小"""
|
"""设置视口大小"""
|
||||||
@@ -410,6 +411,8 @@ def capture_with_agent_browser(
|
|||||||
# 检查是否是反爬虫拦截
|
# 检查是否是反爬虫拦截
|
||||||
if "403" in stdout or "Access Denied" in stdout:
|
if "403" in stdout or "Access Denied" in stdout:
|
||||||
return {"success": False, "error": "网站反爬虫拦截 (403),建议使用 Playwright 后端或手动添加请求头"}
|
return {"success": False, "error": "网站反爬虫拦截 (403),建议使用 Playwright 后端或手动添加请求头"}
|
||||||
|
if "timed out" in stderr.lower() or "timeout" in stderr.lower():
|
||||||
|
return {"success": False, "error": "页面加载超时(可能是慢加载/持续加载页面),已自动切换 Playwright 后端重试;若仍失败请增大等待时间或换其他后端"}
|
||||||
return {"success": False, "error": f"Failed to open URL: {stderr}"}
|
return {"success": False, "error": f"Failed to open URL: {stderr}"}
|
||||||
|
|
||||||
# 等待页面加载
|
# 等待页面加载
|
||||||
@@ -595,7 +598,7 @@ async def capture_with_playwright(
|
|||||||
await stealth_async(page)
|
await stealth_async(page)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
await page.goto(url, wait_until="domcontentloaded", timeout=30000)
|
await page.goto(url, wait_until="domcontentloaded", timeout=60000)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
# 如果 domcontentloaded 超时,尝试 commit
|
# 如果 domcontentloaded 超时,尝试 commit
|
||||||
try:
|
try:
|
||||||
@@ -705,6 +708,14 @@ async def capture_with_cdp(
|
|||||||
"html": html
|
"html": html
|
||||||
})
|
})
|
||||||
|
|
||||||
|
elif action == "text":
|
||||||
|
text = await page.evaluate("document.body.innerText")
|
||||||
|
result_data.append({
|
||||||
|
"url": current_url,
|
||||||
|
"title": title,
|
||||||
|
"text": clean_text(text)
|
||||||
|
})
|
||||||
|
|
||||||
if result_data:
|
if result_data:
|
||||||
return {"success": True, "pages": result_data}
|
return {"success": True, "pages": result_data}
|
||||||
else:
|
else:
|
||||||
@@ -733,6 +744,8 @@ def smart_capture_agent_browser(url, cfg, wait_time, viewport):
|
|||||||
session.set_viewport(vw, vh)
|
session.set_viewport(vw, vh)
|
||||||
success, _, err = session.open(url)
|
success, _, err = session.open(url)
|
||||||
if not success:
|
if not success:
|
||||||
|
if "timed out" in err.lower() or "timeout" in err.lower():
|
||||||
|
return {"success": False, "error": "页面加载超时(可能是慢加载/持续加载页面),已自动切换 Playwright 后端重试;若仍失败请增大等待时间或换其他后端"}
|
||||||
return {"success": False, "error": f"打开网页失败: {err}"}
|
return {"success": False, "error": f"打开网页失败: {err}"}
|
||||||
session.wait(wait_time)
|
session.wait(wait_time)
|
||||||
|
|
||||||
@@ -845,7 +858,7 @@ async def smart_capture_playwright(url, cfg, wait_time, viewport):
|
|||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
try:
|
try:
|
||||||
await page.goto(url, wait_until="domcontentloaded", timeout=30000)
|
await page.goto(url, wait_until="domcontentloaded", timeout=60000)
|
||||||
except Exception:
|
except Exception:
|
||||||
try:
|
try:
|
||||||
await page.goto(url, wait_until="commit", timeout=30000)
|
await page.goto(url, wait_until="commit", timeout=30000)
|
||||||
@@ -942,10 +955,22 @@ async def smart_capture_playwright(url, cfg, wait_time, viewport):
|
|||||||
def smart_capture(url, cfg, wait_time, viewport, backend="auto"):
|
def smart_capture(url, cfg, wait_time, viewport, backend="auto"):
|
||||||
"""按需截图总入口:滚动截图 + 视觉大模型实时判断,返回拼接长图"""
|
"""按需截图总入口:滚动截图 + 视觉大模型实时判断,返回拼接长图"""
|
||||||
if backend == "auto":
|
if backend == "auto":
|
||||||
backend = "agent-browser" if AGENT_BROWSER_AVAILABLE else "playwright"
|
backend = "playwright" if PLAYWRIGHT_AVAILABLE else "agent-browser"
|
||||||
|
|
||||||
if backend == "agent-browser":
|
if backend == "agent-browser":
|
||||||
return smart_capture_agent_browser(url, cfg, wait_time, viewport)
|
result = smart_capture_agent_browser(url, cfg, wait_time, viewport)
|
||||||
|
# 打开页面超时 → 自动切 Playwright 重试(domcontentloaded 更宽容)
|
||||||
|
if (not result.get("success")
|
||||||
|
and "自动切换 Playwright" in (result.get("error") or "")
|
||||||
|
and PLAYWRIGHT_AVAILABLE):
|
||||||
|
try:
|
||||||
|
loop = asyncio.new_event_loop()
|
||||||
|
asyncio.set_event_loop(loop)
|
||||||
|
result = loop.run_until_complete(smart_capture_playwright(url, cfg, wait_time, viewport))
|
||||||
|
loop.close()
|
||||||
|
except Exception as e:
|
||||||
|
return {"success": False, "error": str(e)}
|
||||||
|
return result
|
||||||
elif backend == "playwright":
|
elif backend == "playwright":
|
||||||
try:
|
try:
|
||||||
loop = asyncio.new_event_loop()
|
loop = asyncio.new_event_loop()
|
||||||
@@ -966,20 +991,21 @@ def capture_webpage(
|
|||||||
full_page: bool = False,
|
full_page: bool = False,
|
||||||
viewport: dict = None,
|
viewport: dict = None,
|
||||||
wait_time: int = 2000,
|
wait_time: int = 2000,
|
||||||
backend: str = "auto"
|
backend: str = "auto",
|
||||||
|
cdp_port: int = 9222
|
||||||
):
|
):
|
||||||
"""
|
"""
|
||||||
捕获网页
|
捕获网页
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
backend: "auto", "agent-browser", "playwright"
|
backend: "auto", "agent-browser", "playwright", "chrome-cdp"
|
||||||
"""
|
"""
|
||||||
# 选择后端
|
# 选择后端
|
||||||
if backend == "auto":
|
if backend == "auto":
|
||||||
if AGENT_BROWSER_AVAILABLE:
|
if PLAYWRIGHT_AVAILABLE:
|
||||||
backend = "agent-browser"
|
|
||||||
elif PLAYWRIGHT_AVAILABLE:
|
|
||||||
backend = "playwright"
|
backend = "playwright"
|
||||||
|
elif AGENT_BROWSER_AVAILABLE:
|
||||||
|
backend = "agent-browser"
|
||||||
else:
|
else:
|
||||||
return {"success": False, "error": "No browser backend available. Install agent-browser or playwright."}
|
return {"success": False, "error": "No browser backend available. Install agent-browser or playwright."}
|
||||||
|
|
||||||
@@ -991,6 +1017,22 @@ def capture_webpage(
|
|||||||
url, action, scroll_times, scroll_delay, full_page, viewport, wait_time
|
url, action, scroll_times, scroll_delay, full_page, viewport, wait_time
|
||||||
)
|
)
|
||||||
result["backend"] = "agent-browser"
|
result["backend"] = "agent-browser"
|
||||||
|
# agent-browser 打开页面超时(慢加载/持续加载页)→ 自动切 Playwright 重试(domcontentloaded 更宽容)
|
||||||
|
if (not result.get("success")
|
||||||
|
and "自动切换 Playwright" in (result.get("error") or "")
|
||||||
|
and PLAYWRIGHT_AVAILABLE):
|
||||||
|
try:
|
||||||
|
loop = asyncio.new_event_loop()
|
||||||
|
asyncio.set_event_loop(loop)
|
||||||
|
result = loop.run_until_complete(
|
||||||
|
capture_with_playwright(
|
||||||
|
url, action, scroll_times, scroll_delay, full_page, viewport, wait_time
|
||||||
|
)
|
||||||
|
)
|
||||||
|
loop.close()
|
||||||
|
result["backend"] = "playwright"
|
||||||
|
except Exception as e:
|
||||||
|
return {"success": False, "error": str(e)}
|
||||||
return result
|
return result
|
||||||
|
|
||||||
elif backend == "playwright":
|
elif backend == "playwright":
|
||||||
@@ -1007,9 +1049,46 @@ def capture_webpage(
|
|||||||
)
|
)
|
||||||
loop.close()
|
loop.close()
|
||||||
result["backend"] = "playwright"
|
result["backend"] = "playwright"
|
||||||
|
except Exception as e:
|
||||||
|
result = {"success": False, "error": str(e)}
|
||||||
|
# playwright 失败(auto 默认优先它)→ 自动切 agent-browser 兜底重试
|
||||||
|
if (not result.get("success") and AGENT_BROWSER_AVAILABLE):
|
||||||
|
try:
|
||||||
|
fb = capture_with_agent_browser(
|
||||||
|
url, action, scroll_times, scroll_delay, full_page, viewport, wait_time
|
||||||
|
)
|
||||||
|
fb["backend"] = "agent-browser"
|
||||||
|
result = fb
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
elif backend == "chrome-cdp":
|
||||||
|
# 连接已打开的 Chrome(需 --remote-debugging-port=<cdp_port>),归一化为单结果
|
||||||
|
if not PLAYWRIGHT_AVAILABLE:
|
||||||
|
return {"success": False, "error": "Playwright not installed (chrome-cdp 依赖 playwright)"}
|
||||||
|
try:
|
||||||
|
loop = asyncio.new_event_loop()
|
||||||
|
asyncio.set_event_loop(loop)
|
||||||
|
r = loop.run_until_complete(
|
||||||
|
capture_with_cdp(url_hint=url, action=action, cdp_port=cdp_port)
|
||||||
|
)
|
||||||
|
loop.close()
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
return {"success": False, "error": str(e)}
|
return {"success": False, "error": str(e)}
|
||||||
|
if not r.get("success"):
|
||||||
|
return r
|
||||||
|
if not r.get("pages"):
|
||||||
|
return {"success": False, "error": "chrome-cdp 未找到匹配页面(请确认 Chrome 已开 --remote-debugging-port)"}
|
||||||
|
p0 = r["pages"][0]
|
||||||
|
res = {"success": True, "title": p0.get("title", ""), "backend": "chrome-cdp"}
|
||||||
|
if action == "screenshot":
|
||||||
|
res["file_path"] = p0["file_path"]
|
||||||
|
elif action == "html":
|
||||||
|
res["html"] = p0["html"]
|
||||||
|
elif action == "text":
|
||||||
|
res["text"] = p0["text"]
|
||||||
|
return res
|
||||||
|
|
||||||
else:
|
else:
|
||||||
return {"success": False, "error": f"Unknown backend: {backend}"}
|
return {"success": False, "error": f"Unknown backend: {backend}"}
|
||||||
@@ -1021,6 +1100,38 @@ def index():
|
|||||||
return render_template('index.html')
|
return render_template('index.html')
|
||||||
|
|
||||||
|
|
||||||
|
@app.route('/api/backends')
|
||||||
|
def api_backends():
|
||||||
|
"""返回可用后端及默认选择顺序,供前端渲染选择器"""
|
||||||
|
available = []
|
||||||
|
if AGENT_BROWSER_AVAILABLE:
|
||||||
|
available.append("agent-browser")
|
||||||
|
if PLAYWRIGHT_AVAILABLE:
|
||||||
|
available.append("playwright")
|
||||||
|
# chrome-cdp 依赖 playwright 的 connect_over_cdp
|
||||||
|
available.append("chrome-cdp")
|
||||||
|
unavailable = []
|
||||||
|
if not AGENT_BROWSER_AVAILABLE:
|
||||||
|
unavailable.append("agent-browser")
|
||||||
|
if not PLAYWRIGHT_AVAILABLE:
|
||||||
|
unavailable.append("playwright")
|
||||||
|
unavailable.append("chrome-cdp")
|
||||||
|
return jsonify({
|
||||||
|
"success": True,
|
||||||
|
"default": "auto",
|
||||||
|
"available": available,
|
||||||
|
"unavailable": unavailable,
|
||||||
|
# auto 优先 playwright,其次 agent-browser,再 chrome-cdp
|
||||||
|
"priority": ["playwright", "agent-browser", "chrome-cdp"],
|
||||||
|
"labels": {
|
||||||
|
"auto": "自动(推荐:优先 Playwright,失败自动回退 agent-browser)",
|
||||||
|
"playwright": "Playwright(Python 版,domcontentloaded 更宽容,慢页面更稳,默认首选)",
|
||||||
|
"agent-browser": "agent-browser(Rust 版,反爬强、快,备选)",
|
||||||
|
"chrome-cdp": "Chrome CDP(连接已打开的 Chrome,需 --remote-debugging-port 启动)"
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
@app.route('/api')
|
@app.route('/api')
|
||||||
def api_info():
|
def api_info():
|
||||||
"""API信息"""
|
"""API信息"""
|
||||||
@@ -1033,6 +1144,7 @@ def api_info():
|
|||||||
},
|
},
|
||||||
"endpoints": {
|
"endpoints": {
|
||||||
"/api/capture": "POST - Capture webpage (screenshot/html/text/smart) + 自动入库历史(记录调用者/方式/原始HTML)",
|
"/api/capture": "POST - Capture webpage (screenshot/html/text/smart) + 自动入库历史(记录调用者/方式/原始HTML)",
|
||||||
|
"/api/backends": "GET - 可用后端列表与默认顺序(供前端渲染后端选择器)",
|
||||||
"/api/history": "GET - 历史记录分页列表 (?page&page_size&action&search)",
|
"/api/history": "GET - 历史记录分页列表 (?page&page_size&action&search)",
|
||||||
"/api/history/<id>": "GET - 历史记录详情 / DELETE - 删除记录",
|
"/api/history/<id>": "GET - 历史记录详情 / DELETE - 删除记录",
|
||||||
"/api/history/<id>/file": "GET - 读取历史截图文件",
|
"/api/history/<id>/file": "GET - 读取历史截图文件",
|
||||||
@@ -1287,7 +1399,8 @@ def capture():
|
|||||||
full_page=full_page,
|
full_page=full_page,
|
||||||
viewport=viewport,
|
viewport=viewport,
|
||||||
wait_time=wait_time,
|
wait_time=wait_time,
|
||||||
backend=backend
|
backend=backend,
|
||||||
|
cdp_port=cdp_port
|
||||||
)
|
)
|
||||||
|
|
||||||
backend_used = result.get("backend", backend)
|
backend_used = result.get("backend", backend)
|
||||||
|
|||||||
+37
-4
@@ -624,6 +624,16 @@
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<div class="form-group">
|
||||||
|
<label>捕获后端</label>
|
||||||
|
<select id="backend" style="max-width:100%;">
|
||||||
|
<option value="auto">加载中…</option>
|
||||||
|
</select>
|
||||||
|
<small id="backendHint" style="color: #666; display: block; margin-top: 4px;">
|
||||||
|
自动 = 优先 Playwright,失败自动回退 agent-browser(推荐)
|
||||||
|
</small>
|
||||||
|
</div>
|
||||||
|
|
||||||
<div class="form-group">
|
<div class="form-group">
|
||||||
<label>等待设置(重要!)</label>
|
<label>等待设置(重要!)</label>
|
||||||
<div class="row">
|
<div class="row">
|
||||||
@@ -743,7 +753,7 @@
|
|||||||
"scroll_times": 3, // 可选:加载动态内容
|
"scroll_times": 3, // 可选:加载动态内容
|
||||||
"scroll_delay": 1000, // 可选:滚动间隔
|
"scroll_delay": 1000, // 可选:滚动间隔
|
||||||
"full_page": true, // 可选:全页截图
|
"full_page": true, // 可选:全页截图
|
||||||
"backend": "playwright", // 可选:playwright / agent-browser
|
"backend": "auto", // 可选:auto(默认,优先playwright,失败回退agent-browser) / playwright / agent-browser / chrome-cdp
|
||||||
"caller": "news-tracker", // 可选:调用者标识(历史记录里显示,默认游客)
|
"caller": "news-tracker", // 可选:调用者标识(历史记录里显示,默认游客)
|
||||||
"call_method": "api", // 可选:调用方式 web/api(缺省自动判定)
|
"call_method": "api", // 可选:调用方式 web/api(缺省自动判定)
|
||||||
"viewport": {"width":1280,"height":700},
|
"viewport": {"width":1280,"height":700},
|
||||||
@@ -844,12 +854,35 @@ DELETE /api/history/<id> // 删除历史</code></pre>
|
|||||||
let smartConfig = null;
|
let smartConfig = null;
|
||||||
let lastSmartHistoryId = null;
|
let lastSmartHistoryId = null;
|
||||||
|
|
||||||
// 页面加载时读取历史 + 智能配置
|
// 页面加载时读取历史 + 智能配置 + 可用后端
|
||||||
window.addEventListener('load', () => {
|
window.addEventListener('load', () => {
|
||||||
loadHistory(1);
|
loadHistory(1);
|
||||||
loadSmartConfig();
|
loadSmartConfig();
|
||||||
|
loadBackends();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
async function loadBackends() {
|
||||||
|
const sel = document.getElementById('backend');
|
||||||
|
const hint = document.getElementById('backendHint');
|
||||||
|
try {
|
||||||
|
const res = await fetch('/api/backends');
|
||||||
|
const data = await res.json();
|
||||||
|
if (!data.success) throw new Error(data.error || '加载失败');
|
||||||
|
const labels = data.labels || {};
|
||||||
|
const opts = ['auto'].concat(data.available || []);
|
||||||
|
sel.innerHTML = opts.map(v =>
|
||||||
|
`<option value="${v}">${escapeHtml(v)} — ${escapeHtml(labels[v] || '')}</option>`
|
||||||
|
).join('') + (data.unavailable || []).map(v =>
|
||||||
|
`<option value="${v}" disabled>${escapeHtml(v)} — ${escapeHtml(labels[v] || '')}(不可用)</option>`
|
||||||
|
).join('');
|
||||||
|
sel.value = 'auto';
|
||||||
|
hint.textContent = labels['auto'] || hint.textContent;
|
||||||
|
} catch (err) {
|
||||||
|
sel.innerHTML = '<option value="auto">自动</option>';
|
||||||
|
hint.textContent = '后端列表加载失败:' + err.message;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// 操作类型切换:智能模式显示提示条
|
// 操作类型切换:智能模式显示提示条
|
||||||
document.querySelectorAll('input[name="action"]').forEach(r => {
|
document.querySelectorAll('input[name="action"]').forEach(r => {
|
||||||
r.addEventListener('change', () => {
|
r.addEventListener('change', () => {
|
||||||
@@ -957,6 +990,7 @@ DELETE /api/history/<id> // 删除历史</code></pre>
|
|||||||
scroll_delay: scrollDelay,
|
scroll_delay: scrollDelay,
|
||||||
full_page: fullPage,
|
full_page: fullPage,
|
||||||
wait_time: waitTime,
|
wait_time: waitTime,
|
||||||
|
backend: document.getElementById('backend').value || 'auto',
|
||||||
viewport: {
|
viewport: {
|
||||||
width: viewportWidth,
|
width: viewportWidth,
|
||||||
height: viewportHeight
|
height: viewportHeight
|
||||||
@@ -975,8 +1009,7 @@ DELETE /api/history/<id> // 删除历史</code></pre>
|
|||||||
method: 'POST',
|
method: 'POST',
|
||||||
headers: {
|
headers: {
|
||||||
'Content-Type': 'application/json',
|
'Content-Type': 'application/json',
|
||||||
'X-From': 'web',
|
'X-From': 'web'
|
||||||
'X-Caller': '游客'
|
|
||||||
},
|
},
|
||||||
body: JSON.stringify(currentData)
|
body: JSON.stringify(currentData)
|
||||||
});
|
});
|
||||||
|
|||||||
Reference in New Issue
Block a user