diff --git a/app.py b/app.py index 4283bd2..7a1f560 100644 --- a/app.py +++ b/app.py @@ -259,6 +259,72 @@ def save_smart_config(cfg): return merged +# ===== 网址缓存 + 失败重试 配置 ===== +# cache: enabled=是否启用网址缓存(命中直接返回历史, 不重新提取); ttl_seconds=缓存有效期(秒) +# retry: max_retries=提取失败后的重试次数(默认2, 即最多尝试3次); delay_seconds=每次重试间隔(秒) +DEFAULT_CACHE_CONFIG = {"enabled": True, "ttl_seconds": 3600} +DEFAULT_RETRY_CONFIG = {"max_retries": 2, "delay_seconds": 2} + + +def load_capture_settings(): + """读取 网址缓存+失败重试 配置（缺省回默认；存于 config.json 顶层 cache/retry 段）""" + settings = {"cache": dict(DEFAULT_CACHE_CONFIG), "retry": dict(DEFAULT_RETRY_CONFIG)} + if CONFIG_FILE.exists(): + try: + saved = json.loads(CONFIG_FILE.read_text(encoding="utf-8")) + if isinstance(saved, dict): + for sec in ("cache", "retry"): + if isinstance(saved.get(sec), dict): + for k in settings[sec]: + if k in saved[sec] and saved[sec][k] not in (None, ""): + settings[sec][k] = saved[sec][k] + except Exception: + pass + return settings + + +def save_capture_settings(settings): + """保存 网址缓存+失败重试 配置（只动 config.json 的 cache/retry 段，保留 smart 配置）""" + merged = load_capture_settings() + if isinstance(settings, dict): + for sec in ("cache", "retry"): + if isinstance(settings.get(sec), dict): + for k in merged[sec]: + if k in settings[sec] and settings[sec][k] not in (None, ""): + merged[sec][k] = settings[sec][k] + data = {} + if CONFIG_FILE.exists(): + try: + data = json.loads(CONFIG_FILE.read_text(encoding="utf-8")) + except Exception: + data = {} + if not isinstance(data, dict): + data = {} + data["cache"] = merged["cache"] + data["retry"] = merged["retry"] + CONFIG_FILE.parent.mkdir(exist_ok=True) + CONFIG_FILE.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8") + return merged + + +def find_cache_hit(url, action, ttl_seconds): + """ + 按 url+action 在提取历史中查缓存：取最近一条成功记录，且创建时间在 TTL 内。 + 命中返回 dict(记录)，未命中返回 None。 + """ + from datetime import timedelta + cutoff = (datetime.now() - timedelta(seconds=max(1, int(ttl_seconds)))).strftime("%Y-%m-%d %H:%M:%S") + conn = get_db() + row = conn.execute( + "SELECT id, url, title, backend, file_path, content, html_path, created_at FROM captures " + "WHERE url=? AND action=? AND status='success' AND created_at >= ? " + "ORDER BY id DESC LIMIT 1", + (url, action, cutoff) + ).fetchone() + conn.close() + return dict(row) if row else None + + def extract_json_from_content(content): """从大模型输出中稳健提取 JSON 对象（兼容 ```json 代码块包裹、前后废话）""" if not content: @@ -680,8 +746,16 @@ async def capture_with_playwright( # 如果 domcontentloaded 超时，尝试 commit try: await page.goto(url, wait_until="commit", timeout=30000) - except: - pass + except Exception as e2: + return {"success": False, "error": f"页面加载失败（无法访问 {url}）: {str(e2)[:200]}"} + + # 兜底：检查是否进入了错误页（连接拒绝/DNS失败等会停在 chrome-error:// 或 about:blank） + try: + cur_url = page.url + if cur_url.startswith("chrome-error://") or cur_url == "about:blank": + return {"success": False, "error": f"页面加载失败（无法访问 {url}，浏览器停留在错误页）"} + except Exception: + pass # 固定等待时间（用于验证、加载等过程） if wait_time > 0: @@ -1222,7 +1296,8 @@ def api_info(): "playwright": "available" if PLAYWRIGHT_AVAILABLE else "unavailable" }, "endpoints": { - "/api/capture": "POST - Capture webpage (screenshot/html/text/smart) + 自动入库历史(记录调用者/方式/原始HTML)", + "/api/capture": "POST - Capture webpage (screenshot/html/text/smart) + 自动入库历史(记录调用者/方式/原始HTML); 带 ?refresh=1 绕过缓存; 失败自动重试(次数可配)", + "/api/capture/settings": "GET/POST - 网址缓存(enabled/ttl_seconds) + 失败重试(max_retries/delay_seconds) 配置", "/api/backends": "GET - 可用后端列表与默认顺序(供前端渲染后端选择器)", "/api/smart/config": "GET/POST - 智能截图视觉大模型多接口配置(列表顺序=优先级,失败自动降级)", "/api/smart/test": "POST - 测试指定视觉接口连通性({endpoint_id} / {endpoint} / 旧版{config})", @@ -1484,6 +1559,21 @@ def smart_test_endpoint(): return jsonify({"success": False, "error": str(e)}), 400 +@app.route('/api/capture/settings', methods=['GET', 'POST']) +def capture_settings(): + """ + 网址缓存 + 失败重试 配置 + GET -> {success, settings: {cache:{enabled, ttl_seconds}, retry:{max_retries, delay_seconds}}} + POST -> body {settings: {...}}，保存后返回合并结果 + """ + if request.method == 'GET': + return jsonify({"success": True, "settings": load_capture_settings()}) + data = request.get_json() or {} + settings = data.get("settings") or data + saved = save_capture_settings(settings) + return jsonify({"success": True, "settings": saved}) + + @app.route('/api/capture', methods=['POST']) def capture(): """ @@ -1493,6 +1583,7 @@ def capture(): { "url": "https://example.com", "action": "screenshot" | "html" | "text" | "smart", + "refresh": false, // 可选：true=强制实时提取(绕过网址缓存) "scroll_times": 0, "scroll_delay": 1000, "full_page": false, @@ -1541,46 +1632,84 @@ def capture(): elif call_method not in ("web", "api"): call_method = "api" - # 按需截图：滚动截图 + 视觉大模型实时判断 - if action == "smart": - smart_cfg = load_smart_config() - override = data.get("smart_config") or {} - if isinstance(override, dict): - for k in DEFAULT_SMART_CONFIG: - if k in override and override[k] not in (None, ""): - smart_cfg[k] = override[k] - result = smart_capture(url, smart_cfg, wait_time, viewport, backend) - backend_used = result.get("backend", backend) - title = result.get("title", "") - if not result["success"]: - save_capture(url, title, "smart", backend_used, "failed", error=result.get("error", ""), - caller=caller, call_method=call_method) - return jsonify(result), 400 - # 拼接长图已存持久化目录，直接入库；原始HTML按月归档 - steps_log = json.dumps(result.get("steps", []), ensure_ascii=False) - html_path = save_raw_html(result.get("html", "")) - rid = save_capture(url, title, "smart", backend_used, "success", - file_path=result["file_path"], content=steps_log, - html_path=html_path, caller=caller, call_method=call_method) - result["history_id"] = rid - result["raw_html_path"] = html_path - return jsonify(result) + # ---- 网址缓存：命中直接返回历史，未命中才实时提取 ---------------- + settings = load_capture_settings() + cache_cfg = settings.get("cache", DEFAULT_CACHE_CONFIG) + retry_cfg = settings.get("retry", DEFAULT_RETRY_CONFIG) + force_refresh = bool(data.get("refresh") or data.get("force")) + + if cache_cfg.get("enabled") and not force_refresh: + ttl = int(cache_cfg.get("ttl_seconds", 3600)) + hit = find_cache_hit(url, action, ttl) + if hit: + hit["from_cache"] = True + hit["cached_at"] = hit["created_at"] + hit["cache_ttl_seconds"] = ttl + if action == "screenshot": + if hit["file_path"] and Path(hit["file_path"]).exists(): + return send_file(hit["file_path"], mimetype='image/png') + # 文件已丢则继续实时提取 + elif action == "smart": + if hit["file_path"] and Path(hit["file_path"]).exists(): + try: + steps = json.loads(hit["content"] or "[]") + except Exception: + steps = [] + return jsonify({"success": True, "from_cache": True, "url": url, "action": "smart", + "title": hit["title"], "backend": hit["backend"], + "history_id": hit["id"], "file_path": hit["file_path"], + "steps": steps, "total_steps": len(steps), + "cached_at": hit["created_at"], "cache_ttl_seconds": ttl, + "raw_html_path": hit["html_path"]}) + else: + return jsonify({"success": True, "from_cache": True, "url": url, "action": action, + "title": hit["title"], "backend": hit["backend"], + "history_id": hit["id"], + "content": hit["content"], + "html": hit["content"] if action == "html" else None, + "text": hit["content"] if action == "text" else None, + "cached_at": hit["created_at"], "cache_ttl_seconds": ttl, + "raw_html_path": hit["html_path"]}) + + # ---- 实时提取（带失败重试：次数可配，默认重试 2 次） ---------------- + max_retries = max(0, int(retry_cfg.get("max_retries", 2))) + retry_delay = max(0, int(retry_cfg.get("delay_seconds", 2))) + attempt = 0 + result = None + while True: + attempt += 1 + # 按需截图：滚动截图 + 视觉大模型实时判断 + if action == "smart": + smart_cfg = load_smart_config() + override = data.get("smart_config") or {} + if isinstance(override, dict): + for k in DEFAULT_SMART_CONFIG: + if k in override and override[k] not in (None, ""): + smart_cfg[k] = override[k] + result = smart_capture(url, smart_cfg, wait_time, viewport, backend) + else: + result = capture_webpage( + url=url, + action=action, + scroll_times=scroll_times, + scroll_delay=scroll_delay, + full_page=full_page, + viewport=viewport, + wait_time=wait_time, + backend=backend, + cdp_port=cdp_port + ) + if result.get("success"): + break + if attempt <= max_retries: + time.sleep(retry_delay) # 重试前短暂等待 + continue + break + result["attempts"] = attempt - result = capture_webpage( - url=url, - action=action, - scroll_times=scroll_times, - scroll_delay=scroll_delay, - full_page=full_page, - viewport=viewport, - wait_time=wait_time, - backend=backend, - cdp_port=cdp_port - ) - backend_used = result.get("backend", backend) title = result.get("title", "") - + if not result["success"]: save_capture(url, title, action, backend_used, "failed", error=result.get("error", ""), caller=caller, call_method=call_method) @@ -1588,18 +1717,27 @@ def capture(): # 每次提取后都把最原始 HTML 按月归档到本地 html_path = save_raw_html(result.get("html", "")) - + + if action == "smart": + steps_log = json.dumps(result.get("steps", []), ensure_ascii=False) + rid = save_capture(url, title, "smart", backend_used, "success", + file_path=result["file_path"], content=steps_log, + html_path=html_path, caller=caller, call_method=call_method) + result["history_id"] = rid + result["raw_html_path"] = html_path + return jsonify(result) + if action == "screenshot": persisted = persist_screenshot(result["file_path"]) save_capture(url, title, action, backend_used, "success", file_path=persisted, html_path=html_path, caller=caller, call_method=call_method) return send_file(persisted, mimetype='image/png') - + elif action == "html": save_capture(url, title, action, backend_used, "success", content=result["html"], html_path=html_path, caller=caller, call_method=call_method) return jsonify(result) - + elif action == "text": save_capture(url, title, action, backend_used, "success", content=result["text"], html_path=html_path, caller=caller, call_method=call_method) diff --git a/templates/index.html b/templates/index.html index b2661e1..c71e7fb 100644 --- a/templates/index.html +++ b/templates/index.html @@ -608,6 +608,7 @@ 🧠 按需截图（AI判断滚动） + + + 📦 网址缓存已…（有效期 …），同网址命中缓存直接返回历史结果，不会重复提取。 +
@@ -738,6 +746,8 @@

重要提示：

+ + +