diff --git a/app.py b/app.py index 9ac1518..4283bd2 100644 --- a/app.py +++ b/app.py @@ -163,10 +163,28 @@ init_db() # ===== 按需截图（AI智能判断滚动）配置 ===== CONFIG_FILE = DATA_DIR / "config.json" +# 预置视觉大模型接口（按列表顺序 = 优先级，越靠前越优先；失败自动降级到下一个） +DEFAULT_ENDPOINTS = [ + { + "id": "ep_local_qwen", + "name": "本地 Qwen3.8-27B (18008)", + "base_url": "http://121.40.164.32:18008/v1", + "api_key": "xxxx", + "model": "unsloth/Qwen3.8-27B-NVFP4", + "enabled": True + }, + { + "id": "ep_siliconflow_kimi", + "name": "SiliconFlow Kimi-K2.6", + "base_url": "https://api.siliconflow.cn/v1", + "api_key": "sk-fhpoexpptvjghpnphtaxbkhjwulzovoqfffbckcfscjmwhcg", + "model": "Pro/moonshotai/Kimi-K2.6", + "enabled": True + } +] + DEFAULT_SMART_CONFIG = { - "base_url": "https://www.autodl.art/api/v1", - "api_key": "F9MBfolzuapqTsD4KmUf9qen720rXvUZ3Sp3IrWiCTukqonx", - "model": "qwen3.6-plus", + "endpoints": [json.loads(json.dumps(ep)) for ep in DEFAULT_ENDPOINTS], "prompt": ( "你是网页内容完整性判断助手。下面是一张网页滚动截图的当前视口画面。" "请判断：这个网页的主题内容（正文/主要内容）是否已经完整截取完成，是否还需要继续向下滚动？\n" @@ -182,7 +200,7 @@ DEFAULT_SMART_CONFIG = { def load_smart_config(): - """读取按需截图 LLM 配置（缺省回默认）""" + """读取按需截图 LLM 配置（缺省回默认；旧单接口配置自动迁移为 endpoints[0]）""" cfg = json.loads(json.dumps(DEFAULT_SMART_CONFIG)) if CONFIG_FILE.exists(): try: @@ -193,16 +211,49 @@ def load_smart_config(): cfg[k] = saved[k] except Exception: pass + # 兼容旧版单接口配置：没有 endpoints 时，把 base_url/api_key/model 迁移为第一个接口 + eps = cfg.get("endpoints") or [] + if not eps and cfg.get("base_url"): + eps = [{ + "id": "ep_legacy", + "name": cfg.get("model") or cfg.get("base_url"), + "base_url": cfg["base_url"], + "api_key": cfg.get("api_key", ""), + "model": cfg.get("model", ""), + "enabled": True + }] + cfg["endpoints"] = eps return cfg def save_smart_config(cfg): - """保存按需截图 LLM 配置""" + """保存按需截图 LLM 配置（endpoints 数组顺序 = 优先级）""" merged = json.loads(json.dumps(DEFAULT_SMART_CONFIG)) if isinstance(cfg, dict): - for k in DEFAULT_SMART_CONFIG: + # 滚动/提示词参数 + for k in ("prompt", "max_scrolls", "scroll_ratio", "timeout"): if k in cfg and cfg[k] not in (None, ""): merged[k] = cfg[k] + # 接口列表：保留有效字段，补 id/name/enabled 默认值 + eps = cfg.get("endpoints") + if isinstance(eps, list) and eps: + clean = [] + for i, ep in enumerate(eps): + if not isinstance(ep, dict) or not ep.get("base_url"): + continue + clean.append({ + "id": ep.get("id") or f"ep_{i+1}", + "name": ep.get("name") or ep.get("model") or f"接口{i+1}", + "base_url": ep["base_url"], + "api_key": ep.get("api_key", ""), + "model": ep.get("model", ""), + "enabled": bool(ep.get("enabled", True)) + }) + if clean: + merged["endpoints"] = clean + elif isinstance(cfg.get("endpoints"), list) and not eps: + # 显式传空列表 = 清空 + merged["endpoints"] = [] CONFIG_FILE.parent.mkdir(exist_ok=True) CONFIG_FILE.write_text(json.dumps(merged, ensure_ascii=False, indent=2), encoding="utf-8") return merged @@ -225,17 +276,18 @@ def extract_json_from_content(content): return None -def llm_judge_screenshot(image_path, cfg): +def llm_call_vision(image_path, cfg, endpoint): """ - 调用视觉大模型判断当前截图是否已覆盖主题内容、是否还需滚动 + 用单个视觉大模型接口判断截图 + endpoint: {"base_url", "api_key", "model", "name"} 返回: {"complete": bool, "reason": str} - 异常时抛 ValueError（调用方决定如何处置） + 任何失败（网络/HTTP/解析/格式）都抛 ValueError，由上层降级到下一个接口 """ import requests img_b64 = base64.b64encode(Path(image_path).read_bytes()).decode() - url = cfg["base_url"].rstrip("/") + "/chat/completions" + url = endpoint["base_url"].rstrip("/") + "/chat/completions" payload = { - "model": cfg["model"], + "model": endpoint.get("model", ""), "messages": [{ "role": "user", "content": [ @@ -245,20 +297,45 @@ def llm_judge_screenshot(image_path, cfg): }], "max_tokens": 300 } - headers = {"Authorization": f"Bearer {cfg['api_key']}", "Content-Type": "application/json"} + headers = {"Authorization": f"Bearer {endpoint.get('api_key', '')}", "Content-Type": "application/json"} resp = requests.post(url, headers=headers, json=payload, timeout=int(cfg.get("timeout", 120))) if resp.status_code != 200: - raise ValueError(f"LLM 接口返回 {resp.status_code}: {resp.text[:200]}") + raise ValueError(f"HTTP {resp.status_code}: {resp.text[:200]}") try: content = resp.json()["choices"][0]["message"]["content"] except Exception: - raise ValueError(f"LLM 响应格式异常: {resp.text[:200]}") + raise ValueError(f"响应格式异常: {resp.text[:200]}") parsed = extract_json_from_content(content) if not parsed or "complete" not in parsed: - raise ValueError(f"LLM 判断结果解析失败，原始输出: {content[:200]}") + raise ValueError(f"判断结果解析失败，原始输出: {content[:200]}") return {"complete": bool(parsed["complete"]), "reason": str(parsed.get("reason", ""))} +def llm_judge_screenshot(image_path, cfg): + """ + 调用视觉大模型判断当前截图是否已覆盖主题内容、是否还需滚动。 + 按 endpoints 列表顺序（高→低优先级）逐个尝试，失败/报错/超时自动降级到下一个； + 全部失败才抛 ValueError（调用方保守停止滚动）。 + 返回: {"complete": bool, "reason": str, "model": str, "used_endpoint": str} + """ + import requests # noqa: F401 (确保模块可用，实际在 llm_call_vision 中导入) + endpoints = cfg.get("endpoints") or [] + enabled = [ep for ep in endpoints if ep.get("enabled", True)] + if not enabled: + raise ValueError("未配置可用的视觉大模型接口（请到「⚙️ 智能配置」添加）") + errors = [] + for ep in enabled: + ep_name = ep.get("name") or ep.get("model") or ep.get("base_url", "") + try: + result = llm_call_vision(image_path, cfg, ep) + result["model"] = ep.get("model", "") + result["used_endpoint"] = ep_name + return result + except Exception as e: + errors.append(f"{ep_name}: {e}") + raise ValueError("全部视觉大模型接口调用失败 → " + " | ".join(errors)) + + def stitch_images(image_paths, out_path): """把多张视口截图纵向拼接成一张长图（所有图对齐到最宽宽度）""" from PIL import Image @@ -764,7 +841,8 @@ def smart_capture_agent_browser(url, cfg, wait_time, viewport): # 视觉大模型实时判断 try: judge = llm_judge_screenshot(shot, cfg) - steps.append({"step": step_no, "complete": judge["complete"], "reason": judge["reason"]}) + steps.append({"step": step_no, "complete": judge["complete"], "reason": judge["reason"], + "model": judge.get("model", ""), "endpoint": judge.get("used_endpoint", "")}) except ValueError as ve: # 判断失败：保守停止，保留已截内容 steps.append({"step": step_no, "complete": True, "reason": f"⚠️ 大模型判断失败，停止滚动: {ve}"}) @@ -902,7 +980,8 @@ async def smart_capture_playwright(url, cfg, wait_time, viewport): try: judge = llm_judge_screenshot(shot, cfg) - steps.append({"step": step_no, "complete": judge["complete"], "reason": judge["reason"]}) + steps.append({"step": step_no, "complete": judge["complete"], "reason": judge["reason"], + "model": judge.get("model", ""), "endpoint": judge.get("used_endpoint", "")}) except ValueError as ve: steps.append({"step": step_no, "complete": True, "reason": f"⚠️ 大模型判断失败，停止滚动: {ve}"}) stop_reason = str(ve) @@ -1145,7 +1224,10 @@ def api_info(): "endpoints": { "/api/capture": "POST - Capture webpage (screenshot/html/text/smart) + 自动入库历史(记录调用者/方式/原始HTML)", "/api/backends": "GET - 可用后端列表与默认顺序(供前端渲染后端选择器)", + "/api/smart/config": "GET/POST - 智能截图视觉大模型多接口配置(列表顺序=优先级,失败自动降级)", + "/api/smart/test": "POST - 测试指定视觉接口连通性({endpoint_id} / {endpoint} / 旧版{config})", "/api/history": "GET - 历史记录分页列表 (?page&page_size&action&search)", + "/api/history/stats": "GET - 历史图示统计 (?dimension=day|action|backend|status|caller|method&days=30)", "/api/history/": "GET - 历史记录详情 / DELETE - 删除记录", "/api/history//file": "GET - 读取历史截图文件", "/api/history//html": "GET - 读取保存的原始HTML文件", @@ -1197,6 +1279,78 @@ def history_list(): }) +@app.route('/api/history/stats', methods=['GET']) +def history_stats(): + """ + 提取历史图示统计 + 参数: + dimension: day(按天趋势) | action(操作类型) | backend(后端) | status(状态) | caller(调用者) | method(调用方式) + days: 按天维度的天数范围(1-365, 默认30); 其他维度默认全量(也可传 days 只统计最近N天) + 返回: {success, dimension, days, total, success_count, success_rate, items:[{label, value}]} + """ + dimension = request.args.get('dimension', 'action') + valid = ('day', 'action', 'backend', 'status', 'caller', 'method') + if dimension not in valid: + return jsonify({"success": False, "error": f"dimension 需为 {'/'.join(valid)}"}), 400 + days = min(365, max(1, int(request.args.get('days', 30)))) + + # 时间下限（可选）：非 day 维度也可按 days 过滤 + from datetime import timedelta + cutoff = (datetime.now() - timedelta(days=days)).strftime("%Y-%m-%d %H:%M:%S") + + conn = get_db() + # 总量 / 成功率 + total = conn.execute("SELECT COUNT(*) c FROM captures").fetchone()["c"] + sc = conn.execute("SELECT COUNT(*) c FROM captures WHERE status='success'").fetchone()["c"] + + items = [] + if dimension == 'day': + # 最近 days 天的每日提取量（缺0补0） + rows = conn.execute( + "SELECT substr(created_at,1,10) d, COUNT(*) c FROM captures WHERE created_at >= ? GROUP BY d ORDER BY d", + (cutoff,) + ).fetchall() + m = {r["d"]: r["c"] for r in rows} + date_list = [] + for i in range(days - 1, -1, -1): + date_list.append((datetime.now() - timedelta(days=i)).strftime("%Y-%m-%d")) + items = [{"label": d, "value": m.get(d, 0)} for d in date_list] + else: + col = {"action": "action", "backend": "backend", "status": "status", + "caller": "caller", "method": "call_method"}[dimension] + rows = conn.execute( + f"SELECT {col} k, COUNT(*) c FROM captures WHERE created_at >= ? GROUP BY {col} ORDER BY c DESC", + (cutoff,) + ).fetchall() + # caller 太多时只保留 Top 8 + 其他 + if dimension == 'caller': + top = rows[:8] + rest = sum(r["c"] for r in rows[8:]) + items = [{"label": r["k"] or "未知", "value": r["c"]} for r in top] + if rest: + items.append({"label": "其他", "value": rest}) + else: + for r in rows: + label = r["k"] or "(空)" + if dimension == 'status': + label = "成功" if r["k"] == "success" else ("失败" if r["k"] == "failed" else r["k"]) + elif dimension == 'action': + label = {"screenshot": "📸 截图", "html": "📄 HTML", "text": "📝 文本", "smart": "🧠 按需截图"}.get(r["k"], r["k"]) + items.append({"label": label, "value": r["c"]}) + conn.close() + + # 按天维度去 emoji（保持纯净日期标签） + return jsonify({ + "success": True, + "dimension": dimension, + "days": days, + "total": total, + "success_count": sc, + "success_rate": round(sc / total, 4) if total else 0, + "items": items + }) + + @app.route('/api/history/', methods=['GET']) def history_detail(rid): """历史记录详情（含完整内容）""" @@ -1277,25 +1431,46 @@ def smart_config_endpoint(): @app.route('/api/smart/test', methods=['POST']) def smart_test_endpoint(): - """测试按需截图 LLM 接口连通性（发一条小文本消息验证 base_url/api_key/model）""" + """测试视觉大模型接口连通性（发一条小文本消息验证 base_url/api_key/model）。 + 入参（三选一）： + {endpoint_id: "ep_xxx"} → 测试已保存配置中的某个接口 + {endpoint: {base_url, api_key, model}} → 直接测试传入的接口 + {config: {...}} 或 {base_url,...} → 兼容旧版单接口测试 + """ import requests data = request.get_json() or {} - cfg = dict(load_smart_config()) - if data.get("config"): - for k in DEFAULT_SMART_CONFIG: - if k in data["config"] and data["config"][k] not in (None, ""): - cfg[k] = data["config"][k] - elif data.get("base_url") or data.get("api_key") or data.get("model"): - for k in ("base_url", "api_key", "model"): - if data.get(k): - cfg[k] = data[k] + ep = None - url = cfg["base_url"].rstrip("/") + "/chat/completions" - payload = {"model": cfg["model"], "messages": [{"role": "user", "content": "ping"}], "max_tokens": 5} - headers = {"Authorization": f"Bearer {cfg['api_key']}", "Content-Type": "application/json"} + # 1) endpoint_id：从已保存配置里找 + if data.get("endpoint_id"): + saved = load_smart_config() + for e in (saved.get("endpoints") or []): + if e.get("id") == data["endpoint_id"]: + ep = e + break + if not ep: + return jsonify({"success": False, "error": f"接口 {data['endpoint_id']} 不存在"}), 404 + # 2) endpoint 对象：直接测试 + elif isinstance(data.get("endpoint"), dict) and data["endpoint"].get("base_url"): + ep = data["endpoint"] + # 3) 兼容旧版：config 或平铺字段 + else: + cfg = dict(load_smart_config()) + override = data.get("config") or data + if isinstance(override, dict): + for k in ("base_url", "api_key", "model", "timeout"): + if k in override and override[k] not in (None, ""): + cfg[k] = override[k] + ep = {"base_url": cfg["base_url"], "api_key": cfg.get("api_key", ""), + "model": cfg.get("model", ""), "name": cfg.get("model", "")} + + url = ep["base_url"].rstrip("/") + "/chat/completions" + payload = {"model": ep.get("model", ""), "messages": [{"role": "user", "content": "ping"}], "max_tokens": 5} + headers = {"Authorization": f"Bearer {ep.get('api_key', '')}", "Content-Type": "application/json"} + timeout = int(data.get("timeout") or 60) t0 = time.time() try: - resp = requests.post(url, headers=headers, json=payload, timeout=int(cfg.get("timeout", 120))) + resp = requests.post(url, headers=headers, json=payload, timeout=timeout) cost = round(time.time() - t0, 2) if resp.status_code != 200: return jsonify({"success": False, "error": f"HTTP {resp.status_code}: {resp.text[:200]}", "latency": cost}), 400 @@ -1304,7 +1479,7 @@ def smart_test_endpoint(): reply = d["choices"][0]["message"]["content"][:80] except Exception: reply = "(无内容)" - return jsonify({"success": True, "latency": cost, "model": cfg["model"], "reply": reply}) + return jsonify({"success": True, "latency": cost, "model": ep.get("model", ""), "reply": reply}) except Exception as e: return jsonify({"success": False, "error": str(e)}), 400 diff --git a/templates/index.html b/templates/index.html index 84822e9..b2661e1 100644 --- a/templates/index.html +++ b/templates/index.html @@ -611,7 +611,7 @@
@@ -704,6 +704,7 @@ +
@@ -736,7 +737,8 @@

📚 使用说明

重要提示：

    -
  • 🧠 按需截图：每次截图后用视觉大模型实时判断是否已截全主题内容、是否还需继续滚动，自动滚动到内容结束并拼接成长图（大模型接口可在「⚙️ 智能配置」修改）
    • +
  • 🧠 按需截图：每次截图后用视觉大模型实时判断是否已截全主题内容、是否还需继续滚动，自动滚动到内容结束并拼接成长图；支持多视觉大模型接口配置（「⚙️ 智能配置」中增删改，列表顺序=优先级，失败/报错自动降级到下一个接口）
    • +
  • 📊 提取历史统计：历史区点「📊 统计」，可按天/操作类型/后端/状态/调用者/调用方式等角度切换图表（柱状/饼图可换，支持 7/30/90 天范围，可下载 PNG）
  • 📝 提取文本：自动剔除脚本/样式/标签，得到干净的前端可读文本，适合直接用于分析/喂给大模型
  • ⏱️ 页面加载等待：某些网站需要验证过程（如 Cloudflare），请设置较长时间（10000-30000ms）
  • 🔄 滚动次数：用于加载动态内容（如微博、推特等），建议 3-5 次
    • @@ -757,26 +759,28 @@ "caller": "news-tracker", // 可选：调用者标识（历史记录里显示，默认游客） "call_method": "api", // 可选：调用方式 web/api（缺省自动判定） "viewport": {"width":1280,"height":700}, - "smart_config": { // 可选：临时覆盖按需截图 LLM 配置 - "base_url": "https://www.autodl.art/api/v1", - "api_key": "sk-xxx", - "model": "qwen3.6-plus", + "smart_config": { // 可选：临时覆盖按需截图配置（接口列表按顺序=优先级，失败自动降级） + "endpoints": [ + {"name":"本地Qwen", "base_url":"http://121.40.164.32:18008/v1", "api_key":"xxxx", "model":"unsloth/Qwen3.8-27B-NVFP4", "enabled":true}, + {"name":"Kimi", "base_url":"https://api.siliconflow.cn/v1", "api_key":"sk-xxx", "model":"Pro/moonshotai/Kimi-K2.6", "enabled":true} + ], "max_scrolls": 20, "scroll_ratio": 0.85 } } -GET /api/smart/config // 获取按需截图 LLM 配置 -POST /api/smart/config // 保存配置（body: {config:{...}}） -POST /api/smart/test // 测试连接（body: {config:{...}} 可选） +GET /api/smart/config // 获取智能截图配置（含 endpoints 多接口列表） +POST /api/smart/config // 保存配置（body: {config:{endpoints:[...], prompt, max_scrolls,...}}） +POST /api/smart/test // 测试指定接口（body: {endpoint_id} 或 {endpoint:{...}} 或旧版 {config:{...}}） GET /api/history?page=1&page_size=15&action=all&search=关键词 // 历史分页 +GET /api/history/stats?dimension=day&days=30 // 历史统计（dimension: day/action/backend/status/caller/method） GET /api/history/<id> // 历史详情 GET /api/history/<id>/file // 历史截图文件 DELETE /api/history/<id> // 删除历史 - +