v1.7.1 修复前端捕获报错: fetch headers 不允许非 ISO-8859-1 字符(中文'游客'导致 Failed to read headers)

- 前端 /api/capture 请求头去掉 X-Caller: '游客'（中文在 HTTP 头非法，浏览器 fetch 直接抛错） - 保留 X-From: web（ASCII 安全），调用者缺省由服务端默认游客，行为不变
v1.7.0 历史记录标注调用者/调用方式 + 每次提取自动按月归档原始HTML + 修复agent-browser HTML JSON转义bug
2026-08-31 16:27:04 +08:00 · 2026-08-31 13:12:22 +08:00
2 changed files with 162 additions and 34 deletions
@@ -61,9 +61,11 @@ CAPTURE_DIR.mkdir(exist_ok=True)
 PROJECT_DIR = Path(__file__).parent
 DATA_DIR = PROJECT_DIR / "data"
 CAPTURE_DATA_DIR = DATA_DIR / "captures"
+HTML_DATA_DIR = DATA_DIR / "html"   # 原始HTML按月归档目录
 HISTORY_DB = DATA_DIR / "history.db"
 DATA_DIR.mkdir(exist_ok=True)
 CAPTURE_DATA_DIR.mkdir(exist_ok=True)
+HTML_DATA_DIR.mkdir(exist_ok=True)


 def get_db():
@@ -89,20 +91,31 @@ def init_db():
            file_path TEXT DEFAULT '',
            content TEXT DEFAULT '',
            error TEXT DEFAULT '',
+            caller TEXT DEFAULT '游客',
+            call_method TEXT DEFAULT 'api',
+            html_path TEXT DEFAULT '',
            created_at TEXT NOT NULL
        )
    """)
+    # 兼容旧库：缺列则 ALTER TABLE 补充
+    cols = [row[1] for row in conn.execute("PRAGMA table_info(captures)").fetchall()]
+    for col, ddl in (("caller", "TEXT DEFAULT '游客'"),
+                     ("call_method", "TEXT DEFAULT 'api'"),
+                     ("html_path", "TEXT DEFAULT ''")):
+        if col not in cols:
+            conn.execute(f"ALTER TABLE captures ADD COLUMN {col} {ddl}")
    conn.commit()
    conn.close()


-def save_capture(url, title, action, backend, status, file_path='', content='', error=''):
-    """保存一条提取历史记录"""
+def save_capture(url, title, action, backend, status, file_path='', content='', error='',
+                 caller='游客', call_method='api', html_path=''):
+    """保存一条提取历史记录（含调用者/调用方式/原始HTML路径）"""
    conn = get_db()
    cur = conn.execute(
-        "INSERT INTO captures (url, title, action, backend, status, file_path, content, error, created_at) "
-        "VALUES (?,?,?,?,?,?,?,?,?)",
-        (url, title, action, backend, status, file_path, content, error,
+        "INSERT INTO captures (url, title, action, backend, status, file_path, content, error, caller, call_method, html_path, created_at) "
+        "VALUES (?,?,?,?,?,?,?,?,?,?,?,?)",
+        (url, title, action, backend, status, file_path, content, error, caller, call_method, html_path,
         datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
    )
    conn.commit()
@@ -111,6 +124,22 @@ def save_capture(url, title, action, backend, status, file_path='', content='',
    return rid


+def save_raw_html(html, url=''):
+    """把最原始 HTML 保存到 data/html/<YYYY-MM>/ 按月目录归档；失败返回 ''"""
+    if not html:
+        return ''
+    try:
+        month = datetime.now().strftime("%Y-%m")
+        month_dir = HTML_DATA_DIR / month
+        month_dir.mkdir(parents=True, exist_ok=True)
+        name = f"{datetime.now().strftime('%Y%m%d_%H%M%S')}_{uuid.uuid4().hex[:8]}.html"
+        dest = month_dir / name
+        dest.write_text(html if isinstance(html, str) else str(html), encoding="utf-8")
+        return str(dest)
+    except Exception:
+        return ''
+
+
 def persist_screenshot(src_path):
    """把临时截图复制到持久化目录，供历史记录长期访问"""
    name = f"{datetime.now().strftime('%Y%m%d_%H%M%S')}_{uuid.uuid4().hex[:8]}.png"
@@ -401,21 +430,32 @@ def capture_with_agent_browser(
            success, _, error = session.screenshot(temp_file, full_page=full_page)
            
            if success and temp_file.exists():
-                return {"success": True, "title": title, "file_path": str(temp_file)}
+                res = {"success": True, "title": title, "file_path": str(temp_file)}
+                # 顺带抓取最原始 HTML 存档
+                ok, html_out, _ = session.get_html()
+                if ok and html_out:
+                    res["html"] = session._decode_eval(html_out)
+                return res
            else:
                return {"success": False, "title": title, "error": f"Screenshot failed: {error}"}
        
        elif action == "html":
            success, html, error = session.get_html()
            if success:
-                return {"success": True, "title": title, "html": html}
+                # 修复：eval 结果按 JSON 编码返回，需解码成最原始 HTML
+                return {"success": True, "title": title, "html": session._decode_eval(html)}
            else:
                return {"success": False, "title": title, "error": f"Get HTML failed: {error}"}
        
        elif action == "text":
            success, raw_text, error = session.get_text()
            if success:
-                return {"success": True, "title": title, "text": clean_text(raw_text)}
+                res = {"success": True, "title": title, "text": clean_text(raw_text)}
+                # 顺带抓取最原始 HTML 存档
+                ok, html_out, _ = session.get_html()
+                if ok and html_out:
+                    res["html"] = session._decode_eval(html_out)
+                return res
            else:
                return {"success": False, "title": title, "error": f"Extract text failed: {error}"}
        
@@ -592,6 +632,11 @@ async def capture_with_playwright(
                temp_file = CAPTURE_DIR / f"{session_id}.png"
                await page.screenshot(path=str(temp_file), full_page=full_page)
                result = {"success": True, "title": title, "file_path": str(temp_file)}
+                # 顺带抓取最原始 HTML 存档
+                try:
+                    result["html"] = await page.content()
+                except Exception:
+                    pass
            
            elif action == "html":
                html = await page.content()
@@ -600,6 +645,11 @@ async def capture_with_playwright(
            elif action == "text":
                raw_text = await page.evaluate("document.body.innerText")
                result = {"success": True, "title": title, "text": clean_text(raw_text)}
+                # 顺带抓取最原始 HTML 存档
+                try:
+                    result["html"] = await page.content()
+                except Exception:
+                    pass
            
            else:
                result = {"success": False, "title": title, "error": f"Unknown action: {action}"}
@@ -738,7 +788,7 @@ def smart_capture_agent_browser(url, cfg, wait_time, viewport):
        stitched = stitch_images(shot_paths, out)
        title = session.get_title()

-        return {
+        res = {
            "success": True,
            "title": title,
            "file_path": stitched,
@@ -747,6 +797,11 @@ def smart_capture_agent_browser(url, cfg, wait_time, viewport):
            "stop_reason": stop_reason,
            "backend": "agent-browser"
        }
+        # 顺带抓取最原始 HTML 存档
+        ok, html_out, _ = session.get_html()
+        if ok and html_out:
+            res["html"] = session._decode_eval(html_out)
+        return res
    except Exception as e:
        return {"success": False, "error": str(e)}
    finally:
@@ -864,9 +919,7 @@ async def smart_capture_playwright(url, cfg, wait_time, viewport):
                title = await page.title()
            except Exception:
                title = ""
-            await browser.close()
-
-            return {
+            res = {
                "success": True,
                "title": title,
                "file_path": stitched,
@@ -875,6 +928,13 @@ async def smart_capture_playwright(url, cfg, wait_time, viewport):
                "stop_reason": stop_reason,
                "backend": "playwright"
            }
+            # 顺带抓取最原始 HTML 存档
+            try:
+                res["html"] = await page.content()
+            except Exception:
+                pass
+            await browser.close()
+            return res
    except Exception as e:
        return {"success": False, "error": str(e)}

@@ -972,10 +1032,11 @@ def api_info():
            "playwright": "available" if PLAYWRIGHT_AVAILABLE else "unavailable"
        },
        "endpoints": {
-            "/api/capture": "POST - Capture webpage (screenshot/html/text) + 自动入库历史",
+            "/api/capture": "POST - Capture webpage (screenshot/html/text/smart) + 自动入库历史(记录调用者/方式/原始HTML)",
            "/api/history": "GET - 历史记录分页列表 (?page&page_size&action&search)",
            "/api/history/<id>": "GET - 历史记录详情 / DELETE - 删除记录",
            "/api/history/<id>/file": "GET - 读取历史截图文件",
+            "/api/history/<id>/html": "GET - 读取保存的原始HTML文件",
            "/health": "GET - Health check"
        }
    })
@@ -1006,7 +1067,7 @@ def history_list():
    conn = get_db()
    total = conn.execute(f"SELECT COUNT(*) AS c FROM captures{where_sql}", params).fetchone()["c"]
    rows = conn.execute(
-        f"SELECT id, url, title, action, backend, status, file_path, created_at, "
+        f"SELECT id, url, title, action, backend, status, file_path, caller, call_method, html_path, created_at, "
        f"LENGTH(content) AS content_size, substr(content, 1, 200) AS content_preview, error "
        f"FROM captures{where_sql} ORDER BY id DESC LIMIT ? OFFSET ?",
        params + [page_size, (page - 1) * page_size]
@@ -1046,22 +1107,36 @@ def history_file(rid):
    return send_file(r["file_path"], mimetype='image/png')


+@app.route('/api/history/<int:rid>/html', methods=['GET'])
+def history_html_file(rid):
+    """读取保存的原始 HTML 文件（data/html/<月份>/xxx.html）"""
+    conn = get_db()
+    r = conn.execute("SELECT html_path FROM captures WHERE id=?", (rid,)).fetchone()
+    conn.close()
+    if not r or not r["html_path"] or not Path(r["html_path"]).exists():
+        return jsonify({"success": False, "error": "原始HTML文件不存在或已删除"}), 404
+    return send_file(r["html_path"], mimetype='text/html; charset=utf-8')
+
+
@app.route('/api/history/<int:rid>', methods=['DELETE'])
 def history_delete(rid):
-    """删除历史记录（连带删除截图文件）"""
+    """删除历史记录（连带删除截图文件 / 原始HTML文件）"""
    conn = get_db()
-    r = conn.execute("SELECT file_path FROM captures WHERE id=?", (rid,)).fetchone()
+    r = conn.execute("SELECT file_path, html_path FROM captures WHERE id=?", (rid,)).fetchone()
    if r:
        conn.execute("DELETE FROM captures WHERE id=?", (rid,))
        conn.commit()
    conn.close()
-    if r and r["file_path"]:
-        fp = Path(r["file_path"])
-        if fp.exists() and str(fp).startswith(str(CAPTURE_DATA_DIR)):
-            try:
-                fp.unlink()
-            except Exception:
-                pass
+    if r:
+        for fp_s in (r["file_path"], r["html_path"]):
+            if not fp_s:
+                continue
+            fp = Path(fp_s)
+            if fp.exists() and str(fp).startswith(str(DATA_DIR)):
+                try:
+                    fp.unlink()
+                except Exception:
+                    pass
    return jsonify({"success": True})


@@ -1166,6 +1241,19 @@ def capture():
    cdp_port = int(data.get("cdp_port", 9222))
    url_hint = data.get("url_hint", "")

+    # ---- 调用者与调用方式识别 ----
+    # 调用者：优先请求体 caller，其次请求头 X-Caller/X-Project/X-App，默认游客
+    caller = (data.get("caller") or request.headers.get("X-Caller")
+              or request.headers.get("X-Project") or request.headers.get("X-App")
+              or "游客")
+    # 调用方式：请求体 call_method 或请求头 X-From；缺省按 Referer 判断（本前端=web，否则 api）
+    call_method = data.get("call_method") or request.headers.get("X-From") or ""
+    if not call_method:
+        ref = request.headers.get("Referer") or ""
+        call_method = "web" if (ref and request.host in ref) else "api"
+    elif call_method not in ("web", "api"):
+        call_method = "api"
+
    # 按需截图：滚动截图 + 视觉大模型实时判断
    if action == "smart":
        smart_cfg = load_smart_config()
@@ -1178,13 +1266,17 @@ def capture():
        backend_used = result.get("backend", backend)
        title = result.get("title", "")
        if not result["success"]:
-            save_capture(url, title, "smart", backend_used, "failed", error=result.get("error", ""))
+            save_capture(url, title, "smart", backend_used, "failed", error=result.get("error", ""),
+                         caller=caller, call_method=call_method)
            return jsonify(result), 400
-        # 拼接长图已存持久化目录，直接入库
+        # 拼接长图已存持久化目录，直接入库；原始HTML按月归档
        steps_log = json.dumps(result.get("steps", []), ensure_ascii=False)
+        html_path = save_raw_html(result.get("html", ""))
        rid = save_capture(url, title, "smart", backend_used, "success",
-                           file_path=result["file_path"], content=steps_log)
+                           file_path=result["file_path"], content=steps_log,
+                           html_path=html_path, caller=caller, call_method=call_method)
        result["history_id"] = rid
+        result["raw_html_path"] = html_path
        return jsonify(result)

    result = capture_webpage(
@@ -1202,20 +1294,27 @@ def capture():
    title = result.get("title", "")
    
    if not result["success"]:
-        save_capture(url, title, action, backend_used, "failed", error=result.get("error", ""))
+        save_capture(url, title, action, backend_used, "failed", error=result.get("error", ""),
+                     caller=caller, call_method=call_method)
        return jsonify(result), 400
+
+    # 每次提取后都把最原始 HTML 按月归档到本地
+    html_path = save_raw_html(result.get("html", ""))
    
    if action == "screenshot":
        persisted = persist_screenshot(result["file_path"])
-        save_capture(url, title, action, backend_used, "success", file_path=persisted)
+        save_capture(url, title, action, backend_used, "success", file_path=persisted,
+                     html_path=html_path, caller=caller, call_method=call_method)
        return send_file(persisted, mimetype='image/png')
    
    elif action == "html":
-        save_capture(url, title, action, backend_used, "success", content=result["html"])
+        save_capture(url, title, action, backend_used, "success", content=result["html"],
+                     html_path=html_path, caller=caller, call_method=call_method)
        return jsonify(result)
    
    elif action == "text":
-        save_capture(url, title, action, backend_used, "success", content=result["text"])
+        save_capture(url, title, action, backend_used, "success", content=result["text"],
+                     html_path=html_path, caller=caller, call_method=call_method)
        return jsonify(result)


@@ -370,6 +370,16 @@
            color: #c62828;
        }
        
+        .badge-api {
+            background: #ede7f6;
+            color: #5e35b1;
+        }
+        
+        .badge-web {
+            background: #e0f7fa;
+            color: #00695c;
+        }
+        
        .history-url {
            max-width: 280px;
            overflow: hidden;
@@ -556,7 +566,9 @@
            .history-table th:nth-child(2),
            .history-table td:nth-child(2),
            .history-table th:nth-child(3),
-            .history-table td:nth-child(3) {
+            .history-table td:nth-child(3),
+            .history-table th:nth-child(4),
+            .history-table td:nth-child(4) {
                display: none;
            }
        }
@@ -693,6 +705,8 @@
                        <tr>
                            <th>时间</th>
                            <th>类型</th>
+                            <th>调用者</th>
+                            <th>方式</th>
                            <th>网址</th>
                            <th>标题</th>
                            <th>状态</th>
@@ -717,7 +731,8 @@
                <li>⏱️ <strong>页面加载等待</strong>：某些网站需要验证过程（如 Cloudflare），请设置较长时间（10000-30000ms）</li>
                <li>🔄 <strong>滚动次数</strong>：用于加载动态内容（如微博、推特等），建议 3-5 次</li>
                <li>🎯 <strong>全页截图</strong>：滚动加载后建议开启此选项</li>
-                <li>📜 <strong>提取历史</strong>：每次提取自动入库，支持分页浏览、按类型筛选、搜索与删除</li>
+                <li>📜 <strong>提取历史</strong>：每次提取自动入库，支持分页浏览、按类型筛选、搜索与删除；每条记录会标注<strong>调用者</strong>（游客/项目名）与<strong>调用方式</strong>（web/api），并自动把<strong>最原始 HTML</strong> 按月归档到 <code>data/html/&lt;月份&gt;/</code>（详情里可点「📄 原始HTML文件」查看）</li>
+                <li>👤 <strong>API 调用方标识</strong>：请求体带 <code>caller</code>（或请求头 <code>X-Caller</code>/<code>X-Project</code>）即可在历史里显示项目名；带 <code>call_method</code> 或 <code>X-From</code> 标注调用方式，缺省按 Referer 自动判定 web/api</li>
            </ul>
            <p style="margin-top: 15px;"><strong>API 调用：</strong></p>
            <pre><code>POST /api/capture
@@ -729,6 +744,8 @@
  "scroll_delay": 1000,       // 可选：滚动间隔
  "full_page": true,          // 可选：全页截图
  "backend": "playwright",    // 可选：playwright / agent-browser
+  "caller": "news-tracker",     // 可选：调用者标识（历史记录里显示，默认游客）
+  "call_method": "api",        // 可选：调用方式 web/api（缺省自动判定）
  "viewport": {"width":1280,"height":700},
  "smart_config": {           // 可选：临时覆盖按需截图 LLM 配置
    "base_url": "https://www.autodl.art/api/v1",
@@ -957,7 +974,8 @@ DELETE /api/history/&lt;id&gt;        // 删除历史</code></pre>
                const response = await fetch('/api/capture', {
                    method: 'POST',
                    headers: {
-                        'Content-Type': 'application/json'
+                        'Content-Type': 'application/json',
+                        'X-From': 'web'
                    },
                    body: JSON.stringify(currentData)
                });
@@ -1111,9 +1129,14 @@ DELETE /api/history/&lt;id&gt;        // 删除历史</code></pre>
                const statusText = r.status === 'failed'
                    ? `<span style="color:#c62828;" title="${escapeHtml(r.error || '')}">失败</span>`
                    : '<span style="color:#2e7d32;">成功</span>';
+                const callerText = r.caller || '游客';
+                const methodText = r.call_method || 'api';
+                const methodBadge = methodText === 'web' ? 'badge-web' : 'badge-api';
                return `<tr>
                    <td style="white-space:nowrap;">${escapeHtml(r.created_at)}</td>
                    <td><span class="badge ${escapeHtml(badgeClass)}">${badgeText}</span></td>
+                    <td><span class="history-title" title="${escapeHtml(callerText)}">${escapeHtml(callerText)}</span></td>
+                    <td><span class="badge ${methodBadge}" style="font-size:11px;padding:2px 8px;">${escapeHtml(methodText)}</span></td>
                    <td><a class="history-url" href="${escapeHtml(r.url)}" target="_blank" rel="noopener">${escapeHtml(r.url)}</a></td>
                    <td><span class="history-title" title="${escapeHtml(r.title || '')}">${escapeHtml(r.title || '—')}</span></td>
                    <td>${statusText}</td>
@@ -1173,11 +1196,17 @@ DELETE /api/history/&lt;id&gt;        // 删除历史</code></pre>
            document.getElementById('modalTitle').textContent = (r.title || r.url || '记录详情');
            const badgeClass = r.status === 'failed' ? 'badge-failed' : ('badge-' + r.action);
            const badgeText = r.status === 'failed' ? '❌ 失败' : ({screenshot: '📸 截图', html: '📄 HTML', text: '📝 文本'}[r.action] || r.action);
+            const htmlLink = r.html_path
+                ? `<span><a href="/api/history/${r.id}/html" target="_blank" style="color:#667eea;font-weight:600;" title="${escapeHtml(r.html_path)}">📄 原始HTML文件</a></span>`
+                : '';
            document.getElementById('modalMeta').innerHTML =
                `<span>类型：<span class="badge ${badgeClass}">${badgeText}</span></span>` +
                `<span>时间：${escapeHtml(r.created_at)}</span>` +
+                `<span>调用者：${escapeHtml(r.caller || '游客')}</span>` +
+                `<span>方式：${escapeHtml(r.call_method || 'api')}</span>` +
                `<span>后端：${escapeHtml(r.backend || '—')}</span>` +
-                `<span>状态：${r.status === 'failed' ? '<span style="color:#c62828;">失败</span>' : '<span style="color:#2e7d32;">成功</span>'}</span>`;
+                `<span>状态：${r.status === 'failed' ? '<span style="color:#c62828;">失败</span>' : '<span style="color:#2e7d32;">成功</span>'}</span>` +
+                htmlLink;
            
            const body = document.getElementById('modalBody');
            if (r.status === 'failed') {