Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 hz4th_coder
|
7c00ed2e47 | ||
|
hz4th_coder
|
96b7b5dd7a | ||
|
hz4th_coder
|
4f1b7c341c |
@@ -260,10 +260,10 @@ def save_smart_config(cfg):
|
||||
|
||||
|
||||
# ===== 网址缓存 + 失败重试 配置 =====
|
||||
# cache: enabled=是否启用网址缓存(命中直接返回历史, 不重新提取); ttl_seconds=缓存有效期(秒)
|
||||
# retry: max_retries=提取失败后的重试次数(默认2, 即最多尝试3次); delay_seconds=每次重试间隔(秒)
|
||||
DEFAULT_CACHE_CONFIG = {"enabled": True, "ttl_seconds": 3600}
|
||||
DEFAULT_RETRY_CONFIG = {"max_retries": 2, "delay_seconds": 2}
|
||||
# cache: enabled=是否启用网址缓存(命中直接返回历史, 不重新提取); ttl_seconds=缓存有效期(秒, 默认7天, -1=永久有效)
|
||||
# retry: max_retries=提取失败后的重试次数(默认2, 即最多尝试3次); delay_seconds=每次重试间隔(秒, 默认15)
|
||||
DEFAULT_CACHE_CONFIG = {"enabled": True, "ttl_seconds": 7 * 24 * 3600} # 7天
|
||||
DEFAULT_RETRY_CONFIG = {"max_retries": 2, "delay_seconds": 15}
|
||||
|
||||
|
||||
def load_capture_settings():
|
||||
@@ -310,17 +310,28 @@ def save_capture_settings(settings):
|
||||
def find_cache_hit(url, action, ttl_seconds):
|
||||
"""
|
||||
按 url+action 在提取历史中查缓存：取最近一条成功记录，且创建时间在 TTL 内。
|
||||
ttl_seconds < 0 表示永久有效（不按时间过滤）。
|
||||
命中返回 dict(记录)，未命中返回 None。
|
||||
"""
|
||||
from datetime import timedelta
|
||||
cutoff = (datetime.now() - timedelta(seconds=max(1, int(ttl_seconds)))).strftime("%Y-%m-%d %H:%M:%S")
|
||||
conn = get_db()
|
||||
row = conn.execute(
|
||||
"SELECT id, url, title, backend, file_path, content, html_path, created_at FROM captures "
|
||||
"WHERE url=? AND action=? AND status='success' AND created_at >= ? "
|
||||
"ORDER BY id DESC LIMIT 1",
|
||||
(url, action, cutoff)
|
||||
).fetchone()
|
||||
ttl = int(ttl_seconds)
|
||||
if ttl < 0:
|
||||
# 永久有效
|
||||
row = conn.execute(
|
||||
"SELECT id, url, title, backend, file_path, content, html_path, created_at FROM captures "
|
||||
"WHERE url=? AND action=? AND status='success' "
|
||||
"ORDER BY id DESC LIMIT 1",
|
||||
(url, action)
|
||||
).fetchone()
|
||||
else:
|
||||
from datetime import timedelta
|
||||
cutoff = (datetime.now() - timedelta(seconds=max(1, ttl))).strftime("%Y-%m-%d %H:%M:%S")
|
||||
row = conn.execute(
|
||||
"SELECT id, url, title, backend, file_path, content, html_path, created_at FROM captures "
|
||||
"WHERE url=? AND action=? AND status='success' AND created_at >= ? "
|
||||
"ORDER BY id DESC LIMIT 1",
|
||||
(url, action, cutoff)
|
||||
).fetchone()
|
||||
conn.close()
|
||||
return dict(row) if row else None
|
||||
|
||||
@@ -1584,6 +1595,8 @@ def capture():
|
||||
"url": "https://example.com",
|
||||
"action": "screenshot" | "html" | "text" | "smart",
|
||||
"refresh": false, // 可选：true=强制实时提取(绕过网址缓存)
|
||||
"cache": true, // 可选：false=本次请求禁用缓存
|
||||
"cache_ttl_seconds": 604800, // 可选：本次请求的缓存时效(秒, -1=永久)，覆盖全局设置；命中判定也用它
|
||||
"scroll_times": 0,
|
||||
"scroll_delay": 1000,
|
||||
"full_page": false,
|
||||
@@ -1637,9 +1650,16 @@ def capture():
|
||||
cache_cfg = settings.get("cache", DEFAULT_CACHE_CONFIG)
|
||||
retry_cfg = settings.get("retry", DEFAULT_RETRY_CONFIG)
|
||||
force_refresh = bool(data.get("refresh") or data.get("force"))
|
||||
# per-request 缓存参数：cache_ttl_seconds=本次缓存时效(秒, -1=永久, 覆盖全局)，cache=false / no_cache=true=本次禁用缓存
|
||||
no_cache = data.get("cache") is False or str(data.get("no_cache")).lower() in ("true", "1", "yes")
|
||||
try:
|
||||
v = int(data.get("cache_ttl_seconds"))
|
||||
cache_ttl_override = v if v != 0 else None
|
||||
except Exception:
|
||||
cache_ttl_override = None
|
||||
|
||||
if cache_cfg.get("enabled") and not force_refresh:
|
||||
ttl = int(cache_cfg.get("ttl_seconds", 3600))
|
||||
if cache_cfg.get("enabled") and not force_refresh and not no_cache:
|
||||
ttl = cache_ttl_override or int(cache_cfg.get("ttl_seconds", 3600))
|
||||
hit = find_cache_hit(url, action, ttl)
|
||||
if hit:
|
||||
hit["from_cache"] = True
|
||||
@@ -1757,4 +1777,6 @@ if __name__ == '__main__':
|
||||
print(" playwright: pip install playwright && playwright install chromium")
|
||||
|
||||
print("\n🚀 Server running on http://0.0.0.0:16025")
|
||||
app.run(host='0.0.0.0', port=16025, debug=True)
|
||||
# 生产模式：不启用 debug reloader（reloader 主进程在 worker 意外退出后不会自动拉起，
|
||||
# 曾导致服务静默挂掉），改代码后需手动重启：kill 进程后 nohup 重新启动
|
||||
app.run(host='0.0.0.0', port=16025, debug=False, use_reloader=False, threaded=True)
|
||||
+42
-17
@@ -627,8 +627,10 @@
|
||||
🔄 强制刷新（绕过网址缓存，重新提取）
|
||||
</label>
|
||||
</div>
|
||||
<small style="color: #666; display: block; margin-top: 4px;">
|
||||
📦 网址缓存已<span id="cacheStateBadge">…</span>（有效期 <span id="cacheTtlBadge">…</span>），同网址命中缓存直接返回历史结果，不会重复提取。
|
||||
<small style="color: #666; display: block; margin-top: 6px; line-height: 1.8;">
|
||||
📦 网址缓存：<span id="cacheStateBadge">…</span> ｜ 有效期 <span id="cacheTtlBadge">…</span>
|
||||
<button type="button" class="btn-small" onclick="openCaptureCfg()" style="margin-left:8px;padding:3px 10px;font-size:12px;vertical-align:middle;" title="打开缓存/重试设置">⚙️ 设置时效与开关</button><br>
|
||||
<span style="color:#999;">同网址命中缓存直接返回历史结果，不重复提取；勾选上方「强制刷新」可绕过。</span>
|
||||
</small>
|
||||
</div>
|
||||
|
||||
@@ -746,8 +748,8 @@
|
||||
<p><strong>重要提示：</strong></p>
|
||||
<ul style="margin: 10px 0; line-height: 1.8;">
|
||||
<li>🧠 <strong>按需截图</strong>：每次截图后用视觉大模型实时判断是否已截全主题内容、是否还需继续滚动，自动滚动到内容结束并拼接成长图；支持<strong>多视觉大模型接口配置</strong>（「⚙️ 智能配置」中增删改，列表顺序=优先级，失败/报错自动降级到下一个接口）</li>
|
||||
<li>📦 <strong>网址缓存</strong>：默认开启——同一网址先查提取历史（同动作、成功、在有效期内）直接返回，不重复提取；未命中才实时提取。可「⚙️ 缓存/重试」关开关/改有效期，或请求带 <code>refresh:true</code> 强制绕过</li>
|
||||
<li>🔁 <strong>失败自动重试</strong>：提取失败（网络/超时/浏览器错误）自动重试，次数可设（默认 2 次），响应带 <code>attempts</code> 字段</li>
|
||||
<li>📦 <strong>网址缓存</strong>：默认开启——同一网址先查提取历史（同动作、成功、在有效期内）直接返回，不重复提取；有效期默认 <strong>7 天</strong>，填 <strong>-1 则永久有效</strong>。可「⚙️ 缓存/重试」关开关/改有效期，或请求带 <code>refresh:true</code> 强制绕过</li>
|
||||
<li>🔁 <strong>失败自动重试</strong>：提取失败（网络/超时/浏览器错误）自动重试，次数可设（默认 2 次，间隔 15 秒），响应带 <code>attempts</code> 字段</li>
|
||||
<li>📊 <strong>提取历史统计</strong>：历史区点「📊 统计」，可按天/操作类型/后端/状态/调用者/调用方式等角度切换图表（柱状/饼图可换，支持 7/30/90 天范围，可下载 PNG）</li>
|
||||
<li>📝 <strong>提取文本</strong>：自动剔除脚本/样式/标签，得到干净的前端可读文本，适合直接用于分析/喂给大模型</li>
|
||||
<li>⏱️ <strong>页面加载等待</strong>：某些网站需要验证过程（如 Cloudflare），请设置较长时间（10000-30000ms）</li>
|
||||
@@ -761,7 +763,6 @@
|
||||
{
|
||||
"url": "https://example.com",
|
||||
"action": "smart", // "screenshot" | "html" | "text" | "smart"
|
||||
"refresh": false, // 可选：true=强制实时提取(绕过网址缓存)
|
||||
"wait_time": 15000, // 重要！验证网站需设置较长等待
|
||||
"scroll_times": 3, // 可选：加载动态内容
|
||||
"scroll_delay": 1000, // 可选：滚动间隔
|
||||
@@ -770,6 +771,10 @@
|
||||
"caller": "news-tracker", // 可选：调用者标识（历史记录里显示，默认游客）
|
||||
"call_method": "api", // 可选：调用方式 web/api（缺省自动判定）
|
||||
"viewport": {"width":1280,"height":700},
|
||||
// ---- 📦 网址缓存（默认开启，全局配置在 /api/capture/settings）----
|
||||
"refresh": false, // 可选：true=强制实时提取(绕过缓存)
|
||||
"cache": true, // 可选：false=本次请求禁用缓存
|
||||
"cache_ttl_seconds": 604800, // 可选：本次缓存时效(秒, -1=永久)，覆盖全局有效期，命中判定也用它
|
||||
"smart_config": { // 可选：临时覆盖按需截图配置（接口列表按顺序=优先级，失败自动降级）
|
||||
"endpoints": [
|
||||
{"name":"本地Qwen", "base_url":"http://121.40.164.32:18008/v1", "api_key":"xxxx", "model":"unsloth/Qwen3.8-27B-NVFP4", "enabled":true},
|
||||
@@ -779,11 +784,22 @@
|
||||
"scroll_ratio": 0.85
|
||||
}
|
||||
}
|
||||
// 缓存命中响应（text/html/smart 为 JSON；screenshot 直接返回历史图片文件）：
|
||||
// {success:true, from_cache:true, history_id:123, cached_at:"2026-09-11 22:49:41", cache_ttl_seconds:3600, ...}
|
||||
|
||||
📦 缓存/🔁 重试全局配置（页面「⚙️ 缓存/重试」按钮或以下接口）：
|
||||
GET /api/capture/settings
|
||||
POST /api/capture/settings
|
||||
{
|
||||
"settings": {
|
||||
"cache": {"enabled": true, "ttl_seconds": 604800}, // enabled=缓存开关, ttl_seconds=有效期(秒, 默认7天=604800, -1=永久有效)
|
||||
"retry": {"max_retries": 2, "delay_seconds": 15} // max_retries=失败重试次数(默认2), delay_seconds=重试间隔(秒, 默认15)
|
||||
}
|
||||
}
|
||||
|
||||
GET /api/smart/config // 获取智能截图配置（含 endpoints 多接口列表）
|
||||
POST /api/smart/config // 保存配置（body: {config:{endpoints:[...], prompt, max_scrolls,...}}）
|
||||
POST /api/smart/test // 测试指定接口（body: {endpoint_id} 或 {endpoint:{...}} 或旧版 {config:{...}}）
|
||||
GET /api/capture/settings // 网址缓存+失败重试 配置（{cache:{enabled,ttl_seconds}, retry:{max_retries,delay_seconds}}）
|
||||
GET /api/history?page=1&page_size=15&action=all&search=关键词 // 历史分页
|
||||
GET /api/history/stats?dimension=day&days=30 // 历史统计（dimension: day/action/backend/status/caller/method）
|
||||
GET /api/history/<id> // 历史详情
|
||||
@@ -857,9 +873,9 @@ DELETE /api/history/<id> // 删除历史</code></pre>
|
||||
</div>
|
||||
<div style="display:flex;gap:14px;flex-wrap:wrap;">
|
||||
<div style="flex:1;min-width:180px;">
|
||||
<label for="cacheTtl" style="font-size:13px;">缓存有效期（分钟）</label>
|
||||
<input type="number" id="cacheTtl" min="1" max="10080" value="60" style="width:100%;padding:10px;border:1.5px solid #e0e0e0;border-radius:6px;">
|
||||
<small style="color:#999;display:block;margin-top:4px;">超期后再次提取自动刷新缓存。默认 60 分钟。</small>
|
||||
<label for="cacheTtl" style="font-size:13px;">缓存有效期（分钟，-1 = 永久有效）</label>
|
||||
<input type="number" id="cacheTtl" min="-1" max="525600" value="10080" style="width:100%;padding:10px;border:1.5px solid #e0e0e0;border-radius:6px;">
|
||||
<small style="color:#999;display:block;margin-top:4px;">默认 7 天（10080 分钟）；填 <strong>-1</strong> 则缓存永久有效，只要历史里有该网址的成功记录就一直命中，不再重新提取。</small>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -874,8 +890,8 @@ DELETE /api/history/<id> // 删除历史</code></pre>
|
||||
<small style="color:#999;display:block;margin-top:4px;">0 = 不重试。2 即最多尝试 3 次。</small>
|
||||
</div>
|
||||
<div style="flex:1;min-width:150px;">
|
||||
<label for="retryDelay" style="font-size:13px;">重试间隔（秒）</label>
|
||||
<input type="number" id="retryDelay" min="0" max="60" value="2" style="width:100%;padding:10px;border:1.5px solid #e0e0e0;border-radius:6px;">
|
||||
<label for="retryDelay" style="font-size:13px;">重试间隔（秒，默认 15）</label>
|
||||
<input type="number" id="retryDelay" min="0" max="300" value="15" style="width:100%;padding:10px;border:1.5px solid #e0e0e0;border-radius:6px;">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -957,12 +973,18 @@ DELETE /api/history/<id> // 删除历史</code></pre>
|
||||
loadCaptureSettings();
|
||||
});
|
||||
|
||||
function fmtTtl(sec) {
|
||||
if (sec < 0) return '永久';
|
||||
const min = Math.round(sec / 60);
|
||||
if (min >= 1440) return (min / 1440).toFixed(1).replace(/\.0$/, '') + ' 天';
|
||||
return min + ' 分钟';
|
||||
}
|
||||
|
||||
function updateCacheBadge() {
|
||||
const en = captureSettings.cache && captureSettings.cache.enabled;
|
||||
document.getElementById('cacheStateBadge').textContent = en ? '✅ 开启' : '⛔ 关闭';
|
||||
document.getElementById('cacheStateBadge').style.color = en ? '#2e7d32' : '#c62828';
|
||||
const min = Math.round((captureSettings.cache && captureSettings.cache.ttl_seconds || 3600) / 60);
|
||||
document.getElementById('cacheTtlBadge').textContent = min + ' 分钟';
|
||||
document.getElementById('cacheTtlBadge').textContent = fmtTtl((captureSettings.cache && captureSettings.cache.ttl_seconds) || 3600);
|
||||
}
|
||||
|
||||
async function loadCaptureSettings() {
|
||||
@@ -975,10 +997,11 @@ DELETE /api/history/<id> // 删除历史</code></pre>
|
||||
}
|
||||
|
||||
function openCaptureCfg() {
|
||||
const ttlSec = (captureSettings.cache && captureSettings.cache.ttl_seconds != null) ? captureSettings.cache.ttl_seconds : 604800;
|
||||
document.getElementById('cacheEnabled').checked = !!(captureSettings.cache && captureSettings.cache.enabled);
|
||||
document.getElementById('cacheTtl').value = Math.round(((captureSettings.cache && captureSettings.cache.ttl_seconds) || 3600) / 60);
|
||||
document.getElementById('cacheTtl').value = ttlSec < 0 ? -1 : Math.round(ttlSec / 60);
|
||||
document.getElementById('retryCount').value = (captureSettings.retry && captureSettings.retry.max_retries != null) ? captureSettings.retry.max_retries : 2;
|
||||
document.getElementById('retryDelay').value = (captureSettings.retry && captureSettings.retry.delay_seconds != null) ? captureSettings.retry.delay_seconds : 2;
|
||||
document.getElementById('retryDelay').value = (captureSettings.retry && captureSettings.retry.delay_seconds != null) ? captureSettings.retry.delay_seconds : 15;
|
||||
document.getElementById('captureCfgResult').innerHTML = '';
|
||||
document.getElementById('captureCfgOverlay').classList.add('active');
|
||||
}
|
||||
@@ -989,10 +1012,12 @@ DELETE /api/history/<id> // 删除历史</code></pre>
|
||||
|
||||
async function saveCaptureCfg() {
|
||||
const el = document.getElementById('captureCfgResult');
|
||||
const ttlMin = parseInt(document.getElementById('cacheTtl').value);
|
||||
const ttlSec = ttlMin === -1 ? -1 : (Math.max(1, ttlMin || 1)) * 60;
|
||||
const settings = {
|
||||
cache: {
|
||||
enabled: document.getElementById('cacheEnabled').checked,
|
||||
ttl_seconds: (parseInt(document.getElementById('cacheTtl').value) || 60) * 60
|
||||
ttl_seconds: ttlSec
|
||||
},
|
||||
retry: {
|
||||
max_retries: Math.max(0, parseInt(document.getElementById('retryCount').value) || 0),
|
||||
@@ -1011,7 +1036,7 @@ DELETE /api/history/<id> // 删除历史</code></pre>
|
||||
captureSettings = data.settings;
|
||||
updateCacheBadge();
|
||||
const en = data.settings.cache.enabled;
|
||||
el.innerHTML = `✅ 已保存：网址缓存${en ? '开启' : '关闭'}（有效期 ${Math.round(data.settings.cache.ttl_seconds / 60)} 分钟）｜ 失败重试 ${data.settings.retry.max_retries} 次（间隔 ${data.settings.retry.delay_seconds}s）`;
|
||||
el.innerHTML = `✅ 已保存：网址缓存${en ? '开启' : '关闭'}（有效期 ${fmtTtl(data.settings.cache.ttl_seconds)}）｜ 失败重试 ${data.settings.retry.max_retries} 次（间隔 ${data.settings.retry.delay_seconds}s）`;
|
||||
} else {
|
||||
el.innerHTML = `<span style="color:#c62828;">❌ 保存失败：${escapeHtml(data.error || '未知错误')}</span>`;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user