Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
96b7b5dd7a |
@@ -260,10 +260,10 @@ def save_smart_config(cfg):
|
|||||||
|
|
||||||
|
|
||||||
# ===== 网址缓存 + 失败重试 配置 =====
|
# ===== 网址缓存 + 失败重试 配置 =====
|
||||||
# cache: enabled=是否启用网址缓存(命中直接返回历史, 不重新提取); ttl_seconds=缓存有效期(秒)
|
# cache: enabled=是否启用网址缓存(命中直接返回历史, 不重新提取); ttl_seconds=缓存有效期(秒, 默认7天, -1=永久有效)
|
||||||
# retry: max_retries=提取失败后的重试次数(默认2, 即最多尝试3次); delay_seconds=每次重试间隔(秒)
|
# retry: max_retries=提取失败后的重试次数(默认2, 即最多尝试3次); delay_seconds=每次重试间隔(秒, 默认15)
|
||||||
DEFAULT_CACHE_CONFIG = {"enabled": True, "ttl_seconds": 3600}
|
DEFAULT_CACHE_CONFIG = {"enabled": True, "ttl_seconds": 7 * 24 * 3600} # 7天
|
||||||
DEFAULT_RETRY_CONFIG = {"max_retries": 2, "delay_seconds": 2}
|
DEFAULT_RETRY_CONFIG = {"max_retries": 2, "delay_seconds": 15}
|
||||||
|
|
||||||
|
|
||||||
def load_capture_settings():
|
def load_capture_settings():
|
||||||
@@ -310,17 +310,28 @@ def save_capture_settings(settings):
|
|||||||
def find_cache_hit(url, action, ttl_seconds):
|
def find_cache_hit(url, action, ttl_seconds):
|
||||||
"""
|
"""
|
||||||
按 url+action 在提取历史中查缓存:取最近一条成功记录,且创建时间在 TTL 内。
|
按 url+action 在提取历史中查缓存:取最近一条成功记录,且创建时间在 TTL 内。
|
||||||
|
ttl_seconds < 0 表示永久有效(不按时间过滤)。
|
||||||
命中返回 dict(记录),未命中返回 None。
|
命中返回 dict(记录),未命中返回 None。
|
||||||
"""
|
"""
|
||||||
from datetime import timedelta
|
|
||||||
cutoff = (datetime.now() - timedelta(seconds=max(1, int(ttl_seconds)))).strftime("%Y-%m-%d %H:%M:%S")
|
|
||||||
conn = get_db()
|
conn = get_db()
|
||||||
row = conn.execute(
|
ttl = int(ttl_seconds)
|
||||||
"SELECT id, url, title, backend, file_path, content, html_path, created_at FROM captures "
|
if ttl < 0:
|
||||||
"WHERE url=? AND action=? AND status='success' AND created_at >= ? "
|
# 永久有效
|
||||||
"ORDER BY id DESC LIMIT 1",
|
row = conn.execute(
|
||||||
(url, action, cutoff)
|
"SELECT id, url, title, backend, file_path, content, html_path, created_at FROM captures "
|
||||||
).fetchone()
|
"WHERE url=? AND action=? AND status='success' "
|
||||||
|
"ORDER BY id DESC LIMIT 1",
|
||||||
|
(url, action)
|
||||||
|
).fetchone()
|
||||||
|
else:
|
||||||
|
from datetime import timedelta
|
||||||
|
cutoff = (datetime.now() - timedelta(seconds=max(1, ttl))).strftime("%Y-%m-%d %H:%M:%S")
|
||||||
|
row = conn.execute(
|
||||||
|
"SELECT id, url, title, backend, file_path, content, html_path, created_at FROM captures "
|
||||||
|
"WHERE url=? AND action=? AND status='success' AND created_at >= ? "
|
||||||
|
"ORDER BY id DESC LIMIT 1",
|
||||||
|
(url, action, cutoff)
|
||||||
|
).fetchone()
|
||||||
conn.close()
|
conn.close()
|
||||||
return dict(row) if row else None
|
return dict(row) if row else None
|
||||||
|
|
||||||
@@ -1585,7 +1596,7 @@ def capture():
|
|||||||
"action": "screenshot" | "html" | "text" | "smart",
|
"action": "screenshot" | "html" | "text" | "smart",
|
||||||
"refresh": false, // 可选:true=强制实时提取(绕过网址缓存)
|
"refresh": false, // 可选:true=强制实时提取(绕过网址缓存)
|
||||||
"cache": true, // 可选:false=本次请求禁用缓存
|
"cache": true, // 可选:false=本次请求禁用缓存
|
||||||
"cache_ttl_seconds": 3600, // 可选:本次请求的缓存时效(秒),覆盖全局设置;命中判定也用它
|
"cache_ttl_seconds": 604800, // 可选:本次请求的缓存时效(秒, -1=永久),覆盖全局设置;命中判定也用它
|
||||||
"scroll_times": 0,
|
"scroll_times": 0,
|
||||||
"scroll_delay": 1000,
|
"scroll_delay": 1000,
|
||||||
"full_page": false,
|
"full_page": false,
|
||||||
@@ -1639,10 +1650,11 @@ def capture():
|
|||||||
cache_cfg = settings.get("cache", DEFAULT_CACHE_CONFIG)
|
cache_cfg = settings.get("cache", DEFAULT_CACHE_CONFIG)
|
||||||
retry_cfg = settings.get("retry", DEFAULT_RETRY_CONFIG)
|
retry_cfg = settings.get("retry", DEFAULT_RETRY_CONFIG)
|
||||||
force_refresh = bool(data.get("refresh") or data.get("force"))
|
force_refresh = bool(data.get("refresh") or data.get("force"))
|
||||||
# per-request 缓存参数:cache_ttl_seconds=本次缓存时效(覆盖全局),cache=false / no_cache=true=本次禁用缓存
|
# per-request 缓存参数:cache_ttl_seconds=本次缓存时效(秒, -1=永久, 覆盖全局),cache=false / no_cache=true=本次禁用缓存
|
||||||
no_cache = data.get("cache") is False or str(data.get("no_cache")).lower() in ("true", "1", "yes")
|
no_cache = data.get("cache") is False or str(data.get("no_cache")).lower() in ("true", "1", "yes")
|
||||||
try:
|
try:
|
||||||
cache_ttl_override = max(1, int(data.get("cache_ttl_seconds"))) if data.get("cache_ttl_seconds") is not None else None
|
v = int(data.get("cache_ttl_seconds"))
|
||||||
|
cache_ttl_override = v if v != 0 else None
|
||||||
except Exception:
|
except Exception:
|
||||||
cache_ttl_override = None
|
cache_ttl_override = None
|
||||||
|
|
||||||
|
|||||||
+25
-16
@@ -748,8 +748,8 @@
|
|||||||
<p><strong>重要提示:</strong></p>
|
<p><strong>重要提示:</strong></p>
|
||||||
<ul style="margin: 10px 0; line-height: 1.8;">
|
<ul style="margin: 10px 0; line-height: 1.8;">
|
||||||
<li>🧠 <strong>按需截图</strong>:每次截图后用视觉大模型实时判断是否已截全主题内容、是否还需继续滚动,自动滚动到内容结束并拼接成长图;支持<strong>多视觉大模型接口配置</strong>(「⚙️ 智能配置」中增删改,列表顺序=优先级,失败/报错自动降级到下一个接口)</li>
|
<li>🧠 <strong>按需截图</strong>:每次截图后用视觉大模型实时判断是否已截全主题内容、是否还需继续滚动,自动滚动到内容结束并拼接成长图;支持<strong>多视觉大模型接口配置</strong>(「⚙️ 智能配置」中增删改,列表顺序=优先级,失败/报错自动降级到下一个接口)</li>
|
||||||
<li>📦 <strong>网址缓存</strong>:默认开启——同一网址先查提取历史(同动作、成功、在有效期内)直接返回,不重复提取;未命中才实时提取。可「⚙️ 缓存/重试」关开关/改有效期,或请求带 <code>refresh:true</code> 强制绕过</li>
|
<li>📦 <strong>网址缓存</strong>:默认开启——同一网址先查提取历史(同动作、成功、在有效期内)直接返回,不重复提取;有效期默认 <strong>7 天</strong>,填 <strong>-1 则永久有效</strong>。可「⚙️ 缓存/重试」关开关/改有效期,或请求带 <code>refresh:true</code> 强制绕过</li>
|
||||||
<li>🔁 <strong>失败自动重试</strong>:提取失败(网络/超时/浏览器错误)自动重试,次数可设(默认 2 次),响应带 <code>attempts</code> 字段</li>
|
<li>🔁 <strong>失败自动重试</strong>:提取失败(网络/超时/浏览器错误)自动重试,次数可设(默认 2 次,间隔 15 秒),响应带 <code>attempts</code> 字段</li>
|
||||||
<li>📊 <strong>提取历史统计</strong>:历史区点「📊 统计」,可按天/操作类型/后端/状态/调用者/调用方式等角度切换图表(柱状/饼图可换,支持 7/30/90 天范围,可下载 PNG)</li>
|
<li>📊 <strong>提取历史统计</strong>:历史区点「📊 统计」,可按天/操作类型/后端/状态/调用者/调用方式等角度切换图表(柱状/饼图可换,支持 7/30/90 天范围,可下载 PNG)</li>
|
||||||
<li>📝 <strong>提取文本</strong>:自动剔除脚本/样式/标签,得到干净的前端可读文本,适合直接用于分析/喂给大模型</li>
|
<li>📝 <strong>提取文本</strong>:自动剔除脚本/样式/标签,得到干净的前端可读文本,适合直接用于分析/喂给大模型</li>
|
||||||
<li>⏱️ <strong>页面加载等待</strong>:某些网站需要验证过程(如 Cloudflare),请设置较长时间(10000-30000ms)</li>
|
<li>⏱️ <strong>页面加载等待</strong>:某些网站需要验证过程(如 Cloudflare),请设置较长时间(10000-30000ms)</li>
|
||||||
@@ -774,7 +774,7 @@
|
|||||||
// ---- 📦 网址缓存(默认开启,全局配置在 /api/capture/settings)----
|
// ---- 📦 网址缓存(默认开启,全局配置在 /api/capture/settings)----
|
||||||
"refresh": false, // 可选:true=强制实时提取(绕过缓存)
|
"refresh": false, // 可选:true=强制实时提取(绕过缓存)
|
||||||
"cache": true, // 可选:false=本次请求禁用缓存
|
"cache": true, // 可选:false=本次请求禁用缓存
|
||||||
"cache_ttl_seconds": 3600, // 可选:本次缓存时效(秒),覆盖全局有效期,命中判定也用它
|
"cache_ttl_seconds": 604800, // 可选:本次缓存时效(秒, -1=永久),覆盖全局有效期,命中判定也用它
|
||||||
"smart_config": { // 可选:临时覆盖按需截图配置(接口列表按顺序=优先级,失败自动降级)
|
"smart_config": { // 可选:临时覆盖按需截图配置(接口列表按顺序=优先级,失败自动降级)
|
||||||
"endpoints": [
|
"endpoints": [
|
||||||
{"name":"本地Qwen", "base_url":"http://121.40.164.32:18008/v1", "api_key":"xxxx", "model":"unsloth/Qwen3.8-27B-NVFP4", "enabled":true},
|
{"name":"本地Qwen", "base_url":"http://121.40.164.32:18008/v1", "api_key":"xxxx", "model":"unsloth/Qwen3.8-27B-NVFP4", "enabled":true},
|
||||||
@@ -792,8 +792,8 @@ GET /api/capture/settings
|
|||||||
POST /api/capture/settings
|
POST /api/capture/settings
|
||||||
{
|
{
|
||||||
"settings": {
|
"settings": {
|
||||||
"cache": {"enabled": true, "ttl_seconds": 3600}, // enabled=缓存开关, ttl_seconds=有效期(秒)
|
"cache": {"enabled": true, "ttl_seconds": 604800}, // enabled=缓存开关, ttl_seconds=有效期(秒, 默认7天=604800, -1=永久有效)
|
||||||
"retry": {"max_retries": 2, "delay_seconds": 2} // max_retries=失败重试次数(默认2), delay_seconds=重试间隔(秒)
|
"retry": {"max_retries": 2, "delay_seconds": 15} // max_retries=失败重试次数(默认2), delay_seconds=重试间隔(秒, 默认15)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -873,9 +873,9 @@ DELETE /api/history/<id> // 删除历史</code></pre>
|
|||||||
</div>
|
</div>
|
||||||
<div style="display:flex;gap:14px;flex-wrap:wrap;">
|
<div style="display:flex;gap:14px;flex-wrap:wrap;">
|
||||||
<div style="flex:1;min-width:180px;">
|
<div style="flex:1;min-width:180px;">
|
||||||
<label for="cacheTtl" style="font-size:13px;">缓存有效期(分钟)</label>
|
<label for="cacheTtl" style="font-size:13px;">缓存有效期(分钟,-1 = 永久有效)</label>
|
||||||
<input type="number" id="cacheTtl" min="1" max="10080" value="60" style="width:100%;padding:10px;border:1.5px solid #e0e0e0;border-radius:6px;">
|
<input type="number" id="cacheTtl" min="-1" max="525600" value="10080" style="width:100%;padding:10px;border:1.5px solid #e0e0e0;border-radius:6px;">
|
||||||
<small style="color:#999;display:block;margin-top:4px;">超期后再次提取自动刷新缓存。默认 60 分钟。</small>
|
<small style="color:#999;display:block;margin-top:4px;">默认 7 天(10080 分钟);填 <strong>-1</strong> 则缓存永久有效,只要历史里有该网址的成功记录就一直命中,不再重新提取。</small>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
@@ -890,8 +890,8 @@ DELETE /api/history/<id> // 删除历史</code></pre>
|
|||||||
<small style="color:#999;display:block;margin-top:4px;">0 = 不重试。2 即最多尝试 3 次。</small>
|
<small style="color:#999;display:block;margin-top:4px;">0 = 不重试。2 即最多尝试 3 次。</small>
|
||||||
</div>
|
</div>
|
||||||
<div style="flex:1;min-width:150px;">
|
<div style="flex:1;min-width:150px;">
|
||||||
<label for="retryDelay" style="font-size:13px;">重试间隔(秒)</label>
|
<label for="retryDelay" style="font-size:13px;">重试间隔(秒,默认 15)</label>
|
||||||
<input type="number" id="retryDelay" min="0" max="60" value="2" style="width:100%;padding:10px;border:1.5px solid #e0e0e0;border-radius:6px;">
|
<input type="number" id="retryDelay" min="0" max="300" value="15" style="width:100%;padding:10px;border:1.5px solid #e0e0e0;border-radius:6px;">
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
@@ -973,12 +973,18 @@ DELETE /api/history/<id> // 删除历史</code></pre>
|
|||||||
loadCaptureSettings();
|
loadCaptureSettings();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
function fmtTtl(sec) {
|
||||||
|
if (sec < 0) return '永久';
|
||||||
|
const min = Math.round(sec / 60);
|
||||||
|
if (min >= 1440) return (min / 1440).toFixed(1).replace(/\.0$/, '') + ' 天';
|
||||||
|
return min + ' 分钟';
|
||||||
|
}
|
||||||
|
|
||||||
function updateCacheBadge() {
|
function updateCacheBadge() {
|
||||||
const en = captureSettings.cache && captureSettings.cache.enabled;
|
const en = captureSettings.cache && captureSettings.cache.enabled;
|
||||||
document.getElementById('cacheStateBadge').textContent = en ? '✅ 开启' : '⛔ 关闭';
|
document.getElementById('cacheStateBadge').textContent = en ? '✅ 开启' : '⛔ 关闭';
|
||||||
document.getElementById('cacheStateBadge').style.color = en ? '#2e7d32' : '#c62828';
|
document.getElementById('cacheStateBadge').style.color = en ? '#2e7d32' : '#c62828';
|
||||||
const min = Math.round((captureSettings.cache && captureSettings.cache.ttl_seconds || 3600) / 60);
|
document.getElementById('cacheTtlBadge').textContent = fmtTtl((captureSettings.cache && captureSettings.cache.ttl_seconds) || 3600);
|
||||||
document.getElementById('cacheTtlBadge').textContent = min + ' 分钟';
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async function loadCaptureSettings() {
|
async function loadCaptureSettings() {
|
||||||
@@ -991,10 +997,11 @@ DELETE /api/history/<id> // 删除历史</code></pre>
|
|||||||
}
|
}
|
||||||
|
|
||||||
function openCaptureCfg() {
|
function openCaptureCfg() {
|
||||||
|
const ttlSec = (captureSettings.cache && captureSettings.cache.ttl_seconds != null) ? captureSettings.cache.ttl_seconds : 604800;
|
||||||
document.getElementById('cacheEnabled').checked = !!(captureSettings.cache && captureSettings.cache.enabled);
|
document.getElementById('cacheEnabled').checked = !!(captureSettings.cache && captureSettings.cache.enabled);
|
||||||
document.getElementById('cacheTtl').value = Math.round(((captureSettings.cache && captureSettings.cache.ttl_seconds) || 3600) / 60);
|
document.getElementById('cacheTtl').value = ttlSec < 0 ? -1 : Math.round(ttlSec / 60);
|
||||||
document.getElementById('retryCount').value = (captureSettings.retry && captureSettings.retry.max_retries != null) ? captureSettings.retry.max_retries : 2;
|
document.getElementById('retryCount').value = (captureSettings.retry && captureSettings.retry.max_retries != null) ? captureSettings.retry.max_retries : 2;
|
||||||
document.getElementById('retryDelay').value = (captureSettings.retry && captureSettings.retry.delay_seconds != null) ? captureSettings.retry.delay_seconds : 2;
|
document.getElementById('retryDelay').value = (captureSettings.retry && captureSettings.retry.delay_seconds != null) ? captureSettings.retry.delay_seconds : 15;
|
||||||
document.getElementById('captureCfgResult').innerHTML = '';
|
document.getElementById('captureCfgResult').innerHTML = '';
|
||||||
document.getElementById('captureCfgOverlay').classList.add('active');
|
document.getElementById('captureCfgOverlay').classList.add('active');
|
||||||
}
|
}
|
||||||
@@ -1005,10 +1012,12 @@ DELETE /api/history/<id> // 删除历史</code></pre>
|
|||||||
|
|
||||||
async function saveCaptureCfg() {
|
async function saveCaptureCfg() {
|
||||||
const el = document.getElementById('captureCfgResult');
|
const el = document.getElementById('captureCfgResult');
|
||||||
|
const ttlMin = parseInt(document.getElementById('cacheTtl').value);
|
||||||
|
const ttlSec = ttlMin === -1 ? -1 : (Math.max(1, ttlMin || 1)) * 60;
|
||||||
const settings = {
|
const settings = {
|
||||||
cache: {
|
cache: {
|
||||||
enabled: document.getElementById('cacheEnabled').checked,
|
enabled: document.getElementById('cacheEnabled').checked,
|
||||||
ttl_seconds: (parseInt(document.getElementById('cacheTtl').value) || 60) * 60
|
ttl_seconds: ttlSec
|
||||||
},
|
},
|
||||||
retry: {
|
retry: {
|
||||||
max_retries: Math.max(0, parseInt(document.getElementById('retryCount').value) || 0),
|
max_retries: Math.max(0, parseInt(document.getElementById('retryCount').value) || 0),
|
||||||
@@ -1027,7 +1036,7 @@ DELETE /api/history/<id> // 删除历史</code></pre>
|
|||||||
captureSettings = data.settings;
|
captureSettings = data.settings;
|
||||||
updateCacheBadge();
|
updateCacheBadge();
|
||||||
const en = data.settings.cache.enabled;
|
const en = data.settings.cache.enabled;
|
||||||
el.innerHTML = `✅ 已保存:网址缓存${en ? '开启' : '关闭'}(有效期 ${Math.round(data.settings.cache.ttl_seconds / 60)} 分钟)| 失败重试 ${data.settings.retry.max_retries} 次(间隔 ${data.settings.retry.delay_seconds}s)`;
|
el.innerHTML = `✅ 已保存:网址缓存${en ? '开启' : '关闭'}(有效期 ${fmtTtl(data.settings.cache.ttl_seconds)})| 失败重试 ${data.settings.retry.max_retries} 次(间隔 ${data.settings.retry.delay_seconds}s)`;
|
||||||
} else {
|
} else {
|
||||||
el.innerHTML = `<span style="color:#c62828;">❌ 保存失败:${escapeHtml(data.error || '未知错误')}</span>`;
|
el.innerHTML = `<span style="color:#c62828;">❌ 保存失败:${escapeHtml(data.error || '未知错误')}</span>`;
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user