新增搜索结果缓存功能
- 新增 search_cache 数据库表存储搜索缓存 - 搜索前优先查询缓存,命中则直接返回 - 缓存默认有效期7天,可配置 - 前端支持'使用缓存'开关选项 - 显示缓存状态提示 - 支持自动清理过期缓存
This commit is contained in:
@@ -121,6 +121,20 @@ class Database:
|
|||||||
)
|
)
|
||||||
''')
|
''')
|
||||||
|
|
||||||
|
# 搜索缓存表
|
||||||
|
cursor.execute('''
|
||||||
|
CREATE TABLE IF NOT EXISTS search_cache (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
keyword TEXT NOT NULL,
|
||||||
|
engine TEXT DEFAULT 'bing_cn',
|
||||||
|
results TEXT NOT NULL,
|
||||||
|
result_count INTEGER,
|
||||||
|
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
expires_at DATETIME,
|
||||||
|
UNIQUE(keyword, engine)
|
||||||
|
)
|
||||||
|
''')
|
||||||
|
|
||||||
conn.commit()
|
conn.commit()
|
||||||
|
|
||||||
# ========== 内容库操作 ==========
|
# ========== 内容库操作 ==========
|
||||||
@@ -396,5 +410,52 @@ class Database:
|
|||||||
conn.commit()
|
conn.commit()
|
||||||
return cursor.rowcount
|
return cursor.rowcount
|
||||||
|
|
||||||
|
# ========== 搜索缓存操作 ==========
|
||||||
|
def save_search_cache(self, keyword, engine, results, expire_days=7):
|
||||||
|
"""保存搜索结果缓存"""
|
||||||
|
with self.get_connection() as conn:
|
||||||
|
cursor = conn.cursor()
|
||||||
|
cursor.execute('''
|
||||||
|
INSERT OR REPLACE INTO search_cache (keyword, engine, results, result_count, created_at, expires_at)
|
||||||
|
VALUES (?, ?, ?, ?, CURRENT_TIMESTAMP, datetime('now', '+' || ? || ' days'))
|
||||||
|
''', (keyword, engine, json.dumps(results, ensure_ascii=False), len(results), expire_days))
|
||||||
|
conn.commit()
|
||||||
|
return cursor.lastrowid
|
||||||
|
|
||||||
|
def get_search_cache(self, keyword, engine='bing_cn'):
|
||||||
|
"""获取搜索结果缓存"""
|
||||||
|
with self.get_connection() as conn:
|
||||||
|
cursor = conn.cursor()
|
||||||
|
cursor.execute('''
|
||||||
|
SELECT results, result_count, created_at, expires_at
|
||||||
|
FROM search_cache
|
||||||
|
WHERE keyword = ? AND engine = ? AND expires_at > datetime('now')
|
||||||
|
''', (keyword, engine))
|
||||||
|
row = cursor.fetchone()
|
||||||
|
if row:
|
||||||
|
return {
|
||||||
|
'results': json.loads(row['results']),
|
||||||
|
'count': row['result_count'],
|
||||||
|
'cached_at': row['created_at'],
|
||||||
|
'expires_at': row['expires_at']
|
||||||
|
}
|
||||||
|
return None
|
||||||
|
|
||||||
|
def clear_expired_cache(self):
|
||||||
|
"""清理过期缓存"""
|
||||||
|
with self.get_connection() as conn:
|
||||||
|
cursor = conn.cursor()
|
||||||
|
cursor.execute('DELETE FROM search_cache WHERE expires_at <= datetime("now")')
|
||||||
|
conn.commit()
|
||||||
|
return cursor.rowcount
|
||||||
|
|
||||||
|
def clear_all_cache(self):
|
||||||
|
"""清空所有缓存"""
|
||||||
|
with self.get_connection() as conn:
|
||||||
|
cursor = conn.cursor()
|
||||||
|
cursor.execute('DELETE FROM search_cache')
|
||||||
|
conn.commit()
|
||||||
|
return cursor.rowcount
|
||||||
|
|
||||||
# 全局数据库实例
|
# 全局数据库实例
|
||||||
db = Database()
|
db = Database()
|
||||||
+22
-2
@@ -141,19 +141,39 @@ def internet_search():
|
|||||||
keyword = data.get('keyword', '')
|
keyword = data.get('keyword', '')
|
||||||
max_results = data.get('max_results', 10)
|
max_results = data.get('max_results', 10)
|
||||||
engine = data.get('engine', 'bing_cn') # 默认 Bing 中国
|
engine = data.get('engine', 'bing_cn') # 默认 Bing 中国
|
||||||
|
use_cache = data.get('use_cache', True) # 默认使用缓存
|
||||||
|
cache_days = data.get('cache_days', 7) # 默认缓存7天
|
||||||
|
|
||||||
if not keyword:
|
if not keyword:
|
||||||
return jsonify({'error': '请提供搜索关键词'}), 400
|
return jsonify({'error': '请提供搜索关键词'}), 400
|
||||||
|
|
||||||
|
# 检查是否有缓存
|
||||||
|
cached = None
|
||||||
|
if use_cache:
|
||||||
|
cached = db.get_search_cache(keyword, engine)
|
||||||
|
|
||||||
|
if cached:
|
||||||
|
return jsonify({
|
||||||
|
'success': True,
|
||||||
|
'keyword': keyword,
|
||||||
|
'engine': engine,
|
||||||
|
'results': cached['results'],
|
||||||
|
'count': cached['count'],
|
||||||
|
'cached': True,
|
||||||
|
'cached_at': cached['cached_at'],
|
||||||
|
'expires_at': cached['expires_at']
|
||||||
|
})
|
||||||
|
|
||||||
# 执行互联网搜索
|
# 执行互联网搜索
|
||||||
results = search_service.search_internet(keyword, max_results, engine)
|
results = search_service.search_internet(keyword, max_results, engine, use_cache, cache_days)
|
||||||
|
|
||||||
return jsonify({
|
return jsonify({
|
||||||
'success': True,
|
'success': True,
|
||||||
'keyword': keyword,
|
'keyword': keyword,
|
||||||
'engine': engine,
|
'engine': engine,
|
||||||
'results': results,
|
'results': results,
|
||||||
'count': len(results)
|
'count': len(results),
|
||||||
|
'cached': False
|
||||||
})
|
})
|
||||||
|
|
||||||
@bp.route('/internet-search-and-fetch', methods=['POST'])
|
@bp.route('/internet-search-and-fetch', methods=['POST'])
|
||||||
|
|||||||
@@ -33,7 +33,7 @@ class SearchService:
|
|||||||
)
|
)
|
||||||
return result.stdout, result.stderr, result.returncode
|
return result.stdout, result.stderr, result.returncode
|
||||||
|
|
||||||
def search_internet(self, keyword, max_results=None, engine='bing_cn'):
|
def search_internet(self, keyword, max_results=None, engine='bing_cn', use_cache=True, cache_days=7):
|
||||||
"""
|
"""
|
||||||
从互联网搜索(使用 agent-browser 浏览器自动化)
|
从互联网搜索(使用 agent-browser 浏览器自动化)
|
||||||
|
|
||||||
@@ -42,10 +42,21 @@ class SearchService:
|
|||||||
- bing_global: Bing 国际版
|
- bing_global: Bing 国际版
|
||||||
- google: Google
|
- google: Google
|
||||||
- baidu: 百度
|
- baidu: 百度
|
||||||
|
|
||||||
|
参数:
|
||||||
|
- use_cache: 是否使用缓存(默认True)
|
||||||
|
- cache_days: 缓存有效天数(默认7天)
|
||||||
"""
|
"""
|
||||||
max_results = max_results or self.max_results
|
max_results = max_results or self.max_results
|
||||||
results = []
|
results = []
|
||||||
|
|
||||||
|
# 优先查询缓存
|
||||||
|
if use_cache:
|
||||||
|
cached = db.get_search_cache(keyword, engine)
|
||||||
|
if cached:
|
||||||
|
print(f"使用缓存结果: {keyword} ({engine})")
|
||||||
|
return cached['results']
|
||||||
|
|
||||||
# 根据搜索引擎选择 URL
|
# 根据搜索引擎选择 URL
|
||||||
encoded_keyword = urllib.parse.quote(keyword)
|
encoded_keyword = urllib.parse.quote(keyword)
|
||||||
search_urls = {
|
search_urls = {
|
||||||
@@ -74,7 +85,7 @@ class SearchService:
|
|||||||
return results
|
return results
|
||||||
|
|
||||||
# 3. 解析 JSON 提取搜索结果
|
# 3. 解析 JSON 提取搜索结果
|
||||||
try:
|
try {
|
||||||
data = json.loads(stdout)
|
data = json.loads(stdout)
|
||||||
except json.JSONDecodeError:
|
except json.JSONDecodeError:
|
||||||
print(f"解析 JSON 失败: {stdout[:500]}")
|
print(f"解析 JSON 失败: {stdout[:500]}")
|
||||||
@@ -89,6 +100,10 @@ class SearchService:
|
|||||||
# 5. 关闭浏览器
|
# 5. 关闭浏览器
|
||||||
self._run_browser('close')
|
self._run_browser('close')
|
||||||
|
|
||||||
|
# 6. 保存到缓存
|
||||||
|
if results and use_cache:
|
||||||
|
db.save_search_cache(keyword, engine, results, cache_days)
|
||||||
|
|
||||||
except subprocess.TimeoutExpired:
|
except subprocess.TimeoutExpired:
|
||||||
print(f"搜索超时: {keyword}")
|
print(f"搜索超时: {keyword}")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
|||||||
+10
-1
@@ -23,6 +23,7 @@ async function doSearch() {
|
|||||||
const keyword = document.getElementById('search-keyword').value.trim();
|
const keyword = document.getElementById('search-keyword').value.trim();
|
||||||
const maxResults = parseInt(document.getElementById('search-count').value) || 10;
|
const maxResults = parseInt(document.getElementById('search-count').value) || 10;
|
||||||
const engine = document.getElementById('search-engine').value;
|
const engine = document.getElementById('search-engine').value;
|
||||||
|
const useCache = document.getElementById('use-cache').checked;
|
||||||
const autoFetch = document.getElementById('auto-fetch').checked;
|
const autoFetch = document.getElementById('auto-fetch').checked;
|
||||||
const autoSave = document.getElementById('auto-save').checked;
|
const autoSave = document.getElementById('auto-save').checked;
|
||||||
|
|
||||||
@@ -51,7 +52,7 @@ async function doSearch() {
|
|||||||
const response = await fetch(`${API_BASE}/api/articles/internet-search`, {
|
const response = await fetch(`${API_BASE}/api/articles/internet-search`, {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
headers: { 'Content-Type': 'application/json' },
|
headers: { 'Content-Type': 'application/json' },
|
||||||
body: JSON.stringify({ keyword, max_results: maxResults, engine })
|
body: JSON.stringify({ keyword, max_results: maxResults, engine, use_cache: useCache })
|
||||||
});
|
});
|
||||||
|
|
||||||
const data = await response.json();
|
const data = await response.json();
|
||||||
@@ -65,7 +66,15 @@ async function doSearch() {
|
|||||||
}));
|
}));
|
||||||
|
|
||||||
displayResults();
|
displayResults();
|
||||||
|
|
||||||
|
// 显示缓存状态
|
||||||
|
if (data.cached) {
|
||||||
|
updateStatus('success', `找到 ${searchResults.length} 条结果(使用缓存)`);
|
||||||
|
showToast('使用缓存结果', 'success');
|
||||||
|
} else {
|
||||||
updateStatus('success', `找到 ${searchResults.length} 条结果`);
|
updateStatus('success', `找到 ${searchResults.length} 条结果`);
|
||||||
|
}
|
||||||
|
|
||||||
document.getElementById('results-count').textContent = `${searchResults.length} 条结果`;
|
document.getElementById('results-count').textContent = `${searchResults.length} 条结果`;
|
||||||
document.getElementById('save-all-btn').disabled = searchResults.length === 0;
|
document.getElementById('save-all-btn').disabled = searchResults.length === 0;
|
||||||
|
|
||||||
|
|||||||
@@ -42,6 +42,10 @@
|
|||||||
</button>
|
</button>
|
||||||
</div>
|
</div>
|
||||||
<div class="search-options">
|
<div class="search-options">
|
||||||
|
<label class="checkbox-label">
|
||||||
|
<input type="checkbox" id="use-cache" checked>
|
||||||
|
<span>使用缓存结果(7天有效)</span>
|
||||||
|
</label>
|
||||||
<label class="checkbox-label">
|
<label class="checkbox-label">
|
||||||
<input type="checkbox" id="auto-fetch" checked>
|
<input type="checkbox" id="auto-fetch" checked>
|
||||||
<span>搜索后自动抓取内容</span>
|
<span>搜索后自动抓取内容</span>
|
||||||
|
|||||||
Reference in New Issue
Block a user