diff --git a/models/database.py b/models/database.py index dc99791..f0cdc0c 100644 --- a/models/database.py +++ b/models/database.py @@ -106,6 +106,21 @@ class Database: ) ''') + # 失败的URL记录表 + cursor.execute(''' + CREATE TABLE IF NOT EXISTS failed_urls ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + url TEXT NOT NULL, + title TEXT, + error_message TEXT, + retry_count INTEGER DEFAULT 0, + status TEXT DEFAULT 'failed', + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + last_retry_at DATETIME, + source TEXT DEFAULT 'search' + ) + ''') + conn.commit() # ========== 内容库操作 ========== @@ -309,5 +324,77 @@ class Database: ''', (key, value)) conn.commit() +# ========== 失败URL操作 ========== + def add_failed_url(self, url, title=None, error_message=None, source='search'): + """添加失败的URL""" + with self.get_connection() as conn: + cursor = conn.cursor() + # 先检查是否已存在 + cursor.execute('SELECT id, retry_count FROM failed_urls WHERE url = ?', (url,)) + existing = cursor.fetchone() + + if existing: + # 更新重试次数和错误信息 + cursor.execute(''' + UPDATE failed_urls + SET error_message = ?, last_retry_at = CURRENT_TIMESTAMP, retry_count = retry_count + 1 + WHERE url = ? + ''', (error_message, url)) + else: + # 新增失败记录 + cursor.execute(''' + INSERT INTO failed_urls (url, title, error_message, source) + VALUES (?, ?, ?, ?) + ''', (url, title, error_message, source)) + conn.commit() + return cursor.lastrowid + + def get_failed_urls(self, limit=100, status='failed'): + """获取失败的URL列表""" + with self.get_connection() as conn: + cursor = conn.cursor() + cursor.execute(''' + SELECT * FROM failed_urls + WHERE status = ? + ORDER BY created_at DESC + LIMIT ? + ''', (status, limit)) + return [dict(row) for row in cursor.fetchall()] + + def get_failed_url_count(self): + """获取失败URL数量""" + with self.get_connection() as conn: + cursor = conn.cursor() + cursor.execute('SELECT COUNT(*) FROM failed_urls WHERE status = "failed"') + return cursor.fetchone()[0] + + def mark_url_success(self, url): + """标记URL为成功（已处理）""" + with self.get_connection() as conn: + cursor = conn.cursor() + cursor.execute(''' + UPDATE failed_urls + SET status = 'success', last_retry_at = CURRENT_TIMESTAMP + WHERE url = ? + ''', (url,)) + conn.commit() + return cursor.rowcount > 0 + + def delete_failed_url(self, url_id): + """删除失败URL记录""" + with self.get_connection() as conn: + cursor = conn.cursor() + cursor.execute('DELETE FROM failed_urls WHERE id = ?', (url_id,)) + conn.commit() + return cursor.rowcount > 0 + + def clear_failed_urls(self): + """清空所有失败URL记录""" + with self.get_connection() as conn: + cursor = conn.cursor() + cursor.execute('DELETE FROM failed_urls WHERE status = "failed"') + conn.commit() + return cursor.rowcount + # 全局数据库实例 db = Database() \ No newline at end of file diff --git a/routes/articles.py b/routes/articles.py index b5aac9b..2e4c141 100644 --- a/routes/articles.py +++ b/routes/articles.py @@ -189,4 +189,88 @@ def internet_search_and_fetch(): 'keyword': keyword, 'results': fetched_results, 'count': len(fetched_results) - }) \ No newline at end of file + }) + +# ========== 失败URL管理 ========== + +@bp.route('/failed-urls', methods=['GET']) +def get_failed_urls(): + """获取失败的URL列表""" + limit = request.args.get('limit', 100, type=int) + urls = db.get_failed_urls(limit=limit) + return jsonify({ + 'success': True, + 'urls': urls, + 'count': len(urls) + }) + +@bp.route('/failed-urls', methods=['POST']) +def add_failed_url(): + """记录失败的URL""" + data = request.get_json() + url = data.get('url') + title = data.get('title') + error_message = data.get('error_message') + source = data.get('source', 'search') + + if not url: + return jsonify({'error': '请提供URL'}), 400 + + url_id = db.add_failed_url(url, title, error_message, source) + return jsonify({ + 'success': True, + 'url_id': url_id, + 'message': '失败URL已记录' + }) + +@bp.route('/failed-urls/count', methods=['GET']) +def get_failed_url_count(): + """获取失败URL数量""" + count = db.get_failed_url_count() + return jsonify({ + 'success': True, + 'count': count + }) + +@bp.route('/failed-urls/', methods=['DELETE']) +def delete_failed_url(url_id): + """删除失败URL记录""" + success = db.delete_failed_url(url_id) + if success: + return jsonify({'success': True, 'message': '记录已删除'}) + else: + return jsonify({'error': '记录不存在'}), 404 + +@bp.route('/failed-urls/clear', methods=['POST']) +def clear_failed_urls(): + """清空所有失败URL记录""" + count = db.clear_failed_urls() + return jsonify({ + 'success': True, + 'message': f'已清空 {count} 条记录' + }) + +@bp.route('/failed-urls/retry', methods=['POST']) +def retry_failed_url(): + """重试失败的URL""" + data = request.get_json() + url = data.get('url') + + if not url: + return jsonify({'error': '请提供URL'}), 400 + + # 尝试抓取 + result = search_service.fetch_url_content(url) + + if result and result.get('content'): + # 成功，标记为已处理 + db.mark_url_success(url) + return jsonify({ + 'success': True, + 'message': '抓取成功', + 'data': result + }) + else: + # 仍然失败 + db.add_failed_url(url, error_message='重试失败') + return jsonify({'error': '抓取仍然失败'}), 500 \ No newline at end of file diff --git a/static/css/search.css b/static/css/search.css index 82e74d6..d7aa407 100644 --- a/static/css/search.css +++ b/static/css/search.css @@ -351,4 +351,87 @@ width: 100%; justify-content: space-between; } +} + +/* 失败URL区域 */ +.failed-urls-section { + background: white; + border-radius: 12px; + box-shadow: 0 2px 4px rgba(0,0,0,0.05); + margin-top: 20px; +} + +.failed-urls-section .panel-header { + background: #fef3c7; +} + +.failed-urls-section .panel-header h2 { + color: #92400e; +} + +.failed-count { + color: #92400e; + font-weight: bold; +} + +.failed-urls-list { + display: flex; + flex-direction: column; + gap: 10px; +} + +.failed-url-item { + display: flex; + justify-content: space-between; + align-items: flex-start; + padding: 15px; + border: 1px solid #fcd34d; + border-radius: 8px; + background: #fffbeb; +} + +.failed-url-item.loading { + opacity: 0.5; +} + +.failed-url-info { + flex: 1; +} + +.failed-url-title { + font-weight: 500; + color: #333; + margin-bottom: 5px; +} + +.failed-url-detail { + font-size: 13px; + margin-bottom: 5px; +} + +.failed-url-detail a { + color: #3b82f6; + text-decoration: none; +} + +.failed-url-detail a:hover { + text-decoration: underline; +} + +.failed-error { + color: #dc2626; + margin-left: 10px; + font-size: 12px; +} + +.failed-url-meta { + font-size: 12px; + color: #666; + display: flex; + gap: 15px; +} + +.failed-url-actions { + display: flex; + gap: 8px; } \ No newline at end of file diff --git a/static/js/search.js b/static/js/search.js index b35f601..20aa931 100644 --- a/static/js/search.js +++ b/static/js/search.js @@ -13,6 +13,9 @@ document.addEventListener('DOMContentLoaded', () => { doSearch(); } }); + + // 加载失败URL + loadFailedUrls(); }); // 执行搜索 @@ -199,17 +202,36 @@ async function fetchResult(index) { showToast('抓取成功', 'success'); displayResults(); } else { + // 记录失败URL + await recordFailedUrl(result.url, result.title, data.error); btn.disabled = false; btn.innerHTML = ' 抓取'; + btn.className = 'btn btn-sm btn-danger'; showToast('抓取失败: ' + data.error, 'error'); } } catch (error) { + // 记录失败URL + await recordFailedUrl(result.url, result.title, '抓取出错'); btn.disabled = false; btn.innerHTML = ' 抓取'; + btn.className = 'btn btn-sm btn-danger'; showToast('抓取出错', 'error'); } } +// 记录失败的URL +async function recordFailedUrl(url, title, errorMessage) { + try { + await fetch(`${API_BASE}/api/articles/failed-urls`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ url, title, error_message: errorMessage }) + }); + } catch (error) { + console.error('记录失败URL出错:', error); + } +} + // 保存单个结果 async function saveResult(index) { const result = searchResults[index]; @@ -431,4 +453,121 @@ function escapeHtml(text) { const div = document.createElement('div'); div.textContent = text; return div.innerHTML; +} + +// ========== 失败URL管理 ========== + +// 加载失败URL列表 +async function loadFailedUrls() { + try { + const response = await fetch(`${API_BASE}/api/articles/failed-urls`); + const data = await response.json(); + + if (data.success) { + document.getElementById('failed-count').textContent = `${data.count} 条`; + + const container = document.getElementById('failed-urls-list'); + + if (data.urls.length === 0) { + container.innerHTML = '
暂无失败记录
'; + } else { + container.innerHTML = data.urls.map(url => ` +
+
+
${escapeHtml(url.title || url.url.substring(0, 50))}
+
+ + ${escapeHtml(url.url.substring(0, 60))}${url.url.length > 60 ? '...' : ''} + + ${escapeHtml(url.error_message || '未知错误')} +
+
+ 重试: ${url.retry_count || 0} 次 + ${url.created_at || ''} +
+
+
+ + +
+
+ `).join(''); + } + } + } catch (error) { + console.error('加载失败URL出错:', error); + } +} + +// 重试单个失败URL +async function retryFailedUrl(urlId, url) { + const item = document.getElementById(`failed-${urlId}`); + item.classList.add('loading'); + + try { + const response = await fetch(`${API_BASE}/api/articles/failed-urls/retry`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ url }) + }); + + const data = await response.json(); + + if (data.success) { + showToast('重试成功', 'success'); + loadFailedUrls(); + } else { + showToast('重试失败: ' + data.error, 'error'); + item.classList.remove('loading'); + } + } catch (error) { + showToast('重试出错', 'error'); + item.classList.remove('loading'); + } +} + +// 全部重试 +async function retryAllFailed() { + showToast('正在重试所有失败URL...', ''); + + const response = await fetch(`${API_BASE}/api/articles/failed-urls`); + const data = await response.json(); + + if (data.success && data.urls.length > 0) { + for (const url of data.urls) { + await retryFailedUrl(url.id, url.url); + await new Promise(r => setTimeout(r, 500)); // 避免太快 + } + } +} + +// 删除失败URL记录 +async function deleteFailedUrl(urlId) { + try { + await fetch(`${API_BASE}/api/articles/failed-urls/${urlId}`, { + method: 'DELETE' + }); + loadFailedUrls(); + } catch (error) { + showToast('删除失败', 'error'); + } +} + +// 清空所有失败URL +async function clearFailedUrls() { + if (!confirm('确定要清空所有失败记录吗？')) return; + + try { + await fetch(`${API_BASE}/api/articles/failed-urls/clear`, { + method: 'POST' + }); + showToast('已清空', 'success'); + loadFailedUrls(); + } catch (error) { + showToast('清空失败', 'error'); + } } \ No newline at end of file diff --git a/templates/search.html b/templates/search.html index 2f0e195..bffbe4d 100644 --- a/templates/search.html +++ b/templates/search.html @@ -82,6 +82,30 @@ + +
+
+

抓取失败的网址

+
+ 0 条 + + + +
+
+
+
+
暂无失败记录
+
+
+
+