添加抓取失败网址记录功能

- 新增 failed_urls 数据库表存储失败URL
- 抓取失败时自动记录URL、标题、错误信息
- 搜索页面显示失败URL列表
- 支持重试单个/全部失败URL
- 支持删除和清空失败记录
- 显示重试次数和时间

models/database.py

@@ -106,6 +106,21 @@ class Database:
                 )
             ''')
             
+            # 失败的URL记录表
+            cursor.execute('''
+                CREATE TABLE IF NOT EXISTS failed_urls (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    url TEXT NOT NULL,
+                    title TEXT,
+                    error_message TEXT,
+                    retry_count INTEGER DEFAULT 0,
+                    status TEXT DEFAULT 'failed',
+                    created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+                    last_retry_at DATETIME,
+                    source TEXT DEFAULT 'search'
+                )
+            ''')
+            
             conn.commit()
 
 # ========== 内容库操作 ==========
@@ -309,5 +324,77 @@ class Database:
             ''', (key, value))
             conn.commit()
 
+# ========== 失败URL操作 ==========
+    def add_failed_url(self, url, title=None, error_message=None, source='search'):
+        """添加失败的URL"""
+        with self.get_connection() as conn:
+            cursor = conn.cursor()
+            # 先检查是否已存在
+            cursor.execute('SELECT id, retry_count FROM failed_urls WHERE url = ?', (url,))
+            existing = cursor.fetchone()
+            
+            if existing:
+                # 更新重试次数和错误信息
+                cursor.execute('''
+                    UPDATE failed_urls 
+                    SET error_message = ?, last_retry_at = CURRENT_TIMESTAMP, retry_count = retry_count + 1
+                    WHERE url = ?
+                ''', (error_message, url))
+            else:
+                # 新增失败记录
+                cursor.execute('''
+                    INSERT INTO failed_urls (url, title, error_message, source)
+                    VALUES (?, ?, ?, ?)
+                ''', (url, title, error_message, source))
+            conn.commit()
+            return cursor.lastrowid
+    
+    def get_failed_urls(self, limit=100, status='failed'):
+        """获取失败的URL列表"""
+        with self.get_connection() as conn:
+            cursor = conn.cursor()
+            cursor.execute('''
+                SELECT * FROM failed_urls 
+                WHERE status = ?
+                ORDER BY created_at DESC 
+                LIMIT ?
+            ''', (status, limit))
+            return [dict(row) for row in cursor.fetchall()]
+    
+    def get_failed_url_count(self):
+        """获取失败URL数量"""
+        with self.get_connection() as conn:
+            cursor = conn.cursor()
+            cursor.execute('SELECT COUNT(*) FROM failed_urls WHERE status = "failed"')
+            return cursor.fetchone()[0]
+    
+    def mark_url_success(self, url):
+        """标记URL为成功（已处理）"""
+        with self.get_connection() as conn:
+            cursor = conn.cursor()
+            cursor.execute('''
+                UPDATE failed_urls 
+                SET status = 'success', last_retry_at = CURRENT_TIMESTAMP
+                WHERE url = ?
+            ''', (url,))
+            conn.commit()
+            return cursor.rowcount > 0
+    
+    def delete_failed_url(self, url_id):
+        """删除失败URL记录"""
+        with self.get_connection() as conn:
+            cursor = conn.cursor()
+            cursor.execute('DELETE FROM failed_urls WHERE id = ?', (url_id,))
+            conn.commit()
+            return cursor.rowcount > 0
+    
+    def clear_failed_urls(self):
+        """清空所有失败URL记录"""
+        with self.get_connection() as conn:
+            cursor = conn.cursor()
+            cursor.execute('DELETE FROM failed_urls WHERE status = "failed"')
+            conn.commit()
+            return cursor.rowcount
+
 # 全局数据库实例
 db = Database()

routes/articles.py

@@ -190,3 +190,87 @@ def internet_search_and_fetch():
         'results': fetched_results,
         'count': len(fetched_results)
     })
+
+# ========== 失败URL管理 ==========
+
+@bp.route('/failed-urls', methods=['GET'])
+def get_failed_urls():
+    """获取失败的URL列表"""
+    limit = request.args.get('limit', 100, type=int)
+    urls = db.get_failed_urls(limit=limit)
+    return jsonify({
+        'success': True,
+        'urls': urls,
+        'count': len(urls)
+    })
+
+@bp.route('/failed-urls', methods=['POST'])
+def add_failed_url():
+    """记录失败的URL"""
+    data = request.get_json()
+    url = data.get('url')
+    title = data.get('title')
+    error_message = data.get('error_message')
+    source = data.get('source', 'search')
+    
+    if not url:
+        return jsonify({'error': '请提供URL'}), 400
+    
+    url_id = db.add_failed_url(url, title, error_message, source)
+    return jsonify({
+        'success': True,
+        'url_id': url_id,
+        'message': '失败URL已记录'
+    })
+
+@bp.route('/failed-urls/count', methods=['GET'])
+def get_failed_url_count():
+    """获取失败URL数量"""
+    count = db.get_failed_url_count()
+    return jsonify({
+        'success': True,
+        'count': count
+    })
+
+@bp.route('/failed-urls/<int:url_id>', methods=['DELETE'])
+def delete_failed_url(url_id):
+    """删除失败URL记录"""
+    success = db.delete_failed_url(url_id)
+    if success:
+        return jsonify({'success': True, 'message': '记录已删除'})
+    else:
+        return jsonify({'error': '记录不存在'}), 404
+
+@bp.route('/failed-urls/clear', methods=['POST'])
+def clear_failed_urls():
+    """清空所有失败URL记录"""
+    count = db.clear_failed_urls()
+    return jsonify({
+        'success': True,
+        'message': f'已清空 {count} 条记录'
+    })
+
+@bp.route('/failed-urls/retry', methods=['POST'])
+def retry_failed_url():
+    """重试失败的URL"""
+    data = request.get_json()
+    url = data.get('url')
+    
+    if not url:
+        return jsonify({'error': '请提供URL'}), 400
+    
+    # 尝试抓取
+    result = search_service.fetch_url_content(url)
+    
+    if result and result.get('content'):
+        # 成功，标记为已处理
+        db.mark_url_success(url)
+        return jsonify({
+            'success': True,
+            'message': '抓取成功',
+            'data': result
+        })
+    else:
+        # 仍然失败
+        db.add_failed_url(url, error_message='重试失败')
+        return jsonify({'error': '抓取仍然失败'}), 500

static/css/search.css

@@ -352,3 +352,86 @@
         justify-content: space-between;
     }
 }
+
+/* 失败URL区域 */
+.failed-urls-section {
+    background: white;
+    border-radius: 12px;
+    box-shadow: 0 2px 4px rgba(0,0,0,0.05);
+    margin-top: 20px;
+}
+
+.failed-urls-section .panel-header {
+    background: #fef3c7;
+}
+
+.failed-urls-section .panel-header h2 {
+    color: #92400e;
+}
+
+.failed-count {
+    color: #92400e;
+    font-weight: bold;
+}
+
+.failed-urls-list {
+    display: flex;
+    flex-direction: column;
+    gap: 10px;
+}
+
+.failed-url-item {
+    display: flex;
+    justify-content: space-between;
+    align-items: flex-start;
+    padding: 15px;
+    border: 1px solid #fcd34d;
+    border-radius: 8px;
+    background: #fffbeb;
+}
+
+.failed-url-item.loading {
+    opacity: 0.5;
+}
+
+.failed-url-info {
+    flex: 1;
+}
+
+.failed-url-title {
+    font-weight: 500;
+    color: #333;
+    margin-bottom: 5px;
+}
+
+.failed-url-detail {
+    font-size: 13px;
+    margin-bottom: 5px;
+}
+
+.failed-url-detail a {
+    color: #3b82f6;
+    text-decoration: none;
+}
+
+.failed-url-detail a:hover {
+    text-decoration: underline;
+}
+
+.failed-error {
+    color: #dc2626;
+    margin-left: 10px;
+    font-size: 12px;
+}
+
+.failed-url-meta {
+    font-size: 12px;
+    color: #666;
+    display: flex;
+    gap: 15px;
+}
+
+.failed-url-actions {
+    display: flex;
+    gap: 8px;
+}

static/js/search.js

@@ -13,6 +13,9 @@ document.addEventListener('DOMContentLoaded', () => {
             doSearch();
         }
     });
+    
+    // 加载失败URL
+    loadFailedUrls();
 });
 
 // 执行搜索
@@ -199,17 +202,36 @@ async function fetchResult(index) {
             showToast('抓取成功', 'success');
             displayResults();
         } else {
+            // 记录失败URL
+            await recordFailedUrl(result.url, result.title, data.error);
             btn.disabled = false;
             btn.innerHTML = '<i class="ri-download-line"></i> 抓取';
+            btn.className = 'btn btn-sm btn-danger';
             showToast('抓取失败: ' + data.error, 'error');
         }
     } catch (error) {
+        // 记录失败URL
+        await recordFailedUrl(result.url, result.title, '抓取出错');
         btn.disabled = false;
         btn.innerHTML = '<i class="ri-download-line"></i> 抓取';
+        btn.className = 'btn btn-sm btn-danger';
         showToast('抓取出错', 'error');
     }
 }
 
+// 记录失败的URL
+async function recordFailedUrl(url, title, errorMessage) {
+    try {
+        await fetch(`${API_BASE}/api/articles/failed-urls`, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({ url, title, error_message: errorMessage })
+        });
+    } catch (error) {
+        console.error('记录失败URL出错:', error);
+    }
+}
+
 // 保存单个结果
 async function saveResult(index) {
     const result = searchResults[index];
@@ -432,3 +454,120 @@ function escapeHtml(text) {
     div.textContent = text;
     return div.innerHTML;
 }
+
+// ========== 失败URL管理 ==========
+
+// 加载失败URL列表
+async function loadFailedUrls() {
+    try {
+        const response = await fetch(`${API_BASE}/api/articles/failed-urls`);
+        const data = await response.json();
+        
+        if (data.success) {
+            document.getElementById('failed-count').textContent = `${data.count} 条`;
+            
+            const container = document.getElementById('failed-urls-list');
+            
+            if (data.urls.length === 0) {
+                container.innerHTML = '<div class="empty-text">暂无失败记录</div>';
+            } else {
+                container.innerHTML = data.urls.map(url => `
+                    <div class="failed-url-item" id="failed-${url.id}">
+                        <div class="failed-url-info">
+                            <div class="failed-url-title">${escapeHtml(url.title || url.url.substring(0, 50))}</div>
+                            <div class="failed-url-detail">
+                                <a href="${escapeHtml(url.url)}" target="_blank">
+                                    <i class="ri-external-link-line"></i> ${escapeHtml(url.url.substring(0, 60))}${url.url.length > 60 ? '...' : ''}
+                                </a>
+                                <span class="failed-error">${escapeHtml(url.error_message || '未知错误')}</span>
+                            </div>
+                            <div class="failed-url-meta">
+                                <span>重试: ${url.retry_count || 0} 次</span>
+                                <span>${url.created_at || ''}</span>
+                            </div>
+                        </div>
+                        <div class="failed-url-actions">
+                            <button onclick="retryFailedUrl(${url.id}, '${escapeHtml(url.url)}')" class="btn btn-sm btn-warning">
+                                <i class="ri-restart-line"></i> 重试
+                            </button>
+                            <button onclick="deleteFailedUrl(${url.id})" class="btn btn-sm btn-danger">
+                                <i class="ri-delete-bin-line"></i>
+                            </button>
+                        </div>
+                    </div>
+                `).join('');
+            }
+        }
+    } catch (error) {
+        console.error('加载失败URL出错:', error);
+    }
+}
+
+// 重试单个失败URL
+async function retryFailedUrl(urlId, url) {
+    const item = document.getElementById(`failed-${urlId}`);
+    item.classList.add('loading');
+    
+    try {
+        const response = await fetch(`${API_BASE}/api/articles/failed-urls/retry`, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({ url })
+        });
+        
+        const data = await response.json();
+        
+        if (data.success) {
+            showToast('重试成功', 'success');
+            loadFailedUrls();
+        } else {
+            showToast('重试失败: ' + data.error, 'error');
+            item.classList.remove('loading');
+        }
+    } catch (error) {
+        showToast('重试出错', 'error');
+        item.classList.remove('loading');
+    }
+}
+
+// 全部重试
+async function retryAllFailed() {
+    showToast('正在重试所有失败URL...', '');
+    
+    const response = await fetch(`${API_BASE}/api/articles/failed-urls`);
+    const data = await response.json();
+    
+    if (data.success && data.urls.length > 0) {
+        for (const url of data.urls) {
+            await retryFailedUrl(url.id, url.url);
+            await new Promise(r => setTimeout(r, 500)); // 避免太快
+        }
+    }
+}
+
+// 删除失败URL记录
+async function deleteFailedUrl(urlId) {
+    try {
+        await fetch(`${API_BASE}/api/articles/failed-urls/${urlId}`, {
+            method: 'DELETE'
+        });
+        loadFailedUrls();
+    } catch (error) {
+        showToast('删除失败', 'error');
+    }
+}
+
+// 清空所有失败URL
+async function clearFailedUrls() {
+    if (!confirm('确定要清空所有失败记录吗？')) return;
+    
+    try {
+        await fetch(`${API_BASE}/api/articles/failed-urls/clear`, {
+            method: 'POST'
+        });
+        showToast('已清空', 'success');
+        loadFailedUrls();
+    } catch (error) {
+        showToast('清空失败', 'error');
+    }
+}

templates/search.html

@@ -82,6 +82,30 @@
             </div>
         </div>
 
+        <!-- 失败URL区域 -->
+        <div class="panel failed-urls-section">
+            <div class="panel-header">
+                <h2><i class="ri-error-warning-line"></i> 抓取失败的网址</h2>
+                <div class="panel-actions">
+                    <span id="failed-count" class="failed-count">0 条</span>
+                    <button onclick="loadFailedUrls()" class="btn btn-secondary btn-sm">
+                        <i class="ri-refresh-line"></i> 刷新
+                    </button>
+                    <button onclick="retryAllFailed()" class="btn btn-warning btn-sm">
+                        <i class="ri-restart-line"></i> 全部重试
+                    </button>
+                    <button onclick="clearFailedUrls()" class="btn btn-danger btn-sm">
+                        <i class="ri-delete-bin-line"></i> 清空
+                    </button>
+                </div>
+            </div>
+            <div class="panel-body">
+                <div id="failed-urls-list" class="failed-urls-list">
+                    <div class="empty-text">暂无失败记录</div>
+                </div>
+            </div>
+        </div>
+
         <!-- 保存进度 -->
         <div id="save-progress" class="save-progress" style="display: none;">
             <div class="progress-info">