From c40acab1c980f057256f06388b57e1aed357e63c Mon Sep 17 00:00:00 2001
From: hz4th_coder <hz4th_coder@tphai.com>
Date: Mon, 13 Jul 2026 23:11:54 +0800
Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96URL=E6=8A=93=E5=8F=96?=
 =?UTF-8?q?=EF=BC=9A=E5=A2=9E=E5=8A=A0=E8=B6=85=E6=97=B6=E6=97=B6=E9=97=B4?=
 =?UTF-8?q?=E5=92=8C=E5=A4=87=E7=94=A8=E6=96=B9=E6=A1=88?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- 浏览器超时从20秒增加到60秒
- 浏览器失败时使用 requests 备用方案抓取静态内容
- 解决部分网站加载慢导致超时的问题
---
 services/search_service.py | 53 +++++++++++++++++++++++++++++++++-----
 1 file changed, 47 insertions(+), 6 deletions(-)

diff --git a/services/search_service.py b/services/search_service.py
index b416bd5..be5a40c 100644
--- a/services/search_service.py
+++ b/services/search_service.py
@@ -224,16 +224,17 @@ class SearchService:
         return None
     
     def fetch_url_content(self, url):
-        """抓取网页内容（使用 agent-browser 浏览器方式，绑过反爬虫）"""
+        """抓取网页内容（使用 agent-browser 浏览器方式，绕过反爬虫）"""
         try:
-            # 使用浏览器方式抓取
-            stdout, stderr, code = self._run_browser('open', url, '--timeout', '20000')
+            # 使用浏览器方式抓取，增加超时时间到60秒
+            stdout, stderr, code = self._run_browser('open', url, '--timeout', '60000')
             if code != 0:
                 print(f"打开页面失败: {stderr}")
-                return None
+                # 浏览器失败，尝试使用 requests 备用方案
+                return self._fetch_with_requests(url)
             
-            # 等待页面加载
-            self._run_browser('wait', '5000')
+            # 等待页面加载（增加到10秒）
+            self._run_browser('wait', '10000')
             
             # 获取页面标题
             stdout, stderr, code = self._run_browser('get', 'title', '--timeout', '5000')
@@ -309,6 +310,46 @@ class SearchService:
         
         return result
     
+    def _fetch_with_requests(self, url):
+        """备用方案：使用 requests 抓取静态内容"""
+        try:
+            headers = {
+                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+                'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+                'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8'
+            }
+            response = requests.get(url, headers=headers, timeout=30)
+            response.raise_for_status()
+            
+            soup = BeautifulSoup(response.text, 'html.parser')
+            
+            # 获取标题
+            title = soup.title.string.strip() if soup.title else ''
+            
+            # 移除不需要的标签
+            for tag in soup(['script', 'style', 'nav', 'footer', 'header', 'aside']):
+                tag.decompose()
+            
+            # 获取主要内容
+            text = soup.get_text(separator='\n', strip=True)
+            # 清理多余空白行
+            lines = [line.strip() for line in text.split('\n') if line.strip()]
+            text = '\n'.join(lines)
+            
+            # 提取描述（前200字符）
+            description = text[:200].strip() if text else ''
+            
+            return {
+                'title': title,
+                'description': description,
+                'content': text,
+                'url': url,
+                'fetch_date': datetime.now().isoformat()
+            }
+        except Exception as e:
+            print(f"备用抓取失败: {url}, 错误: {str(e)}")
+            return None
+    
     def search_articles(self, keyword, category=None):
         """从内容库搜索"""
         return db.search_articles(keyword, category)