v1.4.0: 通知日志分页+筛选 / 仪表盘TOP10+更多链接 / 数据源走web-capture-api抓取(获取方式与参数可编辑) / 系统错误邮件通知(频率+静默时段)

crawler.py

@@ -10,6 +10,7 @@
 数据结构与 simulate 保持一致：fetch_source(source) -> list[dict]
   （title/url/content/summary/published_at/domain/entities/source_id/full_text）
 """
+import json
 import re
 import time
 from datetime import datetime
@@ -221,15 +222,92 @@ def extract_links(html, base_url, max_n=10):
             for c in cands[:max_n]]
 
 
+# ---------------- web-capture-api 集成（数据源网页抓取） ----------------
+
+def _webcapture_cfg():
+    cfg = dict(config.WEBCAPTURE_DEFAULTS)
+    cfg.update(db.get_all_settings().get("webcapture", {}))
+    return cfg
+
+
+def _source_capture_params(source):
+    """解析数据源的抓取参数（capture_params JSON -> dict）"""
+    raw = source.get("capture_params") or "{}"
+    try:
+        p = json.loads(raw)
+    except Exception:
+        p = {}
+    return p if isinstance(p, dict) else {}
+
+
+def _call_webcapture(url, action, params=None):
+    """调用 web-capture-api 抓取网页（html/text）。失败抛异常。
+    返回: {"success": True, "title": ..., "html"|"text": ...}
+    """
+    cfg = _webcapture_cfg()
+    base = (cfg.get("api_url") or "").rstrip("/")
+    if not base:
+        raise RuntimeError("web-capture-api 地址未配置（设置页可修改）")
+    payload = {"url": url, "action": action}
+    p = params or {}
+    for k in ("wait_time", "scroll_times", "scroll_delay", "full_page", "backend", "viewport"):
+        if k in p and p[k] not in (None, ""):
+            payload[k] = p[k]
+    r = requests.post(f"{base}/api/capture", json=payload,
+                      timeout=int(cfg.get("timeout", 60)))
+    r.raise_for_status()
+    data = r.json()
+    if not data.get("success"):
+        raise RuntimeError(data.get("error", "web-capture-api 返回失败"))
+    return data
+
+
 # ---------------- 按源采集 ----------------
 
 def _now():
     return datetime.now().strftime("%Y-%m-%d %H:%M:%S")
 
 
+def _fetch_list_via_direct(url):
+    """直接抓取（requests + bs4）：返回 (title, page_text, links)"""
+    per = config.CRAWL_DEFAULTS["per_source_links"]
+    html = fetch_page(url)
+    page_title, page_text = clean_html(html, url)
+    links = extract_links(html, url, max_n=per)
+    return page_title, page_text, links
+
+
+def _fetch_list_via_webcapture(url, params):
+    """走 web-capture-api 抓取列表页：返回 (title, page_text, links)"""
+    per = config.CRAWL_DEFAULTS["per_source_links"]
+    action = (params.get("action") or "html").strip() or "html"
+    data = _call_webcapture(url, action, params)
+    title = data.get("title") or url
+    if action == "text":
+        # 只取正文，无法提取子链接 → 整页作为一条
+        return title, (data.get("text") or ""), []
+    html = data.get("html") or ""
+    page_text = (data.get("text") or "").strip()
+    if not page_text and html:
+        page_text = clean_html(html, url)[1]
+    links = extract_links(html, url, max_n=per) if html else []
+    return title, page_text, links
+
+
+def _fetch_full_via_direct(url):
+    return clean_html(fetch_page(url), url)[1]
+
+
+def _fetch_full_via_webcapture(url, params):
+    data = _call_webcapture(url, "text", params)
+    return data.get("text") or ""
+
+
 def fetch_source(source):
     """采集单个数据源 -> list[item]。
-    example.com 占位源返回模拟数据；真实源抓取失败抛异常（由 fetch_all 捕获并标记 error，不塞模拟数据）。
+    example.com 占位源返回模拟数据；真实源抓取失败抛异常（由调用方捕获并标记 error）。
+    fetch_method：auto=优先 web-capture-api、失败回退直接抓取；webcapture=仅 web-capture-api；
+    direct=直接抓取(requests+bs4)。各源可单独配置 capture_params 抓取参数。
     """
     url = (source.get("url") or "").strip()
     # 模拟源（example.com 占位）→ 用仿真数据填充（补齐 source_id，保证定制监控识别正确）
@@ -238,19 +316,38 @@ def fetch_source(source):
         for it in items:
             it["source_id"] = source["id"]
         return items
+    method = (source.get("fetch_method") or "auto").strip() or "auto"
     per = config.CRAWL_DEFAULTS["per_source_links"]
     full = config.CRAWL_DEFAULTS["full_fetch_links"]
-    html = fetch_page(url)
-    page_title, page_text = clean_html(html, url)
-    links = extract_links(html, url, max_n=per)
+    params = _source_capture_params(source)
+
+    # 抓取列表页（标题 + 正文 + 候选链接）
+    if method == "direct":
+        page_title, page_text, links = _fetch_list_via_direct(url)
+    else:
+        try:
+            page_title, page_text, links = _fetch_list_via_webcapture(url, params)
+        except Exception:
+            if method == "webcapture":
+                raise
+            # auto：回退直接抓取
+            page_title, page_text, links = _fetch_list_via_direct(url)
+
     items = []
     for i, lk in enumerate(links[:per]):
         full_text = ""
         content = lk["summary"]
         if i < full and lk["url"]:
             try:
-                h2 = fetch_page(lk["url"])
-                _, full_text = clean_html(h2, lk["url"])
+                if method == "direct":
+                    full_text = _fetch_full_via_direct(lk["url"])
+                else:
+                    try:
+                        full_text = _fetch_full_via_webcapture(lk["url"], params)
+                    except Exception:
+                        if method == "webcapture":
+                            raise
+                        full_text = _fetch_full_via_direct(lk["url"])
                 if not content or len(content) < len(full_text):
                     content = full_text
             except Exception:
@@ -268,7 +365,7 @@ def fetch_source(source):
             "full_text": full_text or "",
         })
     if not items and page_text:
-        # 页面本身即正文（如单篇/无链接页）→ 整页作为一条
+        # 页面本身即正文（如单篇/无链接页/action=text）→ 整页作为一条
         items.append({
             "title": page_title,
             "url": url,