403 lines
15 KiB
Python
403 lines
15 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""
|
||
新闻智能跟踪系统 - 真实网页采集层
|
||
职责：
|
||
1. 抓取数据源页面（真实 URL），用 readability 风格启发式清洗出「干净可读正文」
|
||
2. 从页面提取候选资讯链接（标题 + 绝对 URL），按文章相似度排序
|
||
3. 对前 N 条候选链接抓取全文，一并入库（存 articles.full_text 便于后期查看）
|
||
4. example.com 等模拟源 / 抓取失败源，回退到 simulate 模拟数据，保证全链路不空转
|
||
|
||
数据结构与 simulate 保持一致：fetch_source(source) -> list[dict]
|
||
（title/url/content/summary/published_at/domain/entities/source_id/full_text）
|
||
"""
|
||
import json
|
||
import re
|
||
import time
|
||
from datetime import datetime
|
||
from urllib.parse import urljoin, urlparse
|
||
|
||
import requests
|
||
from bs4 import BeautifulSoup
|
||
|
||
import config
|
||
import db
|
||
import simulate
|
||
|
||
# 直接丢弃的标签（导航/脚本/广告/评论等）
|
||
_BAD_TAGS = [
|
||
"script", "style", "noscript", "iframe", "form", "button", "select", "option",
|
||
"input", "svg", "canvas", "video", "audio", "nav", "footer", "header", "aside",
|
||
]
|
||
# 出现在 class/id 中即视为噪音块的片段
|
||
_BAD_HINTS = [
|
||
"comment", "share", "social", "advert", "ads", "menu", "sidebar", "footer",
|
||
"nav", "related", "recommend", "promo", "newsletter", "subscribe",
|
||
"cookie", "banner", "breadcrumb", "pagination", "toolbar", "login", "signup",
|
||
"tag-list", "author", "meta", "bottom", "widget", "popup", "overlay",
|
||
]
|
||
# 不是资讯链接的后缀/前缀
|
||
_BAD_HREFS = ("mailto:", "tel:", "javascript:", "#", "?login", "?signup")
|
||
_BAD_EXTS = (".css", ".js", ".ico", ".png", ".jpg", ".jpeg", ".gif", ".webp",
|
||
".pdf", ".zip", ".xml", ".rss", ".atom", ".json")
|
||
# 常见导航词（短标题/纯导航链接，不当作资讯）
|
||
_NAV_WORDS = {
|
||
"research", "business", "developers", "about", "careers", "blog", "contact",
|
||
"sign in", "sign up", "login", "register", "privacy", "terms", "legal", "trust",
|
||
"customer stories", "partners", "docs", "api log in", "skip to main content",
|
||
"home", "news", "newsroom", "press", "company", "safety", "product", "products",
|
||
"pricing", "solutions", "learn more", "read more", "view all", "see all",
|
||
"all news", "all stories", "open a new window", "instagram", "twitter", "facebook",
|
||
"linkedin", "youtube", "github", "login", "log in", "get started", "try now",
|
||
}
|
||
# 标题尾部常见的“分类 + 日期”后缀，清洗掉（如 "Company Aug 27, 2026"）
|
||
_TRAIL_DATE_RE = re.compile(r"\s+\S+\s+[A-Z][a-z]{2}\s+\d{1,2},?\s+\d{4}$")
|
||
|
||
|
||
def fetch_page(url, timeout=None, retries=2):
|
||
"""抓取页面 HTML，带浏览器 UA 与语言头；瞬断自动重试"""
|
||
hdr = {
|
||
"User-Agent": config.CRAWL_DEFAULTS["user_agent"],
|
||
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||
}
|
||
last = None
|
||
for i in range(retries + 1):
|
||
try:
|
||
r = requests.get(url, headers=hdr,
|
||
timeout=timeout or config.CRAWL_DEFAULTS["crawl_timeout"])
|
||
r.raise_for_status()
|
||
return r.text
|
||
except Exception as e:
|
||
last = e
|
||
if i < retries:
|
||
time.sleep(1.5 * (i + 1))
|
||
raise last
|
||
|
||
|
||
# ---------------- 正文清洗（readability 风格） ----------------
|
||
|
||
def _is_noise(tag):
|
||
if tag is None or not hasattr(tag, "name") or tag.name is None:
|
||
return True
|
||
if tag.name in _BAD_TAGS:
|
||
return True
|
||
try:
|
||
tokens = []
|
||
for t in tag.get("class") or []:
|
||
# Tailwind 任意值类（如 [--header-h:...]）只是 CSS 变量，语义噪音，跳过
|
||
if t.startswith("[") and t.endswith("]"):
|
||
continue
|
||
tokens.append(t.lower())
|
||
tokens.append((tag.get("id") or "").lower())
|
||
cls = " ".join(tokens)
|
||
except Exception:
|
||
cls = ""
|
||
return any(h in cls for h in _BAD_HINTS)
|
||
|
||
|
||
def _strip_noise(soup):
|
||
"""删除噪音节点，返回清洗后的 body。先收集再统一删除，避免迭代中改树。"""
|
||
noise = []
|
||
for tag in list(soup.find_all(True)):
|
||
try:
|
||
if tag is None:
|
||
continue
|
||
if _is_noise(tag):
|
||
noise.append(tag)
|
||
continue
|
||
style = (tag.get("style") or "").lower()
|
||
if "display:none" in style or "visibility:hidden" in style:
|
||
noise.append(tag)
|
||
except Exception:
|
||
continue
|
||
for tag in noise:
|
||
try:
|
||
tag.decompose()
|
||
except Exception:
|
||
pass
|
||
return soup.body or soup
|
||
|
||
|
||
def _text_density(tag):
|
||
"""正文块得分：有效文本长度 + 段落/标题数量"""
|
||
text = tag.get_text(" ", strip=True)
|
||
if not text:
|
||
return 0
|
||
paras = len(tag.find_all(["p", "h1", "h2", "h3", "h4", "li", "pre", "blockquote"]))
|
||
links = len(tag.find_all("a"))
|
||
# 链接占比例过高多半是导航/聚合页，降权
|
||
link_penalty = min(1.0, links / max(1, paras) * 0.5)
|
||
return len(text) * (1 - link_penalty) + paras * 30
|
||
|
||
|
||
def _best_content_node(root):
|
||
"""从清洗后的文档里挑选正文块（按文本密度打分）"""
|
||
if root is None:
|
||
return None
|
||
candidates = root.find_all(["article", "main", "div", "section"])
|
||
if not candidates:
|
||
return root
|
||
best, best_score = root, 0
|
||
for c in candidates:
|
||
score = _text_density(c)
|
||
if score > best_score:
|
||
best, best_score = c, score
|
||
return best
|
||
|
||
|
||
def _node_to_lines(node):
|
||
"""把正文块转成干净的按行文本（保留段落结构）"""
|
||
if node is None:
|
||
return []
|
||
lines = []
|
||
for el in node.find_all(["h1", "h2", "h3", "h4", "p", "li", "pre", "blockquote", "td", "th"]):
|
||
if el.find_parent("pre") is not None and el.name != "pre":
|
||
continue
|
||
t = el.get_text(" ", strip=True)
|
||
t = re.sub(r"\s+", " ", t)
|
||
if len(t) >= 2:
|
||
lines.append(t)
|
||
if not lines:
|
||
t = node.get_text(" ", strip=True)
|
||
t = re.sub(r"\s+", " ", t)
|
||
lines = [t] if t else []
|
||
return lines
|
||
|
||
|
||
def clean_html(html, url=""):
|
||
"""抓到的原始 HTML -> (页面标题, 干净可读正文纯文本)"""
|
||
soup = BeautifulSoup(html, "lxml")
|
||
title = (soup.title.get_text(strip=True) if soup.title else "") or url
|
||
body = _strip_noise(soup)
|
||
node = _best_content_node(body)
|
||
text = "\n".join(_node_to_lines(node))
|
||
text = re.sub(r"\n{3,}", "\n\n", text).strip()
|
||
return title, text
|
||
|
||
|
||
# ---------------- 链接提取 ----------------
|
||
|
||
def _is_plausible_article(a, base_url):
|
||
href = (a.get("href") or "").strip()
|
||
if not href or href.startswith(_BAD_HREFS):
|
||
return False
|
||
low = href.lower()
|
||
if any(low.endswith(e) for e in _BAD_EXTS):
|
||
return False
|
||
if urlparse(urljoin(base_url, href)).fragment:
|
||
return False
|
||
text = re.sub(r"\s+", " ", a.get_text(" ", strip=True)).strip()
|
||
if len(text) < 10:
|
||
return False
|
||
if text.lower().strip() in _NAV_WORDS:
|
||
return False
|
||
if re.fullmatch(r"[\d\s·|/\\\-_]+", text):
|
||
return False
|
||
return True
|
||
|
||
|
||
def extract_links(html, base_url, max_n=10):
|
||
"""从页面提取候选资讯链接 -> [{'title','url','summary'}]（按文章相似度排序）"""
|
||
soup = BeautifulSoup(html, "lxml")
|
||
cands, seen = [], set()
|
||
for a in soup.find_all("a", href=True):
|
||
href = urljoin(base_url, a["href"].strip())
|
||
if href in seen or not _is_plausible_article(a, base_url):
|
||
continue
|
||
seen.add(href)
|
||
text = re.sub(r"\s+", " ", a.get_text(" ", strip=True)).strip()
|
||
text = _TRAIL_DATE_RE.sub("", text).strip()
|
||
if len(text) < 10:
|
||
continue
|
||
# 标题在标题标签/文章块内 → 更可能是资讯
|
||
in_head = 1 if a.find_parent(["h1", "h2", "h3", "h4", "article", "main"]) else 0
|
||
parent = a.find_parent(["li", "p", "h1", "h2", "h3", "h4", "article"]) or a.parent
|
||
summary = ""
|
||
if parent is not None and getattr(parent, "get_text", None):
|
||
summary = re.sub(r"\s+", " ", parent.get_text(" ", strip=True))[:200]
|
||
cands.append({"title": text, "url": href, "summary": summary,
|
||
"score": len(text) + in_head * 200})
|
||
cands.sort(key=lambda x: x["score"], reverse=True)
|
||
return [{"title": c["title"], "url": c["url"], "summary": c["summary"]}
|
||
for c in cands[:max_n]]
|
||
|
||
|
||
# ---------------- web-capture-api 集成（数据源网页抓取） ----------------
|
||
|
||
def _webcapture_cfg():
|
||
cfg = dict(config.WEBCAPTURE_DEFAULTS)
|
||
cfg.update(db.get_all_settings().get("webcapture", {}))
|
||
return cfg
|
||
|
||
|
||
def _source_capture_params(source):
|
||
"""解析数据源的抓取参数（capture_params JSON -> dict）"""
|
||
raw = source.get("capture_params") or "{}"
|
||
try:
|
||
p = json.loads(raw)
|
||
except Exception:
|
||
p = {}
|
||
return p if isinstance(p, dict) else {}
|
||
|
||
|
||
def _call_webcapture(url, action, params=None):
|
||
"""调用 web-capture-api 抓取网页（html/text）。失败抛异常。
|
||
返回: {"success": True, "title": ..., "html"|"text": ...}
|
||
"""
|
||
cfg = _webcapture_cfg()
|
||
base = (cfg.get("api_url") or "").rstrip("/")
|
||
if not base:
|
||
raise RuntimeError("web-capture-api 地址未配置（设置页可修改）")
|
||
payload = {"url": url, "action": action}
|
||
p = params or {}
|
||
for k in ("wait_time", "scroll_times", "scroll_delay", "full_page", "backend", "viewport"):
|
||
if k in p and p[k] not in (None, ""):
|
||
payload[k] = p[k]
|
||
r = requests.post(f"{base}/api/capture", json=payload,
|
||
timeout=int(cfg.get("timeout", 60)))
|
||
r.raise_for_status()
|
||
data = r.json()
|
||
if not data.get("success"):
|
||
raise RuntimeError(data.get("error", "web-capture-api 返回失败"))
|
||
return data
|
||
|
||
|
||
# ---------------- 按源采集 ----------------
|
||
|
||
def _now():
|
||
return datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||
|
||
|
||
def _fetch_list_via_direct(url):
|
||
"""直接抓取（requests + bs4）：返回 (title, page_text, links)"""
|
||
per = config.CRAWL_DEFAULTS["per_source_links"]
|
||
html = fetch_page(url)
|
||
page_title, page_text = clean_html(html, url)
|
||
links = extract_links(html, url, max_n=per)
|
||
return page_title, page_text, links
|
||
|
||
|
||
def _fetch_list_via_webcapture(url, params):
|
||
"""走 web-capture-api 抓取列表页：返回 (title, page_text, links)"""
|
||
per = config.CRAWL_DEFAULTS["per_source_links"]
|
||
action = (params.get("action") or "html").strip() or "html"
|
||
data = _call_webcapture(url, action, params)
|
||
title = data.get("title") or url
|
||
if action == "text":
|
||
# 只取正文，无法提取子链接 → 整页作为一条
|
||
return title, (data.get("text") or ""), []
|
||
html = data.get("html") or ""
|
||
page_text = (data.get("text") or "").strip()
|
||
if not page_text and html:
|
||
page_text = clean_html(html, url)[1]
|
||
links = extract_links(html, url, max_n=per) if html else []
|
||
return title, page_text, links
|
||
|
||
|
||
def _fetch_full_via_direct(url):
|
||
return clean_html(fetch_page(url), url)[1]
|
||
|
||
|
||
def _fetch_full_via_webcapture(url, params):
|
||
data = _call_webcapture(url, "text", params)
|
||
return data.get("text") or ""
|
||
|
||
|
||
def fetch_source(source):
|
||
"""采集单个数据源 -> list[item]。
|
||
example.com 占位源返回模拟数据；真实源抓取失败抛异常（由调用方捕获并标记 error）。
|
||
fetch_method：auto=优先 web-capture-api、失败回退直接抓取；webcapture=仅 web-capture-api；
|
||
direct=直接抓取(requests+bs4)。各源可单独配置 capture_params 抓取参数。
|
||
"""
|
||
url = (source.get("url") or "").strip()
|
||
# 模拟源（example.com 占位）→ 用仿真数据填充（补齐 source_id，保证定制监控识别正确）
|
||
if not url or "example.com" in url:
|
||
items = simulate.items_for_source(source["id"], source.get("type", ""))
|
||
for it in items:
|
||
it["source_id"] = source["id"]
|
||
return items
|
||
method = (source.get("fetch_method") or "auto").strip() or "auto"
|
||
per = config.CRAWL_DEFAULTS["per_source_links"]
|
||
full = config.CRAWL_DEFAULTS["full_fetch_links"]
|
||
params = _source_capture_params(source)
|
||
|
||
# 抓取列表页（标题 + 正文 + 候选链接）
|
||
if method == "direct":
|
||
page_title, page_text, links = _fetch_list_via_direct(url)
|
||
else:
|
||
try:
|
||
page_title, page_text, links = _fetch_list_via_webcapture(url, params)
|
||
except Exception:
|
||
if method == "webcapture":
|
||
raise
|
||
# auto：回退直接抓取
|
||
page_title, page_text, links = _fetch_list_via_direct(url)
|
||
|
||
items = []
|
||
for i, lk in enumerate(links[:per]):
|
||
full_text = ""
|
||
content = lk["summary"]
|
||
if i < full and lk["url"]:
|
||
try:
|
||
if method == "direct":
|
||
full_text = _fetch_full_via_direct(lk["url"])
|
||
else:
|
||
try:
|
||
full_text = _fetch_full_via_webcapture(lk["url"], params)
|
||
except Exception:
|
||
if method == "webcapture":
|
||
raise
|
||
full_text = _fetch_full_via_direct(lk["url"])
|
||
if not content or len(content) < len(full_text):
|
||
content = full_text
|
||
except Exception:
|
||
full_text = ""
|
||
items.append({
|
||
"title": lk["title"],
|
||
"url": lk["url"],
|
||
"author": source.get("name", ""),
|
||
"content": (content or page_text)[:4000],
|
||
"summary": (lk["summary"] or content or page_text)[:220],
|
||
"domain": source.get("type") or "",
|
||
"entities": [],
|
||
"source_id": source["id"],
|
||
"published_at": _now(),
|
||
"full_text": full_text or "",
|
||
})
|
||
if not items and page_text:
|
||
# 页面本身即正文（如单篇/无链接页/action=text）→ 整页作为一条
|
||
items.append({
|
||
"title": page_title,
|
||
"url": url,
|
||
"author": source.get("name", ""),
|
||
"content": page_text[:4000],
|
||
"summary": page_text[:220],
|
||
"domain": source.get("type") or "",
|
||
"entities": [],
|
||
"source_id": source["id"],
|
||
"published_at": _now(),
|
||
"full_text": page_text,
|
||
})
|
||
return items
|
||
|
||
|
||
def fetch_all():
|
||
"""采集全部启用数据源 -> (items, {source_id: 条数})
|
||
真实源失败：跳过并标记 sources.status=error（网页可见），不塞模拟数据。"""
|
||
sources = db.list_sources(only_enabled=True)
|
||
items, per, failed = [], {}, []
|
||
for s in sources:
|
||
try:
|
||
got = fetch_source(s)
|
||
per[s["id"]] = len(got)
|
||
items.extend(got)
|
||
except Exception:
|
||
failed.append(s["id"])
|
||
per[s["id"]] = 0
|
||
# 标记各源采集状态
|
||
for sid in failed:
|
||
db.update_source_fetch(sid, status="error", count=0)
|
||
for sid in set(per.keys()) - set(failed):
|
||
db.update_source_fetch(sid, status="ok", count=per.get(sid, 0))
|
||
return items, per
|