v1.4.3 任务结束保存状态前应用排除规则, 防止覆盖运行中配置的队列清理

- engine.url_excluded: 与 filter_links 同逻辑的排除判定辅助函数
- 保存 auto 配置时, 若 exclude 规则变化且任务未运行, 自动从 pending 队列
  剔除命中规则的 URL (visited 已爬标记保留)
- 排除输入框文案优化
- cnblogs-auto 已配置 exclude=[MyComments.html, OtherPosts.html, /comments], 清理 546 条用户中心页

app.py

@@ -15,7 +15,7 @@ from flask import Flask, jsonify, request, send_file, send_from_directory
 import store
 import db
 import notify
-from engine import CrawlJob, probe_links, normalize_url
+from engine import CrawlJob, probe_links, normalize_url, url_excluded
 from scheduler import Scheduler, cron_next, interval_delta
 
 HERE = os.path.dirname(os.path.abspath(__file__))
@@ -345,7 +345,22 @@ def api_update_task(tid):
             if running:
                 job.update_config(body["config"])       # 运行中热更新
         if "auto" in body and task.get("mode") == "auto":
+            old_exclude = task.get("auto", {}).get("exclude") or []
             task["auto"] = {**task.get("auto", {}), **body["auto"]}
+            new_exclude = task["auto"].get("exclude") or []
+            # 排除规则发生变化时, 同步清理待爬队列中已命中的 URL (已爬 visited 保留)
+            if new_exclude and new_exclude != old_exclude and not running:
+                st = store.load_auto_state(tid, task)
+                kept = [p for p in st.get("pending", [])
+                        if not url_excluded(p.get("url", ""), task["auto"])]
+                removed = len(st.get("pending", [])) - len(kept)
+                if removed:
+                    store.save_auto_state(tid, {
+                        "pending": kept,
+                        "visited": st.get("visited", []),
+                    })
+                    if "exclude" in body.get("auto", {}):
+                        task["auto"]["_cleaned"] = removed
         if "schedule" in body and task.get("mode") == "scheduled":
             sch = {**task.get("schedule", {}), **body["schedule"]}
             try:

engine.py

@@ -151,6 +151,21 @@ def normalize_url(url):
         return str(url)
 
 
+def url_excluded(url, auto=None):
+    """判断 URL 是否命中排除规则 (与 filter_links 的 exclude 判定逻辑一致, 供队列清理使用)"""
+    if not auto:
+        return False
+    exclude = auto.get("exclude") or []
+    if not exclude:
+        return False
+    use_regex = bool(auto.get("use_regex"))
+    s = str(url or "")
+    if use_regex:
+        return any(re.search(p, s) for p in exclude)
+    low = s.lower()
+    return any(p.lower() in low for p in exclude)
+
+
 def filter_links(hrefs, seed_url, include=None, exclude=None,
                  same_domain=True, use_regex=False):
     """按规则过滤链接, 返回 (included, excluded); excluded 含排除原因"""
@@ -763,10 +778,15 @@ class CrawlJob:
         finally:
             self._close_browser()
             # 无论完成/停止/异常, 都保存缓存队列与已爬集合, 便于下次继续
+            # 保存前应用当前排除规则过滤, 避免运行中配置的 exclude 被内存快照覆盖
             try:
+                auto_cfg = self.task.get("auto") or {}
+                keep = [(u, d, s) for u, d, s in queue if not url_excluded(u, auto_cfg)]
+                if len(keep) != len(queue):
+                    self._log("info", f"保存状态时按排除规则过滤 {len(queue) - len(keep)} 条")
                 store.save_auto_state(self.task["id"], {
                     "pending": [
-                        {"url": u, "depth": d, "source": s} for u, d, s in queue],
+                        {"url": u, "depth": d, "source": s} for u, d, s in keep],
                     "visited": list(visited),
                 })
                 db.upsert_task_async(self.task)

static/index.html

@@ -126,7 +126,7 @@
             <div class="field"><label>包含规则（每行一个，子串或正则）</label>
               <textarea name="include" rows="3" placeholder="techpowerup.com/review&#10;/news/"></textarea></div>
             <div class="field"><label>排除规则</label>
-              <textarea name="exclude" rows="3" placeholder="login, signup, /tag/, /forum/"></textarea></div>
+              <textarea name="exclude" rows="3" placeholder="每行一个关键词，URL 含任一关键词即不爬取&#10;如: MyComments.html、OtherPosts.html、/comments&#10;勾选正则后按正则匹配"></textarea></div>
           </div>
           <div class="row2">
             <div class="field check"><label><input name="same_domain" type="checkbox" checked> 仅爬同域名</label></div>