From 29176fc66b244f63b18cb8fcbc34be9c054453d9 Mon Sep 17 00:00:00 2001 From: hz4th_coder Date: Tue, 11 Aug 2026 13:02:34 +0800 Subject: [PATCH] =?UTF-8?q?v1.0.1:=20=E5=BC=B9=E7=AA=97=E9=98=B2=E8=AF=AF?= =?UTF-8?q?=E5=85=B3=E7=A1=AE=E8=AE=A4/=E5=B8=83=E5=B1=80=E4=BF=AE?= =?UTF-8?q?=E5=A4=8D/=E5=AE=9A=E6=97=B6=E4=BB=BB=E5=8A=A1=E9=A6=96?= =?UTF-8?q?=E6=AC=A1=E6=89=A7=E8=A1=8C=E6=97=B6=E9=97=B4/=E6=97=A5?= =?UTF-8?q?=E9=97=B4=E5=A4=9C=E9=97=B4=E5=8F=8C=E4=B8=BB=E9=A2=98/?= =?UTF-8?q?=E6=80=BB=E4=BD=93=E7=BB=9F=E8=AE=A1=E5=8C=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 7 ++-- app.py | 86 +++++++++++++++++++++++++++++++++++++---------- static/app.js | 74 +++++++++++++++++++++++++++++++++++++--- static/index.html | 12 +++++++ static/style.css | 79 ++++++++++++++++++++++++++++++++++--------- 5 files changed, 218 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index b6a858a..e96cdfd 100644 --- a/README.md +++ b/README.md @@ -22,9 +22,12 @@ ## 功能总览 ### 1. 前端管理界面 +- **总体统计区**：任务总数 / 运行中 / 累计运行次数 / 成功失败页面 / 图片数 / 磁盘占用 +- **日间/夜间双主题**：右上角按钮一键切换，自动记忆选择 - 任务卡片总览：状态、进度、统计、下次调度时间一目了然 - 一键操作：开始 / 暂停 / 恢复 / 终止 / 编辑 / 删除 - 任务详情：历次运行记录、结果明细表（HTML/TXT 在线预览）、图片缩略图、实时日志 +- **防误关保护**：新建/编辑弹窗有未保存修改时，点窗口外 / ESC / 关闭会先确认 - 运行中的任务参数支持**热更新**（修改后从下一页起生效） ### 2. 批量爬取模式 @@ -36,8 +39,8 @@ - 完成后邮件通知（复用 send_email.py，默认发到 wlq@tphai.com） ### 3. 定时爬取模式 -- **间隔调度**：每 N 分钟 / 小时 / 天 -- **cron 表达式**：5 段式（分 时 日 月 周，周 0/7=周日），如 `0 3 * * *` 每天凌晨 3 点 +- **间隔调度**：每 N 分钟 / 小时 / 天，可指定**首次执行时间**（留空=尽快，已过时间立即执行） +- **cron 表达式**：5 段式（分 时 日 月 周，周 0/7=周日），如 `0 3 * * *` 每天凌晨 3 点，同样支持首次执行时间作为计算起点 - 可启用/停用调度，自动计算下次执行时间；到点自动开跑，跑完自动计算下一次 ### 4. 自动爬取模式 diff --git a/app.py b/app.py index 62c82dc..3a7550b 100644 --- a/app.py +++ b/app.py @@ -38,6 +38,33 @@ def now_str(): return store.now_str() +def _parse_first_run(s): + """解析表单提交的首次执行时间 (datetime-local 格式), 非法返回 None""" + if not s: + return None + try: + return datetime.strptime(str(s), "%Y-%m-%dT%H:%M") + except Exception: + return None + + +def _schedule_next_run(sch): + """根据调度配置 + 首次执行时间计算 next_run (str)""" + first_run = _parse_first_run(sch.get("first_run")) + base = first_run if (first_run and first_run > datetime.now()) else None + if sch.get("type") == "cron": + expr = sch.get("cron") or "0 * * * *" + nn = cron_next(expr, base or datetime.now()) + if not nn: + raise ValueError("cron 表达式在未来一年内无匹配时间") + return nn.strftime("%Y-%m-%d %H:%M:%S") + sch.setdefault("interval_unit", "hours") + sch.setdefault("interval_value", 24) + if base: + return base.strftime("%Y-%m-%d %H:%M:%S") + return (datetime.now() + interval_delta(sch)).strftime("%Y-%m-%d %H:%M:%S") + + def resolve_out_dir(task): cfg = task.get("config", {}) or {} if cfg.get("out_dir", "").strip(): @@ -161,16 +188,7 @@ def api_create_task(): sch.setdefault("enabled", True) sch.setdefault("type", "interval") try: - if sch.get("type") == "cron": - expr = sch.get("cron") or "0 * * * *" - nn = cron_next(expr) - if not nn: - raise ValueError("cron 表达式在未来一年内无匹配时间") - sch["next_run"] = nn.strftime("%Y-%m-%d %H:%M:%S") - else: - sch.setdefault("interval_unit", "hours") - sch.setdefault("interval_value", 24) - sch["next_run"] = (datetime.now() + interval_delta(sch)).strftime("%Y-%m-%d %H:%M:%S") + sch["next_run"] = _schedule_next_run(sch) except ValueError as e: return jsonify({"error": f"调度配置错误: {e}"}), 400 sch.setdefault("last_run", "") @@ -221,13 +239,7 @@ def api_update_task(tid): if "schedule" in body and task.get("mode") == "scheduled": sch = {**task.get("schedule", {}), **body["schedule"]} try: - if sch.get("type") == "cron": - nn = cron_next(sch.get("cron") or "0 * * * *") - if not nn: - raise ValueError("cron 表达式在未来一年内无匹配时间") - sch["next_run"] = nn.strftime("%Y-%m-%d %H:%M:%S") - else: - sch["next_run"] = (datetime.now() + interval_delta(sch)).strftime("%Y-%m-%d %H:%M:%S") + sch["next_run"] = _schedule_next_run(sch) except ValueError as e: return jsonify({"error": f"调度配置错误: {e}"}), 400 task["schedule"] = sch @@ -285,6 +297,46 @@ def api_resume(tid): return jsonify({"error": "任务未在运行"}), 409 +# ---------------- API: 统计 ---------------- + +@app.route("/api/stats") +def api_stats(): + tasks = store.load_tasks() + total_runs = ok = fail = imgs = 0 + for t in tasks: + for r in store.get_runs(t["id"]): + total_runs += 1 + st = r.get("stats") or {} + ok += st.get("ok", 0) + fail += st.get("fail", 0) + imgs += st.get("images", 0) + with JOBS_LOCK: + running = sum(1 for j in JOBS.values() if j.is_running()) + # 统计各任务输出目录的磁盘占用 + size = 0 + seen = set() + for t in tasks: + d = os.path.realpath(resolve_out_dir(t)) + if d in seen or not os.path.isdir(d): + continue + seen.add(d) + for root, _dirs, files in os.walk(d): + for f in files: + try: + size += os.path.getsize(os.path.join(root, f)) + except OSError: + pass + return jsonify({ + "tasks": len(tasks), + "running": running, + "runs": total_runs, + "ok": ok, + "fail": fail, + "images": imgs, + "disk_mb": round(size / 1048576, 1), + }) + + # ---------------- API: 运行记录与文件 ---------------- @app.route("/api/runs/") diff --git a/static/app.js b/static/app.js index 6a4a1aa..e20d662 100644 --- a/static/app.js +++ b/static/app.js @@ -11,6 +11,7 @@ const state = { detail: { task: null, runId: null, logOffset: 0, timer: null }, logTimer: null, }; +let formDirty = false; // 新建/编辑表单是否有未保存修改 /* ---------------- 工具 ---------------- */ function toast(msg, isErr) { @@ -48,6 +49,21 @@ async function loadTasks() { } catch (e) { toast("加载任务失败: " + e.message, true); } + loadStats(); +} + +async function loadStats() { + try { + const s = await api("/api/stats"); + $("stTasks").textContent = s.tasks; + $("stRunning").textContent = s.running; + $("stRuns").textContent = s.runs; + $("stOk").textContent = s.ok; + $("stFail").textContent = s.fail; + $("stImgs").textContent = s.images; + $("stDisk").textContent = s.disk_mb >= 1024 + ? (s.disk_mb / 1024).toFixed(1) + " GB" : s.disk_mb + " MB"; + } catch (e) { /* 统计失败忽略 */ } } function taskStatusBadge(t) { @@ -156,8 +172,17 @@ function switchMode(mode, lock) { $("autoBox").classList.toggle("hidden", mode !== "auto"); } +function toLocalInputVal(iso) { + if (!iso) return ""; + const d = new Date(String(iso).replace(" ", "T")); + if (isNaN(d.getTime())) return ""; + const p = (n) => String(n).padStart(2, "0"); + return `${d.getFullYear()}-${p(d.getMonth() + 1)}-${p(d.getDate())}T${p(d.getHours())}:${p(d.getMinutes())}`; +} + function openCreate() { state.editTask = null; + formDirty = false; $("modalTitle").textContent = "新建爬取任务"; $("taskForm").reset(); $("taskForm").elements["delay_min"].value = 2; @@ -195,6 +220,7 @@ async function openEdit(tid) { f.elements["interval_value"].value = t.schedule.interval_value ?? 24; f.elements["interval_unit"].value = t.schedule.interval_unit || "hours"; f.elements["cron"].value = t.schedule.cron || ""; + f.elements["first_run"].value = toLocalInputVal(t.schedule.next_run || ""); } if (t.mode === "auto" && t.auto) { f.elements["seed_url"].value = t.auto.seed_url || ""; @@ -209,6 +235,7 @@ async function openEdit(tid) { $("formHint").textContent = t.running ? "⚠️ 任务运行中：参数修改将热更新（网址/规则改动下次运行生效）" : ""; + formDirty = false; switchMode(t.mode, true); showModal("taskModal"); } @@ -259,6 +286,7 @@ async function submitForm(e) { interval_unit: f.elements["interval_unit"].value, interval_value: parseInt(f.elements["interval_value"].value) || 1, cron: f.elements["cron"].value.trim(), + first_run: f.elements["first_run"].value || "", }; } try { @@ -275,6 +303,7 @@ async function submitForm(e) { }); toast(`任务「${t.name}」已创建`); } + formDirty = false; hideModal("taskModal"); loadTasks(); } catch (err) { toast(err.message, true); } @@ -457,6 +486,29 @@ function previewFile(tid, path, title) { function showModal(id) { $(id).classList.remove("hidden"); } function hideModal(id) { $(id).classList.add("hidden"); } +/* 关闭任务弹窗: 有未保存修改时先确认 */ +function safeCloseTaskModal() { + if (formDirty && !confirm("有未保存的修改，确定要放弃吗？")) return false; + formDirty = false; + hideModal("taskModal"); + return true; +} + +/* ---------------- 主题切换 ---------------- */ +function applyTheme(theme) { + document.body.dataset.theme = theme; + $("btnTheme").textContent = theme === "light" ? "🌙" : "☀️"; + try { localStorage.setItem("crawler_theme", theme); } catch (e) { /* ignore */ } +} +$("btnTheme").onclick = () => { + applyTheme(document.body.dataset.theme === "light" ? "dark" : "light"); +}; +try { + applyTheme(localStorage.getItem("crawler_theme") || "dark"); +} catch (e) { + applyTheme("dark"); +} + /* ---------------- 事件绑定 ---------------- */ $("btnNew").onclick = openCreate; $("btnNew2").onclick = openCreate; @@ -468,24 +520,36 @@ document.querySelectorAll("#modeTabs .tab").forEach((b) => { $("taskForm").onsubmit = submitForm; $("taskForm").elements["schedule_type"].onchange = syncScheduleUI; -document.querySelectorAll("[data-close]").forEach((b) => b.onclick = () => hideModal("taskModal")); +document.querySelectorAll("[data-close]").forEach((b) => b.onclick = safeCloseTaskModal); document.querySelectorAll("[data-close-detail]").forEach((b) => b.onclick = () => { stopLogPoll(); hideModal("detailModal"); }); document.querySelectorAll("[data-close-preview]").forEach((b) => b.onclick = () => { $("previewFrame").src = "about:blank"; hideModal("previewModal"); }); +/* 表单改动监听 -> 脏标记 */ +$("taskForm").addEventListener("input", () => { formDirty = true; }); +$("taskForm").addEventListener("change", () => { formDirty = true; }); + document.querySelectorAll(".modal-overlay").forEach((ov) => { ov.addEventListener("mousedown", (e) => { if (e.target === ov) { - if (ov.id === "detailModal") stopLogPoll(); - if (ov.id === "previewModal") $("previewFrame").src = "about:blank"; - ov.classList.add("hidden"); + if (ov.id === "taskModal") { + safeCloseTaskModal(); + } else { + if (ov.id === "detailModal") stopLogPoll(); + if (ov.id === "previewModal") $("previewFrame").src = "about:blank"; + ov.classList.add("hidden"); + } } }); }); document.addEventListener("keydown", (e) => { if (e.key === "Escape") { + if (!$("taskModal").classList.contains("hidden")) { + safeCloseTaskModal(); + return; + } stopLogPoll(); $("previewFrame").src = "about:blank"; - ["taskModal", "detailModal", "previewModal"].forEach((id) => $(id).classList.add("hidden")); + ["detailModal", "previewModal"].forEach((id) => $(id).classList.add("hidden")); } }); diff --git a/static/index.html b/static/index.html index b8d2463..af6347e 100644 --- a/static/index.html +++ b/static/index.html @@ -11,12 +11,22 @@
运行中任务: 0 +
+
+
0
📋 任务总数
+
0
🔄 正在运行
+
0
📊 累计运行次数
+
0
✅ 成功页面
+
0
❌ 失败页面
+
0
🖼️ 已爬图片
+
0
💾 磁盘占用
+