689 lines
28 KiB
JavaScript
689 lines
28 KiB
JavaScript
/* 通用爬虫系统 - 前端逻辑 */
|
|
"use strict";
|
|
|
|
const $ = (id) => document.getElementById(id);
|
|
const MODE_LABEL = { batch: "批量", scheduled: "定时", auto: "自动" };
|
|
|
|
const state = {
|
|
tasks: [],
|
|
editTask: null, // 正在编辑的任务
|
|
mode: "batch", // 当前表单模式
|
|
detail: { task: null, runId: null, logOffset: 0, timer: null },
|
|
logTimer: null,
|
|
};
|
|
let formDirty = false; // 新建/编辑表单是否有未保存修改
|
|
|
|
/* ---------------- 工具 ---------------- */
|
|
function toast(msg, isErr) {
|
|
const t = document.createElement("div");
|
|
t.className = "toast" + (isErr ? " error" : "");
|
|
t.textContent = msg;
|
|
document.body.appendChild(t);
|
|
setTimeout(() => t.remove(), 2600);
|
|
}
|
|
|
|
async function api(url, opts) {
|
|
const res = await fetch(url, opts);
|
|
let data = null;
|
|
try { data = await res.json(); } catch (e) { /* ignore */ }
|
|
if (!res.ok) throw new Error((data && data.error) || `HTTP ${res.status}`);
|
|
return data;
|
|
}
|
|
|
|
const esc = (s) => String(s == null ? "" : s).replace(/[&<>"']/g,
|
|
(c) => ({ "&": "&", "<": "<", ">": ">", '"': """, "'": "'" }[c]));
|
|
|
|
function fmtTime(iso) { return iso ? String(iso).replace("T", " ").slice(0, 19) : "—"; }
|
|
|
|
/* ---------------- 任务列表 ---------------- */
|
|
async function loadTasks() {
|
|
try {
|
|
state.tasks = await api("/api/tasks");
|
|
renderTasks();
|
|
const running = state.tasks.filter((t) => t.running).length;
|
|
const pill = $("statusPill");
|
|
pill.textContent = `运行中任务: ${running}`;
|
|
pill.classList.toggle("active", running > 0);
|
|
const st = await api("/api/status");
|
|
$("version").textContent = `v${st.version}`;
|
|
} catch (e) {
|
|
toast("加载任务失败: " + e.message, true);
|
|
}
|
|
loadStats();
|
|
}
|
|
|
|
async function loadStats() {
|
|
try {
|
|
const s = await api("/api/stats");
|
|
$("stTasks").textContent = s.tasks;
|
|
$("stRunning").textContent = s.running;
|
|
$("stRuns").textContent = s.runs;
|
|
$("stOk").textContent = s.ok;
|
|
$("stFail").textContent = s.fail;
|
|
$("stImgs").textContent = s.images;
|
|
$("stDisk").textContent = s.disk_mb >= 1024
|
|
? (s.disk_mb / 1024).toFixed(1) + " GB" : s.disk_mb + " MB";
|
|
} catch (e) { /* 统计失败忽略 */ }
|
|
}
|
|
|
|
function taskStatusBadge(t) {
|
|
if (t.running) return '<span class="badge running">● 运行中</span>';
|
|
const r = t.latest_run;
|
|
if (r && r.status === "paused") return '<span class="badge paused">⏸ 已暂停</span>';
|
|
if (t.mode === "scheduled" && t.schedule && t.schedule.enabled) {
|
|
const next = t.schedule.next_run ? ` · 下次 ${t.schedule.next_run.slice(11)}` : "";
|
|
return `<span class="badge scheduled-wait">⏰ 等待调度${esc(next)}</span>`;
|
|
}
|
|
if (r) return `<span class="badge ${r.status}">${({ completed: "✅ 已完成", failed: "❌ 失败", stopped: "⏹ 已停止" })[r.status] || esc(r.status)}</span>`;
|
|
return '<span class="badge">未运行</span>';
|
|
}
|
|
|
|
function renderTasks() {
|
|
const box = $("taskList");
|
|
$("emptyState").classList.toggle("hidden", state.tasks.length > 0);
|
|
box.innerHTML = state.tasks.map(taskCard).join("");
|
|
}
|
|
|
|
function taskCard(t) {
|
|
const r = t.latest_run;
|
|
const cfg = t.config || {};
|
|
const prog = r && (r.status === "running" || r.status === "paused") ? `
|
|
<div class="progress"><div style="width:${r.progress.percent || 0}%"></div></div>
|
|
<div class="progress-label">
|
|
<span>${r.progress.done || 0} / ${r.progress.total || "?"}</span>
|
|
<span class="url-cell" style="max-width:220px">${esc(r.progress.current_url || "")}</span>
|
|
</div>` : "";
|
|
const stats = r ? `
|
|
<div class="stats">
|
|
<span class="ok">✅ ${r.stats.ok || 0}</span>
|
|
<span class="fail">❌ ${r.stats.fail || 0}</span>
|
|
<span class="img">🖼️ ${r.stats.images || 0}</span>
|
|
</div>` : "";
|
|
const urlCount = (t.urls || []).length;
|
|
const line2 = t.mode === "auto"
|
|
? `起始: <b>${esc((t.auto && t.auto.seed_url) || "")}</b>`
|
|
: `网址: <b>${urlCount}</b> 个`;
|
|
const line3 = t.mode === "scheduled"
|
|
? `调度: <b>${t.schedule.type === "cron" ? esc(t.schedule.cron) : "每 " + (t.schedule.interval_value || "-") + " " + ({ minutes: "分钟", hours: "小时", days: "天" }[t.schedule.interval_unit] || "")}</b>`
|
|
: "";
|
|
const outDir = cfg.out_dir || "out/" + t.id;
|
|
|
|
let acts = "";
|
|
if (t.running) {
|
|
acts = `
|
|
<button class="btn sm" onclick="actTask('${t.id}','pause')">⏸ 暂停</button>
|
|
<button class="btn sm" onclick="actTask('${t.id}','stop')">⏹ 终止</button>`;
|
|
} else {
|
|
const canStart = t.mode !== "scheduled" || (t.urls || []).length > 0;
|
|
acts = `<button class="btn sm primary" ${canStart ? "" : "disabled"} onclick="actTask('${t.id}','start')">▶ 开始</button>`;
|
|
}
|
|
acts += `
|
|
<button class="btn sm" onclick="openEdit('${t.id}')">✏️ 编辑</button>
|
|
<button class="btn sm" onclick="openDetail('${t.id}')">📋 详情</button>
|
|
<button class="btn sm danger" onclick="delTask('${t.id}')">🗑️</button>`;
|
|
|
|
return `
|
|
<div class="card">
|
|
<div class="card-head">
|
|
<span class="card-name" title="${esc(t.name)}">${esc(t.name)}</span>
|
|
${taskStatusBadge(t)}
|
|
</div>
|
|
<div class="card-meta">
|
|
<span class="badge ${t.mode}">${MODE_LABEL[t.mode]}</span>
|
|
<span class="card-line">${line2}</span>
|
|
</div>
|
|
${line3 ? `<div class="card-line">${line3}</div>` : ""}
|
|
<div class="card-line">输出: <b>${esc(outDir)}</b></div>
|
|
${prog}
|
|
${stats}
|
|
<div class="card-actions">${acts}</div>
|
|
</div>`;
|
|
}
|
|
|
|
/* ---------------- 任务操作 ---------------- */
|
|
async function actTask(tid, act) {
|
|
try {
|
|
const r = await api(`/api/tasks/${tid}/${act}`, { method: "POST" });
|
|
if (r.msg) toast(r.msg);
|
|
loadTasks();
|
|
if (state.detail.task && state.detail.task.id === tid) openDetail(tid);
|
|
} catch (e) { toast(e.message, true); }
|
|
}
|
|
|
|
async function delTask(tid) {
|
|
const t = state.tasks.find((x) => x.id === tid);
|
|
if (!confirm(`确定删除任务「${t ? t.name : tid}」?\n运行记录与任务配置将被删除(已爬取的文件保留在磁盘上)。`)) return;
|
|
try {
|
|
await api(`/api/tasks/${tid}`, { method: "DELETE" });
|
|
toast("已删除");
|
|
loadTasks();
|
|
} catch (e) { toast(e.message, true); }
|
|
}
|
|
|
|
/* ---------------- 新建 / 编辑表单 ---------------- */
|
|
function switchMode(mode, lock) {
|
|
state.mode = mode;
|
|
document.querySelectorAll("#modeTabs .tab").forEach((b) => {
|
|
b.classList.toggle("active", b.dataset.mode === mode);
|
|
b.disabled = !!lock;
|
|
});
|
|
$("urlsField").classList.toggle("hidden", mode !== "batch" && mode !== "scheduled");
|
|
$("scheduleBox").classList.toggle("hidden", mode !== "scheduled");
|
|
$("autoBox").classList.toggle("hidden", mode !== "auto");
|
|
}
|
|
|
|
function toLocalInputVal(iso) {
|
|
if (!iso) return "";
|
|
const d = new Date(String(iso).replace(" ", "T"));
|
|
if (isNaN(d.getTime())) return "";
|
|
const p = (n) => String(n).padStart(2, "0");
|
|
return `${d.getFullYear()}-${p(d.getMonth() + 1)}-${p(d.getDate())}T${p(d.getHours())}:${p(d.getMinutes())}`;
|
|
}
|
|
|
|
function openCreate() {
|
|
state.editTask = null;
|
|
formDirty = false;
|
|
$("modalTitle").textContent = "新建爬取任务";
|
|
$("taskForm").reset();
|
|
$("taskForm").elements["delay_min"].value = 2;
|
|
$("taskForm").elements["delay_max"].value = 5;
|
|
$("taskForm").elements["timeout"].value = 60;
|
|
$("taskForm").elements["retry_count"].value = 2;
|
|
$("taskForm").elements["retry_interval"].value = 3;
|
|
$("taskForm").elements["notify_email"].value = "wlq@tphai.com";
|
|
$("formHint").textContent = "";
|
|
switchMode("batch", false);
|
|
showModal("taskModal");
|
|
}
|
|
|
|
async function openEdit(tid) {
|
|
const t = state.tasks.find((x) => x.id === tid) || await api(`/api/tasks/${tid}`);
|
|
state.editTask = t;
|
|
$("modalTitle").textContent = `编辑任务「${t.name}」`;
|
|
const f = $("taskForm");
|
|
f.reset();
|
|
const cfg = t.config || {};
|
|
f.elements["name"].value = t.name || "";
|
|
f.elements["out_dir"].value = cfg.out_dir || "";
|
|
f.elements["delay_min"].value = cfg.delay_min ?? 2;
|
|
f.elements["delay_max"].value = cfg.delay_max ?? 5;
|
|
f.elements["timeout"].value = cfg.timeout ?? 60;
|
|
f.elements["retry_count"].value = cfg.retry_count ?? 2;
|
|
f.elements["retry_interval"].value = cfg.retry_interval ?? 3;
|
|
f.elements["crawl_images"].checked = !!cfg.crawl_images;
|
|
f.elements["notify"].checked = !!cfg.notify;
|
|
f.elements["notify_email"].value = cfg.notify_email || "wlq@tphai.com";
|
|
f.elements["urls"].value = (t.urls || []).join("\n");
|
|
if (t.mode === "scheduled" && t.schedule) {
|
|
f.elements["schedule_type"].value = t.schedule.type || "interval";
|
|
f.elements["schedule_enabled"].checked = t.schedule.enabled !== false;
|
|
f.elements["interval_value"].value = t.schedule.interval_value ?? 24;
|
|
f.elements["interval_unit"].value = t.schedule.interval_unit || "hours";
|
|
f.elements["cron"].value = t.schedule.cron || "";
|
|
f.elements["first_run"].value = toLocalInputVal(t.schedule.next_run || "");
|
|
}
|
|
if (t.mode === "auto" && t.auto) {
|
|
f.elements["seed_url"].value = t.auto.seed_url || "";
|
|
f.elements["include"].value = (t.auto.include || []).join("\n");
|
|
f.elements["exclude"].value = (t.auto.exclude || []).join("\n");
|
|
f.elements["same_domain"].checked = t.auto.same_domain !== false;
|
|
f.elements["use_regex"].checked = !!t.auto.use_regex;
|
|
f.elements["max_pages"].value = t.auto.max_pages ?? 50;
|
|
f.elements["max_depth"].value = t.auto.max_depth ?? 2;
|
|
}
|
|
syncScheduleUI();
|
|
$("formHint").textContent = t.running
|
|
? "⚠️ 任务运行中:参数修改将热更新(网址/规则改动下次运行生效)"
|
|
: "";
|
|
formDirty = false;
|
|
switchMode(t.mode, true);
|
|
showModal("taskModal");
|
|
}
|
|
|
|
function syncScheduleUI() {
|
|
const f = $("taskForm");
|
|
const t = f.elements["schedule_type"].value;
|
|
$("intervalBox").classList.toggle("hidden", t !== "interval");
|
|
$("cronBox").classList.toggle("hidden", t !== "cron");
|
|
}
|
|
|
|
function splitLines(v) { return String(v || "").split("\n").map((s) => s.trim()).filter(Boolean); }
|
|
|
|
async function submitForm(e) {
|
|
e.preventDefault();
|
|
const f = $("taskForm");
|
|
const name = f.elements["name"].value.trim();
|
|
if (!name) { toast("请填写项目名称", true); return; }
|
|
const mode = state.mode;
|
|
const config = {
|
|
out_dir: f.elements["out_dir"].value.trim(),
|
|
delay_min: parseFloat(f.elements["delay_min"].value) || 2,
|
|
delay_max: parseFloat(f.elements["delay_max"].value) || 5,
|
|
timeout: parseInt(f.elements["timeout"].value) || 60,
|
|
retry_count: parseInt(f.elements["retry_count"].value) || 0,
|
|
retry_interval: parseFloat(f.elements["retry_interval"].value) || 3,
|
|
crawl_images: f.elements["crawl_images"].checked,
|
|
notify: f.elements["notify"].checked,
|
|
notify_email: f.elements["notify_email"].value.trim() || "wlq@tphai.com",
|
|
};
|
|
const urls = splitLines(f.elements["urls"].value);
|
|
const payload = { name, mode, config, urls };
|
|
if (mode === "auto") {
|
|
payload.auto = {
|
|
seed_url: f.elements["seed_url"].value.trim(),
|
|
include: splitLines(f.elements["include"].value),
|
|
exclude: splitLines(f.elements["exclude"].value),
|
|
same_domain: f.elements["same_domain"].checked,
|
|
use_regex: f.elements["use_regex"].checked,
|
|
max_pages: parseInt(f.elements["max_pages"].value) || 50,
|
|
max_depth: parseInt(f.elements["max_depth"].value) || 2,
|
|
};
|
|
}
|
|
if (mode === "scheduled") {
|
|
payload.schedule = {
|
|
type: f.elements["schedule_type"].value,
|
|
enabled: f.elements["schedule_enabled"].checked,
|
|
interval_unit: f.elements["interval_unit"].value,
|
|
interval_value: parseInt(f.elements["interval_value"].value) || 1,
|
|
cron: f.elements["cron"].value.trim(),
|
|
first_run: f.elements["first_run"].value || "",
|
|
};
|
|
}
|
|
try {
|
|
if (state.editTask) {
|
|
await api(`/api/tasks/${state.editTask.id}`, {
|
|
method: "PUT", headers: { "Content-Type": "application/json" },
|
|
body: JSON.stringify(payload),
|
|
});
|
|
toast("任务已更新");
|
|
} else {
|
|
const t = await api("/api/tasks", {
|
|
method: "POST", headers: { "Content-Type": "application/json" },
|
|
body: JSON.stringify(payload),
|
|
});
|
|
toast(`任务「${t.name}」已创建`);
|
|
}
|
|
formDirty = false;
|
|
hideModal("taskModal");
|
|
loadTasks();
|
|
} catch (err) { toast(err.message, true); }
|
|
}
|
|
|
|
/* ---------------- 详情 ---------------- */
|
|
async function openDetail(tid) {
|
|
try {
|
|
const t = await api(`/api/tasks/${tid}`);
|
|
state.detail.task = t;
|
|
const runs = t.runs || [];
|
|
state.detail.runId = runs.find((r) => r.status === "running")
|
|
? runs[0].id : (runs[0] ? runs[0].id : null);
|
|
state.detail.logOffset = 0;
|
|
$("detailTitle").textContent = `任务详情 · ${t.name}`;
|
|
renderDetail();
|
|
showModal("detailModal");
|
|
startLogPoll();
|
|
} catch (e) { toast(e.message, true); }
|
|
}
|
|
|
|
function renderDetail() {
|
|
const t = state.detail.task;
|
|
const runs = t.runs || [];
|
|
const run = runs.find((r) => r.id === state.detail.runId) || runs[0] || null;
|
|
state.detail.runId = run ? run.id : null;
|
|
const cfg = t.config || {};
|
|
|
|
const meta = `
|
|
<div class="detail-meta">
|
|
<span>模式: <b>${MODE_LABEL[t.mode] || t.mode}</b></span>
|
|
<span>任务ID: <b>${t.id}</b></span>
|
|
<span>创建: <b>${fmtTime(t.created_at)}</b></span>
|
|
<span>输出: <b>${esc(cfg.out_dir || "out/" + t.id)}</b></span>
|
|
${t.mode === "auto" ? `<span>起始: <b>${esc((t.auto && t.auto.seed_url) || "")}</b></span>` : ""}
|
|
</div>`;
|
|
|
|
let sched = "";
|
|
if (t.mode === "scheduled" && t.schedule) {
|
|
const s = t.schedule;
|
|
sched = `<div class="detail-meta">
|
|
<span>调度: <b>${s.type === "cron" ? esc(s.cron) : "每 " + (s.interval_value || "-") + " " + ({ minutes: "分钟", hours: "小时", days: "天" }[s.interval_unit] || "")}</b></span>
|
|
<span>启用: <b>${s.enabled ? "是" : "否"}</b></span>
|
|
<span>下次: <b>${esc(s.next_run || "—")}</b></span>
|
|
<span>上次: <b>${esc(s.last_run || "—")}</b></span>
|
|
<span>已运行: <b>${s.runs_count || 0}</b> 次</span>
|
|
</div>`;
|
|
}
|
|
|
|
const acts = `
|
|
<div class="detail-actions">
|
|
${t.running ? `
|
|
<button class="btn" onclick="actTask('${t.id}','pause')">⏸ 暂停</button>
|
|
<button class="btn danger" onclick="actTask('${t.id}','stop')">⏹ 终止</button>`
|
|
: `<button class="btn primary" onclick="actTask('${t.id}','start')">▶ 立即执行</button>`}
|
|
${t.mode === "auto" ? `<button class="btn" onclick="probeTask('${t.id}')">🧪 试爬取</button>` : ""}
|
|
<button class="btn" onclick="openEdit('${t.id}')">✏️ 编辑配置</button>
|
|
</div>`;
|
|
|
|
const chips = runs.map((r) => `
|
|
<span class="run-chip ${r.id === state.detail.runId ? "active" : ""}"
|
|
onclick="selectRun('${r.id}')">
|
|
${fmtTime(r.started_at).slice(5)} · ${r.status} · ${r.progress.done || 0}/${r.progress.total || "?"}
|
|
</span>`).join("");
|
|
|
|
const body = run ? renderRunPanel(t, run) : '<div class="card-line">暂无运行记录,点击「立即执行」开始第一次爬取。</div>';
|
|
|
|
$("detailBody").innerHTML = `
|
|
${meta}
|
|
${sched}
|
|
${acts}
|
|
<div class="runs-bar">${chips || '<span class="card-line">暂无运行记录</span>'}</div>
|
|
${body}`;
|
|
}
|
|
|
|
function renderRunPanel(t, run) {
|
|
const s = run.stats || { ok: 0, fail: 0, images: 0 };
|
|
const prog = run.status === "running" || run.status === "paused" ? `
|
|
<div class="progress"><div style="width:${run.progress.percent || 0}%"></div></div>` : "";
|
|
const results = run.results || [];
|
|
const rows = results.map((r, i) => {
|
|
const statusCls = r.status === "OK" ? "t-ok" : "t-fail";
|
|
const html = r.html_file ? `<span class="file-link" onclick="previewFile('${t.id}','${esc(r.html_file)}','HTML: ${esc(r.url)}')">HTML</span>` : "—";
|
|
const txt = r.txt_file ? `<span class="file-link" onclick="previewFile('${t.id}','${esc(r.txt_file)}','文本: ${esc(r.url)}')">TXT</span>` : "—";
|
|
const imgs = (r.images || []).length
|
|
? `<span class="file-link" onclick="scrollThumbs()">🖼️ ${(r.images || []).length}</span>` : "—";
|
|
const ctime = r.crawl_time ? `<span title="${esc(r.crawl_time)}">${esc(r.crawl_time.slice(5, 19))}</span>` : "—";
|
|
const meta = r.meta_file
|
|
? `<span class="file-link" onclick="showMeta('${t.id}','${esc(r.meta_file)}')">📋 元数据</span>` : "—";
|
|
return `<tr>
|
|
<td>${i + 1}</td>
|
|
<td class="${statusCls}">${r.status}</td>
|
|
<td class="title-cell" title="${esc(r.title)}">${esc(r.title)}</td>
|
|
<td class="url-cell" title="${esc(r.url)}">${esc(r.url)}</td>
|
|
<td>${ctime}</td>
|
|
<td>${html}</td><td>${txt}</td><td>${imgs}</td>
|
|
<td>${meta}</td>
|
|
<td title="${esc(r.error || "")}">${esc((r.error || "").slice(0, 50))}</td>
|
|
</tr>`;
|
|
}).join("");
|
|
const thumbs = results.flatMap((r) => r.images || []).slice(0, 60);
|
|
const thumbHtml = thumbs.length ? `
|
|
<div id="thumbsBox" class="thumbs">
|
|
${thumbs.map((im) => `<img src="/api/file?task_id=${t.id}&path=${esc(im.file)}" loading="lazy"
|
|
onclick="previewFile('${t.id}','${esc(im.file)}','图片预览')">`).join("")}
|
|
</div>` : "";
|
|
|
|
return `
|
|
<div class="run-panel">
|
|
<div class="run-stats">
|
|
<span>运行ID: <b>${run.id}</b></span>
|
|
<span>状态: <b class="${run.status === "running" ? "t-ok" : ""}">${run.status}</b></span>
|
|
<span>开始: <b>${fmtTime(run.started_at)}</b></span>
|
|
<span>结束: <b>${fmtTime(run.finished_at)}</b></span>
|
|
<span class="ok">✅ ${s.ok}</span>
|
|
<span class="fail">❌ ${s.fail}</span>
|
|
<span class="img">🖼️ ${s.images}</span>
|
|
</div>
|
|
${prog}
|
|
<div class="card-line">当前: <b>${esc(run.progress.current_url || "")}</b></div>
|
|
${results.length ? `
|
|
<div class="table-wrap">
|
|
<table>
|
|
<thead><tr><th>#</th><th>状态</th><th>标题</th><th>网址</th><th>爬取时间</th><th>HTML</th><th>TXT</th><th>图片</th><th>元数据</th><th>错误</th></tr></thead>
|
|
<tbody>${rows}</tbody>
|
|
</table>
|
|
</div>` : '<div class="card-line">暂无结果</div>'}
|
|
${thumbHtml}
|
|
<div class="logs" id="logBox"></div>
|
|
</div>`;
|
|
}
|
|
|
|
function scrollThumbs() {
|
|
const box = $("thumbsBox");
|
|
if (box) box.scrollIntoView({ behavior: "smooth", block: "center" });
|
|
}
|
|
|
|
async function selectRun(rid) {
|
|
state.detail.runId = rid;
|
|
state.detail.logOffset = 0;
|
|
renderDetail();
|
|
startLogPoll();
|
|
}
|
|
|
|
async function startLogPoll() {
|
|
stopLogPoll();
|
|
const t = state.detail.task;
|
|
if (!t || !state.detail.runId) return;
|
|
const rid = state.detail.runId;
|
|
const tick = async () => {
|
|
try {
|
|
const d = await api(`/api/runs/${rid}/logs?offset=${state.detail.logOffset}`);
|
|
if (d.logs && d.logs.length) {
|
|
const box = $("logBox");
|
|
if (box) {
|
|
const atBottom = box.scrollHeight - box.scrollTop - box.clientHeight < 40;
|
|
d.logs.forEach((l) => {
|
|
const div = document.createElement("div");
|
|
div.className = l.level;
|
|
div.textContent = `[${l.ts}] ${l.msg}`;
|
|
box.appendChild(div);
|
|
});
|
|
if (atBottom) box.scrollTop = box.scrollHeight;
|
|
state.detail.logOffset += d.logs.length;
|
|
}
|
|
}
|
|
} catch (e) { /* ignore */ }
|
|
loadTasks(); // 后台保持列表刷新
|
|
};
|
|
tick();
|
|
state.logTimer = setInterval(tick, 2500);
|
|
}
|
|
|
|
function stopLogPoll() {
|
|
if (state.logTimer) { clearInterval(state.logTimer); state.logTimer = null; }
|
|
}
|
|
|
|
/* ---------------- 试爬取 ---------------- */
|
|
function collectProbeFromForm() {
|
|
const f = $("taskForm");
|
|
return {
|
|
seed_url: f.elements["seed_url"].value.trim(),
|
|
include: splitLines(f.elements["include"].value),
|
|
exclude: splitLines(f.elements["exclude"].value),
|
|
same_domain: f.elements["same_domain"].checked,
|
|
use_regex: f.elements["use_regex"].checked,
|
|
timeout: parseInt(f.elements["timeout"].value) || 60,
|
|
};
|
|
}
|
|
|
|
async function runProbe(params, btn) {
|
|
if (btn) { btn.disabled = true; btn.textContent = "⏳ 探测中..."; }
|
|
try {
|
|
const r = await api("/api/probe", {
|
|
method: "POST", headers: { "Content-Type": "application/json" },
|
|
body: JSON.stringify(params),
|
|
});
|
|
renderProbe(r);
|
|
} catch (e) { toast(e.message, true); }
|
|
finally {
|
|
if (btn) { btn.disabled = false; btn.textContent = "🧪 试爬取"; }
|
|
}
|
|
}
|
|
|
|
async function probeTask(tid) {
|
|
try {
|
|
const r = await api(`/api/tasks/${tid}/probe`, { method: "POST" });
|
|
renderProbe(r);
|
|
} catch (e) { toast(e.message, true); }
|
|
}
|
|
|
|
function renderProbe(r) {
|
|
const body = $("probeBody");
|
|
if (!r.ok) {
|
|
body.innerHTML = `<div class="card-line">❌ ${esc(r.error || "试爬取失败")}</div>`;
|
|
showModal("probeModal");
|
|
return;
|
|
}
|
|
const included = r.included || [];
|
|
const excluded = r.excluded || [];
|
|
const incHtml = included.length
|
|
? included.map((u) => `<div class="probe-item included"><span class="u">${esc(u)}</span></div>`).join("")
|
|
: '<div class="card-line">没有符合条件的链接,请调整规则</div>';
|
|
const excHtml = excluded.length
|
|
? excluded.map((x) => `<div class="probe-item excluded"><span class="u">${esc(x.url)}</span><span class="reason">${esc(x.reason)}</span></div>`).join("")
|
|
: '<div class="card-line">无</div>';
|
|
const moreInc = r.total_included > included.length
|
|
? `<div class="card-line">... 还有 ${r.total_included - included.length} 条未显示</div>` : "";
|
|
const moreExc = r.total_excluded > excluded.length
|
|
? `<div class="card-line">... 还有 ${r.total_excluded - excluded.length} 条未显示</div>` : "";
|
|
body.innerHTML = `
|
|
<div class="probe-summary">
|
|
<span>页面标题: <b>${esc(r.title)}</b></span>
|
|
<span>抓取时间: <b>${esc(r.crawled_at)}</b></span>
|
|
</div>
|
|
<div class="probe-summary">
|
|
<span>发现链接: <b>${r.total_links}</b> 条</span>
|
|
<span class="t-ok">✅ 将爬取: <b>${r.total_included}</b> 条</span>
|
|
<span class="t-fail">🚫 被排除: <b>${r.total_excluded}</b> 条</span>
|
|
</div>
|
|
<div class="card-line" style="font-weight:600">✅ 符合规则、将爬取的链接(显示前 ${included.length} 条):</div>
|
|
<div class="probe-list">${incHtml}${moreInc}</div>
|
|
<div class="card-line" style="font-weight:600;margin-top:8px">🚫 被排除的链接及原因(显示前 ${excluded.length} 条):</div>
|
|
<div class="probe-list">${excHtml}${moreExc}</div>`;
|
|
showModal("probeModal");
|
|
}
|
|
|
|
/* ---------------- 元数据查看 ---------------- */
|
|
const META_FIELDS = [
|
|
["type", "资源类型"], ["mode", "爬取模式"], ["task_name", "任务名称"],
|
|
["task_id", "任务ID"], ["run_id", "运行ID"], ["crawl_time", "爬取时间"],
|
|
["url", "爬取网址"], ["source_url", "来源链接"], ["depth", "爬取深度"],
|
|
["title", "页面标题"], ["status", "状态"], ["attempts", "尝试次数"],
|
|
["error", "错误信息"], ["html_file", "HTML文件"], ["txt_file", "文本文件"],
|
|
["page_url", "所属页面"],
|
|
];
|
|
|
|
async function showMeta(taskId, metaFile) {
|
|
try {
|
|
const data = await api(`/api/file?task_id=${taskId}&path=${encodeURIComponent(metaFile)}`);
|
|
const rows = META_FIELDS.filter(([k]) => data[k] !== undefined && data[k] !== "" && data[k] !== null)
|
|
.map(([k, label]) => {
|
|
let v = data[k];
|
|
if (k === "status") v = `<span class="${v === "OK" ? "t-ok" : "t-fail"}">${esc(v)}</span>`;
|
|
if (k === "mode") v = `<span class="badge ${esc(v)}">${MODE_LABEL[v] || esc(v)}</span>`;
|
|
if (k === "type") v = v === "images" ? "🖼️ 图片集" : "📄 网页";
|
|
if (k === "depth") v = v === 0 ? "0(起始页)" : esc(v);
|
|
return `<tr><td>${label}</td><td>${v}</td></tr>`;
|
|
}).join("");
|
|
// 来源链接空值也显示(起始页无来源)
|
|
const srcRow = `<tr><td>来源链接</td><td>${data.source_url ? esc(data.source_url) : "—(起始页,无来源)"}</td></tr>`;
|
|
const finalRows = rows.includes(srcRow) ? rows : rows + srcRow;
|
|
let imgs = "";
|
|
if (data.images && data.images.length) {
|
|
imgs = `
|
|
<div class="card-line" style="font-weight:600;margin-top:8px">🖼️ 图片明细(${data.images.length} 张):</div>
|
|
<div class="table-wrap" style="max-height:200px">
|
|
<table><thead><tr><th>文件</th><th>原始URL</th><th>大小</th><th>下载时间</th></tr></thead><tbody>
|
|
${data.images.map((im) => `<tr>
|
|
<td>${esc(im.file)}</td>
|
|
<td class="url-cell" title="${esc(im.url)}">${esc(im.url)}</td>
|
|
<td>${im.size ? Math.round(im.size / 1024) + " KB" : "—"}</td>
|
|
<td>${esc(im.download_time || "")}</td></tr>`).join("")}
|
|
</tbody></table>
|
|
</div>`;
|
|
}
|
|
$("metaBody").innerHTML = `<table class="meta-table"><tbody>${finalRows}</tbody></table>${imgs}`;
|
|
showModal("metaModal");
|
|
} catch (e) { toast(e.message, true); }
|
|
}
|
|
|
|
/* ---------------- 文件预览 ---------------- */
|
|
function previewFile(tid, path, title) {
|
|
$("previewTitle").textContent = title || "预览";
|
|
$("previewFrame").src = `/api/file?task_id=${tid}&path=${encodeURIComponent(path)}`;
|
|
showModal("previewModal");
|
|
}
|
|
|
|
/* ---------------- 弹窗控制 ---------------- */
|
|
function showModal(id) { $(id).classList.remove("hidden"); }
|
|
function hideModal(id) { $(id).classList.add("hidden"); }
|
|
|
|
/* 关闭任务弹窗: 有未保存修改时先确认 */
|
|
function safeCloseTaskModal() {
|
|
if (formDirty && !confirm("有未保存的修改,确定要放弃吗?")) return false;
|
|
formDirty = false;
|
|
hideModal("taskModal");
|
|
return true;
|
|
}
|
|
|
|
/* ---------------- 主题切换 ---------------- */
|
|
function applyTheme(theme) {
|
|
document.body.dataset.theme = theme;
|
|
$("btnTheme").textContent = theme === "light" ? "🌙" : "☀️";
|
|
try { localStorage.setItem("crawler_theme", theme); } catch (e) { /* ignore */ }
|
|
}
|
|
$("btnTheme").onclick = () => {
|
|
applyTheme(document.body.dataset.theme === "light" ? "dark" : "light");
|
|
};
|
|
try {
|
|
applyTheme(localStorage.getItem("crawler_theme") || "dark");
|
|
} catch (e) {
|
|
applyTheme("dark");
|
|
}
|
|
|
|
/* ---------------- 事件绑定 ---------------- */
|
|
$("btnNew").onclick = openCreate;
|
|
$("btnNew2").onclick = openCreate;
|
|
$("btnRefresh").onclick = loadTasks;
|
|
$("btnProbe").onclick = () => {
|
|
const p = collectProbeFromForm();
|
|
if (!p.seed_url) { toast("请先填写起始网址", true); return; }
|
|
runProbe(p, $("btnProbe"));
|
|
};
|
|
|
|
document.querySelectorAll("#modeTabs .tab").forEach((b) => {
|
|
b.onclick = () => { switchMode(b.dataset.mode, false); };
|
|
});
|
|
$("taskForm").onsubmit = submitForm;
|
|
$("taskForm").elements["schedule_type"].onchange = syncScheduleUI;
|
|
|
|
document.querySelectorAll("[data-close]").forEach((b) => b.onclick = safeCloseTaskModal);
|
|
document.querySelectorAll("[data-close-detail]").forEach((b) => b.onclick = () => { stopLogPoll(); hideModal("detailModal"); });
|
|
document.querySelectorAll("[data-close-preview]").forEach((b) => b.onclick = () => { $("previewFrame").src = "about:blank"; hideModal("previewModal"); });
|
|
document.querySelectorAll("[data-close-probe]").forEach((b) => b.onclick = () => hideModal("probeModal"));
|
|
document.querySelectorAll("[data-close-meta]").forEach((b) => b.onclick = () => hideModal("metaModal"));
|
|
|
|
/* 表单改动监听 -> 脏标记 */
|
|
$("taskForm").addEventListener("input", () => { formDirty = true; });
|
|
$("taskForm").addEventListener("change", () => { formDirty = true; });
|
|
|
|
document.querySelectorAll(".modal-overlay").forEach((ov) => {
|
|
ov.addEventListener("mousedown", (e) => {
|
|
if (e.target === ov) {
|
|
if (ov.id === "taskModal") {
|
|
safeCloseTaskModal();
|
|
} else {
|
|
if (ov.id === "detailModal") stopLogPoll();
|
|
if (ov.id === "previewModal") $("previewFrame").src = "about:blank";
|
|
ov.classList.add("hidden");
|
|
}
|
|
}
|
|
});
|
|
});
|
|
document.addEventListener("keydown", (e) => {
|
|
if (e.key === "Escape") {
|
|
if (!$("taskModal").classList.contains("hidden")) {
|
|
safeCloseTaskModal();
|
|
return;
|
|
}
|
|
stopLogPoll();
|
|
$("previewFrame").src = "about:blank";
|
|
["detailModal", "probeModal", "metaModal", "previewModal"].forEach((id) => $(id).classList.add("hidden"));
|
|
}
|
|
});
|
|
|
|
/* ---------------- 启动 ---------------- */
|
|
loadTasks();
|
|
setInterval(() => {
|
|
const detailOpen = !$("detailModal").classList.contains("hidden");
|
|
if (!detailOpen) loadTasks();
|
|
}, 4000);
|