/* 通用爬虫系统 - 前端逻辑 */
"use strict";
const $ = (id) => document.getElementById(id);
const MODE_LABEL = { batch: "批量", scheduled: "定时", auto: "自动" };
const DETAIL_PAGE_SIZE = 100; // 详情结果每页条数 (默认)
const state = {
tasks: [],
editTask: null, // 正在编辑的任务
mode: "batch", // 当前表单模式
detail: { task: null, runId: null, logOffset: 0, timer: null, pageSize: DETAIL_PAGE_SIZE },
logTimer: null,
};
let formDirty = false; // 新建/编辑表单是否有未保存修改
/* ---------------- 工具 ---------------- */
function toast(msg, isErr) {
const t = document.createElement("div");
t.className = "toast" + (isErr ? " error" : "");
t.textContent = msg;
document.body.appendChild(t);
setTimeout(() => t.remove(), 2600);
}
async function api(url, opts) {
const res = await fetch(url, opts);
let data = null;
try { data = await res.json(); } catch (e) { /* ignore */ }
if (!res.ok) throw new Error((data && data.error) || `HTTP ${res.status}`);
return data;
}
const esc = (s) => String(s == null ? "" : s).replace(/[&<>"']/g,
(c) => ({ "&": "&", "<": "<", ">": ">", '"': """, "'": "'" }[c]));
function fmtTime(iso) { return iso ? String(iso).replace("T", " ").slice(0, 19) : "—"; }
/* ---------------- 任务列表 ---------------- */
async function loadTasks() {
try {
state.tasks = await api("/api/tasks");
renderTasks();
const running = state.tasks.filter((t) => t.running).length;
const pill = $("statusPill");
pill.textContent = `运行中任务: ${running}`;
pill.classList.toggle("active", running > 0);
const st = await api("/api/status");
$("version").textContent = st && st.version ? `v${st.version}` : "v-";
} catch (e) {
toast("加载任务失败: " + e.message, true);
}
loadStats();
}
const STAT_IDS = ["stTasks", "stRunning", "stRuns", "stOk", "stFail", "stImgs", "stDisk"];
async function loadStats() {
// 未读到真实数据前, 一律显示 "-"
STAT_IDS.forEach((id) => { $(id).textContent = "-"; });
$("trashCount").textContent = "-";
try {
const s = await api("/api/stats");
$("stTasks").textContent = s.tasks ?? "-";
$("stRunning").textContent = s.running ?? "-";
$("stRuns").textContent = s.runs ?? "-";
$("stOk").textContent = s.ok ?? "-";
$("stFail").textContent = s.fail ?? "-";
$("stImgs").textContent = s.images ?? "-";
const mb = Number(s.disk_mb);
$("stDisk").textContent = Number.isFinite(mb) && s.disk_mb != null
? (mb >= 1024 ? (mb / 1024).toFixed(1) + " GB" : mb + " MB") : "-";
$("trashCount").textContent = s.trash ?? "-";
} catch (e) { /* 统计失败保持 "-" */ }
}
function taskStatusBadge(t) {
if (t.running) return '● 运行中';
const r = t.latest_run;
if (r && r.status === "paused") return '⏸ 已暂停';
if (t.mode === "scheduled" && t.schedule && t.schedule.enabled) {
const next = t.schedule.next_run ? ` · 下次 ${t.schedule.next_run.slice(11)}` : "";
return `⏰ 等待调度${esc(next)}`;
}
if (r) return `${({ completed: "✅ 已完成", failed: "❌ 失败", stopped: "⏹ 已停止" })[r.status] || esc(r.status)}`;
return '未运行';
}
const GROUPS = [
{ key: "batch", label: "📄 批量爬取", desc: "一次性爬取指定网址列表" },
{ key: "scheduled", label: "⏰ 定时爬取", desc: "按间隔或 cron 表达式定时执行" },
{ key: "auto", label: "🤖 自动爬取", desc: "从起始网址自动发现链接,持续递归爬取到无新链接为止" },
];
function renderTasks() {
const box = $("taskList");
$("emptyState").classList.toggle("hidden", state.tasks.length > 0);
box.innerHTML = GROUPS.map((g) => {
const items = state.tasks.filter((t) => t.mode === g.key);
if (!items.length) return "";
return `
${g.label}
${g.desc}
${items.length} 个任务
${items.map(taskCard).join("")}
`;
}).join("");
}
function taskCard(t) {
const r = t.latest_run;
const cfg = t.config || {};
const prog = r && (r.status === "running" || r.status === "paused") ? `
${r.progress.done || 0} / ${r.progress.total || "?"}
${esc(r.progress.current_url || "")}
` : "";
const stats = r ? `
✅ ${r.stats.ok || 0}
❌ ${r.stats.fail || 0}
🖼️ ${r.stats.images || 0}
` : "";
const urlCount = (t.urls || []).length;
const line2 = t.mode === "auto"
? `起始: ${esc((t.auto && t.auto.seed_url) || "")}`
: `网址: ${urlCount} 个`;
const line3 = t.mode === "scheduled"
? `调度: ${t.schedule.type === "cron" ? esc(t.schedule.cron) : "每 " + (t.schedule.interval_value || "-") + " " + ({ minutes: "分钟", hours: "小时", days: "天" }[t.schedule.interval_unit] || "")}`
: "";
const outDir = cfg.out_dir || "out/" + t.id;
let acts = "";
if (t.running) {
acts = `
`;
} else {
const canStart = t.mode !== "scheduled" || (t.urls || []).length > 0;
acts = ``;
}
acts += `
`;
return `
${esc(t.name)}
${taskStatusBadge(t)}
${MODE_LABEL[t.mode]}
${line2}
${line3 ? `
${line3}
` : ""}
输出: ${esc(outDir)}
${prog}
${stats}
${acts}
`;
}
/* ---------------- 任务操作 ---------------- */
async function actTask(tid, act) {
try {
const r = await api(`/api/tasks/${tid}/${act}`, { method: "POST" });
if (r.msg) toast(r.msg);
loadTasks();
if (state.detail.task && state.detail.task.id === tid) openDetail(tid);
} catch (e) { toast(e.message, true); }
}
async function delTask(tid) {
const t = state.tasks.find((x) => x.id === tid);
if (!confirm(`确定删除任务「${t ? t.name : tid}」?\n任务将移入回收站,可在回收站中恢复或彻底删除。`)) return;
try {
const r = await api(`/api/tasks/${tid}`, { method: "DELETE" });
toast(r.msg || "已删除");
loadTasks();
} catch (e) { toast(e.message, true); }
}
/* ---------------- 网址文件导入 ---------------- */
const MAX_IMPORT_SIZE = 2 * 1024 * 1024; // 2MB 上限
async function readFileSmart(file) {
/* 自动识别 UTF-8 / GBK 编码 */
const buf = await file.arrayBuffer();
let text = new TextDecoder("utf-8").decode(buf);
if (text.includes("\uFFFD")) {
try { text = new TextDecoder("gbk").decode(buf); } catch (e) { /* 保留 utf-8 结果 */ }
}
return text;
}
function importUrlsText(text, mode) {
const ta = $("taskForm").elements["urls"];
const clean = (s) => s.split(" #")[0].trim(); // 去掉行内注释 (URL 不含空格, 安全)
const existing = new Set(
mode === "overwrite" ? [] : ta.value.split(/\r?\n/).map(clean).filter(Boolean)
);
const fresh = text.split(/\r?\n/).map(clean).filter(Boolean);
let added = 0;
for (const line of fresh) {
if (!existing.has(line)) { existing.add(line); added++; }
}
ta.value = Array.from(existing).join("\n");
return { total: fresh.length, added };
}
$("btnImportUrls").onclick = () => $("urlFileInput").click();
$("urlFileInput").onchange = async (e) => {
const file = e.target.files && e.target.files[0];
e.target.value = ""; // 允许重复选择同一文件
if (!file) return;
if (file.size > MAX_IMPORT_SIZE) {
toast("文件过大(上限 2MB),请拆分后导入", true);
return;
}
try {
const text = await readFileSmart(file);
const mode = $("importMode").value;
const r = importUrlsText(text, mode);
formDirty = true;
if (mode === "overwrite") {
toast(`已导入 ${file.name}:共 ${r.total} 行(覆盖原列表,新增 ${r.added} 条)`);
} else {
toast(`已导入 ${file.name}:共 ${r.total} 行,新增 ${r.added} 条网址(追加)`);
}
} catch (err) {
toast("文件读取失败: " + err.message, true);
}
};
/* ---------------- 搜索 ---------------- */
async function doSearch() {
const q = $("searchInput").value.trim();
if (!q) { toast("请输入搜索关键词", true); return; }
try {
const d = await api(`/api/search?q=${encodeURIComponent(q)}`);
renderSearch(d);
showModal("searchModal");
} catch (e) { toast(e.message, true); }
}
function renderSearch(d) {
const body = $("searchBody");
if (!d.results.length) {
body.innerHTML = ``;
return;
}
const items = d.results.map((r) => {
const badge = `${MODE_LABEL[r.mode] || esc(r.mode)}`;
if (r.type === "task") {
return `
${badge}
📋 ${esc(r.task_name)}
任务
${r.seed_url ? "起始: " + esc(r.seed_url) : "网址 " + (r.urls_count || 0) + " 个"} · 创建于 ${esc(r.created_at || "")}
`;
}
const stCls = r.status === "OK" ? "t-ok" : "t-fail";
const html = r.html_file ? `HTML` : "";
const txt = r.txt_file ? `TXT` : "";
const meta = r.meta_file ? `📋 元数据` : "";
return `
${badge}
${esc(r.title) || "(无标题)"}
${r.status}
${esc(r.run_status)}
${esc(r.crawl_time || "")}
🌐 ${esc(r.url)}${r.images ? ` · 🖼️ ${r.images}` : ""}
${html}${txt}${meta}
`;
}).join("");
body.innerHTML = `关键词: ${esc(d.q)}共 ${d.count} 条结果
${items}`;
}
/* 搜索结果的跳转动作: 先关搜索弹窗再打开目标 */
function searchOpenDetail(tid) {
hideModal("searchModal");
openDetail(tid);
}
function searchPreview(tid, path, title) {
hideModal("searchModal");
previewFile(tid, path, title);
}
function searchMeta(tid, metaFile) {
hideModal("searchModal");
showMeta(tid, metaFile);
}
/* ---------------- 回收站 ---------------- */
async function openTrash() {
try {
const items = await api("/api/trash");
state.trashItems = items;
renderTrash(items);
showModal("trashModal");
} catch (e) { toast(e.message, true); }
}
function renderTrash(items) {
const body = $("trashBody");
if (!items.length) {
body.innerHTML = ``;
return;
}
body.innerHTML = items.map((t) => `
${esc(t.name)}
${MODE_LABEL[t.mode] || esc(t.mode)}
删除时间: ${esc(t.deleted_at)} · 运行次数: ${t.runs_count || 0}
输出目录: ${esc(t.out_dir || "")}
`).join("")
+ ``;
}
async function restoreTrash(tid) {
try {
const r = await api(`/api/trash/${tid}/restore`, { method: "POST" });
toast(r.msg || "已恢复");
openTrash();
loadTasks();
} catch (e) { toast(e.message, true); }
}
async function purgeTrashTask(tid) {
const t = state.trashItems && state.trashItems.find((x) => x.id === tid);
if (!confirm(`彻底删除任务「${t ? t.name : tid}」?\n任务配置与所有运行记录将永久删除,不可恢复!`)) return;
try {
const r = await api(`/api/trash/${tid}`, { method: "DELETE" });
toast(r.files_removed ? "已彻底删除(含输出文件)" : "已彻底删除(自定义输出目录已保留)");
openTrash();
loadTasks();
} catch (e) { toast(e.message, true); }
}
async function clearTrash() {
if (!confirm(`确定清空回收站?\n回收站中所有任务将永久删除,不可恢复!`)) return;
try {
const r = await api("/api/trash", { method: "DELETE" });
toast(`已清空 ${r.purged} 项`);
openTrash();
loadTasks();
} catch (e) { toast(e.message, true); }
}
/* ---------------- 新建 / 编辑表单 ---------------- */
function switchMode(mode, lock) {
state.mode = mode;
document.querySelectorAll("#modeTabs .tab").forEach((b) => {
b.classList.toggle("active", b.dataset.mode === mode);
b.disabled = !!lock;
});
$("urlsField").classList.toggle("hidden", mode !== "batch" && mode !== "scheduled");
$("scheduleBox").classList.toggle("hidden", mode !== "scheduled");
$("autoBox").classList.toggle("hidden", mode !== "auto");
}
function toLocalInputVal(iso) {
if (!iso) return "";
const d = new Date(String(iso).replace(" ", "T"));
if (isNaN(d.getTime())) return "";
const p = (n) => String(n).padStart(2, "0");
return `${d.getFullYear()}-${p(d.getMonth() + 1)}-${p(d.getDate())}T${p(d.getHours())}:${p(d.getMinutes())}`;
}
function openCreate() {
state.editTask = null;
formDirty = false;
$("modalTitle").textContent = "新建爬取任务";
$("taskForm").reset();
$("taskForm").elements["delay_min"].value = 2;
$("taskForm").elements["delay_max"].value = 5;
$("taskForm").elements["timeout"].value = 60;
$("taskForm").elements["retry_count"].value = 2;
$("taskForm").elements["retry_interval"].value = 3;
$("taskForm").elements["notify_email"].value = "wlq@tphai.com";
$("formHint").textContent = "";
switchMode("batch", false);
showModal("taskModal");
}
async function openEdit(tid) {
const t = state.tasks.find((x) => x.id === tid) || await api(`/api/tasks/${tid}`);
state.editTask = t;
$("modalTitle").textContent = `编辑任务「${t.name}」`;
const f = $("taskForm");
f.reset();
const cfg = t.config || {};
f.elements["name"].value = t.name || "";
f.elements["out_dir"].value = cfg.out_dir || "";
f.elements["delay_min"].value = cfg.delay_min ?? 2;
f.elements["delay_max"].value = cfg.delay_max ?? 5;
f.elements["timeout"].value = cfg.timeout ?? 60;
f.elements["retry_count"].value = cfg.retry_count ?? 2;
f.elements["retry_interval"].value = cfg.retry_interval ?? 3;
f.elements["crawl_images"].checked = !!cfg.crawl_images;
f.elements["notify"].checked = !!cfg.notify;
f.elements["notify_email"].value = cfg.notify_email || "wlq@tphai.com";
f.elements["urls"].value = (t.urls || []).join("\n");
if (t.mode === "scheduled" && t.schedule) {
f.elements["schedule_type"].value = t.schedule.type || "interval";
f.elements["schedule_enabled"].checked = t.schedule.enabled !== false;
f.elements["interval_value"].value = t.schedule.interval_value ?? 24;
f.elements["interval_unit"].value = t.schedule.interval_unit || "hours";
f.elements["cron"].value = t.schedule.cron || "";
f.elements["first_run"].value = toLocalInputVal(t.schedule.next_run || "");
}
if (t.mode === "auto" && t.auto) {
f.elements["seed_url"].value = t.auto.seed_url || "";
f.elements["include"].value = (t.auto.include || []).join("\n");
f.elements["exclude"].value = (t.auto.exclude || []).join("\n");
f.elements["same_domain"].checked = t.auto.same_domain !== false;
f.elements["use_regex"].checked = !!t.auto.use_regex;
f.elements["max_pages"].value = t.auto.max_pages ?? 1000;
f.elements["max_depth"].value = t.auto.max_depth ?? 0;
}
syncScheduleUI();
$("formHint").textContent = t.running
? "⚠️ 任务运行中:参数修改将热更新(网址/规则改动下次运行生效)"
: "";
formDirty = false;
switchMode(t.mode, true);
showModal("taskModal");
}
function syncScheduleUI() {
const f = $("taskForm");
const t = f.elements["schedule_type"].value;
$("intervalBox").classList.toggle("hidden", t !== "interval");
$("cronBox").classList.toggle("hidden", t !== "cron");
}
function splitLines(v) { return String(v || "").split("\n").map((s) => s.trim()).filter(Boolean); }
async function submitForm(e) {
e.preventDefault();
const f = $("taskForm");
const name = f.elements["name"].value.trim();
if (!name) { toast("请填写项目名称", true); return; }
const mode = state.mode;
const config = {
out_dir: f.elements["out_dir"].value.trim(),
delay_min: parseFloat(f.elements["delay_min"].value) || 2,
delay_max: parseFloat(f.elements["delay_max"].value) || 5,
timeout: parseInt(f.elements["timeout"].value) || 60,
retry_count: parseInt(f.elements["retry_count"].value) || 0,
retry_interval: parseFloat(f.elements["retry_interval"].value) || 3,
crawl_images: f.elements["crawl_images"].checked,
notify: f.elements["notify"].checked,
notify_email: f.elements["notify_email"].value.trim() || "wlq@tphai.com",
};
const urls = splitLines(f.elements["urls"].value);
const payload = { name, mode, config, urls };
if (mode === "auto") {
payload.auto = {
seed_url: f.elements["seed_url"].value.trim(),
include: splitLines(f.elements["include"].value),
exclude: splitLines(f.elements["exclude"].value),
same_domain: f.elements["same_domain"].checked,
use_regex: f.elements["use_regex"].checked,
max_pages: parseInt(f.elements["max_pages"].value) || 0,
max_depth: parseInt(f.elements["max_depth"].value) || 0,
};
}
if (mode === "scheduled") {
payload.schedule = {
type: f.elements["schedule_type"].value,
enabled: f.elements["schedule_enabled"].checked,
interval_unit: f.elements["interval_unit"].value,
interval_value: parseInt(f.elements["interval_value"].value) || 1,
cron: f.elements["cron"].value.trim(),
first_run: f.elements["first_run"].value || "",
};
}
try {
if (state.editTask) {
await api(`/api/tasks/${state.editTask.id}`, {
method: "PUT", headers: { "Content-Type": "application/json" },
body: JSON.stringify(payload),
});
toast("任务已更新");
} else {
const t = await api("/api/tasks", {
method: "POST", headers: { "Content-Type": "application/json" },
body: JSON.stringify(payload),
});
toast(`任务「${t.name}」已创建`);
}
formDirty = false;
hideModal("taskModal");
loadTasks();
} catch (err) { toast(err.message, true); }
}
/* ---------------- 详情 ---------------- */
async function openDetail(tid) {
try {
const t = await api(`/api/tasks/${tid}?page_size=${DETAIL_PAGE_SIZE}`);
state.detail.task = t;
const runs = t.runs || [];
const cur = t.run;
state.detail.runId = cur ? cur.id : (runs[0] ? runs[0].id : null);
state.detail.pageSize = (t.run && t.run.run_page_size) || DETAIL_PAGE_SIZE;
state.detail.logOffset = 0;
$("detailTitle").textContent = `任务详情 · ${t.name}`;
renderDetail();
showModal("detailModal");
startLogPoll();
} catch (e) { toast(e.message, true); }
}
async function loadRunPage(tid, rid, page, size) {
const ps = size || state.detail.pageSize || DETAIL_PAGE_SIZE;
return api(`/api/tasks/${tid}?run=${rid}&page=${page}&page_size=${ps}`);
}
async function selectRun(rid) {
const t = state.detail.task;
if (!t) return;
try {
const d = await loadRunPage(t.id, rid, 1);
state.detail.task = d;
state.detail.runId = d.run ? d.run.id : null;
state.detail.pageSize = (d.run && d.run.run_page_size) || DETAIL_PAGE_SIZE;
state.detail.logOffset = 0;
renderDetail();
startLogPoll();
} catch (e) { toast(e.message, true); }
}
async function goRunPage(p) {
const t = state.detail.task;
if (!t || !state.detail.runId) return;
const ps = (t.run && t.run.run_page_size) || state.detail.pageSize || DETAIL_PAGE_SIZE;
try {
const d = await loadRunPage(t.id, state.detail.runId, p, ps);
state.detail.task = d;
state.detail.pageSize = (d.run && d.run.run_page_size) || ps;
renderDetail();
} catch (e) { toast(e.message, true); }
}
/* 跳转到指定页码 (输入框回车/点击 GO) */
function jumpRunPage() {
const t = state.detail.task;
if (!t || !state.detail.runId || !t.run) return;
const inp = $("pagerJump");
const pages = t.run.run_pages || 1;
let p = parseInt(inp.value, 10);
if (!p || isNaN(p)) p = 1;
p = Math.min(Math.max(p, 1), pages);
inp.value = p;
goRunPage(p);
}
/* 切换每页条数 */
async function changePageSize(size) {
const t = state.detail.task;
if (!t || !state.detail.runId) return;
try {
const d = await loadRunPage(t.id, state.detail.runId, 1, parseInt(size, 10) || 100);
state.detail.task = d;
state.detail.pageSize = (d.run && d.run.run_page_size) || 100;
renderDetail();
} catch (e) { toast(e.message, true); }
}
/* 生成页码窗口列表, 0 表示省略号: 1 2 3 … 97 98 99 100 */
function pageWindow(cur, pages, width) {
width = width || 5;
const win = [];
if (pages <= width + 2) {
for (let i = 1; i <= pages; i++) win.push(i);
return win;
}
win.push(1);
let lo = Math.max(2, cur - Math.floor(width / 2));
let hi = Math.min(pages - 1, cur + Math.floor(width / 2));
if (hi - lo < width - 1) {
if (lo <= 2) hi = lo + width - 1;
else lo = hi - width + 1;
}
if (lo > 2) win.push(0);
for (let i = lo; i <= hi; i++) win.push(i);
if (hi < pages - 1) win.push(0);
win.push(pages);
return win;
}
/* 重爬失败页: 注入待爬队列并自动继续爬取 */
async function retryFailed() {
const t = state.detail.task;
if (!t || !state.detail.runId) return;
if (!confirm("将把本次运行失败的页面重新加入待爬队列并立即继续爬取,确定?")) return;
try {
const d = await api(`/api/tasks/${t.id}/retry-failed`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ run: state.detail.runId }),
});
toast(`已注入 ${d.injected} 个失败页,开始继续爬取…`);
if ((d.injected || 0) > 0) {
await api(`/api/tasks/${t.id}/continue`, { method: "POST" });
}
loadTasks();
openDetail(t.id);
} catch (e) { toast(e.message, true); }
}
function renderDetail() {
const t = state.detail.task;
const runs = t.runs || [];
const run = t.run || null;
state.detail.runId = run ? run.id : (runs[0] ? runs[0].id : null);
const cfg = t.config || {};
const meta = `
模式: ${MODE_LABEL[t.mode] || t.mode}
任务ID: ${t.id}
创建: ${fmtTime(t.created_at)}
输出: ${esc(cfg.out_dir || "out/" + t.id)}
${t.mode === "auto" ? `
起始: ${esc((t.auto && t.auto.seed_url) || "")}
待爬缓存: ${t.auto_pending_count ?? "-"} 条(停止后可继续爬取)
已爬: ${t.auto_visited_count ?? "-"} 页` : ""}
`;
let sched = "";
if (t.mode === "scheduled" && t.schedule) {
const s = t.schedule;
sched = `
调度: ${s.type === "cron" ? esc(s.cron) : "每 " + (s.interval_value || "-") + " " + ({ minutes: "分钟", hours: "小时", days: "天" }[s.interval_unit] || "")}
启用: ${s.enabled ? "是" : "否"}
下次: ${esc(s.next_run || "—")}
上次: ${esc(s.last_run || "—")}
已运行: ${s.runs_count || 0} 次
`;
}
const acts = `
${t.running ? `
`
: `
${t.mode === "auto" && (t.auto_pending_count || 0) > 0 ? `
` : ""}`}
${t.mode === "auto" ? `
` : ""}
`;
const chips = runs.map((r) => `
${fmtTime(r.started_at).slice(5)} · ${r.status} · ${r.progress.done || 0}/${r.progress.total || "?"}
`).join("");
const body = run ? renderRunPanel(t, run) : '暂无运行记录,点击「立即执行」开始第一次爬取。
';
$("detailBody").innerHTML = `
${meta}
${sched}
${acts}
${chips || '暂无运行记录'}
${body}`;
}
function renderRunPanel(t, run) {
const s = run.stats || { ok: 0, fail: 0, images: 0 };
const prog = run.status === "running" || run.status === "paused" ? `
` : "";
const results = run.results || [];
const rows = results.map((r, i) => {
const statusCls = r.status === "OK" ? "t-ok" : "t-fail";
const html = r.html_file ? `HTML` : "—";
const txt = r.txt_file ? `TXT` : "—";
const imgs = (r.images || []).length
? `🖼️ ${(r.images || []).length}` : "—";
const ctime = r.crawl_time ? `${esc(r.crawl_time.slice(5, 19))}` : "—";
const meta = r.meta_file
? `📋 元数据` : "—";
return `
| ${(run.results_total ? (run.run_page - 1) * DETAIL_PAGE_SIZE : 0) + i + 1} |
${r.status} |
${esc(r.title)} |
${esc(r.url)} |
${ctime} |
${html} | ${txt} | ${imgs} |
${meta} |
${esc((r.error || "").slice(0, 50))} |
`;
}).join("");
const thumbs = results.flatMap((r) => r.images || []).slice(0, 60);
const thumbHtml = thumbs.length ? `
${thumbs.map((im) => `
})
`).join("")}
` : "";
const total = run.results_total || results.length;
const pages = run.run_pages || 1;
const page = run.run_page || 1;
const pageSize = run.run_page_size || DETAIL_PAGE_SIZE;
const multi = pages > 1;
const pager = total > 0 ? `
` : "";
return `
运行ID: ${run.id}
状态: ${run.status}
开始: ${fmtTime(run.started_at)}
结束: ${fmtTime(run.finished_at)}
✅ ${s.ok}
❌ ${s.fail}
🖼️ ${s.images}
${t.mode === "auto" && (s.fail || 0) > 0 && run.status !== "running" && run.status !== "paused" ? `
` : ""}
${prog}
当前: ${esc(run.progress.current_url || "")}
${results.length ? `
| # | 状态 | 标题 | 网址 | 爬取时间 | HTML | TXT | 图片 | 元数据 | 错误 |
${rows}
${pager}` : '
暂无结果
'}
${thumbHtml}
`;
}
function scrollThumbs() {
const box = $("thumbsBox");
if (box) box.scrollIntoView({ behavior: "smooth", block: "center" });
}
async function startLogPoll() {
stopLogPoll();
const t = state.detail.task;
if (!t || !state.detail.runId) return;
const rid = state.detail.runId;
const tick = async () => {
try {
const d = await api(`/api/runs/${rid}/logs?offset=${state.detail.logOffset}`);
if (d.logs && d.logs.length) {
const box = $("logBox");
if (box) {
const atBottom = box.scrollHeight - box.scrollTop - box.clientHeight < 40;
d.logs.forEach((l) => {
const div = document.createElement("div");
div.className = l.level;
div.textContent = `[${l.ts}] ${l.msg}`;
box.appendChild(div);
});
if (atBottom) box.scrollTop = box.scrollHeight;
state.detail.logOffset += d.logs.length;
}
}
} catch (e) { /* ignore */ }
loadTasks(); // 后台保持列表刷新
};
tick();
state.logTimer = setInterval(tick, 2500);
}
function stopLogPoll() {
if (state.logTimer) { clearInterval(state.logTimer); state.logTimer = null; }
}
async function contTask(tid) {
try {
const r = await api(`/api/tasks/${tid}/continue`, { method: "POST" });
toast("已开始继续爬取缓存队列");
loadTasks();
if (state.detail.task && state.detail.task.id === tid) openDetail(tid);
} catch (e) { toast(e.message, true); }
}
/* ---------------- 打包导出 ---------------- */
async function exportZip(tid) {
const btn = window.event && window.event.target;
if (btn) { btn.disabled = true; btn.textContent = "⏳ 打包中..."; }
try {
const res = await fetch(`/api/export?task_id=${tid}`);
if (!res.ok) {
let d = null;
try { d = await res.json(); } catch (e) { /* ignore */ }
throw new Error((d && d.error) || `HTTP ${res.status}`);
}
let fname = "export.zip";
const disp = res.headers.get("Content-Disposition") || "";
const m = disp.match(/filename\*?=(?:UTF-8'')?"?([^";]+)"?/i);
if (m) { try { fname = decodeURIComponent(m[1]); } catch (e) { fname = m[1]; } }
const blob = await res.blob();
const a = document.createElement("a");
a.href = URL.createObjectURL(blob);
a.download = fname;
document.body.appendChild(a);
a.click();
a.remove();
setTimeout(() => URL.revokeObjectURL(a.href), 5000);
toast(`打包下载完成 ${fname} (${(blob.size / 1048576).toFixed(1)} MB)`);
} catch (e) {
toast("打包失败: " + e.message, true);
} finally {
if (btn) { btn.disabled = false; btn.textContent = "📦 打包下载"; }
}
}
async function exportEmail(tid) {
const t = (state.detail.task && state.detail.task.id === tid)
? state.detail.task : state.tasks.find((x) => x.id === tid);
const def = (t && t.config && t.config.notify_email) || "wlq@tphai.com";
const email = prompt("发送到邮箱(留空默认 wlq@tphai.com):", def);
if (email === null) return; // 用户取消
const btn = window.event && window.event.target;
if (btn) { btn.disabled = true; btn.textContent = "⏳ 打包发送中..."; }
try {
const r = await api("/api/export/email", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ task_id: tid, email: email.trim() || "wlq@tphai.com" }),
});
toast(r.msg || "已发送");
} catch (e) {
toast("发送失败: " + e.message, true);
} finally {
if (btn) { btn.disabled = false; btn.textContent = "📧 发邮箱"; }
}
}
/* ---------------- 试爬取 ---------------- */
function collectProbeFromForm() {
const f = $("taskForm");
return {
seed_url: f.elements["seed_url"].value.trim(),
include: splitLines(f.elements["include"].value),
exclude: splitLines(f.elements["exclude"].value),
same_domain: f.elements["same_domain"].checked,
use_regex: f.elements["use_regex"].checked,
timeout: parseInt(f.elements["timeout"].value) || 60,
};
}
async function runProbe(params, btn) {
if (btn) { btn.disabled = true; btn.textContent = "⏳ 探测中..."; }
try {
const r = await api("/api/probe", {
method: "POST", headers: { "Content-Type": "application/json" },
body: JSON.stringify(params),
});
renderProbe(r);
} catch (e) { toast(e.message, true); }
finally {
if (btn) { btn.disabled = false; btn.textContent = "🧪 试爬取"; }
}
}
async function probeTask(tid) {
try {
const r = await api(`/api/tasks/${tid}/probe`, { method: "POST" });
renderProbe(r);
} catch (e) { toast(e.message, true); }
}
async function clearCache(tid) {
if (!confirm("清空待爬缓存与已爬记录?\n下次运行将从起始网址重新开始爬取。")) return;
try {
const r = await api(`/api/tasks/${tid}/clear_cache`, { method: "POST" });
toast(r.msg || "已清空");
openDetail(tid);
} catch (e) { toast(e.message, true); }
}
function renderProbe(r) {
const body = $("probeBody");
if (!r.ok) {
body.innerHTML = `❌ ${esc(r.error || "试爬取失败")}
`;
showModal("probeModal");
return;
}
const included = r.included || [];
const excluded = r.excluded || [];
const incHtml = included.length
? included.map((u) => `${esc(u)}
`).join("")
: '没有符合条件的链接,请调整规则
';
const excHtml = excluded.length
? excluded.map((x) => `${esc(x.url)}${esc(x.reason)}
`).join("")
: '无
';
const moreInc = r.total_included > included.length
? `... 还有 ${r.total_included - included.length} 条未显示
` : "";
const moreExc = r.total_excluded > excluded.length
? `... 还有 ${r.total_excluded - excluded.length} 条未显示
` : "";
body.innerHTML = `
页面标题: ${esc(r.title)}
抓取时间: ${esc(r.crawled_at)}
发现链接: ${r.total_links} 条
✅ 将爬取: ${r.total_included} 条
🚫 被排除: ${r.total_excluded} 条
✅ 符合规则、将爬取的链接(显示前 ${included.length} 条):
${incHtml}${moreInc}
🚫 被排除的链接及原因(显示前 ${excluded.length} 条):
${excHtml}${moreExc}
`;
showModal("probeModal");
}
/* ---------------- 元数据查看 ---------------- */
const META_FIELDS = [
["type", "资源类型"], ["mode", "爬取模式"], ["task_name", "任务名称"],
["task_id", "任务ID"], ["run_id", "运行ID"], ["crawl_time", "爬取时间"],
["url", "爬取网址"], ["source_url", "来源链接"], ["depth", "爬取深度"],
["title", "页面标题"], ["status", "状态"], ["attempts", "尝试次数"],
["error", "错误信息"], ["html_file", "HTML文件"], ["txt_file", "文本文件"],
["page_url", "所属页面"],
];
async function showMeta(taskId, metaFile) {
try {
const data = await api(`/api/file?task_id=${taskId}&path=${encodeURIComponent(metaFile)}`);
const rows = META_FIELDS.filter(([k]) => data[k] !== undefined && data[k] !== "" && data[k] !== null)
.map(([k, label]) => {
let v = data[k];
if (k === "status") v = `${esc(v)}`;
if (k === "mode") v = `${MODE_LABEL[v] || esc(v)}`;
if (k === "type") v = v === "images" ? "🖼️ 图片集" : "📄 网页";
if (k === "depth") v = v === 0 ? "0(起始页)" : esc(v);
return `| ${label} | ${v} |
`;
}).join("");
// 来源链接空值也显示(起始页无来源)
const srcRow = `| 来源链接 | ${data.source_url ? esc(data.source_url) : "—(起始页,无来源)"} |
`;
const finalRows = rows.includes(srcRow) ? rows : rows + srcRow;
let imgs = "";
if (data.images && data.images.length) {
imgs = `
🖼️ 图片明细(${data.images.length} 张):
| 文件 | 原始URL | 大小 | 下载时间 |
${data.images.map((im) => `
| ${esc(im.file)} |
${esc(im.url)} |
${im.size ? Math.round(im.size / 1024) + " KB" : "—"} |
${esc(im.download_time || "")} |
`).join("")}
`;
}
$("metaBody").innerHTML = `${imgs}`;
showModal("metaModal");
} catch (e) { toast(e.message, true); }
}
/* ---------------- 文件预览 ---------------- */
function previewFile(tid, path, title) {
$("previewTitle").textContent = title || "预览";
$("previewFrame").src = `/api/file?task_id=${tid}&path=${encodeURIComponent(path)}`;
showModal("previewModal");
}
/* ---------------- 弹窗控制 ---------------- */
function showModal(id) { $(id).classList.remove("hidden"); }
function hideModal(id) { $(id).classList.add("hidden"); }
/* 关闭任务弹窗: 有未保存修改时先确认 */
function safeCloseTaskModal() {
if (formDirty && !confirm("有未保存的修改,确定要放弃吗?")) return false;
formDirty = false;
hideModal("taskModal");
return true;
}
/* ---------------- 主题切换 ---------------- */
function applyTheme(theme) {
document.body.dataset.theme = theme;
$("btnTheme").textContent = theme === "light" ? "🌙" : "☀️";
try { localStorage.setItem("crawler_theme", theme); } catch (e) { /* ignore */ }
}
$("btnTheme").onclick = () => {
applyTheme(document.body.dataset.theme === "light" ? "dark" : "light");
};
try {
applyTheme(localStorage.getItem("crawler_theme") || "dark");
} catch (e) {
applyTheme("dark");
}
/* ---------------- 事件绑定 ---------------- */
$("btnNew").onclick = openCreate;
$("btnNew2").onclick = openCreate;
$("btnRefresh").onclick = loadTasks;
$("btnTrash").onclick = openTrash;
$("btnSearch").onclick = doSearch;
$("searchInput").addEventListener("keydown", (e) => { if (e.key === "Enter") doSearch(); });
$("btnProbe").onclick = () => {
const p = collectProbeFromForm();
if (!p.seed_url) { toast("请先填写起始网址", true); return; }
runProbe(p, $("btnProbe"));
};
document.querySelectorAll("#modeTabs .tab").forEach((b) => {
b.onclick = () => { switchMode(b.dataset.mode, false); };
});
$("taskForm").onsubmit = submitForm;
$("taskForm").elements["schedule_type"].onchange = syncScheduleUI;
document.querySelectorAll("[data-close]").forEach((b) => b.onclick = safeCloseTaskModal);
document.querySelectorAll("[data-close-detail]").forEach((b) => b.onclick = () => { stopLogPoll(); hideModal("detailModal"); });
document.querySelectorAll("[data-close-preview]").forEach((b) => b.onclick = () => { $("previewFrame").src = "about:blank"; hideModal("previewModal"); });
document.querySelectorAll("[data-close-probe]").forEach((b) => b.onclick = () => hideModal("probeModal"));
document.querySelectorAll("[data-close-meta]").forEach((b) => b.onclick = () => hideModal("metaModal"));
document.querySelectorAll("[data-close-trash]").forEach((b) => b.onclick = () => hideModal("trashModal"));
document.querySelectorAll("[data-close-search]").forEach((b) => b.onclick = () => hideModal("searchModal"));
/* 表单改动监听 -> 脏标记 */
$("taskForm").addEventListener("input", () => { formDirty = true; });
$("taskForm").addEventListener("change", () => { formDirty = true; });
document.querySelectorAll(".modal-overlay").forEach((ov) => {
ov.addEventListener("mousedown", (e) => {
if (e.target === ov) {
if (ov.id === "taskModal") {
safeCloseTaskModal();
} else {
if (ov.id === "detailModal") stopLogPoll();
if (ov.id === "previewModal") $("previewFrame").src = "about:blank";
ov.classList.add("hidden");
}
}
});
});
document.addEventListener("keydown", (e) => {
if (e.key === "Escape") {
if (!$("taskModal").classList.contains("hidden")) {
safeCloseTaskModal();
return;
}
stopLogPoll();
$("previewFrame").src = "about:blank";
["detailModal", "probeModal", "metaModal", "trashModal", "searchModal", "previewModal"].forEach((id) => $(id).classList.add("hidden"));
}
});
/* ---------------- 启动 ---------------- */
loadTasks();
setInterval(() => {
const detailOpen = !$("detailModal").classList.contains("hidden");
if (!detailOpen) loadTasks();
}, 4000);