v1.0.2: 自动爬取试爬取功能 + 每个页面/图片生成操作信息元数据(模式/时间/网址/来源链接/深度等)
This commit is contained in:
+130
-3
@@ -359,6 +359,7 @@ function renderDetail() {
|
||||
<button class="btn" onclick="actTask('${t.id}','pause')">⏸ 暂停</button>
|
||||
<button class="btn danger" onclick="actTask('${t.id}','stop')">⏹ 终止</button>`
|
||||
: `<button class="btn primary" onclick="actTask('${t.id}','start')">▶ 立即执行</button>`}
|
||||
${t.mode === "auto" ? `<button class="btn" onclick="probeTask('${t.id}')">🧪 试爬取</button>` : ""}
|
||||
<button class="btn" onclick="openEdit('${t.id}')">✏️ 编辑配置</button>
|
||||
</div>`;
|
||||
|
||||
@@ -389,13 +390,18 @@ function renderRunPanel(t, run) {
|
||||
const txt = r.txt_file ? `<span class="file-link" onclick="previewFile('${t.id}','${esc(r.txt_file)}','文本: ${esc(r.url)}')">TXT</span>` : "—";
|
||||
const imgs = (r.images || []).length
|
||||
? `<span class="file-link" onclick="scrollThumbs()">🖼️ ${(r.images || []).length}</span>` : "—";
|
||||
const ctime = r.crawl_time ? `<span title="${esc(r.crawl_time)}">${esc(r.crawl_time.slice(5, 19))}</span>` : "—";
|
||||
const meta = r.meta_file
|
||||
? `<span class="file-link" onclick="showMeta('${t.id}','${esc(r.meta_file)}')">📋 元数据</span>` : "—";
|
||||
return `<tr>
|
||||
<td>${i + 1}</td>
|
||||
<td class="${statusCls}">${r.status}</td>
|
||||
<td class="title-cell" title="${esc(r.title)}">${esc(r.title)}</td>
|
||||
<td class="url-cell" title="${esc(r.url)}">${esc(r.url)}</td>
|
||||
<td>${ctime}</td>
|
||||
<td>${html}</td><td>${txt}</td><td>${imgs}</td>
|
||||
<td title="${esc(r.error || "")}">${esc((r.error || "").slice(0, 60))}</td>
|
||||
<td>${meta}</td>
|
||||
<td title="${esc(r.error || "")}">${esc((r.error || "").slice(0, 50))}</td>
|
||||
</tr>`;
|
||||
}).join("");
|
||||
const thumbs = results.flatMap((r) => r.images || []).slice(0, 60);
|
||||
@@ -421,7 +427,7 @@ function renderRunPanel(t, run) {
|
||||
${results.length ? `
|
||||
<div class="table-wrap">
|
||||
<table>
|
||||
<thead><tr><th>#</th><th>状态</th><th>标题</th><th>网址</th><th>HTML</th><th>TXT</th><th>图片</th><th>错误</th></tr></thead>
|
||||
<thead><tr><th>#</th><th>状态</th><th>标题</th><th>网址</th><th>爬取时间</th><th>HTML</th><th>TXT</th><th>图片</th><th>元数据</th><th>错误</th></tr></thead>
|
||||
<tbody>${rows}</tbody>
|
||||
</table>
|
||||
</div>` : '<div class="card-line">暂无结果</div>'}
|
||||
@@ -475,6 +481,120 @@ function stopLogPoll() {
|
||||
if (state.logTimer) { clearInterval(state.logTimer); state.logTimer = null; }
|
||||
}
|
||||
|
||||
/* ---------------- 试爬取 ---------------- */
|
||||
function collectProbeFromForm() {
|
||||
const f = $("taskForm");
|
||||
return {
|
||||
seed_url: f.elements["seed_url"].value.trim(),
|
||||
include: splitLines(f.elements["include"].value),
|
||||
exclude: splitLines(f.elements["exclude"].value),
|
||||
same_domain: f.elements["same_domain"].checked,
|
||||
use_regex: f.elements["use_regex"].checked,
|
||||
timeout: parseInt(f.elements["timeout"].value) || 60,
|
||||
};
|
||||
}
|
||||
|
||||
async function runProbe(params, btn) {
|
||||
if (btn) { btn.disabled = true; btn.textContent = "⏳ 探测中..."; }
|
||||
try {
|
||||
const r = await api("/api/probe", {
|
||||
method: "POST", headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(params),
|
||||
});
|
||||
renderProbe(r);
|
||||
} catch (e) { toast(e.message, true); }
|
||||
finally {
|
||||
if (btn) { btn.disabled = false; btn.textContent = "🧪 试爬取"; }
|
||||
}
|
||||
}
|
||||
|
||||
async function probeTask(tid) {
|
||||
try {
|
||||
const r = await api(`/api/tasks/${tid}/probe`, { method: "POST" });
|
||||
renderProbe(r);
|
||||
} catch (e) { toast(e.message, true); }
|
||||
}
|
||||
|
||||
function renderProbe(r) {
|
||||
const body = $("probeBody");
|
||||
if (!r.ok) {
|
||||
body.innerHTML = `<div class="card-line">❌ ${esc(r.error || "试爬取失败")}</div>`;
|
||||
showModal("probeModal");
|
||||
return;
|
||||
}
|
||||
const included = r.included || [];
|
||||
const excluded = r.excluded || [];
|
||||
const incHtml = included.length
|
||||
? included.map((u) => `<div class="probe-item included"><span class="u">${esc(u)}</span></div>`).join("")
|
||||
: '<div class="card-line">没有符合条件的链接，请调整规则</div>';
|
||||
const excHtml = excluded.length
|
||||
? excluded.map((x) => `<div class="probe-item excluded"><span class="u">${esc(x.url)}</span><span class="reason">${esc(x.reason)}</span></div>`).join("")
|
||||
: '<div class="card-line">无</div>';
|
||||
const moreInc = r.total_included > included.length
|
||||
? `<div class="card-line">... 还有 ${r.total_included - included.length} 条未显示</div>` : "";
|
||||
const moreExc = r.total_excluded > excluded.length
|
||||
? `<div class="card-line">... 还有 ${r.total_excluded - excluded.length} 条未显示</div>` : "";
|
||||
body.innerHTML = `
|
||||
<div class="probe-summary">
|
||||
<span>页面标题: <b>${esc(r.title)}</b></span>
|
||||
<span>抓取时间: <b>${esc(r.crawled_at)}</b></span>
|
||||
</div>
|
||||
<div class="probe-summary">
|
||||
<span>发现链接: <b>${r.total_links}</b> 条</span>
|
||||
<span class="t-ok">✅ 将爬取: <b>${r.total_included}</b> 条</span>
|
||||
<span class="t-fail">🚫 被排除: <b>${r.total_excluded}</b> 条</span>
|
||||
</div>
|
||||
<div class="card-line" style="font-weight:600">✅ 符合规则、将爬取的链接（显示前 ${included.length} 条）:</div>
|
||||
<div class="probe-list">${incHtml}${moreInc}</div>
|
||||
<div class="card-line" style="font-weight:600;margin-top:8px">🚫 被排除的链接及原因（显示前 ${excluded.length} 条）:</div>
|
||||
<div class="probe-list">${excHtml}${moreExc}</div>`;
|
||||
showModal("probeModal");
|
||||
}
|
||||
|
||||
/* ---------------- 元数据查看 ---------------- */
|
||||
const META_FIELDS = [
|
||||
["type", "资源类型"], ["mode", "爬取模式"], ["task_name", "任务名称"],
|
||||
["task_id", "任务ID"], ["run_id", "运行ID"], ["crawl_time", "爬取时间"],
|
||||
["url", "爬取网址"], ["source_url", "来源链接"], ["depth", "爬取深度"],
|
||||
["title", "页面标题"], ["status", "状态"], ["attempts", "尝试次数"],
|
||||
["error", "错误信息"], ["html_file", "HTML文件"], ["txt_file", "文本文件"],
|
||||
["page_url", "所属页面"],
|
||||
];
|
||||
|
||||
async function showMeta(taskId, metaFile) {
|
||||
try {
|
||||
const data = await api(`/api/file?task_id=${taskId}&path=${encodeURIComponent(metaFile)}`);
|
||||
const rows = META_FIELDS.filter(([k]) => data[k] !== undefined && data[k] !== "" && data[k] !== null)
|
||||
.map(([k, label]) => {
|
||||
let v = data[k];
|
||||
if (k === "status") v = `<span class="${v === "OK" ? "t-ok" : "t-fail"}">${esc(v)}</span>`;
|
||||
if (k === "mode") v = `<span class="badge ${esc(v)}">${MODE_LABEL[v] || esc(v)}</span>`;
|
||||
if (k === "type") v = v === "images" ? "🖼️ 图片集" : "📄 网页";
|
||||
if (k === "depth") v = v === 0 ? "0（起始页）" : esc(v);
|
||||
return `<tr><td>${label}</td><td>${v}</td></tr>`;
|
||||
}).join("");
|
||||
// 来源链接空值也显示(起始页无来源)
|
||||
const srcRow = `<tr><td>来源链接</td><td>${data.source_url ? esc(data.source_url) : "—（起始页，无来源）"}</td></tr>`;
|
||||
const finalRows = rows.includes(srcRow) ? rows : rows + srcRow;
|
||||
let imgs = "";
|
||||
if (data.images && data.images.length) {
|
||||
imgs = `
|
||||
<div class="card-line" style="font-weight:600;margin-top:8px">🖼️ 图片明细（${data.images.length} 张）:</div>
|
||||
<div class="table-wrap" style="max-height:200px">
|
||||
<table><thead><tr><th>文件</th><th>原始URL</th><th>大小</th><th>下载时间</th></tr></thead><tbody>
|
||||
${data.images.map((im) => `<tr>
|
||||
<td>${esc(im.file)}</td>
|
||||
<td class="url-cell" title="${esc(im.url)}">${esc(im.url)}</td>
|
||||
<td>${im.size ? Math.round(im.size / 1024) + " KB" : "—"}</td>
|
||||
<td>${esc(im.download_time || "")}</td></tr>`).join("")}
|
||||
</tbody></table>
|
||||
</div>`;
|
||||
}
|
||||
$("metaBody").innerHTML = `<table class="meta-table"><tbody>${finalRows}</tbody></table>${imgs}`;
|
||||
showModal("metaModal");
|
||||
} catch (e) { toast(e.message, true); }
|
||||
}
|
||||
|
||||
/* ---------------- 文件预览 ---------------- */
|
||||
function previewFile(tid, path, title) {
|
||||
$("previewTitle").textContent = title || "预览";
|
||||
@@ -513,6 +633,11 @@ try {
|
||||
$("btnNew").onclick = openCreate;
|
||||
$("btnNew2").onclick = openCreate;
|
||||
$("btnRefresh").onclick = loadTasks;
|
||||
$("btnProbe").onclick = () => {
|
||||
const p = collectProbeFromForm();
|
||||
if (!p.seed_url) { toast("请先填写起始网址", true); return; }
|
||||
runProbe(p, $("btnProbe"));
|
||||
};
|
||||
|
||||
document.querySelectorAll("#modeTabs .tab").forEach((b) => {
|
||||
b.onclick = () => { switchMode(b.dataset.mode, false); };
|
||||
@@ -523,6 +648,8 @@ $("taskForm").elements["schedule_type"].onchange = syncScheduleUI;
|
||||
document.querySelectorAll("[data-close]").forEach((b) => b.onclick = safeCloseTaskModal);
|
||||
document.querySelectorAll("[data-close-detail]").forEach((b) => b.onclick = () => { stopLogPoll(); hideModal("detailModal"); });
|
||||
document.querySelectorAll("[data-close-preview]").forEach((b) => b.onclick = () => { $("previewFrame").src = "about:blank"; hideModal("previewModal"); });
|
||||
document.querySelectorAll("[data-close-probe]").forEach((b) => b.onclick = () => hideModal("probeModal"));
|
||||
document.querySelectorAll("[data-close-meta]").forEach((b) => b.onclick = () => hideModal("metaModal"));
|
||||
|
||||
/* 表单改动监听 -> 脏标记 */
|
||||
$("taskForm").addEventListener("input", () => { formDirty = true; });
|
||||
@@ -549,7 +676,7 @@ document.addEventListener("keydown", (e) => {
|
||||
}
|
||||
stopLogPoll();
|
||||
$("previewFrame").src = "about:blank";
|
||||
["detailModal", "previewModal"].forEach((id) => $(id).classList.add("hidden"));
|
||||
["detailModal", "probeModal", "metaModal", "previewModal"].forEach((id) => $(id).classList.add("hidden"));
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
+27
-1
@@ -102,7 +102,11 @@
|
||||
|
||||
<div id="autoBox" class="hidden box">
|
||||
<div class="field"><label>起始网址 *（系统将自动发现符合规则的链接并爬取）</label>
|
||||
<input name="seed_url" placeholder="https://example.com/news"></div>
|
||||
<div class="row2">
|
||||
<input name="seed_url" placeholder="https://example.com/news" style="flex:1">
|
||||
<button type="button" class="btn sm" id="btnProbe">🧪 试爬取</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="row2">
|
||||
<div class="field"><label>包含规则（每行一个，子串或正则）</label>
|
||||
<textarea name="include" rows="3" placeholder="techpowerup.com/review /news/"></textarea></div>
|
||||
@@ -139,6 +143,28 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 试爬取结果弹窗 -->
|
||||
<div id="probeModal" class="modal-overlay hidden">
|
||||
<div class="modal wide">
|
||||
<div class="modal-head">
|
||||
<span>🧪 试爬取结果（规则筛选预览）</span>
|
||||
<button class="btn ghost sm" data-close-probe>✕</button>
|
||||
</div>
|
||||
<div id="probeBody" class="detail-wrap"></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 元数据弹窗 -->
|
||||
<div id="metaModal" class="modal-overlay hidden">
|
||||
<div class="modal">
|
||||
<div class="modal-head">
|
||||
<span>📋 资源操作信息（元数据）</span>
|
||||
<button class="btn ghost sm" data-close-meta>✕</button>
|
||||
</div>
|
||||
<div id="metaBody" class="detail-wrap"></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 文件预览弹窗 -->
|
||||
<div id="previewModal" class="modal-overlay hidden">
|
||||
<div class="modal wide tall">
|
||||
|
||||
@@ -215,6 +215,21 @@ td.title-cell { max-width: 220px; overflow: hidden; text-overflow: ellipsis; }
|
||||
/* ---------- preview ---------- */
|
||||
#previewFrame { flex: 1; border: none; background: var(--preview-bg); border-radius: 0 0 14px 14px; }
|
||||
|
||||
/* ---------- 试爬取结果 / 元数据 ---------- */
|
||||
.probe-summary { display: flex; flex-wrap: wrap; gap: 16px; font-size: 13px; }
|
||||
.probe-list {
|
||||
border: 1px solid var(--border); border-radius: 8px; max-height: 260px;
|
||||
overflow: auto; padding: 8px 12px; display: flex; flex-direction: column;
|
||||
gap: 5px; font-size: 12px; background: var(--log-bg);
|
||||
}
|
||||
.probe-item { display: flex; gap: 10px; align-items: baseline; }
|
||||
.probe-item .u { color: var(--text); word-break: break-all; }
|
||||
.probe-item.included .u { color: var(--green); }
|
||||
.probe-item.excluded .u { color: var(--muted); text-decoration: line-through; }
|
||||
.probe-item .reason { color: var(--yellow); font-size: 11px; flex-shrink: 0; }
|
||||
.meta-table td { white-space: normal; word-break: break-all; }
|
||||
.meta-table td:first-child { width: 110px; color: var(--muted); }
|
||||
|
||||
/* ---------- misc ---------- */
|
||||
.toast {
|
||||
position: fixed; top: 70px; left: 50%; transform: translateX(-50%);
|
||||
|
||||
Reference in New Issue
Block a user