v1.0.9: 爬取记录入库MySQL(crawler库: 任务/运行/结果三表, 内容不入库只存元数据, OK/FAIL状态标记)

app.py

@@ -10,6 +10,7 @@ from datetime import datetime
 from flask import Flask, jsonify, request, send_file, send_from_directory
 
 import store
+import db
 from engine import CrawlJob, probe_links
 from scheduler import Scheduler, cron_next, interval_delta
 
@@ -93,6 +94,10 @@ def persist_cb(task_id, run):
     done = run["progress"].get("done") or 0
     run["progress"]["percent"] = round(done * 100 / total) if total else 0
     store.save_run(task_id, run)
+    try:
+        db.sync_run(run, lambda tid, r: store.save_run(tid, r))
+    except Exception as e:
+        print(f"[db] 同步失败: {e}", flush=True)
 
 
 def start_run(task):
@@ -106,6 +111,7 @@ def start_run(task):
         job = CrawlJob(task, run, persist_cb)
         JOBS[task["id"]] = job
         job.start()
+        db.upsert_task(task)              # 确保任务在库中
         return run, None
 
 
@@ -214,6 +220,7 @@ def api_create_task():
             return jsonify({"error": "请至少填写一个网址"}), 400
 
     store.upsert_task(task)
+    db.upsert_task(task)
     return jsonify(task), 201
 
 
@@ -260,6 +267,7 @@ def api_update_task(tid):
             task["schedule"] = sch
         task["updated_at"] = now_str()
         store.upsert_task(task)
+        db.upsert_task(task)
     return jsonify(task)
 
 
@@ -275,6 +283,7 @@ def api_delete_task(tid):
     with JOBS_LOCK:
         JOBS.pop(tid, None)
     store.soft_delete_task(tid, now_str())
+    db.upsert_task(store.get_task(tid))
     return jsonify({"ok": True, "msg": "已移入回收站"})
 
 
@@ -309,6 +318,7 @@ def api_trash_restore(tid):
     if not task or not task.get("deleted_at"):
         return jsonify({"error": "任务不在回收站中"}), 404
     store.restore_task(tid)
+    db.upsert_task(store.get_task(tid))
     return jsonify({"ok": True, "msg": "已恢复"})
 
 
@@ -320,6 +330,7 @@ def api_trash_purge(tid):
     out_dir = resolve_out_dir(task)
     removed = _purge_out_dir(out_dir)
     store.purge_task(tid)
+    db.purge_task_db(tid)
     return jsonify({"ok": True, "purged": True, "files_removed": removed, "out_dir": out_dir})
 
 
@@ -328,6 +339,7 @@ def api_trash_clear():
     items = store.list_trash()
     dirs = [resolve_out_dir(t) for t in items]
     store.purge_trash()
+    db.purge_trash_db()
     removed = sum(1 for d in dirs if _purge_out_dir(d))
     return jsonify({"ok": True, "purged": len(items), "dirs_removed": removed})
 
@@ -563,6 +575,10 @@ scheduler = Scheduler(start_run)
 def main():
     os.makedirs(os.path.join(HERE, "data"), exist_ok=True)
     os.makedirs(os.path.join(HERE, "out"), exist_ok=True)
+    db.init_db()
+    # 全量同步存量任务到数据库
+    for t in store.load_tasks():
+        db.upsert_task(t)
     scheduler.start()
     print(f"[universal-crawler] 启动完成, 管理界面: http://0.0.0.0:{PORT}/")
     app.run(host="0.0.0.0", port=PORT, threaded=True, debug=False)