diff --git a/apps/desktop/src/commands.rs b/apps/desktop/src/commands.rs index f3093d7..8ca7f97 100644 --- a/apps/desktop/src/commands.rs +++ b/apps/desktop/src/commands.rs @@ -1632,6 +1632,8 @@ pub fn kb_import_documents( file_size: bytes.len() as i64, char_count: text.chars().count() as i64, chunk_count: chunks.len() as i64, + file_path: String::new(), + source_id: None, created_at: String::new(), updated_at: String::new(), }; @@ -1684,6 +1686,193 @@ pub fn kb_search( .map_err(|e| e.to_string()) } +#[derive(Deserialize)] +pub struct KbSourceInput { + pub kb_id: String, + pub path: String, + #[serde(default)] + pub extensions: String, + #[serde(default = "default_true")] + pub recursive: bool, +} + +#[derive(Serialize)] +pub struct KbSourceScanResult { + pub imported: usize, + pub ignored: usize, + pub errors: Vec, +} + +#[tauri::command] +pub fn kb_list_sources( + state: State<'_, App>, + kb_id: String, +) -> Result, String> { + let db = state.core.db.lock().unwrap(); + kb_db::list_sources(&db, &kb_id).map_err(|e| e.to_string()) +} + +/// 添加目录来源并立即扫描导入：只收录文本 / 图片视频 / 音频三类文件，其余忽略。 +#[tauri::command] +pub fn kb_add_source( + state: State<'_, App>, + input: KbSourceInput, +) -> Result { + let dir = std::path::PathBuf::from(input.path.trim()); + if !dir.is_dir() { + return Err(format!("目录不存在或不是文件夹：{}", dir.display())); + } + let exts = crate::knowledge::normalize_extensions(&input.extensions); + let db = state.core.db.lock().unwrap(); + let kb = kb_db::get_knowledge_base(&db, &input.kb_id) + .map_err(|e| e.to_string())? + .ok_or_else(|| "知识库不存在".to_string())?; + let source_id = uuid::Uuid::new_v4().to_string(); + let source = xianren_core::KbSource { + id: source_id.clone(), + kb_id: input.kb_id.clone(), + path: dir.to_string_lossy().to_string(), + extensions: input.extensions.trim().to_string(), + recursive: input.recursive, + created_at: String::new(), + }; + kb_db::upsert_source(&db, &source).map_err(|e| e.to_string())?; + // 同目录重复添加时沿用已有来源 id，避免文档归属错乱 + let source_id = kb_db::list_sources(&db, &input.kb_id) + .map_err(|e| e.to_string())? + .into_iter() + .find(|s| s.path == source.path && s.kb_id == source.kb_id) + .map(|s| s.id) + .unwrap_or(source_id); + Ok(scan_and_import(&db, &kb, &source_id, &dir, &exts, input.recursive)) +} + +/// 重新扫描该知识库的全部目录来源，导入新增文件（按文件路径去重）。 +#[tauri::command] +pub fn kb_scan_sources( + state: State<'_, App>, + kb_id: String, +) -> Result { + let db = state.core.db.lock().unwrap(); + let kb = kb_db::get_knowledge_base(&db, &kb_id) + .map_err(|e| e.to_string())? + .ok_or_else(|| "知识库不存在".to_string())?; + let sources = kb_db::list_sources(&db, &kb_id).map_err(|e| e.to_string())?; + let mut total = KbSourceScanResult { + imported: 0, + ignored: 0, + errors: Vec::new(), + }; + for src in sources { + let dir = std::path::PathBuf::from(&src.path); + if !dir.is_dir() { + total.errors.push(format!("目录不存在：{}", src.path)); + continue; + } + let exts = crate::knowledge::normalize_extensions(&src.extensions); + let r = scan_and_import(&db, &kb, &src.id, &dir, &exts, src.recursive); + total.imported += r.imported; + total.ignored += r.ignored; + total.errors.extend(r.errors); + } + Ok(total) +} + +/// 删除目录来源及其导入的全部文档。 +#[tauri::command] +pub fn kb_remove_source(state: State<'_, App>, source_id: String) -> Result { + let db = state.core.db.lock().unwrap(); + kb_db::get_source(&db, &source_id) + .map_err(|e| e.to_string())? + .ok_or_else(|| "目录来源不存在".to_string())?; + let removed = kb_db::delete_docs_by_source(&db, &source_id).map_err(|e| e.to_string())?; + kb_db::delete_source(&db, &source_id).map_err(|e| e.to_string())?; + Ok(removed) +} + +/// 扫描目录并导入文档（文本提取切块；图片/视频/音频以文件名为内容登记；其他忽略）。 +fn scan_and_import( + db: &rusqlite::Connection, + kb: &xianren_core::KnowledgeBase, + source_id: &str, + dir: &std::path::Path, + extensions: &[String], + recursive: bool, +) -> KbSourceScanResult { + let mut result = KbSourceScanResult { + imported: 0, + ignored: 0, + errors: Vec::new(), + }; + let files = crate::knowledge::collect_files(dir, extensions, recursive); + for file in files { + let name = file + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or("") + .to_string(); + let file_path = file.to_string_lossy().to_string(); + let category = crate::knowledge::classify(&name); + if category == crate::knowledge::FileCategory::Other { + result.ignored += 1; + continue; + } + if kb_db::document_exists(db, &kb.id, &file_path).unwrap_or(false) { + continue; + } + let bytes = match std::fs::read(&file) { + Ok(b) => b, + Err(e) => { + result.errors.push(format!("{name}: 读取失败 {e}")); + continue; + } + }; + let content = if category == crate::knowledge::FileCategory::Text { + match crate::knowledge::extract_text(&name, &bytes) { + Ok(t) => t, + Err(e) => { + result.errors.push(format!("{name}: {e}")); + continue; + } + } + } else { + // 图片 / 视频 / 音频：以文件名为内容登记，便于按名检索 + name.clone() + }; + let chunks = if category == crate::knowledge::FileCategory::Text { + kb_db::chunk_text(&content, kb.chunk_size as usize, kb.chunk_overlap as usize) + } else { + vec![content.clone()] + }; + if chunks.is_empty() { + result.errors.push(format!("{name}: 切分后没有有效分块")); + continue; + } + let doc = xianren_core::KbDocument { + id: uuid::Uuid::new_v4().to_string(), + kb_id: kb.id.clone(), + name, + file_type: file + .extension() + .map(|e| e.to_string_lossy().to_lowercase()) + .unwrap_or_default(), + file_size: bytes.len() as i64, + char_count: content.chars().count() as i64, + chunk_count: chunks.len() as i64, + file_path, + source_id: Some(source_id.to_string()), + created_at: String::new(), + updated_at: String::new(), + }; + if let Err(e) = kb_db::insert_document_with_chunks(db, &doc, &content, &chunks) { + result.errors.push(format!("{}: 写入失败 {e}", doc.name)); + continue; + } + result.imported += 1; + } + result +} + #[tauri::command] pub fn list_conversations( state: State<'_, App>, diff --git a/apps/desktop/src/knowledge.rs b/apps/desktop/src/knowledge.rs index 522ec36..58d8be6 100644 --- a/apps/desktop/src/knowledge.rs +++ b/apps/desktop/src/knowledge.rs @@ -1,10 +1,95 @@ -/// 按扩展名从文件字节提取纯文本：文本类文件直接按 UTF-8 读取，PDF 走 pdf-extract。 -pub fn extract_text(name: &str, bytes: &[u8]) -> Result { - let ext = name - .rsplit('.') +use std::path::Path; + +/// 文件分类：可读文本 / 图片视频 / 音频 / 其他（其他一律忽略）。 +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FileCategory { + Text, + Media, + Other, +} + +const TEXT_EXTS: &[&str] = &[ + "txt", "md", "markdown", "json", "csv", "tsv", "log", "yml", "yaml", "toml", "ini", + "conf", "cfg", "xml", "html", "htm", "py", "rs", "ts", "tsx", "js", "jsx", "java", "c", + "cpp", "h", "hpp", "go", "php", "rb", "sh", "bat", "ps1", "sql", "css", "scss", "sass", + "pdf", +]; + +const IMAGE_EXTS: &[&str] = &[ + "png", "jpg", "jpeg", "gif", "webp", "bmp", "svg", "tiff", "tif", "ico", "avif", "heic", +]; + +const VIDEO_EXTS: &[&str] = &[ + "mp4", "mkv", "avi", "mov", "webm", "flv", "wmv", "m4v", "mpg", "mpeg", "3gp", +]; + +const AUDIO_EXTS: &[&str] = &[ + "mp3", "wav", "flac", "ogg", "oga", "m4a", "aac", "wma", "opus", "aiff", "aif", +]; + +fn ext_of(name: &str) -> String { + name.rsplit('.') .next() .map(|e| e.to_lowercase()) - .unwrap_or_default(); + .unwrap_or_default() +} + +/// 分类文件：文本（可提取内容）/ 媒体（图片、视频、音频）/ 其他。 +pub fn classify(name: &str) -> FileCategory { + let ext = ext_of(name); + if TEXT_EXTS.contains(&ext.as_str()) { + FileCategory::Text + } else if IMAGE_EXTS.contains(&ext.as_str()) + || VIDEO_EXTS.contains(&ext.as_str()) + || AUDIO_EXTS.contains(&ext.as_str()) + { + FileCategory::Media + } else { + FileCategory::Other + } +} + +/// 规范化用户输入的后缀列表：`.txt, .md；PDF` → ["txt", "md", "pdf"]；空串表示全部。 +pub fn normalize_extensions(raw: &str) -> Vec { + raw.split([',', ';', '，', '；', ' ']) + .map(|s| s.trim().trim_start_matches('.').to_lowercase()) + .filter(|s| !s.is_empty()) + .collect::>() +} + +fn has_matching_ext(name: &str, extensions: &[String]) -> bool { + let ext = ext_of(name); + extensions.iter().any(|e| e == &ext) +} + +/// 递归/非递归收集目录下符合后缀条件的文件，按路径排序。 +pub fn collect_files(path: &Path, extensions: &[String], recursive: bool) -> Vec { + let mut out = Vec::new(); + fn walk(dir: &Path, exts: &[String], recursive: bool, out: &mut Vec) { + let Ok(entries) = std::fs::read_dir(dir) else { + return; + }; + for entry in entries.flatten() { + let p = entry.path(); + if p.is_dir() { + if recursive { + walk(&p, exts, recursive, out); + } + } else if let Some(name) = p.file_name().and_then(|n| n.to_str()) { + if exts.is_empty() || has_matching_ext(name, exts) { + out.push(p); + } + } + } + } + walk(path, extensions, recursive, &mut out); + out.sort(); + out +} + +/// 按扩展名从文件字节提取纯文本：文本类文件直接按 UTF-8 读取，PDF 走 pdf-extract。 +pub fn extract_text(name: &str, bytes: &[u8]) -> Result { + let ext = ext_of(name); let text = match ext.as_str() { "pdf" => pdf_extract::extract_text_from_mem(bytes) .map_err(|e| format!("PDF 解析失败：{e}"))?, @@ -31,3 +116,48 @@ pub fn extract_text(name: &str, bytes: &[u8]) -> Result { } Ok(cleaned) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn classify_three_categories() { + assert_eq!(classify("readme.md"), FileCategory::Text); + assert_eq!(classify("手册.pdf"), FileCategory::Text); + assert_eq!(classify("photo.PNG"), FileCategory::Media); + assert_eq!(classify("clip.mp4"), FileCategory::Media); + assert_eq!(classify("song.mp3"), FileCategory::Media); + assert_eq!(classify("app.exe"), FileCategory::Other); + assert_eq!(classify("archive.zip"), FileCategory::Other); + assert_eq!(classify("noext"), FileCategory::Other); + } + + #[test] + fn normalize_extensions_handles_dots_and_separators() { + assert_eq!( + normalize_extensions(".txt, .md；PDF, png"), + vec!["txt", "md", "pdf", "png"] + ); + assert!(normalize_extensions(" ").is_empty()); + } + + #[test] + fn collect_files_filters_and_recurses() { + let base = std::env::temp_dir().join(format!("kb_scan_test_{}", uuid::Uuid::new_v4())); + std::fs::create_dir_all(base.join("sub")).unwrap(); + std::fs::write(base.join("a.txt"), "hello").unwrap(); + std::fs::write(base.join("b.exe"), "x").unwrap(); + std::fs::write(base.join("sub/c.md"), "hi").unwrap(); + std::fs::write(base.join("sub/d.png"), "img").unwrap(); + + let all = collect_files(&base, &[], true); + assert_eq!(all.len(), 4); + let txt_only = collect_files(&base, &["txt".to_string()], true); + assert_eq!(txt_only.len(), 1); + let non_recursive = collect_files(&base, &[], false); + assert_eq!(non_recursive.len(), 2); + + std::fs::remove_dir_all(&base).unwrap(); + } +} diff --git a/apps/desktop/src/lib.rs b/apps/desktop/src/lib.rs index 0caf707..7e88681 100644 --- a/apps/desktop/src/lib.rs +++ b/apps/desktop/src/lib.rs @@ -195,6 +195,10 @@ pub fn run() { commands::remove_kb_document, commands::kb_rechunk_document, commands::kb_search, + commands::kb_list_sources, + commands::kb_add_source, + commands::kb_scan_sources, + commands::kb_remove_source, commands::list_models, commands::import_model, commands::remove_model, diff --git a/crates/core/src/app.rs b/crates/core/src/app.rs index 5191cca..292841b 100644 --- a/crates/core/src/app.rs +++ b/crates/core/src/app.rs @@ -123,6 +123,8 @@ fn open_db(path: &Path) -> Result { ensure_column(&conn, "scheduled_tasks", "email_mode", "TEXT NOT NULL DEFAULT 'fixed'")?; ensure_column(&conn, "scheduled_tasks", "email_subject", "TEXT NOT NULL DEFAULT ''")?; ensure_column(&conn, "scheduled_tasks", "email_body", "TEXT NOT NULL DEFAULT ''")?; + ensure_column(&conn, "kb_documents", "file_path", "TEXT NOT NULL DEFAULT ''")?; + ensure_column(&conn, "kb_documents", "source_id", "TEXT")?; Ok(conn) } diff --git a/crates/core/src/knowledge_base.rs b/crates/core/src/knowledge_base.rs index b0a52bf..94858e9 100644 --- a/crates/core/src/knowledge_base.rs +++ b/crates/core/src/knowledge_base.rs @@ -24,10 +24,22 @@ pub struct KbDocument { pub file_size: i64, pub char_count: i64, pub chunk_count: i64, + pub file_path: String, + pub source_id: Option, pub created_at: String, pub updated_at: String, } +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct KbSource { + pub id: String, + pub kb_id: String, + pub path: String, + pub extensions: String, + pub recursive: bool, + pub created_at: String, +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct KbDocumentDetail { #[serde(flatten)] @@ -45,7 +57,8 @@ pub struct KbSearchHit { } const KB_COLUMNS: &str = "id, name, description, chunk_size, chunk_overlap, created_at, updated_at"; -const DOC_COLUMNS: &str = "id, kb_id, name, file_type, file_size, char_count, chunk_count, created_at, updated_at"; +const DOC_COLUMNS: &str = "id, kb_id, name, file_type, file_size, char_count, chunk_count, file_path, source_id, created_at, updated_at"; +const SOURCE_COLUMNS: &str = "id, kb_id, path, extensions, recursive, created_at"; // ---------- 知识库 ---------- @@ -149,7 +162,7 @@ pub fn get_document_detail(db: &Connection, id: &str) -> Result Result<()> { Ok(()) } +/// 同一知识库下是否已存在来自该文件路径的文档（目录导入去重）。 +pub fn document_exists(db: &Connection, kb_id: &str, file_path: &str) -> Result { + if file_path.is_empty() { + return Ok(false); + } + let exists: bool = db.query_row( + "SELECT EXISTS(SELECT 1 FROM kb_documents WHERE kb_id = ?1 AND file_path = ?2)", + params![kb_id, file_path], + |row| row.get(0), + )?; + Ok(exists) +} + +// ---------- 目录来源 ---------- + +pub fn list_sources(db: &Connection, kb_id: &str) -> Result> { + let mut stmt = db.prepare(&format!( + "SELECT {SOURCE_COLUMNS} FROM kb_sources WHERE kb_id = ?1 ORDER BY created_at ASC" + ))?; + let rows = stmt.query_map(params![kb_id], row_to_source)?; + let mut out = Vec::new(); + for row in rows { + out.push(row?); + } + Ok(out) +} + +pub fn get_source(db: &Connection, id: &str) -> Result> { + let mut stmt = db.prepare(&format!( + "SELECT {SOURCE_COLUMNS} FROM kb_sources WHERE id = ?1" + ))?; + let mut rows = stmt.query_map(params![id], row_to_source)?; + match rows.next() { + Some(row) => Ok(Some(row?)), + None => Ok(None), + } +} + +pub fn upsert_source(db: &Connection, source: &KbSource) -> Result<()> { + db.execute( + "INSERT INTO kb_sources (id, kb_id, path, extensions, recursive) + VALUES (?1, ?2, ?3, ?4, ?5) + ON CONFLICT(kb_id, path) DO UPDATE SET + extensions = excluded.extensions, recursive = excluded.recursive", + params![ + source.id, + source.kb_id, + source.path, + source.extensions, + source.recursive as i32, + ], + )?; + Ok(()) +} + +pub fn delete_source(db: &Connection, id: &str) -> Result<()> { + db.execute("DELETE FROM kb_sources WHERE id = ?1", params![id])?; + Ok(()) +} + +/// 删除某个目录来源导入的全部文档（分块随外键级联）。 +pub fn delete_docs_by_source(db: &Connection, source_id: &str) -> Result { + let n = db.execute( + "DELETE FROM kb_documents WHERE source_id = ?1", + params![source_id], + )?; + Ok(n) +} + /// 重新切分文档：删除旧分块后按新参数重分并重建索引。 pub fn rechunk_document( db: &Connection, @@ -328,8 +412,22 @@ fn row_to_doc(row: &rusqlite::Row<'_>) -> rusqlite::Result { file_size: row.get(4)?, char_count: row.get(5)?, chunk_count: row.get(6)?, - created_at: row.get(7)?, - updated_at: row.get(8)?, + file_path: row.get(7)?, + source_id: row.get(8)?, + created_at: row.get(9)?, + updated_at: row.get(10)?, + }) +} + +fn row_to_source(row: &rusqlite::Row<'_>) -> rusqlite::Result { + let recursive: i32 = row.get(4)?; + Ok(KbSource { + id: row.get(0)?, + kb_id: row.get(1)?, + path: row.get(2)?, + extensions: row.get(3)?, + recursive: recursive != 0, + created_at: row.get(5)?, }) } @@ -363,6 +461,8 @@ mod tests { file_size: content.len() as i64, char_count: content.chars().count() as i64, chunk_count: 0, + file_path: String::new(), + source_id: None, created_at: String::new(), updated_at: String::new(), }; diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index 1be26e1..ac16343 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -12,7 +12,7 @@ pub mod workflows; pub use app::CoreApp; pub use error::{CoreError, Result}; -pub use knowledge_base::{KbDocument, KbDocumentDetail, KbSearchHit, KnowledgeBase}; +pub use knowledge_base::{KbDocument, KbDocumentDetail, KbSearchHit, KbSource, KnowledgeBase}; pub use models::ModelInfo; pub use agents::Agent; pub use scheduled_tasks::ScheduledTask; diff --git a/crates/core/src/schema.sql b/crates/core/src/schema.sql index 2be1ebe..11bd491 100644 --- a/crates/core/src/schema.sql +++ b/crates/core/src/schema.sql @@ -147,6 +147,8 @@ CREATE TABLE IF NOT EXISTS kb_documents ( char_count INTEGER NOT NULL DEFAULT 0, chunk_count INTEGER NOT NULL DEFAULT 0, content TEXT NOT NULL DEFAULT '', + file_path TEXT NOT NULL DEFAULT '', + source_id TEXT, created_at TEXT NOT NULL DEFAULT (datetime('now')), updated_at TEXT NOT NULL DEFAULT (datetime('now')) ); @@ -160,6 +162,17 @@ CREATE TABLE IF NOT EXISTS kb_chunks ( created_at TEXT NOT NULL DEFAULT (datetime('now')) ); +CREATE TABLE IF NOT EXISTS kb_sources ( + id TEXT PRIMARY KEY, + kb_id TEXT NOT NULL REFERENCES knowledge_bases(id) ON DELETE CASCADE, + path TEXT NOT NULL, + extensions TEXT NOT NULL DEFAULT '', + recursive INTEGER NOT NULL DEFAULT 1, + created_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_kb_sources_kb_path ON kb_sources(kb_id, path); + -- 全文检索索引（trigram 分词，对中文/短文本更友好；content='' 为 contentless-delete 模式） CREATE VIRTUAL TABLE IF NOT EXISTS kb_chunks_fts USING fts5(content, content = '', tokenize = 'trigram'); diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index f03012b..4c7c941 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -12,7 +12,7 @@ ``` xianren_studio/ -├── apps/desktop/ Tauri 桌面壳（Rust 命令层、tauri.conf.json、推荐模型/预制智能体/预制工作流默认 JSON、工作流执行引擎 workflow.rs、邮件发送 mail.rs、文档提取 knowledge.rs） +├── apps/desktop/ Tauri 桌面壳（Rust 命令层、tauri.conf.json、推荐模型/预制智能体/预制工作流默认 JSON、工作流执行引擎 workflow.rs、邮件发送 mail.rs、文档提取与目录扫描 knowledge.rs） ├── crates/core/ 领域核心：SQLite（模型注册表、智能体、工作流、知识库、定时计划、会话、消息、设置） ├── crates/engine/ llama-server 生命周期 + 流式/非流式聊天（本地与远程 OpenAI 兼容） ├── crates/download/ 分片断点续传下载器 @@ -111,6 +111,7 @@ ui/src/ | 定时计划 | `list_scheduled_tasks`、`add_scheduled_task`、`update_scheduled_task`、`remove_scheduled_task`、`set_scheduled_task_enabled`、`run_scheduled_task_now` | | 邮件 | `mail_test`（用当前 SMTP 配置发送测试邮件） | | 知识库 | `list_knowledge_bases`、`add_knowledge_base`、`update_knowledge_base`、`remove_knowledge_base`、`list_kb_documents`、`get_kb_document`、`kb_import_documents`、`remove_kb_document`、`kb_rechunk_document`、`kb_search` | +| 知识库·目录来源 | `kb_list_sources`、`kb_add_source`、`kb_scan_sources`、`kb_remove_source` | | 应用/设置 | `app_info`、`autostart_status`、`autostart_set`、`settings_get`、`settings_set` | | 模型 | `list_models`、`import_model`、`remove_model`、`set_model_enabled`、`scan_models`、`add_remote_model` | | 模型广场 | `search_models`、`list_model_files`、`list_recommended_models`、`import_recommendations`、`fetch_model_page` | @@ -143,6 +144,7 @@ ui/src/ - `send_task_email` / `load_smtp_config` / `split_recipients`：读取设置页「邮件」SMTP 配置、拆分多收件人并调用 `mail.rs` 发送。 - `kb_import_documents`：base64 解码 → `knowledge.rs::extract_text` 提取文本（文本类直接 UTF-8，PDF 走 pdf-extract）→ `kb_db::chunk_text` 切块（按知识库 chunk_size/overlap）→ `insert_document_with_chunks` 事务写入文档与分块，逐文件返回成功/失败结果。 - `kb_search` / `kb_rechunk_document`：检索走 `kb_db::search`（FTS5 trigram + BM25，短查询回退 LIKE）；重切按知识库最新分块设置重建该文档分块与索引。 +- `kb_add_source` / `kb_scan_sources` / `kb_remove_source`：目录来源注册与扫描——`knowledge.rs::collect_files` 按后缀过滤（空=全部）与递归开关收集文件，`classify` 分成文本 / 图片视频 / 音频三类（其他忽略），文本提取切块入库、媒体以文件名为内容登记，`kb_db::document_exists` 按（kb_id + file_path）去重；删除来源级联删除其导入文档。 **邮件发送模块（`apps/desktop/src/mail.rs`）：** @@ -185,13 +187,14 @@ ui/src/ | `skills` | `name`、`description`、`content`、`enabled` | 技能工具库 | | `mcp_servers` | `name`、`description`、`url`、`auth_token`、`enabled` | MCP 服务配置 | | `knowledge_bases` | `name`、`description`、`chunk_size`、`chunk_overlap` | 知识库及分块设置 | -| `kb_documents` | `kb_id`（级联删除）、`name`、`file_type`、`file_size`、`char_count`、`chunk_count`、`content`（提取后的纯文本） | 知识库文档；删除/重切分块由 FTS 触发器同步 | +| `kb_documents` | `kb_id`（级联删除）、`name`、`file_type`、`file_size`、`char_count`、`chunk_count`、`content`（提取后的纯文本/媒体文件名）、`file_path`、`source_id` | 知识库文档；目录导入记录文件路径与来源，`file_path` 用于去重；删除/重切分块由 FTS 触发器同步 | | `kb_chunks` | `kb_id`、`document_id`（级联删除）、`seq`、`content` | 文档分块 | | `kb_chunks_fts` | FTS5 虚拟表（contentless-delete + trigram 分词） | 全文检索索引，由 `kb_chunks` 触发器维护 | +| `kb_sources` | `kb_id`（级联删除）、`path`、`extensions`、`recursive`；`(kb_id, path)` 唯一 | 目录来源（多个）；添加时立即扫描，可重新扫描增量导入 | 迁移清单（`ensure_column`）：`models.kind/base_url/api_key/api_model/enabled`、`messages.elapsed_ms/first_token_ms/images_json/model_id`、`conversations.pinned/favorite/tools_json/agent_id`。 -新增列迁移：`scheduled_tasks.email_enabled / email_to / email_mode / email_subject / email_body`（`ensure_column` 幂等补齐）。 +新增列迁移：`scheduled_tasks.email_enabled / email_to / email_mode / email_subject / email_body`、`kb_documents.file_path / source_id`（`ensure_column` 幂等补齐）。 ### 5.3 `models.rs` @@ -219,7 +222,7 @@ ui/src/ ### 5.9 `knowledge_base.rs` -知识库 CRUD（`list/get/insert/update/delete`）、文档 CRUD（`list/get/get_detail`）、`insert_document_with_chunks`（文档 + 分块同事务写入，FTS 触发器自动建索引）、`delete_document`（级联删除分块并同步索引）、`rechunk_document`、`chunk_text`（按字符切块 + 重叠）、`search`（FTS5 trigram BM25 排序，查询 <3 字或 MATCH 失败回退 LIKE）。 +知识库 CRUD（`list/get/insert/update/delete`）、文档 CRUD（`list/get/get_detail`）、`insert_document_with_chunks`（文档 + 分块同事务写入，FTS 触发器自动建索引）、`delete_document`（级联删除分块并同步索引）、`document_exists`（目录导入按文件路径去重）、`rechunk_document`、`chunk_text`（按字符切块 + 重叠）、`search`（FTS5 trigram BM25 排序，查询 <3 字或 MATCH 失败回退 LIKE）；目录来源 `list/get/upsert/delete_sources` 与 `delete_docs_by_source`。 --- diff --git a/docs/FEATURES.md b/docs/FEATURES.md index d008237..06c89dd 100644 --- a/docs/FEATURES.md +++ b/docs/FEATURES.md @@ -133,6 +133,11 @@ - 左侧导航新增「知识库」选项卡，页面分左右两栏：左侧知识库列表，右侧选中库的管理区。 - **知识库管理**：新建 / 编辑 / 删除知识库，字段含名称、描述、**分块大小**（100–10000 字，默认 500）与**块间重叠**（默认 50）；删除知识库会级联删除其文档与分块。 - **文档导入**：支持多文件上传，格式覆盖文本类文件（txt / md / json / csv / 代码 / HTML 等）与 **PDF**（后端提取文本）；导入后按知识库分块设置自动切块并写入全文索引。 +- **目录来源（批量导入）**：可为知识库添加多个本地目录，扫描目录批量导入文档： + - **后缀过滤**：可指定只导入的后缀列表（如 `.txt,.md,.pdf`），留空默认全部； + - **递归子目录**：可开关是否递归扫描所有子目录； + - **类型过滤**：只收录**可读文本**（提取内容并分块检索）、**图片 / 视频**、**音频**三类文件（图片视频音频以文件名为内容登记，可按文件名检索），检测到其他类型自动忽略； + - 同一目录重复添加自动更新配置不产生重复来源；按文件路径去重，可一键「重新扫描」增量导入新增文件；删除目录来源会同时删除它导入的文档。 - **文档操作**：文档列表展示类型 / 大小 / 字数 / 分块数；支持全文预览、「重切」（按最新分块设置重新切分，适用于修改设置后）与删除。 - **检索**：全文检索基于 SQLite FTS5（trigram 分词，对中文友好），按 BM25 相关性排序；支持单库检索（也可全库检索），结果展示来源文档与分块序号，可一键复制分块内容用于对话。 - 数据表：`knowledge_bases`、`kb_documents`（含提取后的纯文本）、`kb_chunks`（分块）、`kb_chunks_fts`（FTS5 全文索引，contentless-delete 模式 + 触发器同步）。 @@ -224,6 +229,7 @@ ### 2026-08-17 +- 知识库新增「目录来源」：支持添加多个本地目录批量导入（后缀过滤默认全部、可开关递归子目录），只收录文本 / 图片视频 / 音频三类文件其余忽略，支持重新扫描增量导入与按来源删除。 - 新增「知识库」选项卡与完整操作界面：知识库管理（新建/编辑/删除）、多格式文档上传（文本类 + PDF）、自动分块与全文检索（FTS5 trigram + BM25）、文档预览 / 重新切分 / 删除、检索结果一键复制。 - 计划任务支持发送邮件：任务设置中可开启邮件并填写收件人，内容支持固定内容与大模型生成两种模式；设置页新增「邮件」SMTP 配置与测试发送。 - 设置页新增「启动」：开机启动开关（写入 Windows 注册表启动项）；「自动加载模型」列表（多个本地模型按顺序在启动时逐个加载，最后一个保持运行）。 diff --git a/ui/src/api.ts b/ui/src/api.ts index d38a302..9a67fbd 100644 --- a/ui/src/api.ts +++ b/ui/src/api.ts @@ -245,6 +245,28 @@ export interface KbImportResult { error: string | null; } +export interface KbSource { + id: string; + kb_id: string; + path: string; + extensions: string; + recursive: boolean; + created_at: string; +} + +export interface KbSourceInput { + kb_id: string; + path: string; + extensions: string; + recursive: boolean; +} + +export interface KbSourceScanResult { + imported: number; + ignored: number; + errors: string[]; +} + export interface McpServer { id: string; name: string; @@ -483,6 +505,13 @@ export const api = { invoke("kb_rechunk_document", { documentId }), kbSearch: (kbId: string | null, query: string, limit?: number) => invoke("kb_search", { kbId, query, limit: limit ?? 10 }), + kbListSources: (kbId: string) => invoke("kb_list_sources", { kbId }), + kbAddSource: (input: KbSourceInput) => + invoke("kb_add_source", { input }), + kbScanSources: (kbId: string) => + invoke("kb_scan_sources", { kbId }), + kbRemoveSource: (sourceId: string) => + invoke("kb_remove_source", { sourceId }), listModels: () => invoke("list_models"), importModel: (path: string) => invoke("import_model", { path }), removeModel: (id: string) => invoke("remove_model", { id }), diff --git a/ui/src/pages/KnowledgeBasePage.tsx b/ui/src/pages/KnowledgeBasePage.tsx index 9fe2ac9..854c683 100644 --- a/ui/src/pages/KnowledgeBasePage.tsx +++ b/ui/src/pages/KnowledgeBasePage.tsx @@ -4,6 +4,8 @@ import { KbDocument, KbDocumentDetail, KbSearchHit, + KbSource, + KbSourceScanResult, KnowledgeBase, KnowledgeBaseInput, } from "../api"; @@ -56,6 +58,9 @@ export default function KnowledgeBasePage() { const [query, setQuery] = useState(""); const [searching, setSearching] = useState(false); const [searchResults, setSearchResults] = useState(null); + const [sources, setSources] = useState([]); + const [sourceModal, setSourceModal] = useState(false); + const [sourceMsg, setSourceMsg] = useState(null); const uploadInputRef = useRef(null); const selected = useMemo( @@ -88,6 +93,10 @@ export default function KnowledgeBasePage() { .listKbDocuments(selectedId) .then(setDocs) .catch((e) => setMsg(`加载文档失败：${String(e)}`)); + api + .kbListSources(selectedId) + .then(setSources) + .catch((e) => setMsg(`加载目录来源失败：${String(e)}`)); }, [selectedId]); async function handleSaveKb(input: KnowledgeBaseInput) { @@ -189,6 +198,76 @@ export default function KnowledgeBasePage() { } } + async function refreshSourcesAndDocs() { + if (!selected) return; + const [d, s] = await Promise.all([ + api.listKbDocuments(selected.id), + api.kbListSources(selected.id), + ]); + setDocs(d); + setSources(s); + await refreshKbs(); + } + + function formatScanMsg(label: string, r: KbSourceScanResult) { + const errPart = + r.errors.length > 0 + ? `，${r.errors.length} 个错误（${r.errors + .slice(0, 3) + .join("；")}${r.errors.length > 3 ? "…" : ""}）` + : ""; + return `${label}完成：导入 ${r.imported} 个，忽略 ${r.ignored} 个${errPart}`; + } + + async function handleAddSource(path: string, extensions: string, recursive: boolean) { + if (!selected) return; + setBusy(true); + setSourceMsg(null); + try { + const r = await api.kbAddSource({ + kb_id: selected.id, + path, + extensions, + recursive, + }); + await refreshSourcesAndDocs(); + setSourceMsg(formatScanMsg("添加目录", r)); + } catch (e) { + setSourceMsg(`添加目录失败：${String(e)}`); + } finally { + setBusy(false); + setSourceModal(false); + } + } + + async function handleScanAll() { + if (!selected) return; + setBusy(true); + setSourceMsg(null); + try { + const r = await api.kbScanSources(selected.id); + await refreshSourcesAndDocs(); + setSourceMsg(formatScanMsg("重新扫描", r)); + } catch (e) { + setSourceMsg(`重新扫描失败：${String(e)}`); + } finally { + setBusy(false); + } + } + + async function handleRemoveSource(src: KbSource) { + if (!confirm(`确定删除目录来源「${src.path}」吗？将同时删除由它导入的文档。`)) return; + try { + const n = await api.kbRemoveSource(src.id); + setSourceMsg(`已删除目录来源及其 ${n} 个文档`); + if (selected) { + await refreshSourcesAndDocs(); + } + } catch (e) { + setSourceMsg(`删除失败：${String(e)}`); + } + } + return (
{/* 左侧：知识库列表 */} @@ -343,6 +422,81 @@ export default function KnowledgeBasePage() { ) : null} + {/* 目录来源 */} +
+
+
+ 目录来源（{sources.length}） +
+
+ + +
+
+

+ 按目录批量导入文档：只收录可读文本、图片视频、音频三类文件，其余自动忽略；可指定后缀过滤（默认全部），可选是否递归子目录，同一知识库可添加多个目录 +

+ {sourceMsg ? ( +
+ {sourceMsg} +
+ ) : null} + {sources.length === 0 ? ( +
+ 还没有目录来源，点击「添加目录」批量导入文件夹中的文档 +
+ ) : ( +
+ {sources.map((src) => ( +
+ +
+
+ {src.path} +
+
+ + {src.extensions.trim() + ? src.extensions + .split(/[,;，；\s]+/) + .filter(Boolean) + .map((e) => (e.startsWith(".") ? e : `.${e}`)) + .join(" ") + : "全部后缀"} + + + {src.recursive ? "递归子目录" : "仅当前目录"} + +
+
+ +
+ ))} +
+ )} +
+ {/* 文档列表 */}
@@ -381,6 +535,10 @@ export default function KnowledgeBasePage() { /> ) : null} + {sourceModal ? ( + setSourceModal(false)} /> + ) : null} + {preview ? (
); } + +function SourceModal({ + busy, + onSave, + onClose, +}: { + busy: boolean; + onSave: (path: string, extensions: string, recursive: boolean) => void; + onClose: () => void; +}) { + const [path, setPath] = useState(""); + const [extensions, setExtensions] = useState(""); + const [recursive, setRecursive] = useState(true); + const valid = path.trim().length > 0; + + return ( +
+
e.stopPropagation()} + > +
+ 添加目录来源 + +
+
+
+ + setPath(e.target.value)} + /> +
+
+ + setExtensions(e.target.value)} + /> +
+
+ setRecursive(v)} + /> + + {recursive ? "递归扫描所有子目录" : "仅扫描当前目录"} + +
+

+ 导入时只收录可读文本、图片视频、音频三类文件，检测到其他类型会自动忽略；图片 / 视频 / 音频以文件名为内容登记，可按文件名检索。 +

+
+
+ + +
+
+
+ ); +} + +function Toggle({ + checked, + onChange, +}: { + checked: boolean; + onChange: (v: boolean) => void; +}) { + return ( + + ); +}