From ab734478e81d91b7a3f9abbbd6995195f70ff188 Mon Sep 17 00:00:00 2001 From: xianrenge <34586161+xianrenge@users.noreply.github.com> Date: Mon, 17 Aug 2026 14:13:09 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=96=B0=E5=A2=9E=E7=9F=A5=E8=AF=86?= =?UTF-8?q?=E5=BA=93=E9=80=89=E9=A1=B9=E5=8D=A1=E4=B8=8E=E5=AE=8C=E6=95=B4?= =?UTF-8?q?=E6=93=8D=E4=BD=9C=E7=95=8C=E9=9D=A2=E2=80=94=E2=80=94=E5=BA=93?= =?UTF-8?q?=E7=AE=A1=E7=90=86=E3=80=81=E5=A4=9A=E6=A0=BC=E5=BC=8F=E6=96=87?= =?UTF-8?q?=E6=A1=A3=E4=B8=8A=E4=BC=A0=EF=BC=88=E6=96=87=E6=9C=AC/PDF?= =?UTF-8?q?=EF=BC=89=E3=80=81=E8=87=AA=E5=8A=A8=E5=88=86=E5=9D=97=E3=80=81?= =?UTF-8?q?FTS=20=E5=85=A8=E6=96=87=E6=A3=80=E7=B4=A2=E3=80=81=E9=A2=84?= =?UTF-8?q?=E8=A7=88/=E9=87=8D=E5=88=87/=E5=88=A0=E9=99=A4=E3=80=81?= =?UTF-8?q?=E7=BB=93=E6=9E=9C=E5=A4=8D=E5=88=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.lock | 132 ++++++- README.md | 1 + apps/desktop/Cargo.toml | 2 + apps/desktop/src/commands.rs | 243 ++++++++++++ apps/desktop/src/knowledge.rs | 33 ++ apps/desktop/src/lib.rs | 11 + crates/core/src/knowledge_base.rs | 474 ++++++++++++++++++++++ crates/core/src/lib.rs | 2 + crates/core/src/schema.sql | 48 +++ docs/ARCHITECTURE.md | 18 +- docs/FEATURES.md | 27 +- ui/src/App.tsx | 3 + ui/src/api.ts | 72 ++++ ui/src/components/Icon.tsx | 25 ++ ui/src/pages/KnowledgeBasePage.tsx | 607 +++++++++++++++++++++++++++++ 15 files changed, 1686 insertions(+), 12 deletions(-) create mode 100644 apps/desktop/src/knowledge.rs create mode 100644 crates/core/src/knowledge_base.rs create mode 100644 ui/src/pages/KnowledgeBasePage.tsx diff --git a/Cargo.lock b/Cargo.lock index a00c3e6..4f86ef0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,15 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "adobe-cmap-parser" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae8abfa9a4688de8fc9f42b3f013b6fffec18ed8a554f5f113577e0b9b3212a3" +dependencies = [ + "pom", +] + [[package]] name = "ahash" version = "0.8.12" @@ -1073,6 +1082,15 @@ version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ef6b89e5b37196644d8796de5268852ff179b44e96276cf4290264843743bb7" +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + [[package]] name = "endi" version = "1.1.1" @@ -1127,6 +1145,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "euclid" +version = "0.20.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bb7ef65b3777a325d1eeefefab5b6d4959da54747e33bd6258e789640f307ad" +dependencies = [ + "num-traits", +] + [[package]] name = "event-listener" version = "5.4.2" @@ -2218,7 +2245,7 @@ dependencies = [ "httpdate", "idna", "mime", - "nom", + "nom 8.0.0", "percent-encoding", "quoted_printable", "rustls", @@ -2325,6 +2352,24 @@ version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +[[package]] +name = "lopdf" +version = "0.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5c8ecfc6c72051981c0459f75ccc585e7ff67c70829560cda8e647882a9abff" +dependencies = [ + "encoding_rs", + "flate2", + "indexmap 2.14.0", + "itoa", + "log", + "md-5", + "nom 7.1.3", + "rangemap", + "time", + "weezl", +] + [[package]] name = "lru-slab" version = "0.1.2" @@ -2357,6 +2402,16 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + [[package]] name = "memchr" version = "2.8.3" @@ -2378,6 +2433,12 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.8.9" @@ -2450,6 +2511,16 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "nom" version = "8.0.0" @@ -2787,6 +2858,21 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "pdf-extract" +version = "0.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbb3a5387b94b9053c1e69d8abfd4dd6dae7afda65a5c5279bc1f42ab39df575" +dependencies = [ + "adobe-cmap-parser", + "encoding_rs", + "euclid", + "lopdf", + "postscript", + "type1-encoding-parser", + "unicode-normalization", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -2922,6 +3008,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "pom" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60f6ce597ecdcc9a098e7fddacb1065093a3d66446fa16c675e7e71d1b5c28e6" + [[package]] name = "portable-atomic" version = "1.15.0" @@ -2937,6 +3029,12 @@ dependencies = [ "portable-atomic", ] +[[package]] +name = "postscript" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78451badbdaebaf17f053fd9152b3ffb33b516104eacb45e7864aaa9c712f306" + [[package]] name = "potential_utf" version = "0.1.5" @@ -3138,6 +3236,12 @@ dependencies = [ "rand_core", ] +[[package]] +name = "rangemap" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a611d15b50743feb4c76b7d03edcb0e64f399c26961e4efe6975bc398be6aa3d" + [[package]] name = "raw-window-handle" version = "0.6.2" @@ -4690,6 +4794,15 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "type1-encoding-parser" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa10c302f5a53b7ad27fd42a3996e23d096ba39b5b8dd6d9e683a05b01bee749" +dependencies = [ + "pom", +] + [[package]] name = "typeid" version = "1.0.3" @@ -4760,6 +4873,15 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + [[package]] name = "unicode-segmentation" version = "1.13.3" @@ -5095,6 +5217,12 @@ dependencies = [ "windows-core 0.61.2", ] +[[package]] +name = "weezl" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" + [[package]] name = "winapi" version = "0.3.9" @@ -5672,9 +5800,11 @@ dependencies = [ name = "xianren-desktop" version = "0.1.0" dependencies = [ + "base64 0.22.1", "dirs 5.0.1", "futures", "lettre", + "pdf-extract", "reqwest 0.12.28", "rusqlite", "serde", diff --git a/README.md b/README.md index f7f4b9d..3731381 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ - **聊天**：流式输出、Markdown/代码高亮、采样参数调节、多会话管理；右侧栏「可选大模型服务」只列出有部署状态的大模型（运行中/启动中/出错，彩色状态点）与已启用的在线 API 模型，模型名不显示 `.gguf` 后缀；每条回答下方显示所用模型名；回答完成后自动预测用户可能说的话（可点击填入输入框，条数与开关可在设置调整）；切换选项卡后保持会话与大模型等状态 - **智能体**：内置预制智能体（通用助手、代码专家、写作助手、翻译官、数据分析师、提示词优化师）并支持自定义，每个智能体拥有独立人设（系统提示词）与专属会话区，可单独对话干活；支持本地与在线大模型，可恢复被删除的预制智能体 - **工作流**：画布式可视化编排（开始 → 大模型/文本 → 输出，节点连线传参），内置翻译、总结、两步润色、写作助手等预制工作流并支持自定义；一键运行，支持本地与在线大模型，可在画布上实时查看每个节点的输出 +- **知识库**：上传 txt / md / PDF 等文档自动分块建索引，支持全文检索（中文友好）、文档预览 / 重新切分 / 删除，检索结果可一键复制 - **对话细节**：空会话复用（已有无消息的新建对话时不再新建）、回答重新生成/版本历史/编辑重提、思考过程展示、会话置顶收藏与导入导出 - **本地 API 服务**：OpenAI 兼容端点（/v1/models、/v1/chat/completions、/v1/embeddings），仅本机监听，可选 API Key diff --git a/apps/desktop/Cargo.toml b/apps/desktop/Cargo.toml index e114229..effce44 100644 --- a/apps/desktop/Cargo.toml +++ b/apps/desktop/Cargo.toml @@ -29,6 +29,8 @@ tokio.workspace = true reqwest.workspace = true futures.workspace = true lettre = { version = "0.11", default-features = false, features = ["smtp-transport", "builder", "tokio1", "tokio1-rustls-tls", "hostname"] } +base64 = "0.22" +pdf-extract = "0.7" uuid.workspace = true dirs.workspace = true rusqlite.workspace = true diff --git a/apps/desktop/src/commands.rs b/apps/desktop/src/commands.rs index ce94e7b..f3093d7 100644 --- a/apps/desktop/src/commands.rs +++ b/apps/desktop/src/commands.rs @@ -1,6 +1,7 @@ use crate::App; use crate::mcp_client; use crate::tools::{self, MarkerFilter, ToolDef}; +use base64::Engine; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::path::PathBuf; @@ -10,6 +11,7 @@ use tauri::{AppHandle, Emitter, Manager, State}; use tokio::sync::RwLock; use xianren_api::ApiState; use xianren_core::agents as agents_db; +use xianren_core::knowledge_base as kb_db; use xianren_core::models as models_db; use xianren_core::mcp_servers as mcp_db; use xianren_core::scheduled_tasks as scheduled_db; @@ -1441,6 +1443,247 @@ fn resolve_scheduled_model( Err("没有可用的大模型：请先在模型管理部署本地模型或启用在线 API 模型".into()) } +// ---------- 知识库 ---------- + +#[derive(Deserialize)] +pub struct KnowledgeBaseInput { + pub name: String, + #[serde(default)] + pub description: String, + #[serde(default = "default_chunk_size")] + pub chunk_size: i64, + #[serde(default = "default_chunk_overlap")] + pub chunk_overlap: i64, +} + +fn default_chunk_size() -> i64 { + 500 +} + +fn default_chunk_overlap() -> i64 { + 50 +} + +#[derive(Deserialize)] +pub struct KbFileInput { + pub name: String, + pub data_base64: String, +} + +#[derive(Serialize)] +pub struct KbImportResult { + pub name: String, + pub ok: bool, + pub char_count: i64, + pub chunk_count: i64, + pub error: Option, +} + +fn validate_kb_input(input: &KnowledgeBaseInput) -> Result<(), String> { + if input.name.trim().is_empty() { + return Err("知识库名称不能为空".into()); + } + if input.chunk_size < 100 || input.chunk_size > 10000 { + return Err("分块大小需在 100–10000 字之间".into()); + } + if input.chunk_overlap < 0 || input.chunk_overlap >= input.chunk_size { + return Err("重叠字数需大于等于 0 且小于分块大小".into()); + } + Ok(()) +} + +#[tauri::command] +pub fn list_knowledge_bases( + state: State<'_, App>, +) -> Result, String> { + let db = state.core.db.lock().unwrap(); + kb_db::list_knowledge_bases(&db).map_err(|e| e.to_string()) +} + +#[tauri::command] +pub fn add_knowledge_base( + state: State<'_, App>, + input: KnowledgeBaseInput, +) -> Result { + validate_kb_input(&input)?; + let db = state.core.db.lock().unwrap(); + let id = uuid::Uuid::new_v4().to_string(); + let kb = xianren_core::KnowledgeBase { + id, + name: input.name.trim().to_string(), + description: input.description.trim().to_string(), + chunk_size: input.chunk_size, + chunk_overlap: input.chunk_overlap, + doc_count: 0, + chunk_count: 0, + created_at: String::new(), + updated_at: String::new(), + }; + kb_db::insert_knowledge_base(&db, &kb).map_err(|e| e.to_string())?; + kb_db::get_knowledge_base(&db, &kb.id) + .map_err(|e| e.to_string())? + .ok_or_else(|| "knowledge base not found".to_string()) +} + +#[tauri::command] +pub fn update_knowledge_base( + state: State<'_, App>, + id: String, + input: KnowledgeBaseInput, +) -> Result<(), String> { + validate_kb_input(&input)?; + let db = state.core.db.lock().unwrap(); + let existing = kb_db::get_knowledge_base(&db, &id) + .map_err(|e| e.to_string())? + .ok_or_else(|| "知识库不存在".to_string())?; + let kb = xianren_core::KnowledgeBase { + id, + name: input.name.trim().to_string(), + description: input.description.trim().to_string(), + chunk_size: input.chunk_size, + chunk_overlap: input.chunk_overlap, + doc_count: existing.doc_count, + chunk_count: existing.chunk_count, + created_at: existing.created_at, + updated_at: String::new(), + }; + kb_db::update_knowledge_base(&db, &kb).map_err(|e| e.to_string()) +} + +#[tauri::command] +pub fn remove_knowledge_base(state: State<'_, App>, id: String) -> Result<(), String> { + let db = state.core.db.lock().unwrap(); + kb_db::delete_knowledge_base(&db, &id).map_err(|e| e.to_string()) +} + +#[tauri::command] +pub fn list_kb_documents( + state: State<'_, App>, + kb_id: String, +) -> Result, String> { + let db = state.core.db.lock().unwrap(); + kb_db::list_documents(&db, &kb_id).map_err(|e| e.to_string()) +} + +#[tauri::command] +pub fn get_kb_document( + state: State<'_, App>, + document_id: String, +) -> Result { + let db = state.core.db.lock().unwrap(); + kb_db::get_document_detail(&db, &document_id) + .map_err(|e| e.to_string())? + .ok_or_else(|| "文档不存在".to_string()) +} + +/// 导入文档：解码 base64 → 提取文本 → 按知识库分块设置切块 → 写入索引。 +#[tauri::command] +pub fn kb_import_documents( + state: State<'_, App>, + kb_id: String, + files: Vec, +) -> Result, String> { + let db = state.core.db.lock().unwrap(); + let kb = kb_db::get_knowledge_base(&db, &kb_id) + .map_err(|e| e.to_string())? + .ok_or_else(|| "知识库不存在".to_string())?; + let mut results = Vec::new(); + for file in files { + let name = file.name.trim().to_string(); + let mut result = KbImportResult { + name: name.clone(), + ok: false, + char_count: 0, + chunk_count: 0, + error: None, + }; + let bytes = match base64::engine::general_purpose::STANDARD.decode(&file.data_base64) { + Ok(b) => b, + Err(e) => { + result.error = Some(format!("base64 解码失败：{e}")); + results.push(result); + continue; + } + }; + let text = match crate::knowledge::extract_text(&name, &bytes) { + Ok(t) => t, + Err(e) => { + result.error = Some(e); + results.push(result); + continue; + } + }; + let chunks = + kb_db::chunk_text(&text, kb.chunk_size as usize, kb.chunk_overlap as usize); + if chunks.is_empty() { + result.error = Some("切分后没有有效分块".into()); + results.push(result); + continue; + } + let doc = xianren_core::KbDocument { + id: uuid::Uuid::new_v4().to_string(), + kb_id: kb.id.clone(), + name: name.clone(), + file_type: name + .rsplit('.') + .next() + .unwrap_or("") + .to_lowercase(), + file_size: bytes.len() as i64, + char_count: text.chars().count() as i64, + chunk_count: chunks.len() as i64, + created_at: String::new(), + updated_at: String::new(), + }; + if let Err(e) = kb_db::insert_document_with_chunks(&db, &doc, &text, &chunks) { + result.error = Some(format!("写入失败：{e}")); + results.push(result); + continue; + } + result.ok = true; + result.char_count = doc.char_count; + result.chunk_count = doc.chunk_count; + results.push(result); + } + Ok(results) +} + +#[tauri::command] +pub fn remove_kb_document(state: State<'_, App>, document_id: String) -> Result<(), String> { + let db = state.core.db.lock().unwrap(); + kb_db::delete_document(&db, &document_id).map_err(|e| e.to_string()) +} + +/// 按知识库最新分块设置重新切分指定文档。 +#[tauri::command] +pub fn kb_rechunk_document( + state: State<'_, App>, + document_id: String, +) -> Result { + let db = state.core.db.lock().unwrap(); + let detail = kb_db::get_document_detail(&db, &document_id) + .map_err(|e| e.to_string())? + .ok_or_else(|| "文档不存在".to_string())?; + let kb = kb_db::get_knowledge_base(&db, &detail.doc.kb_id) + .map_err(|e| e.to_string())? + .ok_or_else(|| "知识库不存在".to_string())?; + kb_db::rechunk_document(&db, &document_id, kb.chunk_size, kb.chunk_overlap) + .map_err(|e| e.to_string()) +} + +/// 检索知识库（kb_id 为空时检索全部知识库）。 +#[tauri::command] +pub fn kb_search( + state: State<'_, App>, + kb_id: Option, + query: String, + limit: Option, +) -> Result, String> { + let db = state.core.db.lock().unwrap(); + kb_db::search(&db, kb_id.as_deref(), &query, limit.unwrap_or(10)) + .map_err(|e| e.to_string()) +} + #[tauri::command] pub fn list_conversations( state: State<'_, App>, diff --git a/apps/desktop/src/knowledge.rs b/apps/desktop/src/knowledge.rs new file mode 100644 index 0000000..522ec36 --- /dev/null +++ b/apps/desktop/src/knowledge.rs @@ -0,0 +1,33 @@ +/// 按扩展名从文件字节提取纯文本：文本类文件直接按 UTF-8 读取，PDF 走 pdf-extract。 +pub fn extract_text(name: &str, bytes: &[u8]) -> Result { + let ext = name + .rsplit('.') + .next() + .map(|e| e.to_lowercase()) + .unwrap_or_default(); + let text = match ext.as_str() { + "pdf" => pdf_extract::extract_text_from_mem(bytes) + .map_err(|e| format!("PDF 解析失败：{e}"))?, + "txt" | "md" | "markdown" | "json" | "csv" | "tsv" | "log" | "yml" | "yaml" + | "toml" | "ini" | "conf" | "cfg" | "xml" | "html" | "htm" | "py" | "rs" + | "ts" | "tsx" | "js" | "jsx" | "java" | "c" | "cpp" | "h" | "hpp" | "go" + | "php" | "rb" | "sh" | "bat" | "ps1" | "sql" | "css" | "scss" | "sass" => { + String::from_utf8_lossy(bytes).to_string() + } + _ => { + return Err(format!( + "暂不支持的文件类型：{}（支持文本类文件与 PDF）", + if ext.is_empty() { + "无扩展名" + } else { + &ext + } + )); + } + }; + let cleaned = text.replace('\0', "").trim().to_string(); + if cleaned.is_empty() { + return Err("文件内容为空或无法提取文本".into()); + } + Ok(cleaned) +} diff --git a/apps/desktop/src/lib.rs b/apps/desktop/src/lib.rs index 7c45776..0caf707 100644 --- a/apps/desktop/src/lib.rs +++ b/apps/desktop/src/lib.rs @@ -1,4 +1,5 @@ mod commands; +mod knowledge; mod mail; mod mcp_client; mod tools; @@ -184,6 +185,16 @@ pub fn run() { commands::autostart_status, commands::autostart_set, commands::mail_test, + commands::list_knowledge_bases, + commands::add_knowledge_base, + commands::update_knowledge_base, + commands::remove_knowledge_base, + commands::list_kb_documents, + commands::get_kb_document, + commands::kb_import_documents, + commands::remove_kb_document, + commands::kb_rechunk_document, + commands::kb_search, commands::list_models, commands::import_model, commands::remove_model, diff --git a/crates/core/src/knowledge_base.rs b/crates/core/src/knowledge_base.rs new file mode 100644 index 0000000..b0a52bf --- /dev/null +++ b/crates/core/src/knowledge_base.rs @@ -0,0 +1,474 @@ +use crate::error::Result; +use rusqlite::{params, Connection}; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct KnowledgeBase { + pub id: String, + pub name: String, + pub description: String, + pub chunk_size: i64, + pub chunk_overlap: i64, + pub doc_count: i64, + pub chunk_count: i64, + pub created_at: String, + pub updated_at: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct KbDocument { + pub id: String, + pub kb_id: String, + pub name: String, + pub file_type: String, + pub file_size: i64, + pub char_count: i64, + pub chunk_count: i64, + pub created_at: String, + pub updated_at: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct KbDocumentDetail { + #[serde(flatten)] + pub doc: KbDocument, + pub content: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct KbSearchHit { + pub chunk_id: String, + pub document_id: String, + pub document_name: String, + pub seq: i64, + pub content: String, +} + +const KB_COLUMNS: &str = "id, name, description, chunk_size, chunk_overlap, created_at, updated_at"; +const DOC_COLUMNS: &str = "id, kb_id, name, file_type, file_size, char_count, chunk_count, created_at, updated_at"; + +// ---------- 知识库 ---------- + +pub fn list_knowledge_bases(db: &Connection) -> Result> { + let mut stmt = db.prepare(&format!( + "SELECT {KB_COLUMNS}, + (SELECT COUNT(*) FROM kb_documents d WHERE d.kb_id = kb.id), + (SELECT COUNT(*) FROM kb_chunks c WHERE c.kb_id = kb.id) + FROM knowledge_bases kb ORDER BY kb.updated_at DESC, kb.created_at DESC" + ))?; + let rows = stmt.query_map([], row_to_kb)?; + let mut out = Vec::new(); + for row in rows { + out.push(row?); + } + Ok(out) +} + +pub fn get_knowledge_base(db: &Connection, id: &str) -> Result> { + let mut stmt = db.prepare(&format!( + "SELECT {KB_COLUMNS}, + (SELECT COUNT(*) FROM kb_documents d WHERE d.kb_id = kb.id), + (SELECT COUNT(*) FROM kb_chunks c WHERE c.kb_id = kb.id) + FROM knowledge_bases kb WHERE kb.id = ?1" + ))?; + let mut rows = stmt.query_map(params![id], row_to_kb)?; + match rows.next() { + Some(row) => Ok(Some(row?)), + None => Ok(None), + } +} + +pub fn insert_knowledge_base(db: &Connection, kb: &KnowledgeBase) -> Result<()> { + db.execute( + "INSERT INTO knowledge_bases (id, name, description, chunk_size, chunk_overlap) + VALUES (?1, ?2, ?3, ?4, ?5)", + params![ + kb.id, + kb.name, + kb.description, + kb.chunk_size.clamp(100, 10000), + kb.chunk_overlap.clamp(0, 10000), + ], + )?; + Ok(()) +} + +pub fn update_knowledge_base(db: &Connection, kb: &KnowledgeBase) -> Result<()> { + db.execute( + "UPDATE knowledge_bases SET name = ?1, description = ?2, + chunk_size = ?3, chunk_overlap = ?4, updated_at = datetime('now') + WHERE id = ?5", + params![ + kb.name, + kb.description, + kb.chunk_size.clamp(100, 10000), + kb.chunk_overlap.clamp(0, 10000), + kb.id, + ], + )?; + Ok(()) +} + +pub fn delete_knowledge_base(db: &Connection, id: &str) -> Result<()> { + db.execute( + "DELETE FROM knowledge_bases WHERE id = ?1", + params![id], + )?; + Ok(()) +} + +// ---------- 文档 ---------- + +pub fn list_documents(db: &Connection, kb_id: &str) -> Result> { + let mut stmt = db.prepare(&format!( + "SELECT {DOC_COLUMNS} FROM kb_documents + WHERE kb_id = ?1 ORDER BY updated_at DESC, created_at DESC" + ))?; + let rows = stmt.query_map(params![kb_id], row_to_doc)?; + let mut out = Vec::new(); + for row in rows { + out.push(row?); + } + Ok(out) +} + +pub fn get_document(db: &Connection, id: &str) -> Result> { + let mut stmt = db.prepare(&format!( + "SELECT {DOC_COLUMNS} FROM kb_documents WHERE id = ?1" + ))?; + let mut rows = stmt.query_map(params![id], row_to_doc)?; + match rows.next() { + Some(row) => Ok(Some(row?)), + None => Ok(None), + } +} + +pub fn get_document_detail(db: &Connection, id: &str) -> Result> { + let mut stmt = db.prepare(&format!( + "SELECT {DOC_COLUMNS}, content FROM kb_documents WHERE id = ?1" + ))?; + let mut rows = stmt.query_map(params![id], |row| { + let doc = row_to_doc(row)?; + let content: String = row.get(9)?; + Ok(KbDocumentDetail { doc, content }) + })?; + match rows.next() { + Some(row) => Ok(Some(row?)), + None => Ok(None), + } +} + +/// 插入文档及其分块（同一事务内完成，成功后自动写入 FTS 索引）。 +pub fn insert_document_with_chunks( + db: &Connection, + doc: &KbDocument, + content: &str, + chunks: &[String], +) -> Result<()> { + let tx = db.unchecked_transaction()?; + tx.execute( + "INSERT INTO kb_documents + (id, kb_id, name, file_type, file_size, char_count, chunk_count, content) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)", + params![ + doc.id, + doc.kb_id, + doc.name, + doc.file_type, + doc.file_size, + doc.char_count, + chunks.len() as i64, + content, + ], + )?; + for (i, chunk) in chunks.iter().enumerate() { + tx.execute( + "INSERT INTO kb_chunks (id, kb_id, document_id, seq, content) + VALUES (?1, ?2, ?3, ?4, ?5)", + params![ + uuid::Uuid::new_v4().to_string(), + doc.kb_id, + doc.id, + (i + 1) as i64, + chunk, + ], + )?; + } + tx.commit()?; + Ok(()) +} + +pub fn delete_document(db: &Connection, id: &str) -> Result<()> { + db.execute("DELETE FROM kb_documents WHERE id = ?1", params![id])?; + Ok(()) +} + +/// 重新切分文档：删除旧分块后按新参数重分并重建索引。 +pub fn rechunk_document( + db: &Connection, + id: &str, + chunk_size: i64, + chunk_overlap: i64, +) -> Result { + let detail = get_document_detail(db, id)? + .ok_or_else(|| crate::error::CoreError::Other("document not found".into()))?; + let chunks = chunk_text(&detail.content, chunk_size as usize, chunk_overlap as usize); + let tx = db.unchecked_transaction()?; + tx.execute("DELETE FROM kb_chunks WHERE document_id = ?1", params![id])?; + for (i, chunk) in chunks.iter().enumerate() { + tx.execute( + "INSERT INTO kb_chunks (id, kb_id, document_id, seq, content) + VALUES (?1, ?2, ?3, ?4, ?5)", + params![ + uuid::Uuid::new_v4().to_string(), + detail.doc.kb_id, + id, + (i + 1) as i64, + chunk, + ], + )?; + } + tx.execute( + "UPDATE kb_documents SET chunk_count = ?1, updated_at = datetime('now') WHERE id = ?2", + params![chunks.len() as i64, id], + )?; + tx.commit()?; + Ok(chunks.len()) +} + +/// 把文本按字符数切块，相邻块之间保留 overlap 字符的重叠。 +pub fn chunk_text(text: &str, chunk_size: usize, chunk_overlap: usize) -> Vec { + let size = chunk_size.max(100); + let overlap = chunk_overlap.min(size.saturating_sub(1)); + let chars: Vec = text.chars().collect(); + let mut chunks = Vec::new(); + let mut start = 0usize; + while start < chars.len() { + let end = (start + size).min(chars.len()); + let piece: String = chars[start..end].iter().collect(); + if !piece.trim().is_empty() { + chunks.push(piece); + } + if end >= chars.len() { + break; + } + start = end.saturating_sub(overlap); + } + chunks +} + +// ---------- 检索 ---------- + +/// 在知识库中检索：优先 FTS5（trigram）BM25 排序，短查询或解析失败回退 LIKE。 +pub fn search( + db: &Connection, + kb_id: Option<&str>, + query: &str, + limit: usize, +) -> Result> { + let q = query.trim(); + if q.is_empty() { + return Ok(Vec::new()); + } + let limit = limit.clamp(1, 100) as i64; + if q.chars().count() >= 3 { + let fts_q = format!("\"{}\"", q.replace('"', "\"\"")); + let result = db.prepare( + "SELECT c.id, c.document_id, d.name, c.seq, c.content + FROM kb_chunks c + JOIN kb_documents d ON d.id = c.document_id + JOIN kb_chunks_fts ON kb_chunks_fts.rowid = c.rowid + WHERE (?1 IS NULL OR c.kb_id = ?1) AND kb_chunks_fts MATCH ?2 + ORDER BY bm25(kb_chunks_fts) LIMIT ?3", + ) + .and_then(|mut stmt| { + let rows = stmt.query_map(params![kb_id, fts_q, limit], row_to_hit)?; + rows.collect::, _>>() + }); + if let Ok(hits) = result { + return Ok(hits); + } + } + // 回退：LIKE 子串匹配 + let mut stmt = db.prepare( + "SELECT c.id, c.document_id, d.name, c.seq, c.content + FROM kb_chunks c + JOIN kb_documents d ON d.id = c.document_id + WHERE (?1 IS NULL OR c.kb_id = ?1) AND c.content LIKE '%' || ?2 || '%' + ORDER BY c.rowid LIMIT ?3", + )?; + let rows = stmt.query_map(params![kb_id, q, limit], row_to_hit)?; + let mut out = Vec::new(); + for row in rows { + out.push(row?); + } + Ok(out) +} + +fn row_to_kb(row: &rusqlite::Row<'_>) -> rusqlite::Result { + Ok(KnowledgeBase { + id: row.get(0)?, + name: row.get(1)?, + description: row.get(2)?, + chunk_size: row.get(3)?, + chunk_overlap: row.get(4)?, + doc_count: row.get(7)?, + chunk_count: row.get(8)?, + created_at: row.get(5)?, + updated_at: row.get(6)?, + }) +} + +fn row_to_doc(row: &rusqlite::Row<'_>) -> rusqlite::Result { + Ok(KbDocument { + id: row.get(0)?, + kb_id: row.get(1)?, + name: row.get(2)?, + file_type: row.get(3)?, + file_size: row.get(4)?, + char_count: row.get(5)?, + chunk_count: row.get(6)?, + created_at: row.get(7)?, + updated_at: row.get(8)?, + }) +} + +fn row_to_hit(row: &rusqlite::Row<'_>) -> rusqlite::Result { + Ok(KbSearchHit { + chunk_id: row.get(0)?, + document_id: row.get(1)?, + document_name: row.get(2)?, + seq: row.get(3)?, + content: row.get(4)?, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use rusqlite::Connection; + + fn test_conn() -> Connection { + let conn = Connection::open_in_memory().unwrap(); + conn.execute_batch(include_str!("schema.sql")).unwrap(); + conn + } + + fn insert_doc(conn: &Connection, id: &str, content: &str) { + let doc = KbDocument { + id: id.to_string(), + kb_id: "kb1".to_string(), + name: format!("{id}.md"), + file_type: "md".to_string(), + file_size: content.len() as i64, + char_count: content.chars().count() as i64, + chunk_count: 0, + created_at: String::new(), + updated_at: String::new(), + }; + let chunks = chunk_text(content, 200, 20); + insert_document_with_chunks(conn, &doc, content, &chunks).unwrap(); + } + + #[test] + fn chunk_and_search_roundtrip() { + let conn = test_conn(); + let kb = KnowledgeBase { + id: "kb1".to_string(), + name: "测试库".to_string(), + description: String::new(), + chunk_size: 200, + chunk_overlap: 20, + doc_count: 0, + chunk_count: 0, + created_at: String::new(), + updated_at: String::new(), + }; + insert_knowledge_base(&conn, &kb).unwrap(); + insert_doc(&conn, "d1", "仙人工作室是一个本地大模型桌面应用，支持聊天、智能体与工作流。"); + + let kbs = list_knowledge_bases(&conn).unwrap(); + assert_eq!(kbs.len(), 1); + assert!(kbs[0].chunk_count >= 1); + + let hits = search(&conn, Some("kb1"), "智能体", 10).unwrap(); + assert_eq!(hits.len(), 1); + assert!(hits[0].content.contains("智能体")); + } + + #[test] + fn short_query_falls_back_to_like() { + let conn = test_conn(); + insert_knowledge_base( + &conn, + &KnowledgeBase { + id: "kb1".to_string(), + name: "测试库".to_string(), + description: String::new(), + chunk_size: 200, + chunk_overlap: 20, + doc_count: 0, + chunk_count: 0, + created_at: String::new(), + updated_at: String::new(), + }, + ) + .unwrap(); + insert_doc(&conn, "d1", "今天是星期一，天气晴朗，适合出去散步。"); + let hits = search(&conn, Some("kb1"), "散步", 10).unwrap(); + assert_eq!(hits.len(), 1); + } + + #[test] + fn delete_document_removes_chunks() { + let conn = test_conn(); + insert_knowledge_base( + &conn, + &KnowledgeBase { + id: "kb1".to_string(), + name: "测试库".to_string(), + description: String::new(), + chunk_size: 200, + chunk_overlap: 20, + doc_count: 0, + chunk_count: 0, + created_at: String::new(), + updated_at: String::new(), + }, + ) + .unwrap(); + insert_doc(&conn, "d1", "知识库测试内容，用于验证删除级联。"); + delete_document(&conn, "d1").unwrap(); + assert!(get_document(&conn, "d1").unwrap().is_none()); + let hits = search(&conn, Some("kb1"), "知识库", 10).unwrap(); + assert!(hits.is_empty()); + } + + #[test] + fn rechunk_changes_chunk_count() { + let conn = test_conn(); + insert_knowledge_base( + &conn, + &KnowledgeBase { + id: "kb1".to_string(), + name: "测试库".to_string(), + description: String::new(), + chunk_size: 200, + chunk_overlap: 20, + doc_count: 0, + chunk_count: 0, + created_at: String::new(), + updated_at: String::new(), + }, + ) + .unwrap(); + let long = "这是一段很长很长的知识库测试文本。".repeat(100); + insert_doc(&conn, "d1", &long); + let before = get_document(&conn, "d1").unwrap().unwrap().chunk_count; + let n = rechunk_document(&conn, "d1", 500, 50).unwrap(); + let after = get_document(&conn, "d1").unwrap().unwrap().chunk_count; + assert_eq!(n as i64, after); + assert!(after < before); + assert!(!search(&conn, Some("kb1"), "测试文本", 5).unwrap().is_empty()); + } +} diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index f005d3d..1be26e1 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -1,6 +1,7 @@ pub mod agents; pub mod app; pub mod error; +pub mod knowledge_base; pub mod mcp_servers; pub mod models; pub mod scheduled_tasks; @@ -11,6 +12,7 @@ pub mod workflows; pub use app::CoreApp; pub use error::{CoreError, Result}; +pub use knowledge_base::{KbDocument, KbDocumentDetail, KbSearchHit, KnowledgeBase}; pub use models::ModelInfo; pub use agents::Agent; pub use scheduled_tasks::ScheduledTask; diff --git a/crates/core/src/schema.sql b/crates/core/src/schema.sql index 6d508de..2be1ebe 100644 --- a/crates/core/src/schema.sql +++ b/crates/core/src/schema.sql @@ -127,3 +127,51 @@ CREATE TABLE IF NOT EXISTS scheduled_tasks ( created_at TEXT NOT NULL DEFAULT (datetime('now')), updated_at TEXT NOT NULL DEFAULT (datetime('now')) ); + +CREATE TABLE IF NOT EXISTS knowledge_bases ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + description TEXT NOT NULL DEFAULT '', + chunk_size INTEGER NOT NULL DEFAULT 500, + chunk_overlap INTEGER NOT NULL DEFAULT 50, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE TABLE IF NOT EXISTS kb_documents ( + id TEXT PRIMARY KEY, + kb_id TEXT NOT NULL REFERENCES knowledge_bases(id) ON DELETE CASCADE, + name TEXT NOT NULL, + file_type TEXT NOT NULL DEFAULT '', + file_size INTEGER NOT NULL DEFAULT 0, + char_count INTEGER NOT NULL DEFAULT 0, + chunk_count INTEGER NOT NULL DEFAULT 0, + content TEXT NOT NULL DEFAULT '', + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE TABLE IF NOT EXISTS kb_chunks ( + id TEXT PRIMARY KEY, + kb_id TEXT NOT NULL REFERENCES knowledge_bases(id) ON DELETE CASCADE, + document_id TEXT NOT NULL REFERENCES kb_documents(id) ON DELETE CASCADE, + seq INTEGER NOT NULL DEFAULT 1, + content TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +-- 全文检索索引（trigram 分词，对中文/短文本更友好；content='' 为 contentless-delete 模式） +CREATE VIRTUAL TABLE IF NOT EXISTS kb_chunks_fts USING fts5(content, content = '', tokenize = 'trigram'); + +CREATE TRIGGER IF NOT EXISTS kb_chunks_ai AFTER INSERT ON kb_chunks BEGIN + INSERT INTO kb_chunks_fts(rowid, content) VALUES (new.rowid, new.content); +END; + +CREATE TRIGGER IF NOT EXISTS kb_chunks_ad AFTER DELETE ON kb_chunks BEGIN + INSERT INTO kb_chunks_fts(kb_chunks_fts, rowid) VALUES ('delete', old.rowid); +END; + +CREATE TRIGGER IF NOT EXISTS kb_chunks_au AFTER UPDATE ON kb_chunks BEGIN + INSERT INTO kb_chunks_fts(kb_chunks_fts, rowid) VALUES ('delete', old.rowid); + INSERT INTO kb_chunks_fts(rowid, content) VALUES (new.rowid, new.content); +END; diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 76fabdf..f03012b 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -12,8 +12,8 @@ ``` xianren_studio/ -├── apps/desktop/ Tauri 桌面壳（Rust 命令层、tauri.conf.json、推荐模型/预制智能体/预制工作流默认 JSON、工作流执行引擎 workflow.rs、邮件发送 mail.rs） -├── crates/core/ 领域核心：SQLite（模型注册表、智能体、工作流、定时计划、会话、消息、设置） +├── apps/desktop/ Tauri 桌面壳（Rust 命令层、tauri.conf.json、推荐模型/预制智能体/预制工作流默认 JSON、工作流执行引擎 workflow.rs、邮件发送 mail.rs、文档提取 knowledge.rs） +├── crates/core/ 领域核心：SQLite（模型注册表、智能体、工作流、知识库、定时计划、会话、消息、设置） ├── crates/engine/ llama-server 生命周期 + 流式/非流式聊天（本地与远程 OpenAI 兼容） ├── crates/download/ 分片断点续传下载器 ├── crates/api/ OpenAI 兼容本地 API 服务（axum） @@ -48,7 +48,7 @@ apps/desktop/src/commands.rs ← 所有 Tauri 命令（invoke_handler 注册于 ``` ui/src/ ├── main.tsx 入口 -├── App.tsx 导航栏、路由（9 个页面）、全局事件订阅（引擎/下载/模型/建议/定时计划/会话） +├── App.tsx 导航栏、路由（10 个页面）、全局事件订阅（引擎/下载/模型/建议/定时计划/会话） ├── api.ts Tauri invoke 封装 + 全部请求/事件 TypeScript 类型 ├── store.ts zustand 全局状态（见下） ├── styles.css 全局样式 @@ -59,6 +59,7 @@ ui/src/ ├── AgentChatPage.tsx 智能体会话页（按路由加载智能体并渲染 ChatPage 智能体模式） ├── WorkflowsPage.tsx 工作流列表（预制/自定义分组、新建/删除/恢复预制） ├── WorkflowCanvasPage.tsx 工作流画布（节点拖入/拖拽/连线、滚轮缩放、空白平移、节点/连线右键菜单、导入导出、运行与结果展示） + ├── KnowledgeBasePage.tsx 知识库（库管理、文档上传/预览/重切、FTS 检索与复制引用） ├── ModelsPage.tsx 模型管理（本地部署 + 在线 API 启用开关） ├── ModelPlazaPage.tsx 模型广场（HF/ModelScope 搜索与下载） ├── TasksPage.tsx 任务（下载/部署进度） @@ -109,6 +110,7 @@ ui/src/ | 工作流 | `list_workflows`、`add_workflow`、`update_workflow`、`remove_workflow`、`set_workflow_enabled`、`restore_preset_workflows`、`run_workflow` | | 定时计划 | `list_scheduled_tasks`、`add_scheduled_task`、`update_scheduled_task`、`remove_scheduled_task`、`set_scheduled_task_enabled`、`run_scheduled_task_now` | | 邮件 | `mail_test`（用当前 SMTP 配置发送测试邮件） | +| 知识库 | `list_knowledge_bases`、`add_knowledge_base`、`update_knowledge_base`、`remove_knowledge_base`、`list_kb_documents`、`get_kb_document`、`kb_import_documents`、`remove_kb_document`、`kb_rechunk_document`、`kb_search` | | 应用/设置 | `app_info`、`autostart_status`、`autostart_set`、`settings_get`、`settings_set` | | 模型 | `list_models`、`import_model`、`remove_model`、`set_model_enabled`、`scan_models`、`add_remote_model` | | 模型广场 | `search_models`、`list_model_files`、`list_recommended_models`、`import_recommendations`、`fetch_model_page` | @@ -139,6 +141,8 @@ ui/src/ - `autostart_status` / `autostart_set`：查询 / 设置开机启动（`tauri-plugin-autostart` 的 `ManagerExt`）。 - `run_scheduled_task_once`：大模型执行完成后，若任务开启邮件，按内容模式发送——固定模式用填写的主题/正文，LLM 模式用智能体回答作正文（主题留空取回答首行）；邮件发送失败时任务标记失败但保留回答内容。 - `send_task_email` / `load_smtp_config` / `split_recipients`：读取设置页「邮件」SMTP 配置、拆分多收件人并调用 `mail.rs` 发送。 +- `kb_import_documents`：base64 解码 → `knowledge.rs::extract_text` 提取文本（文本类直接 UTF-8，PDF 走 pdf-extract）→ `kb_db::chunk_text` 切块（按知识库 chunk_size/overlap）→ `insert_document_with_chunks` 事务写入文档与分块，逐文件返回成功/失败结果。 +- `kb_search` / `kb_rechunk_document`：检索走 `kb_db::search`（FTS5 trigram + BM25，短查询回退 LIKE）；重切按知识库最新分块设置重建该文档分块与索引。 **邮件发送模块（`apps/desktop/src/mail.rs`）：** @@ -180,6 +184,10 @@ ui/src/ | `message_versions` | `content`、`tokens_out`、`seq` | 重新生成前的旧版本 | | `skills` | `name`、`description`、`content`、`enabled` | 技能工具库 | | `mcp_servers` | `name`、`description`、`url`、`auth_token`、`enabled` | MCP 服务配置 | +| `knowledge_bases` | `name`、`description`、`chunk_size`、`chunk_overlap` | 知识库及分块设置 | +| `kb_documents` | `kb_id`（级联删除）、`name`、`file_type`、`file_size`、`char_count`、`chunk_count`、`content`（提取后的纯文本） | 知识库文档；删除/重切分块由 FTS 触发器同步 | +| `kb_chunks` | `kb_id`、`document_id`（级联删除）、`seq`、`content` | 文档分块 | +| `kb_chunks_fts` | FTS5 虚拟表（contentless-delete + trigram 分词） | 全文检索索引，由 `kb_chunks` 触发器维护 | 迁移清单（`ensure_column`）：`models.kind/base_url/api_key/api_model/enabled`、`messages.elapsed_ms/first_token_ms/images_json/model_id`、`conversations.pinned/favorite/tools_json/agent_id`。 @@ -209,6 +217,10 @@ ui/src/ 计划任务 CRUD 与运行状态落库：`list/get/insert/update/delete/set_enabled`（停用时清空 `next_run_at`，启用时重新计算）、`mark_running` / `finish_run`（写执行结果并推进下次执行时间）、`list_due`（到点且未运行中的启用计划）、`reset_stale_running`（应用重启后把中断的「运行中」标记为失败）；任务行含邮件配置字段（`email_enabled/email_to/email_mode/email_subject/email_body`）。 +### 5.9 `knowledge_base.rs` + +知识库 CRUD（`list/get/insert/update/delete`）、文档 CRUD（`list/get/get_detail`）、`insert_document_with_chunks`（文档 + 分块同事务写入，FTS 触发器自动建索引）、`delete_document`（级联删除分块并同步索引）、`rechunk_document`、`chunk_text`（按字符切块 + 重叠）、`search`（FTS5 trigram BM25 排序，查询 <3 字或 MATCH 失败回退 LIKE）。 + --- ## 6. 引擎与远程调用（`crates/engine`） diff --git a/docs/FEATURES.md b/docs/FEATURES.md index 306f341..d008237 100644 --- a/docs/FEATURES.md +++ b/docs/FEATURES.md @@ -12,13 +12,14 @@ ## 1. 页面总览 -应用共 9 个选项卡（导航左侧栏）： +应用共 10 个选项卡（导航左侧栏）： | 选项卡 | 路由 | 页面 | | --- | --- | --- | | 对话 | `/chat` | `ui/src/pages/ChatPage.tsx` | | 智能体 | `/agents`、`/agents/:agentId` | `ui/src/pages/AgentsPage.tsx`、`ui/src/pages/AgentChatPage.tsx` | | 工作流 | `/workflows`、`/workflows/:workflowId` | `ui/src/pages/WorkflowsPage.tsx`、`ui/src/pages/WorkflowCanvasPage.tsx` | +| 知识库 | `/knowledge` | `ui/src/pages/KnowledgeBasePage.tsx` | | 模型管理 | `/` | `ui/src/pages/ModelsPage.tsx` | | 模型广场 | `/plaza` | `ui/src/pages/ModelPlazaPage.tsx` | | 任务 | `/tasks` | `ui/src/pages/TasksPage.tsx` | @@ -127,7 +128,16 @@ --- -## 5. 模型管理页（`/`） +## 5. 知识库页（`/knowledge`） + +- 左侧导航新增「知识库」选项卡，页面分左右两栏：左侧知识库列表，右侧选中库的管理区。 +- **知识库管理**：新建 / 编辑 / 删除知识库，字段含名称、描述、**分块大小**（100–10000 字，默认 500）与**块间重叠**（默认 50）；删除知识库会级联删除其文档与分块。 +- **文档导入**：支持多文件上传，格式覆盖文本类文件（txt / md / json / csv / 代码 / HTML 等）与 **PDF**（后端提取文本）；导入后按知识库分块设置自动切块并写入全文索引。 +- **文档操作**：文档列表展示类型 / 大小 / 字数 / 分块数；支持全文预览、「重切」（按最新分块设置重新切分，适用于修改设置后）与删除。 +- **检索**：全文检索基于 SQLite FTS5（trigram 分词，对中文友好），按 BM25 相关性排序；支持单库检索（也可全库检索），结果展示来源文档与分块序号，可一键复制分块内容用于对话。 +- 数据表：`knowledge_bases`、`kb_documents`（含提取后的纯文本）、`kb_chunks`（分块）、`kb_chunks_fts`（FTS5 全文索引，contentless-delete 模式 + 触发器同步）。 + +## 6. 模型管理页（`/`） - **本地模型**：启动时自动扫描模型目录（可手动重新扫描）；支持导入本地 GGUF 文件；可「部署」（后台加载 llama-server 并显示进度）或「停止」；可打开所在目录、移除。 - **在线 API 模型**（OpenAI 兼容）：添加时填写显示名称、Base URL、API Key（可选）、上游模型 ID。 @@ -136,7 +146,7 @@ --- -## 6. 模型广场页（`/plaza`） +## 7. 模型广场页（`/plaza`） - 搜索 Hugging Face / ModelScope 上的 GGUF 模型，支持热门榜（空关键词）。 - 查看仓库的 GGUF 量化版本文件列表，选择版本一键下载（ModelScope 文件自动附带 SHA256 校验）。 @@ -145,7 +155,7 @@ --- -## 7. 任务页（`/tasks`） +## 8. 任务页（`/tasks`） ### 7.1 计划任务 @@ -169,7 +179,7 @@ --- -## 8. 工具页（`/tools`） +## 9. 工具页（`/tools`） - **联网搜索**：配置 Tavily API Key，支持手动测试搜索；对话中的「联网搜索」开关依赖该配置。 - **技能（Skill）**： @@ -183,14 +193,14 @@ --- -## 9. 服务管理页（`/server`） +## 10. 服务管理页（`/server`） - OpenAI 兼容的本地 API 服务：`/v1/models`、`/v1/chat/completions`（SSE 流式）、`/v1/embeddings`。 - 可设置端口与 API Key，仅本机监听；可启动/停止并查看状态。 --- -## 10. 设置页（`/settings`） +## 11. 设置页（`/settings`） - **引擎与路径**：模型目录、llama-server 路径、模型下载源（hf-mirror / huggingface.co / modelscope.cn）、默认后端（auto/cpu/cuda/vulkan）、上传大小上限。 - **启动**： @@ -208,12 +218,13 @@ --- -## 11. 改动记录 +## 12. 改动记录 > 按时间倒序追加；每次改动功能都要在此登记。 ### 2026-08-17 +- 新增「知识库」选项卡与完整操作界面：知识库管理（新建/编辑/删除）、多格式文档上传（文本类 + PDF）、自动分块与全文检索（FTS5 trigram + BM25）、文档预览 / 重新切分 / 删除、检索结果一键复制。 - 计划任务支持发送邮件：任务设置中可开启邮件并填写收件人，内容支持固定内容与大模型生成两种模式；设置页新增「邮件」SMTP 配置与测试发送。 - 设置页新增「启动」：开机启动开关（写入 Windows 注册表启动项）；「自动加载模型」列表（多个本地模型按顺序在启动时逐个加载，最后一个保持运行）。 - 任务页新增「定时计划」：可指定执行智能体（默认通用助手）、执行内容与间隔（分钟 / 小时 / 天），后台到点自动在智能体会话中执行并记录结果；支持立即执行 / 编辑 / 删除 / 启用停用。 diff --git a/ui/src/App.tsx b/ui/src/App.tsx index 073f2b6..49ecd0a 100644 --- a/ui/src/App.tsx +++ b/ui/src/App.tsx @@ -20,11 +20,13 @@ import AgentsPage from "./pages/AgentsPage"; import AgentChatPage from "./pages/AgentChatPage"; import WorkflowsPage from "./pages/WorkflowsPage"; import WorkflowCanvasPage from "./pages/WorkflowCanvasPage"; +import KnowledgeBasePage from "./pages/KnowledgeBasePage"; const navItems = [ { to: "/chat", label: "对话", icon: "chat" }, { to: "/agents", label: "智能体", icon: "bot" }, { to: "/workflows", label: "工作流", icon: "workflow" }, + { to: "/knowledge", label: "知识库", icon: "book" }, { to: "/", label: "模型管理", icon: "box", end: true }, { to: "/plaza", label: "模型广场", icon: "plaza" }, { to: "/tasks", label: "任务", icon: "tasks" }, @@ -250,6 +252,7 @@ export default function App() { } /> } /> } /> + } /> } /> } /> } /> diff --git a/ui/src/api.ts b/ui/src/api.ts index e73657b..d38a302 100644 --- a/ui/src/api.ts +++ b/ui/src/api.ts @@ -194,6 +194,57 @@ export interface ScheduledTaskInput { email_body: string; } +export interface KnowledgeBase { + id: string; + name: string; + description: string; + chunk_size: number; + chunk_overlap: number; + doc_count: number; + chunk_count: number; + created_at: string; + updated_at: string; +} + +export interface KnowledgeBaseInput { + name: string; + description: string; + chunk_size: number; + chunk_overlap: number; +} + +export interface KbDocument { + id: string; + kb_id: string; + name: string; + file_type: string; + file_size: number; + char_count: number; + chunk_count: number; + created_at: string; + updated_at: string; +} + +export interface KbDocumentDetail extends KbDocument { + content: string; +} + +export interface KbSearchHit { + chunk_id: string; + document_id: string; + document_name: string; + seq: number; + content: string; +} + +export interface KbImportResult { + name: string; + ok: boolean; + char_count: number; + chunk_count: number; + error: string | null; +} + export interface McpServer { id: string; name: string; @@ -411,6 +462,27 @@ export const api = { runScheduledTaskNow: (id: string) => invoke("run_scheduled_task_now", { id }), mailTest: (to: string) => invoke("mail_test", { to }), + listKnowledgeBases: () => invoke("list_knowledge_bases"), + addKnowledgeBase: (input: KnowledgeBaseInput) => + invoke("add_knowledge_base", { input }), + updateKnowledgeBase: (id: string, input: KnowledgeBaseInput) => + invoke("update_knowledge_base", { id, input }), + removeKnowledgeBase: (id: string) => + invoke("remove_knowledge_base", { id }), + listKbDocuments: (kbId: string) => + invoke("list_kb_documents", { kbId }), + getKbDocument: (documentId: string) => + invoke("get_kb_document", { documentId }), + kbImportDocuments: ( + kbId: string, + files: { name: string; data_base64: string }[], + ) => invoke("kb_import_documents", { kbId, files }), + removeKbDocument: (documentId: string) => + invoke("remove_kb_document", { documentId }), + kbRechunkDocument: (documentId: string) => + invoke("kb_rechunk_document", { documentId }), + kbSearch: (kbId: string | null, query: string, limit?: number) => + invoke("kb_search", { kbId, query, limit: limit ?? 10 }), listModels: () => invoke("list_models"), importModel: (path: string) => invoke("import_model", { path }), removeModel: (id: string) => invoke("remove_model", { id }), diff --git a/ui/src/components/Icon.tsx b/ui/src/components/Icon.tsx index 7e4c92b..21a9d48 100644 --- a/ui/src/components/Icon.tsx +++ b/ui/src/components/Icon.tsx @@ -128,6 +128,31 @@ const paths: Record = { ), + book: ( + <> + + + + ), + search: ( + <> + + + + ), + upload: ( + <> + + + + + ), + file: ( + <> + + + + ), }; export default function Icon({ diff --git a/ui/src/pages/KnowledgeBasePage.tsx b/ui/src/pages/KnowledgeBasePage.tsx new file mode 100644 index 0000000..9fe2ac9 --- /dev/null +++ b/ui/src/pages/KnowledgeBasePage.tsx @@ -0,0 +1,607 @@ +import { useEffect, useMemo, useRef, useState } from "react"; +import { + api, + KbDocument, + KbDocumentDetail, + KbSearchHit, + KnowledgeBase, + KnowledgeBaseInput, +} from "../api"; +import Icon from "../components/Icon"; + +function fmtSize(bytes: number) { + if (bytes >= 1024 * 1024) return `${(bytes / 1024 / 1024).toFixed(1)} MB`; + if (bytes >= 1024) return `${(bytes / 1024).toFixed(0)} KB`; + return `${bytes} B`; +} + +function fmtTime(s: string | null | undefined) { + if (!s) return "—"; + const d = new Date(s.replace(" ", "T") + "Z"); + if (Number.isNaN(d.getTime())) return s; + return d.toLocaleString("zh-CN", { + month: "2-digit", + day: "2-digit", + hour: "2-digit", + minute: "2-digit", + hour12: false, + }); +} + +function fileToBase64(file: File): Promise { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onload = () => { + const result = reader.result as string; + const idx = result.indexOf(","); + resolve(idx >= 0 ? result.slice(idx + 1) : result); + }; + reader.onerror = () => reject(reader.error); + reader.readAsDataURL(file); + }); +} + +export default function KnowledgeBasePage() { + const [kbs, setKbs] = useState([]); + const [selectedId, setSelectedId] = useState(null); + const [docs, setDocs] = useState([]); + const [msg, setMsg] = useState(null); + const [busy, setBusy] = useState(false); + const [kbModal, setKbModal] = useState<{ + open: boolean; + editing: KnowledgeBase | null; + }>({ open: false, editing: null }); + const [uploadMsg, setUploadMsg] = useState(null); + const [preview, setPreview] = useState(null); + const [query, setQuery] = useState(""); + const [searching, setSearching] = useState(false); + const [searchResults, setSearchResults] = useState(null); + const uploadInputRef = useRef(null); + + const selected = useMemo( + () => kbs.find((k) => k.id === selectedId) ?? null, + [kbs, selectedId], + ); + + async function refreshKbs(keepSelected = true) { + const list = await api.listKnowledgeBases(); + setKbs(list); + if (!list.some((k) => k.id === selectedId)) { + setSelectedId(keepSelected ? (list[0]?.id ?? null) : null); + } else if (!selectedId && list.length > 0) { + setSelectedId(list[0].id); + } + } + + useEffect(() => { + refreshKbs().catch((e) => setMsg(String(e))); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); + + useEffect(() => { + if (!selectedId) { + setDocs([]); + setSearchResults(null); + return; + } + api + .listKbDocuments(selectedId) + .then(setDocs) + .catch((e) => setMsg(`加载文档失败：${String(e)}`)); + }, [selectedId]); + + async function handleSaveKb(input: KnowledgeBaseInput) { + setBusy(true); + try { + if (kbModal.editing) { + await api.updateKnowledgeBase(kbModal.editing.id, input); + setMsg("知识库已更新"); + } else { + const kb = await api.addKnowledgeBase(input); + setSelectedId(kb.id); + setMsg("知识库已创建"); + } + await refreshKbs(); + setKbModal({ open: false, editing: null }); + } catch (e) { + setMsg(`保存失败：${String(e)}`); + } finally { + setBusy(false); + } + } + + async function handleDeleteKb(kb: KnowledgeBase) { + if (!confirm(`确定删除知识库「${kb.name}」吗？将同时删除其中 ${kb.doc_count} 个文档与 ${kb.chunk_count} 个分块。`)) { + return; + } + try { + await api.removeKnowledgeBase(kb.id); + setSelectedId(null); + await refreshKbs(false); + setMsg("知识库已删除"); + } catch (e) { + setMsg(`删除失败：${String(e)}`); + } + } + + async function handleUpload(files: FileList | null) { + if (!files || files.length === 0 || !selected) return; + setBusy(true); + setUploadMsg(null); + try { + const payload = []; + for (const file of Array.from(files)) { + payload.push({ name: file.name, data_base64: await fileToBase64(file) }); + } + const results = await api.kbImportDocuments(selected.id, payload); + const okCount = results.filter((r) => r.ok).length; + const failed = results.filter((r) => !r.ok); + setUploadMsg( + `导入完成：成功 ${okCount} 个${failed.length > 0 ? `，失败 ${failed.length} 个（${failed.map((f) => `${f.name}: ${f.error}`).join("；")}）` : ""}`, + ); + await refreshKbs(); + setDocs(await api.listKbDocuments(selected.id)); + } catch (e) { + setUploadMsg(`导入失败：${String(e)}`); + } finally { + setBusy(false); + if (uploadInputRef.current) uploadInputRef.current.value = ""; + } + } + + async function handleRechunk(doc: KbDocument) { + if (!confirm(`按知识库当前分块设置重新切分「${doc.name}」吗？`)) return; + try { + const n = await api.kbRechunkDocument(doc.id); + setMsg(`已重新切分为 ${n} 个分块`); + if (selected) { + setDocs(await api.listKbDocuments(selected.id)); + await refreshKbs(); + } + } catch (e) { + setMsg(`重新切分失败：${String(e)}`); + } + } + + async function handleDeleteDoc(doc: KbDocument) { + if (!confirm(`确定删除文档「${doc.name}」吗？`)) return; + try { + await api.removeKbDocument(doc.id); + setMsg("文档已删除"); + if (selected) { + setDocs(await api.listKbDocuments(selected.id)); + await refreshKbs(); + } + } catch (e) { + setMsg(`删除失败：${String(e)}`); + } + } + + async function handleSearch() { + if (!query.trim()) return; + setSearching(true); + try { + setSearchResults(await api.kbSearch(selectedId, query.trim(), 20)); + } catch (e) { + setMsg(`检索失败：${String(e)}`); + } finally { + setSearching(false); + } + } + + return ( +
+ {/* 左侧：知识库列表 */} + + + {/* 右侧：主区域 */} +
+ {!selected ? ( +
+ 从左侧选择一个知识库，或点击「+ 新建」创建一个知识库 +
+ ) : ( +
+
+
+

+ + {selected.name} +

+

+ {selected.description || "（无描述）"} +

+
+ {selected.doc_count} 个文档 · {selected.chunk_count} 个分块 · + 分块 {selected.chunk_size} 字 / 重叠 {selected.chunk_overlap} 字 +
+
+
+ + + + handleUpload(e.target.files)} + /> +
+
+ + {msg ? ( +
+ {msg} + +
+ ) : null} + {uploadMsg ? ( +
+ {uploadMsg} +
+ ) : null} + + {/* 检索 */} +
+
检索知识库
+
+ setQuery(e.target.value)} + onKeyDown={(e) => { + if (e.key === "Enter") handleSearch(); + }} + /> + +
+ {searchResults ? ( + searchResults.length === 0 ? ( +
没有匹配的内容
+ ) : ( +
+ {searchResults.map((hit) => ( + + ))} +
+ ) + ) : null} +
+ + {/* 文档列表 */} +
+
+ 文档（{docs.length}） +
+ {docs.length === 0 ? ( +
+ 还没有文档，点击右上角「上传文档」导入 txt / md / PDF 等文件 +
+ ) : ( +
+ {docs.map((doc) => ( + { + api.getKbDocument(doc.id).then(setPreview).catch((e) => setMsg(String(e))); + }} + onRechunk={() => handleRechunk(doc)} + onDelete={() => handleDeleteDoc(doc)} + /> + ))} +
+ )} +
+
+ )} +
+ + {kbModal.open ? ( + setKbModal({ open: false, editing: null })} + /> + ) : null} + + {preview ? ( +
setPreview(null)} + > +
e.stopPropagation()} + > +
+
+
{preview.name}
+
+ {preview.char_count.toLocaleString()} 字 · {preview.chunk_count} 个分块 · + 上传于 {fmtTime(preview.created_at)} +
+
+ +
+ 
+              {preview.content}
+
+
+
+ ) : null} +
+ ); +} + +function SearchHitRow({ hit }: { hit: KbSearchHit }) { + const [copied, setCopied] = useState(false); + return ( +
+
+ + + {hit.document_name} · 分块 {hit.seq} + + +
+
{hit.content}
+
+ ); +} + +function DocRow({ + doc, + onPreview, + onRechunk, + onDelete, +}: { + doc: KbDocument; + onPreview: () => void; + onRechunk: () => void; + onDelete: () => void; +}) { + return ( +
+
+ +
+
+
+ {doc.name} + {doc.file_type ? ( + + {doc.file_type} + + ) : null} +
+
+ {fmtSize(doc.file_size)} · {doc.char_count.toLocaleString()} 字 ·{" "} + {doc.chunk_count} 个分块 · {fmtTime(doc.updated_at)} +
+
+ + + +
+ ); +} + +function KbModal({ + editing, + busy, + onSave, + onClose, +}: { + editing: KnowledgeBase | null; + busy: boolean; + onSave: (input: KnowledgeBaseInput) => void; + onClose: () => void; +}) { + const [name, setName] = useState(editing?.name ?? ""); + const [description, setDescription] = useState(editing?.description ?? ""); + const [chunkSize, setChunkSize] = useState(editing?.chunk_size ?? 500); + const [chunkOverlap, setChunkOverlap] = useState(editing?.chunk_overlap ?? 50); + const valid = + name.trim() && + chunkSize >= 100 && + chunkSize <= 10000 && + chunkOverlap >= 0 && + chunkOverlap < chunkSize; + + return ( +
+
e.stopPropagation()} + > +
+ + {editing ? "编辑知识库" : "新建知识库"} + + +
+
+
+ + setName(e.target.value)} + /> +
+
+ + setDescription(e.target.value)} + /> +
+
+
+ + setChunkSize(Number(e.target.value) || 500)} + /> +
+
+ + setChunkOverlap(Number(e.target.value) || 0)} + /> +
+
+

+ 分块设置影响检索粒度：块越小定位越精准，重叠避免关键信息被截断。修改后可用文档「重切」按钮按新设置重新切分。 +

+
+
+ + +
+
+
+ ); +}