feat: 知识库支持目录来源——多个目录批量导入、后缀过滤(默认全部)、可开关递归子目录,只收录文本/图片视频/音频三类其余忽略

This commit is contained in:
xianrenge
2026-08-17 21:39:06 +08:00
parent ab734478e8
commit 06564babae
11 changed files with 754 additions and 16 deletions
+2
View File
@@ -123,6 +123,8 @@ fn open_db(path: &Path) -> Result<Connection> {
ensure_column(&conn, "scheduled_tasks", "email_mode", "TEXT NOT NULL DEFAULT 'fixed'")?;
ensure_column(&conn, "scheduled_tasks", "email_subject", "TEXT NOT NULL DEFAULT ''")?;
ensure_column(&conn, "scheduled_tasks", "email_body", "TEXT NOT NULL DEFAULT ''")?;
ensure_column(&conn, "kb_documents", "file_path", "TEXT NOT NULL DEFAULT ''")?;
ensure_column(&conn, "kb_documents", "source_id", "TEXT")?;
Ok(conn)
}
+106 -6
View File
@@ -24,10 +24,22 @@ pub struct KbDocument {
pub file_size: i64,
pub char_count: i64,
pub chunk_count: i64,
pub file_path: String,
pub source_id: Option<String>,
pub created_at: String,
pub updated_at: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct KbSource {
pub id: String,
pub kb_id: String,
pub path: String,
pub extensions: String,
pub recursive: bool,
pub created_at: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct KbDocumentDetail {
#[serde(flatten)]
@@ -45,7 +57,8 @@ pub struct KbSearchHit {
}
const KB_COLUMNS: &str = "id, name, description, chunk_size, chunk_overlap, created_at, updated_at";
const DOC_COLUMNS: &str = "id, kb_id, name, file_type, file_size, char_count, chunk_count, created_at, updated_at";
const DOC_COLUMNS: &str = "id, kb_id, name, file_type, file_size, char_count, chunk_count, file_path, source_id, created_at, updated_at";
const SOURCE_COLUMNS: &str = "id, kb_id, path, extensions, recursive, created_at";
// ---------- 知识库 ----------
@@ -149,7 +162,7 @@ pub fn get_document_detail(db: &Connection, id: &str) -> Result<Option<KbDocumen
))?;
let mut rows = stmt.query_map(params![id], |row| {
let doc = row_to_doc(row)?;
let content: String = row.get(9)?;
let content: String = row.get(11)?;
Ok(KbDocumentDetail { doc, content })
})?;
match rows.next() {
@@ -168,8 +181,8 @@ pub fn insert_document_with_chunks(
let tx = db.unchecked_transaction()?;
tx.execute(
"INSERT INTO kb_documents
(id, kb_id, name, file_type, file_size, char_count, chunk_count, content)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)",
(id, kb_id, name, file_type, file_size, char_count, chunk_count, content, file_path, source_id)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)",
params![
doc.id,
doc.kb_id,
@@ -179,6 +192,8 @@ pub fn insert_document_with_chunks(
doc.char_count,
chunks.len() as i64,
content,
doc.file_path,
doc.source_id,
],
)?;
for (i, chunk) in chunks.iter().enumerate() {
@@ -203,6 +218,75 @@ pub fn delete_document(db: &Connection, id: &str) -> Result<()> {
Ok(())
}
/// 同一知识库下是否已存在来自该文件路径的文档(目录导入去重)。
pub fn document_exists(db: &Connection, kb_id: &str, file_path: &str) -> Result<bool> {
if file_path.is_empty() {
return Ok(false);
}
let exists: bool = db.query_row(
"SELECT EXISTS(SELECT 1 FROM kb_documents WHERE kb_id = ?1 AND file_path = ?2)",
params![kb_id, file_path],
|row| row.get(0),
)?;
Ok(exists)
}
// ---------- 目录来源 ----------
pub fn list_sources(db: &Connection, kb_id: &str) -> Result<Vec<KbSource>> {
let mut stmt = db.prepare(&format!(
"SELECT {SOURCE_COLUMNS} FROM kb_sources WHERE kb_id = ?1 ORDER BY created_at ASC"
))?;
let rows = stmt.query_map(params![kb_id], row_to_source)?;
let mut out = Vec::new();
for row in rows {
out.push(row?);
}
Ok(out)
}
pub fn get_source(db: &Connection, id: &str) -> Result<Option<KbSource>> {
let mut stmt = db.prepare(&format!(
"SELECT {SOURCE_COLUMNS} FROM kb_sources WHERE id = ?1"
))?;
let mut rows = stmt.query_map(params![id], row_to_source)?;
match rows.next() {
Some(row) => Ok(Some(row?)),
None => Ok(None),
}
}
pub fn upsert_source(db: &Connection, source: &KbSource) -> Result<()> {
db.execute(
"INSERT INTO kb_sources (id, kb_id, path, extensions, recursive)
VALUES (?1, ?2, ?3, ?4, ?5)
ON CONFLICT(kb_id, path) DO UPDATE SET
extensions = excluded.extensions, recursive = excluded.recursive",
params![
source.id,
source.kb_id,
source.path,
source.extensions,
source.recursive as i32,
],
)?;
Ok(())
}
pub fn delete_source(db: &Connection, id: &str) -> Result<()> {
db.execute("DELETE FROM kb_sources WHERE id = ?1", params![id])?;
Ok(())
}
/// 删除某个目录来源导入的全部文档(分块随外键级联)。
pub fn delete_docs_by_source(db: &Connection, source_id: &str) -> Result<usize> {
let n = db.execute(
"DELETE FROM kb_documents WHERE source_id = ?1",
params![source_id],
)?;
Ok(n)
}
/// 重新切分文档:删除旧分块后按新参数重分并重建索引。
pub fn rechunk_document(
db: &Connection,
@@ -328,8 +412,22 @@ fn row_to_doc(row: &rusqlite::Row<'_>) -> rusqlite::Result<KbDocument> {
file_size: row.get(4)?,
char_count: row.get(5)?,
chunk_count: row.get(6)?,
created_at: row.get(7)?,
updated_at: row.get(8)?,
file_path: row.get(7)?,
source_id: row.get(8)?,
created_at: row.get(9)?,
updated_at: row.get(10)?,
})
}
fn row_to_source(row: &rusqlite::Row<'_>) -> rusqlite::Result<KbSource> {
let recursive: i32 = row.get(4)?;
Ok(KbSource {
id: row.get(0)?,
kb_id: row.get(1)?,
path: row.get(2)?,
extensions: row.get(3)?,
recursive: recursive != 0,
created_at: row.get(5)?,
})
}
@@ -363,6 +461,8 @@ mod tests {
file_size: content.len() as i64,
char_count: content.chars().count() as i64,
chunk_count: 0,
file_path: String::new(),
source_id: None,
created_at: String::new(),
updated_at: String::new(),
};
+1 -1
View File
@@ -12,7 +12,7 @@ pub mod workflows;
pub use app::CoreApp;
pub use error::{CoreError, Result};
pub use knowledge_base::{KbDocument, KbDocumentDetail, KbSearchHit, KnowledgeBase};
pub use knowledge_base::{KbDocument, KbDocumentDetail, KbSearchHit, KbSource, KnowledgeBase};
pub use models::ModelInfo;
pub use agents::Agent;
pub use scheduled_tasks::ScheduledTask;
+13
View File
@@ -147,6 +147,8 @@ CREATE TABLE IF NOT EXISTS kb_documents (
char_count INTEGER NOT NULL DEFAULT 0,
chunk_count INTEGER NOT NULL DEFAULT 0,
content TEXT NOT NULL DEFAULT '',
file_path TEXT NOT NULL DEFAULT '',
source_id TEXT,
created_at TEXT NOT NULL DEFAULT (datetime('now')),
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
);
@@ -160,6 +162,17 @@ CREATE TABLE IF NOT EXISTS kb_chunks (
created_at TEXT NOT NULL DEFAULT (datetime('now'))
);
CREATE TABLE IF NOT EXISTS kb_sources (
id TEXT PRIMARY KEY,
kb_id TEXT NOT NULL REFERENCES knowledge_bases(id) ON DELETE CASCADE,
path TEXT NOT NULL,
extensions TEXT NOT NULL DEFAULT '',
recursive INTEGER NOT NULL DEFAULT 1,
created_at TEXT NOT NULL DEFAULT (datetime('now'))
);
CREATE UNIQUE INDEX IF NOT EXISTS idx_kb_sources_kb_path ON kb_sources(kb_id, path);
-- 全文检索索引(trigram 分词,对中文/短文本更友好;content='' 为 contentless-delete 模式)
CREATE VIRTUAL TABLE IF NOT EXISTS kb_chunks_fts USING fts5(content, content = '', tokenize = 'trigram');