feat: 知识库支持目录来源——多个目录批量导入、后缀过滤(默认全部)、可开关递归子目录,只收录文本/图片视频/音频三类其余忽略
This commit is contained in:
@@ -123,6 +123,8 @@ fn open_db(path: &Path) -> Result<Connection> {
|
||||
ensure_column(&conn, "scheduled_tasks", "email_mode", "TEXT NOT NULL DEFAULT 'fixed'")?;
|
||||
ensure_column(&conn, "scheduled_tasks", "email_subject", "TEXT NOT NULL DEFAULT ''")?;
|
||||
ensure_column(&conn, "scheduled_tasks", "email_body", "TEXT NOT NULL DEFAULT ''")?;
|
||||
ensure_column(&conn, "kb_documents", "file_path", "TEXT NOT NULL DEFAULT ''")?;
|
||||
ensure_column(&conn, "kb_documents", "source_id", "TEXT")?;
|
||||
Ok(conn)
|
||||
}
|
||||
|
||||
|
||||
@@ -24,10 +24,22 @@ pub struct KbDocument {
|
||||
pub file_size: i64,
|
||||
pub char_count: i64,
|
||||
pub chunk_count: i64,
|
||||
pub file_path: String,
|
||||
pub source_id: Option<String>,
|
||||
pub created_at: String,
|
||||
pub updated_at: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct KbSource {
|
||||
pub id: String,
|
||||
pub kb_id: String,
|
||||
pub path: String,
|
||||
pub extensions: String,
|
||||
pub recursive: bool,
|
||||
pub created_at: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct KbDocumentDetail {
|
||||
#[serde(flatten)]
|
||||
@@ -45,7 +57,8 @@ pub struct KbSearchHit {
|
||||
}
|
||||
|
||||
const KB_COLUMNS: &str = "id, name, description, chunk_size, chunk_overlap, created_at, updated_at";
|
||||
const DOC_COLUMNS: &str = "id, kb_id, name, file_type, file_size, char_count, chunk_count, created_at, updated_at";
|
||||
const DOC_COLUMNS: &str = "id, kb_id, name, file_type, file_size, char_count, chunk_count, file_path, source_id, created_at, updated_at";
|
||||
const SOURCE_COLUMNS: &str = "id, kb_id, path, extensions, recursive, created_at";
|
||||
|
||||
// ---------- 知识库 ----------
|
||||
|
||||
@@ -149,7 +162,7 @@ pub fn get_document_detail(db: &Connection, id: &str) -> Result<Option<KbDocumen
|
||||
))?;
|
||||
let mut rows = stmt.query_map(params![id], |row| {
|
||||
let doc = row_to_doc(row)?;
|
||||
let content: String = row.get(9)?;
|
||||
let content: String = row.get(11)?;
|
||||
Ok(KbDocumentDetail { doc, content })
|
||||
})?;
|
||||
match rows.next() {
|
||||
@@ -168,8 +181,8 @@ pub fn insert_document_with_chunks(
|
||||
let tx = db.unchecked_transaction()?;
|
||||
tx.execute(
|
||||
"INSERT INTO kb_documents
|
||||
(id, kb_id, name, file_type, file_size, char_count, chunk_count, content)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)",
|
||||
(id, kb_id, name, file_type, file_size, char_count, chunk_count, content, file_path, source_id)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)",
|
||||
params![
|
||||
doc.id,
|
||||
doc.kb_id,
|
||||
@@ -179,6 +192,8 @@ pub fn insert_document_with_chunks(
|
||||
doc.char_count,
|
||||
chunks.len() as i64,
|
||||
content,
|
||||
doc.file_path,
|
||||
doc.source_id,
|
||||
],
|
||||
)?;
|
||||
for (i, chunk) in chunks.iter().enumerate() {
|
||||
@@ -203,6 +218,75 @@ pub fn delete_document(db: &Connection, id: &str) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 同一知识库下是否已存在来自该文件路径的文档(目录导入去重)。
|
||||
pub fn document_exists(db: &Connection, kb_id: &str, file_path: &str) -> Result<bool> {
|
||||
if file_path.is_empty() {
|
||||
return Ok(false);
|
||||
}
|
||||
let exists: bool = db.query_row(
|
||||
"SELECT EXISTS(SELECT 1 FROM kb_documents WHERE kb_id = ?1 AND file_path = ?2)",
|
||||
params![kb_id, file_path],
|
||||
|row| row.get(0),
|
||||
)?;
|
||||
Ok(exists)
|
||||
}
|
||||
|
||||
// ---------- 目录来源 ----------
|
||||
|
||||
pub fn list_sources(db: &Connection, kb_id: &str) -> Result<Vec<KbSource>> {
|
||||
let mut stmt = db.prepare(&format!(
|
||||
"SELECT {SOURCE_COLUMNS} FROM kb_sources WHERE kb_id = ?1 ORDER BY created_at ASC"
|
||||
))?;
|
||||
let rows = stmt.query_map(params![kb_id], row_to_source)?;
|
||||
let mut out = Vec::new();
|
||||
for row in rows {
|
||||
out.push(row?);
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
pub fn get_source(db: &Connection, id: &str) -> Result<Option<KbSource>> {
|
||||
let mut stmt = db.prepare(&format!(
|
||||
"SELECT {SOURCE_COLUMNS} FROM kb_sources WHERE id = ?1"
|
||||
))?;
|
||||
let mut rows = stmt.query_map(params![id], row_to_source)?;
|
||||
match rows.next() {
|
||||
Some(row) => Ok(Some(row?)),
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn upsert_source(db: &Connection, source: &KbSource) -> Result<()> {
|
||||
db.execute(
|
||||
"INSERT INTO kb_sources (id, kb_id, path, extensions, recursive)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5)
|
||||
ON CONFLICT(kb_id, path) DO UPDATE SET
|
||||
extensions = excluded.extensions, recursive = excluded.recursive",
|
||||
params![
|
||||
source.id,
|
||||
source.kb_id,
|
||||
source.path,
|
||||
source.extensions,
|
||||
source.recursive as i32,
|
||||
],
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn delete_source(db: &Connection, id: &str) -> Result<()> {
|
||||
db.execute("DELETE FROM kb_sources WHERE id = ?1", params![id])?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 删除某个目录来源导入的全部文档(分块随外键级联)。
|
||||
pub fn delete_docs_by_source(db: &Connection, source_id: &str) -> Result<usize> {
|
||||
let n = db.execute(
|
||||
"DELETE FROM kb_documents WHERE source_id = ?1",
|
||||
params![source_id],
|
||||
)?;
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
/// 重新切分文档:删除旧分块后按新参数重分并重建索引。
|
||||
pub fn rechunk_document(
|
||||
db: &Connection,
|
||||
@@ -328,8 +412,22 @@ fn row_to_doc(row: &rusqlite::Row<'_>) -> rusqlite::Result<KbDocument> {
|
||||
file_size: row.get(4)?,
|
||||
char_count: row.get(5)?,
|
||||
chunk_count: row.get(6)?,
|
||||
created_at: row.get(7)?,
|
||||
updated_at: row.get(8)?,
|
||||
file_path: row.get(7)?,
|
||||
source_id: row.get(8)?,
|
||||
created_at: row.get(9)?,
|
||||
updated_at: row.get(10)?,
|
||||
})
|
||||
}
|
||||
|
||||
fn row_to_source(row: &rusqlite::Row<'_>) -> rusqlite::Result<KbSource> {
|
||||
let recursive: i32 = row.get(4)?;
|
||||
Ok(KbSource {
|
||||
id: row.get(0)?,
|
||||
kb_id: row.get(1)?,
|
||||
path: row.get(2)?,
|
||||
extensions: row.get(3)?,
|
||||
recursive: recursive != 0,
|
||||
created_at: row.get(5)?,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -363,6 +461,8 @@ mod tests {
|
||||
file_size: content.len() as i64,
|
||||
char_count: content.chars().count() as i64,
|
||||
chunk_count: 0,
|
||||
file_path: String::new(),
|
||||
source_id: None,
|
||||
created_at: String::new(),
|
||||
updated_at: String::new(),
|
||||
};
|
||||
|
||||
@@ -12,7 +12,7 @@ pub mod workflows;
|
||||
|
||||
pub use app::CoreApp;
|
||||
pub use error::{CoreError, Result};
|
||||
pub use knowledge_base::{KbDocument, KbDocumentDetail, KbSearchHit, KnowledgeBase};
|
||||
pub use knowledge_base::{KbDocument, KbDocumentDetail, KbSearchHit, KbSource, KnowledgeBase};
|
||||
pub use models::ModelInfo;
|
||||
pub use agents::Agent;
|
||||
pub use scheduled_tasks::ScheduledTask;
|
||||
|
||||
@@ -147,6 +147,8 @@ CREATE TABLE IF NOT EXISTS kb_documents (
|
||||
char_count INTEGER NOT NULL DEFAULT 0,
|
||||
chunk_count INTEGER NOT NULL DEFAULT 0,
|
||||
content TEXT NOT NULL DEFAULT '',
|
||||
file_path TEXT NOT NULL DEFAULT '',
|
||||
source_id TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
@@ -160,6 +162,17 @@ CREATE TABLE IF NOT EXISTS kb_chunks (
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS kb_sources (
|
||||
id TEXT PRIMARY KEY,
|
||||
kb_id TEXT NOT NULL REFERENCES knowledge_bases(id) ON DELETE CASCADE,
|
||||
path TEXT NOT NULL,
|
||||
extensions TEXT NOT NULL DEFAULT '',
|
||||
recursive INTEGER NOT NULL DEFAULT 1,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_kb_sources_kb_path ON kb_sources(kb_id, path);
|
||||
|
||||
-- 全文检索索引(trigram 分词,对中文/短文本更友好;content='' 为 contentless-delete 模式)
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS kb_chunks_fts USING fts5(content, content = '', tokenize = 'trigram');
|
||||
|
||||
|
||||
Reference in New Issue
Block a user