V1.0.0: 基于索引的知识检索系统
核心功能: - 文档索引：使用LLM分析提取关键词/摘要/主题/实体 - 查询处理：LLM分析查询意图并扩展关键词 - BM25检索：基于倒排索引的相关性排序 - RAG问答：检索增强生成 技术栈: - Flask + SQLAlchemy - OpenAI API兼容LLM - BM25算法 特点: 不依赖向量模型和向量库
This commit is contained in:
@@ -0,0 +1,275 @@
|
||||
"""
|
||||
数据库模型定义
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
from flask_sqlalchemy import SQLAlchemy
|
||||
import json
|
||||
|
||||
db = SQLAlchemy()
|
||||
|
||||
|
||||
class Document(db.Model):
|
||||
"""文档表"""
|
||||
__tablename__ = 'documents'
|
||||
|
||||
id = db.Column(db.Integer, primary_key=True)
|
||||
|
||||
# 文件信息
|
||||
filename = db.Column(db.String(255), nullable=False)
|
||||
filepath = db.Column(db.String(512), nullable=False)
|
||||
file_type = db.Column(db.String(20), nullable=False)
|
||||
file_size = db.Column(db.Integer, default=0)
|
||||
|
||||
# 文档元数据
|
||||
title = db.Column(db.String(500), nullable=True)
|
||||
author = db.Column(db.String(100), nullable=True)
|
||||
source = db.Column(db.String(255), nullable=True)
|
||||
|
||||
# 处理状态
|
||||
status = db.Column(db.String(20), default='pending') # pending, processing, indexed, failed
|
||||
error_message = db.Column(db.Text, nullable=True)
|
||||
|
||||
# 文档内容（可选存储原文）
|
||||
content = db.Column(db.Text, nullable=True)
|
||||
|
||||
# 文档摘要（LLM生成）
|
||||
summary = db.Column(db.Text, nullable=True)
|
||||
|
||||
# 主要关键词（JSON数组）
|
||||
keywords = db.Column(db.Text, nullable=True)
|
||||
|
||||
# 文档分类/主题
|
||||
category = db.Column(db.String(100), nullable=True)
|
||||
topics = db.Column(db.Text, nullable=True) # JSON数组
|
||||
|
||||
# 文档实体（人物、地点、组织等）
|
||||
entities = db.Column(db.Text, nullable=True) # JSON对象
|
||||
|
||||
# 统计信息
|
||||
chunk_count = db.Column(db.Integer, default=0) # 分块数量
|
||||
word_count = db.Column(db.Integer, default=0) # 字数
|
||||
|
||||
# 时间戳
|
||||
created_at = db.Column(db.DateTime, default=datetime.utcnow)
|
||||
indexed_at = db.Column(db.DateTime, nullable=True)
|
||||
updated_at = db.Column(db.DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
||||
|
||||
# 关系
|
||||
chunks = db.relationship('DocumentChunk', backref='document', lazy=True, cascade='all, delete-orphan')
|
||||
|
||||
def get_keywords(self):
|
||||
"""获取关键词列表"""
|
||||
if self.keywords:
|
||||
return json.loads(self.keywords)
|
||||
return []
|
||||
|
||||
def set_keywords(self, keywords):
|
||||
"""设置关键词"""
|
||||
self.keywords = json.dumps(keywords, ensure_ascii=False)
|
||||
|
||||
def get_topics(self):
|
||||
"""获取主题列表"""
|
||||
if self.topics:
|
||||
return json.loads(self.topics)
|
||||
return []
|
||||
|
||||
def set_topics(self, topics):
|
||||
"""设置主题"""
|
||||
self.topics = json.dumps(topics, ensure_ascii=False)
|
||||
|
||||
def get_entities(self):
|
||||
"""获取实体"""
|
||||
if self.entities:
|
||||
return json.loads(self.entities)
|
||||
return {}
|
||||
|
||||
def set_entities(self, entities):
|
||||
"""设置实体"""
|
||||
self.entities = json.dumps(entities, ensure_ascii=False)
|
||||
|
||||
def to_dict(self):
|
||||
return {
|
||||
'id': self.id,
|
||||
'filename': self.filename,
|
||||
'title': self.title,
|
||||
'status': self.status,
|
||||
'summary': self.summary,
|
||||
'keywords': self.get_keywords(),
|
||||
'category': self.category,
|
||||
'chunk_count': self.chunk_count,
|
||||
'word_count': self.word_count,
|
||||
'created_at': self.created_at.isoformat() if self.created_at else None,
|
||||
'indexed_at': self.indexed_at.isoformat() if self.indexed_at else None,
|
||||
}
|
||||
|
||||
|
||||
class DocumentChunk(db.Model):
|
||||
"""文档分块表"""
|
||||
__tablename__ = 'document_chunks'
|
||||
|
||||
id = db.Column(db.Integer, primary_key=True)
|
||||
document_id = db.Column(db.Integer, db.ForeignKey('documents.id'), nullable=False)
|
||||
|
||||
# 分块信息
|
||||
chunk_index = db.Column(db.Integer, default=0) # 块序号
|
||||
content = db.Column(db.Text, nullable=False) # 块内容
|
||||
|
||||
# LLM生成的索引信息
|
||||
summary = db.Column(db.Text, nullable=True) # 块摘要
|
||||
keywords = db.Column(db.Text, nullable=True) # 关键词JSON数组
|
||||
topics = db.Column(db.Text, nullable=True) # 主题JSON数组
|
||||
|
||||
# 位置信息
|
||||
start_char = db.Column(db.Integer, default=0)
|
||||
end_char = db.Column(db.Integer, default=0)
|
||||
|
||||
# 词频统计（用于BM25）
|
||||
term_freq = db.Column(db.Text, nullable=True) # JSON对象 {term: count}
|
||||
|
||||
created_at = db.Column(db.DateTime, default=datetime.utcnow)
|
||||
|
||||
def get_keywords(self):
|
||||
if self.keywords:
|
||||
return json.loads(self.keywords)
|
||||
return []
|
||||
|
||||
def set_keywords(self, keywords):
|
||||
self.keywords = json.dumps(keywords, ensure_ascii=False)
|
||||
|
||||
def get_term_freq(self):
|
||||
if self.term_freq:
|
||||
return json.loads(self.term_freq)
|
||||
return {}
|
||||
|
||||
def set_term_freq(self, tf):
|
||||
self.term_freq = json.dumps(tf, ensure_ascii=False)
|
||||
|
||||
def to_dict(self):
|
||||
return {
|
||||
'id': self.id,
|
||||
'document_id': self.document_id,
|
||||
'chunk_index': self.chunk_index,
|
||||
'content': self.content[:200] + '...' if len(self.content) > 200 else self.content,
|
||||
'summary': self.summary,
|
||||
'keywords': self.get_keywords(),
|
||||
}
|
||||
|
||||
|
||||
class InvertedIndex(db.Model):
|
||||
"""倒排索引表"""
|
||||
__tablename__ = 'inverted_index'
|
||||
|
||||
id = db.Column(db.Integer, primary_key=True)
|
||||
|
||||
# 索引项
|
||||
term = db.Column(db.String(100), nullable=False, index=True) # 关键词
|
||||
term_type = db.Column(db.String(20), default='keyword') # keyword, entity, topic
|
||||
|
||||
# 文档频率
|
||||
doc_freq = db.Column(db.Integer, default=0) # 包含该词的文档数
|
||||
|
||||
# 倒排列表（JSON：[{doc_id, chunk_id, tf, positions}]）
|
||||
postings = db.Column(db.Text, nullable=False)
|
||||
|
||||
# 统计
|
||||
total_freq = db.Column(db.Integer, default=0) # 总词频
|
||||
|
||||
updated_at = db.Column(db.DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
||||
|
||||
def get_postings(self):
|
||||
if self.postings:
|
||||
return json.loads(self.postings)
|
||||
return []
|
||||
|
||||
def set_postings(self, postings):
|
||||
self.postings = json.dumps(postings, ensure_ascii=False)
|
||||
self.doc_freq = len(set(p['doc_id'] for p in postings))
|
||||
self.total_freq = sum(p.get('tf', 1) for p in postings)
|
||||
|
||||
@staticmethod
|
||||
def get_or_create(term, term_type='keyword'):
|
||||
"""获取或创建索引项"""
|
||||
index = InvertedIndex.query.filter_by(term=term, term_type=term_type).first()
|
||||
if not index:
|
||||
index = InvertedIndex(term=term, term_type=term_type, postings='[]')
|
||||
db.session.add(index)
|
||||
return index
|
||||
|
||||
|
||||
class QueryLog(db.Model):
|
||||
"""查询日志表"""
|
||||
__tablename__ = 'query_logs'
|
||||
|
||||
id = db.Column(db.Integer, primary_key=True)
|
||||
|
||||
# 查询信息
|
||||
original_query = db.Column(db.Text, nullable=False) # 原始查询
|
||||
processed_query = db.Column(db.Text, nullable=True) # 处理后的查询
|
||||
expanded_terms = db.Column(db.Text, nullable=True) # 扩展词JSON数组
|
||||
|
||||
# 查询意图
|
||||
intent = db.Column(db.String(50), nullable=True) # 查询意图
|
||||
entities = db.Column(db.Text, nullable=True) # 识别的实体
|
||||
|
||||
# 检索结果
|
||||
result_count = db.Column(db.Integer, default=0)
|
||||
top_doc_ids = db.Column(db.Text, nullable=True) # 返回的文档ID JSON数组
|
||||
|
||||
# 性能
|
||||
retrieval_time = db.Column(db.Float, default=0) # 检索耗时(秒)
|
||||
total_time = db.Column(db.Float, default=0) # 总耗时(秒)
|
||||
|
||||
# 用户反馈
|
||||
rating = db.Column(db.Integer, nullable=True) # 评分1-5
|
||||
feedback = db.Column(db.Text, nullable=True) # 反馈文本
|
||||
|
||||
created_at = db.Column(db.DateTime, default=datetime.utcnow)
|
||||
|
||||
def to_dict(self):
|
||||
return {
|
||||
'id': self.id,
|
||||
'query': self.original_query,
|
||||
'result_count': self.result_count,
|
||||
'retrieval_time': self.retrieval_time,
|
||||
'created_at': self.created_at.isoformat() if self.created_at else None,
|
||||
}
|
||||
|
||||
|
||||
class IndexStats(db.Model):
|
||||
"""索引统计表"""
|
||||
__tablename__ = 'index_stats'
|
||||
|
||||
id = db.Column(db.Integer, primary_key=True)
|
||||
|
||||
# 统计信息
|
||||
total_documents = db.Column(db.Integer, default=0)
|
||||
total_chunks = db.Column(db.Integer, default=0)
|
||||
total_terms = db.Column(db.Integer, default=0)
|
||||
total_words = db.Column(db.Integer, default=0)
|
||||
|
||||
# 索引大小
|
||||
index_size_mb = db.Column(db.Float, default=0)
|
||||
|
||||
# 最后更新
|
||||
last_indexed_at = db.Column(db.DateTime, nullable=True)
|
||||
|
||||
updated_at = db.Column(db.DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
||||
|
||||
@staticmethod
|
||||
def get_stats():
|
||||
stats = IndexStats.query.first()
|
||||
if not stats:
|
||||
stats = IndexStats()
|
||||
db.session.add(stats)
|
||||
db.session.commit()
|
||||
return stats
|
||||
|
||||
def update_stats(self):
|
||||
"""更新统计信息"""
|
||||
self.total_documents = Document.query.filter_by(status='indexed').count()
|
||||
self.total_chunks = DocumentChunk.query.count()
|
||||
self.total_terms = InvertedIndex.query.count()
|
||||
self.total_words = db.session.query(db.func.sum(Document.word_count)).scalar() or 0
|
||||
self.last_indexed_at = datetime.utcnow()
|
||||
db.session.commit()
|
||||
Reference in New Issue
Block a user