初始化参数数据自动化管理系统
功能: - 文章内容库管理 - 待处理产品列表管理 - 自动处理流程 - 智能搜索和数据提取 - ParamHub API集成 - 定时任务调度 部署端口: 16043
This commit is contained in:
@@ -0,0 +1,412 @@
|
||||
"""
|
||||
数据处理服务 - 核心处理逻辑
|
||||
"""
|
||||
import json
|
||||
import re
|
||||
from datetime import datetime
|
||||
from config import Config
|
||||
from models.database import db
|
||||
from services.search_service import search_service
|
||||
from services.paramhub_client import paramhub_client
|
||||
|
||||
class DataProcessService:
|
||||
def __init__(self):
|
||||
self.config = Config
|
||||
|
||||
def process_product(self, product_info):
|
||||
"""
|
||||
处理单个产品的完整流程
|
||||
|
||||
Args:
|
||||
product_info: {
|
||||
'product_name': str,
|
||||
'category': str (可选),
|
||||
'subcategory': str (可选)
|
||||
}
|
||||
|
||||
Returns:
|
||||
{
|
||||
'success': bool,
|
||||
'message': str,
|
||||
'review_id': str (如果成功提交),
|
||||
'new_products': list (发现的新产品)
|
||||
}
|
||||
"""
|
||||
product_name = product_info['product_name']
|
||||
category = product_info.get('category')
|
||||
subcategory = product_info.get('subcategory')
|
||||
|
||||
result = {
|
||||
'success': False,
|
||||
'message': '',
|
||||
'review_id': None,
|
||||
'new_products': []
|
||||
}
|
||||
|
||||
try:
|
||||
# 1. 从内容库和互联网搜索原始数据
|
||||
print(f"[处理] 开始处理产品: {product_name}")
|
||||
search_results = search_service.search_all(
|
||||
keyword=product_name,
|
||||
category=category,
|
||||
include_internet=True
|
||||
)
|
||||
|
||||
if search_results['total'] == 0:
|
||||
# 没有找到数据，发送通知
|
||||
paramhub_client.send_notification(f"未找到产品 '{product_name}' 的相关数据")
|
||||
result['message'] = '未找到相关数据'
|
||||
return result
|
||||
|
||||
# 2. 提取对应产品的具体内容（排除无关产品）
|
||||
extracted_data = self.extract_product_data(
|
||||
product_name,
|
||||
search_results,
|
||||
category,
|
||||
subcategory
|
||||
)
|
||||
|
||||
if not extracted_data:
|
||||
result['message'] = '无法提取有效数据'
|
||||
return result
|
||||
|
||||
# 3. 按照类别和子类别字段填充内容
|
||||
filled_data = self.fill_product_fields(
|
||||
extracted_data,
|
||||
category,
|
||||
subcategory
|
||||
)
|
||||
|
||||
if not filled_data:
|
||||
result['message'] = '填充数据失败'
|
||||
return result
|
||||
|
||||
# 4. 提交到待审核区
|
||||
category_type = self.get_category_type(category)
|
||||
success, review_id_or_error = paramhub_client.submit_for_review(
|
||||
category_type,
|
||||
filled_data,
|
||||
subcategory
|
||||
)
|
||||
|
||||
if success:
|
||||
# 记录处理历史
|
||||
db.add_process_history(
|
||||
product_name=product_name,
|
||||
category=category,
|
||||
subcategory=subcategory,
|
||||
status='submitted',
|
||||
review_id=review_id_or_error,
|
||||
details={'data': filled_data}
|
||||
)
|
||||
|
||||
result['success'] = True
|
||||
result['message'] = f'已提交审核，review_id: {review_id_or_error}'
|
||||
result['review_id'] = review_id_or_error
|
||||
|
||||
print(f"[处理] 产品 {product_name} 提交成功")
|
||||
else:
|
||||
result['message'] = f'提交失败: {review_id_or_error}'
|
||||
db.add_process_history(
|
||||
product_name=product_name,
|
||||
category=category,
|
||||
subcategory=subcategory,
|
||||
status='failed',
|
||||
details={'error': review_id_or_error}
|
||||
)
|
||||
|
||||
# 5. 检查是否发现新的未处理产品
|
||||
new_products = self.discover_new_products(
|
||||
product_name,
|
||||
search_results,
|
||||
category
|
||||
)
|
||||
result['new_products'] = new_products
|
||||
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
error_msg = f"处理失败: {str(e)}"
|
||||
print(f"[错误] {error_msg}")
|
||||
result['message'] = error_msg
|
||||
db.add_process_history(
|
||||
product_name=product_name,
|
||||
category=category,
|
||||
subcategory=subcategory,
|
||||
status='error',
|
||||
details={'error': str(e)}
|
||||
)
|
||||
return result
|
||||
|
||||
def extract_product_data(self, target_product, search_results, category, subcategory):
|
||||
"""
|
||||
从搜索结果中提取目标产品的具体内容
|
||||
排除无关产品
|
||||
"""
|
||||
extracted = {
|
||||
'name': target_product,
|
||||
'category': category,
|
||||
'subcategory': subcategory,
|
||||
'raw_data': []
|
||||
}
|
||||
|
||||
# 收集所有相关内容
|
||||
all_content = []
|
||||
|
||||
# 从内容库结果中提取
|
||||
for article in search_results.get('articles', []):
|
||||
product_names = json.loads(article.get('product_names', '[]'))
|
||||
|
||||
# 检查是否包含目标产品
|
||||
if self.is_product_match(target_product, product_names):
|
||||
all_content.append({
|
||||
'source': article.get('source'),
|
||||
'url': article.get('url'),
|
||||
'summary': article.get('summary'),
|
||||
'content': article.get('content'),
|
||||
'keywords': json.loads(article.get('keywords', '[]'))
|
||||
})
|
||||
|
||||
# 从互联网搜索结果中提取
|
||||
for item in search_results.get('internet', []):
|
||||
if self.is_product_match(target_product, [item.get('title', '')]):
|
||||
all_content.append({
|
||||
'source': 'internet',
|
||||
'url': item.get('url'),
|
||||
'content': item.get('content')
|
||||
})
|
||||
|
||||
if not all_content:
|
||||
return None
|
||||
|
||||
extracted['raw_data'] = all_content
|
||||
return extracted
|
||||
|
||||
def fill_product_fields(self, extracted_data, category, subcategory):
|
||||
"""
|
||||
根据分类字段配置，填充产品数据
|
||||
严格按照来源数据，不创造内容
|
||||
"""
|
||||
if not extracted_data:
|
||||
return None
|
||||
|
||||
# 获取分类字段配置
|
||||
category_info = paramhub_client.get_category_fields(subcategory) if subcategory else None
|
||||
|
||||
# 基础字段
|
||||
filled_data = {
|
||||
'name': extracted_data['name'],
|
||||
'visible': True,
|
||||
'is_pinned': False
|
||||
}
|
||||
|
||||
# 根据分类类型填充字段
|
||||
category_type = self.get_category_type(category)
|
||||
|
||||
if category_type == 'model':
|
||||
filled_data.update(self.extract_model_fields(extracted_data['raw_data']))
|
||||
elif category_type == 'gpu':
|
||||
filled_data.update(self.extract_gpu_fields(extracted_data['raw_data']))
|
||||
elif category_type == 'cpu':
|
||||
filled_data.update(self.extract_cpu_fields(extracted_data['raw_data']))
|
||||
else:
|
||||
# 动态分类，使用分类字段配置
|
||||
if category_info and 'fields' in category_info:
|
||||
for field in category_info['fields']:
|
||||
value = self.extract_field_from_data(field, extracted_data['raw_data'])
|
||||
if value:
|
||||
filled_data[field] = value
|
||||
|
||||
# 添加数据来源
|
||||
filled_data['_source'] = 'auto_manager'
|
||||
filled_data['_extracted_at'] = datetime.now().isoformat()
|
||||
|
||||
return filled_data
|
||||
|
||||
def extract_model_fields(self, raw_data):
|
||||
"""提取AI模型字段"""
|
||||
fields = {}
|
||||
|
||||
for data in raw_data:
|
||||
content = data.get('content', '') or ''
|
||||
summary = data.get('summary', '') or ''
|
||||
text = f"{summary}\n{content}"
|
||||
|
||||
# 提取参数量
|
||||
if 'parameters' not in fields:
|
||||
params_match = re.search(r'(\d+(?:\.\d+)?)\s*[Bb]', text)
|
||||
if params_match:
|
||||
fields['parameters'] = f"{params_match.group(1)}B"
|
||||
|
||||
# 提取上下文长度
|
||||
if 'context_length' not in fields:
|
||||
ctx_match = re.search(r'context[:\s]+(\d+)', text, re.I)
|
||||
if ctx_match:
|
||||
fields['context_length'] = int(ctx_match.group(1))
|
||||
|
||||
# 提取组织
|
||||
if 'organization' not in fields:
|
||||
org_match = re.search(r'(?:by\s+|from\s+|developed\s+by\s+)([\w\s]+?)(?:\s|,|\.|$)', text, re.I)
|
||||
if org_match:
|
||||
fields['organization'] = org_match.group(1).strip()
|
||||
|
||||
# 提取发布日期
|
||||
if 'publish_date' not in fields:
|
||||
date_match = re.search(r'(\d{4}[-/]\d{1,2}[-/]\d{1,2})', text)
|
||||
if date_match:
|
||||
fields['publish_date'] = date_match.group(1).replace('/', '-')
|
||||
|
||||
return fields
|
||||
|
||||
def extract_gpu_fields(self, raw_data):
|
||||
"""提取GPU字段"""
|
||||
fields = {}
|
||||
|
||||
for data in raw_data:
|
||||
content = data.get('content', '') or ''
|
||||
summary = data.get('summary', '') or ''
|
||||
text = f"{summary}\n{content}"
|
||||
|
||||
# 提取显存
|
||||
if 'memory_gb' not in fields:
|
||||
mem_match = re.search(r'(\d+)\s*GB', text)
|
||||
if mem_match:
|
||||
fields['memory_gb'] = int(mem_match.group(1))
|
||||
|
||||
# 提取CUDA核心
|
||||
if 'cuda_cores' not in fields:
|
||||
cuda_match = re.search(r'(\d+)\s*(?:CUDA|cuda)\s*(?:cores|core)', text, re.I)
|
||||
if cuda_match:
|
||||
fields['cuda_cores'] = int(cuda_match.group(1))
|
||||
|
||||
# 提取价格
|
||||
if 'price_usd' not in fields:
|
||||
price_match = re.search(r'\$(\d+(?:,\d+)*)', text)
|
||||
if price_match:
|
||||
fields['price_usd'] = int(price_match.group(1).replace(',', ''))
|
||||
|
||||
return fields
|
||||
|
||||
def extract_cpu_fields(self, raw_data):
|
||||
"""提取CPU字段"""
|
||||
fields = {}
|
||||
|
||||
for data in raw_data:
|
||||
content = data.get('content', '') or ''
|
||||
summary = data.get('summary', '') or ''
|
||||
text = f"{summary}\n{content}"
|
||||
|
||||
# 提取核心数
|
||||
if 'cores' not in fields:
|
||||
cores_match = re.search(r'(\d+)\s*(?:cores?|Cores?)', text, re.I)
|
||||
if cores_match:
|
||||
fields['cores'] = int(cores_match.group(1))
|
||||
|
||||
# 提取线程数
|
||||
if 'threads' not in fields:
|
||||
threads_match = re.search(r'(\d+)\s*(?:threads?|Threads?)', text, re.I)
|
||||
if threads_match:
|
||||
fields['threads'] = int(threads_match.group(1))
|
||||
|
||||
# 提取频率
|
||||
if 'base_clock' not in fields:
|
||||
clock_match = re.search(r'(\d+(?:\.\d+)?)\s*GHz', text)
|
||||
if clock_match:
|
||||
fields['base_clock'] = float(clock_match.group(1))
|
||||
|
||||
return fields
|
||||
|
||||
def extract_field_from_data(self, field_name, raw_data):
|
||||
"""从数据中提取指定字段"""
|
||||
# 通用字段提取逻辑
|
||||
for data in raw_data:
|
||||
content = data.get('content', '') or ''
|
||||
summary = data.get('summary', '') or ''
|
||||
text = f"{summary}\n{content}"
|
||||
|
||||
# 尝试直接匹配字段名
|
||||
pattern = rf'{field_name}[:\s]+([^\n]+)'
|
||||
match = re.search(pattern, text, re.I)
|
||||
if match:
|
||||
return match.group(1).strip()
|
||||
|
||||
return None
|
||||
|
||||
def get_category_type(self, category):
|
||||
"""获取分类类型"""
|
||||
if not category:
|
||||
return 'dynamic'
|
||||
|
||||
category_lower = category.lower()
|
||||
if 'model' in category_lower or 'ai' in category_lower or 'llm' in category_lower:
|
||||
return 'model'
|
||||
elif 'gpu' in category_lower:
|
||||
return 'gpu'
|
||||
elif 'cpu' in category_lower:
|
||||
return 'cpu'
|
||||
else:
|
||||
return 'dynamic'
|
||||
|
||||
def is_product_match(self, target_product, product_names):
|
||||
"""检查产品名称是否匹配"""
|
||||
target = target_product.lower().strip()
|
||||
|
||||
for name in product_names:
|
||||
name_lower = name.lower().strip()
|
||||
|
||||
# 完全匹配
|
||||
if target == name_lower:
|
||||
return True
|
||||
|
||||
# 包含匹配
|
||||
if target in name_lower or name_lower in target:
|
||||
return True
|
||||
|
||||
# 关键词匹配（去掉型号后缀）
|
||||
target_base = re.sub(r'[-\d]+$', '', target)
|
||||
name_base = re.sub(r'[-\d]+$', '', name_lower)
|
||||
if target_base == name_base:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def discover_new_products(self, current_product, search_results, category):
|
||||
"""
|
||||
从搜索结果中发现新的未处理产品
|
||||
"""
|
||||
new_products = []
|
||||
|
||||
# 收集所有产品名称
|
||||
all_product_names = set()
|
||||
|
||||
for article in search_results.get('articles', []):
|
||||
product_names = json.loads(article.get('product_names', '[]'))
|
||||
all_product_names.update(product_names)
|
||||
|
||||
# 检查是否在待处理列表中
|
||||
for product_name in all_product_names:
|
||||
# 排除当前产品
|
||||
if product_name.lower() == current_product.lower():
|
||||
continue
|
||||
|
||||
# 检查是否已在待处理列表或已处理
|
||||
pending_count = db.get_pending_count()
|
||||
pending_products = [p['product_name'] for p in db.get_pending_products(limit=1000)]
|
||||
|
||||
if product_name not in pending_products:
|
||||
# 检查是否已处理过
|
||||
history = db.get_history_by_product(product_name)
|
||||
if not history:
|
||||
# 添加到待处理列表
|
||||
db.add_pending_product(
|
||||
product_name=product_name,
|
||||
category=category,
|
||||
source='discovered',
|
||||
priority=1 # 发现的产品优先级较低
|
||||
)
|
||||
new_products.append(product_name)
|
||||
|
||||
return new_products
|
||||
|
||||
# 全局处理服务实例
|
||||
process_service = DataProcessService()
|
||||
Reference in New Issue
Block a user