hz4th_coder
9a95b71631
- agents.py: supervisor/review/debate 三模式编排引擎，步骤流水全记录 - eval.py: eval 数据集 + LLM-as-Judge 评分 + 任务产出自动沉淀 + Worker 排行榜 - templates.py: 任务/项目/团队三类模板，占位符渲染(JSON安全转义)，一键应用 - enterprise.py: 用户体系/RBAC、OIDC+LDAP SSO、审计日志、数据导出/保留期/PII脱敏 - db.py: V2 表结构 + 迁移 + 启动恢复(stale running→failed) - app.py: V2 API 路由 + 登录升级(账号体系)+ 审计钩子 - 前端: 新增多Agent协作/自动评估/模板市场/企业版 4 页面 - Dockerfile: 企业版私有化部署
242 lines
11 KiB
Python
242 lines
11 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""
|
||
V2 自动评估体系
|
||
- eval 数据集：用例（input/expected）+ 评分标准（rubric），随任务运行自动沉淀
|
||
- 评估运行：Worker 逐条作答 → LLM-as-Judge 按 rubric 打分 → 平均分/排行榜
|
||
- 一键沉淀：把已验收通过的任务产出转为 eval 用例
|
||
"""
|
||
import json
|
||
import time
|
||
import threading
|
||
import traceback
|
||
import db
|
||
import llm_gateway
|
||
import notify
|
||
|
||
JUDGE_PROMPT = (
|
||
'你是严格、公正的评估裁判（LLM-as-Judge）。请对执行结果按评分标准打分。\n'
|
||
'输出严格 JSON：{{"score": 0到100的整数, "judgment": "评分理由（50字内）"}}\n'
|
||
'只输出 JSON。\n\n'
|
||
'评分标准：{rubric}\n'
|
||
'测试输入：{input}\n'
|
||
'参考答案（预期）：{expected}\n'
|
||
'执行结果：{output}'
|
||
)
|
||
|
||
DEFAULT_RUBRIC = (
|
||
'内容准确性 40%（与参考答案一致性、事实正确性），'
|
||
'完整性 30%（是否覆盖输入要求的所有要点），'
|
||
'清晰度与格式 30%（结构清晰、语言通顺）。'
|
||
)
|
||
|
||
BUILTIN_DATASETS = [
|
||
{
|
||
'name': '通用指令遵循',
|
||
'description': '基础指令遵循与回答质量基准集（内置）',
|
||
'rubric': '指令遵循度 50%（是否按要求格式/要点作答），准确性 30%，简洁性 20%。',
|
||
'tags': ['基础', '指令遵循'],
|
||
'cases': [
|
||
{'input': '用 3 句话向非技术人员解释什么是"大语言模型（LLM）"。',
|
||
'expected': '包含：基于海量文本训练、预测下一个词/生成文本、可对话问答等要点'},
|
||
{'input': '列出数据库事务的 ACID 四个特性，并各用一句话解释。',
|
||
'expected': '原子性、一致性、隔离性、持久性，且各有解释'},
|
||
{'input': '写一段 50 字以内的产品文案，推广一款 AI 项目管理工具。',
|
||
'expected': '≤50字、突出AI提效、有行动号召'},
|
||
],
|
||
},
|
||
{
|
||
'name': '代码生成',
|
||
'description': '代码生成正确性基准集（内置）',
|
||
'rubric': '功能正确性 60%，代码质量 25%（可读性/健壮性），格式规范 15%。',
|
||
'tags': ['代码'],
|
||
'cases': [
|
||
{'input': '用 Python 写一个函数 fibonacci(n)，返回斐波那契数列前 n 项。',
|
||
'expected': '正确实现、处理 n<=0、含示例'},
|
||
{'input': '写一个 SQL 查询：统计每个部门的员工人数，按人数降序。',
|
||
'expected': 'GROUP BY department, ORDER BY count DESC'},
|
||
{'input': '用 Python 写一个装饰器 @retry(times=3)，失败自动重试。',
|
||
'expected': '捕获异常重试、最多 times 次、可传参'},
|
||
],
|
||
},
|
||
]
|
||
|
||
|
||
def _extract_judge(text):
|
||
"""解析裁判 JSON，容错兜底"""
|
||
try:
|
||
t = text.strip()
|
||
if t.startswith('```'):
|
||
t = t.strip('`')
|
||
if t.startswith('json'):
|
||
t = t[4:]
|
||
t = t.strip()
|
||
start = t.find('{')
|
||
end = t.rfind('}') + 1
|
||
data = json.loads(t[start:end]) if end > start else {}
|
||
score = float(data.get('score', data.get('总分', 0)))
|
||
judgment = data.get('judgment') or data.get('评分理由') or ''
|
||
return max(0.0, min(100.0, score)), str(judgment)[:500]
|
||
except Exception:
|
||
import re
|
||
m = re.search(r'(\d{1,3})\s*[/／]\s*100', text)
|
||
if m:
|
||
return float(m.group(1)), text[:300]
|
||
m = re.search(r'"score"\s*:\s*(\d{1,3})', text)
|
||
if m:
|
||
return float(m.group(1)), text[:300]
|
||
return 0.0, '裁判输出解析失败'
|
||
|
||
|
||
def create_builtin_datasets():
|
||
"""首次启动注入内置数据集"""
|
||
c = db.q('SELECT COUNT(*) c FROM eval_datasets WHERE is_builtin=1')[0]['c']
|
||
if c > 0:
|
||
return
|
||
for ds in BUILTIN_DATASETS:
|
||
did = db.w(
|
||
'INSERT INTO eval_datasets (name, description, rubric, tags, is_builtin, created_at, updated_at) '
|
||
'VALUES (?,?,?,?,1,?,?)',
|
||
(ds['name'], ds['description'], ds['rubric'], json.dumps(ds['tags']), db.now(), db.now()))
|
||
for case in ds['cases']:
|
||
db.w('INSERT INTO eval_cases (dataset_id, input, expected, tags, created_at) VALUES (?,?,?,?,?)',
|
||
(did, case['input'], case['expected'], json.dumps(ds['tags']), db.now()))
|
||
|
||
|
||
def sink_done_tasks():
|
||
"""沉淀：把最近验收通过（done 且无打回）且尚未沉淀的任务转为 eval 用例（幂等）"""
|
||
# 幂等标记：记录在 settings
|
||
sunk = set(json.loads(db.get_setting('eval_sunk_task_ids', '[]')))
|
||
rows = db.q(
|
||
'SELECT id, title, description, output_text, updated_at FROM tasks '
|
||
'WHERE status="done" AND rejection_count=0 AND output_text!="" ORDER BY id DESC LIMIT 200')
|
||
created = 0
|
||
for t in rows:
|
||
if t['id'] in sunk:
|
||
continue
|
||
# 找到/创建"任务沉淀"数据集
|
||
ds = db.q('SELECT id FROM eval_datasets WHERE name=?', ('任务产出沉淀',), one=True)
|
||
if not ds:
|
||
ds_id = db.w(
|
||
'INSERT INTO eval_datasets (name, description, rubric, tags, is_builtin, created_at, updated_at) '
|
||
'VALUES (?,?,?,?,0,?,?)',
|
||
('任务产出沉淀', '从已验收任务自动沉淀的高质量问答对', DEFAULT_RUBRIC,
|
||
json.dumps(['沉淀']), db.now(), db.now()))
|
||
else:
|
||
ds_id = ds['id']
|
||
# 避免重复用例
|
||
dup = db.q('SELECT id FROM eval_cases WHERE dataset_id=? AND input=?', (ds_id, t['description'] or t['title']), one=True)
|
||
if dup:
|
||
sunk.add(t['id'])
|
||
continue
|
||
db.w('INSERT INTO eval_cases (dataset_id, input, expected, tags, created_at) VALUES (?,?,?,?,?)',
|
||
(ds_id, (t['description'] or t['title'])[:2000], t['output_text'][:4000],
|
||
json.dumps(['沉淀', f'task#{t["id"]}']), db.now()))
|
||
sunk.add(t['id'])
|
||
created += 1
|
||
db.set_setting('eval_sunk_task_ids', json.dumps(list(sunk)[-2000:]))
|
||
return created
|
||
|
||
|
||
def evaluate_case(dataset, case, worker):
|
||
"""单条用例：Worker 作答 + Judge 打分。返回 (output, score, judgment, latency, cost, tokens)"""
|
||
t0 = time.time()
|
||
# 1) Worker 作答
|
||
r = llm_gateway.chat(
|
||
worker['provider'], worker['model'],
|
||
[{'role': 'system', 'content': worker['system_prompt'] or '你是待评估的执行 Agent，请直接回答问题。'},
|
||
{'role': 'user', 'content': case['input']}],
|
||
temperature=worker['temperature'], max_tokens=worker['max_tokens'] or 2000,
|
||
base_url=worker['base_url'] or None, api_key=worker['api_key'] or None)
|
||
output = r['text']
|
||
latency = int((time.time() - t0) * 1000)
|
||
tokens = r['total_tokens']
|
||
|
||
# 2) Judge 打分
|
||
judge_cost = 0.0
|
||
try:
|
||
j = llm_gateway.chat(
|
||
worker['provider'], worker['model'],
|
||
[{'role': 'system', 'content': '你只输出 JSON。'},
|
||
{'role': 'user', 'content': JUDGE_PROMPT.format(
|
||
rubric=dataset['rubric'] or DEFAULT_RUBRIC,
|
||
input=case['input'][:2000],
|
||
expected=(case['expected'] or '无参考答案，凭专业判断')[:3000],
|
||
output=output[:4000])}],
|
||
temperature=0.1, max_tokens=600,
|
||
base_url=worker['base_url'] or None, api_key=worker['api_key'] or None)
|
||
score, judgment = _extract_judge(j['text'])
|
||
judge_cost = j['cost']
|
||
tokens += j['total_tokens']
|
||
except Exception as e:
|
||
score, judgment = 0.0, f'裁判调用失败: {e}'
|
||
total_cost = round(r['cost'] + judge_cost, 6)
|
||
return output, score, judgment, latency, total_cost, tokens
|
||
|
||
|
||
def run_eval(run_id):
|
||
"""后台线程：执行一次完整评估"""
|
||
run = db.q('SELECT * FROM eval_runs WHERE id=?', (run_id,), one=True)
|
||
if not run:
|
||
return
|
||
ds = db.q('SELECT * FROM eval_datasets WHERE id=?', (run['dataset_id'],), one=True)
|
||
worker = db.q('SELECT * FROM workers WHERE id=?', (run['worker_id'],), one=True)
|
||
if not ds or not worker:
|
||
db.w('UPDATE eval_runs SET status="failed", finished_at=? WHERE id=?', (db.now(), run_id))
|
||
return
|
||
cases = db.q('SELECT * FROM eval_cases WHERE dataset_id=? ORDER BY id', (ds['id'],))
|
||
total_tokens, total_cost, scores, done = 0, 0.0, [], 0
|
||
try:
|
||
for case in cases:
|
||
try:
|
||
output, score, judgment, latency, cost, tokens = evaluate_case(ds, case, worker)
|
||
db.w(
|
||
'INSERT INTO eval_results (run_id, case_id, worker_id, output, score, judgment, '
|
||
'latency_ms, cost, created_at) VALUES (?,?,?,?,?,?,?,?,?)',
|
||
(run_id, case['id'], worker['id'], output, score, judgment, latency, cost, db.now()))
|
||
total_cost += cost
|
||
total_tokens += tokens
|
||
scores.append(score)
|
||
done += 1
|
||
db.w('UPDATE eval_runs SET cases_done=? WHERE id=?', (done, run_id))
|
||
except Exception as e:
|
||
db.w(
|
||
'INSERT INTO eval_results (run_id, case_id, worker_id, output, score, judgment, '
|
||
'latency_ms, cost, created_at) VALUES (?,?,?,?,?,?,?,?,?)',
|
||
(run_id, case['id'], worker['id'], '', 0, f'执行失败: {str(e)[:200]}', 0, 0, db.now()))
|
||
done += 1
|
||
db.w('UPDATE eval_runs SET cases_done=? WHERE id=?', (done, run_id))
|
||
avg = round(sum(scores) / len(scores), 2) if scores else 0
|
||
db.w('UPDATE eval_runs SET status="done", score=?, total_tokens=?, cost=?, cases_done=?, finished_at=? '
|
||
'WHERE id=?', (avg, total_tokens, round(total_cost, 6), done, db.now(), run_id))
|
||
try:
|
||
notify.notify('task_done', f'评估完成：{ds["name"]}',
|
||
f'Worker「{worker["name"]}」在数据集「{ds["name"]}」上平均分 {avg}/100，'
|
||
f'共 {done}/{len(cases)} 条用例，成本 ¥{total_cost:.4f}',
|
||
save_alert=True, level='info', atype='eval_done')
|
||
except Exception:
|
||
pass
|
||
except Exception as e:
|
||
db.w('UPDATE eval_runs SET status="failed", error=?, finished_at=? WHERE id=?',
|
||
(str(e)[:300], db.now(), run_id))
|
||
_trace(e)
|
||
|
||
|
||
def _trace(e):
|
||
print('[eval]', traceback.format_exc())
|
||
|
||
|
||
class EvalRunner:
|
||
def __init__(self):
|
||
self._threads = {}
|
||
|
||
def submit(self, run_id):
|
||
if run_id in self._threads and self._threads[run_id].is_alive():
|
||
return False
|
||
t = threading.Thread(target=run_eval, args=(run_id,), daemon=True)
|
||
self._threads[run_id] = t
|
||
t.start()
|
||
return True
|
||
|
||
|
||
runner = EvalRunner()
|