commit 17f32385a6545e17bd00b5c18e6a4aefd7979506 Author: hz4th_coder Date: Sun Aug 23 10:36:31 2026 +0800 LLM速度测试台 v1.0.0：多提供商(OpenAI/Anthropic/Gemini)速度测试，支持上文长度/生成长度/采样次数/避免缓存，实时日志+指标(首字延迟/预填充/解码速度)+历史留存 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..88f9c4a --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +data/ +logs/ +*.pyc +__pycache__/ diff --git a/app.py b/app.py new file mode 100644 index 0000000..c1c1ebd --- /dev/null +++ b/app.py @@ -0,0 +1,143 @@ +# -*- coding: utf-8 -*- +"""LLM 速度测试台 - Flask 主应用""" +import json + +from flask import Flask, jsonify, request, send_from_directory + +import config +import database as db +from llm_providers import DEFAULT_URLS, ProviderError, call_stream +from tester import TestRunner + +app = Flask(__name__, static_folder="static", static_url_path="") +app.json.ensure_ascii = False + +db.init_db() + +RUNNERS = {} # test_id -> TestRunner + + +@app.route("/") +def index(): + return send_from_directory(app.static_folder, "index.html") + + +@app.route("/api/health") +def health(): + running = [tid for tid, r in RUNNERS.items() if r.is_alive()] + return jsonify({"ok": True, "port": config.PORT, "running_tests": running}) + + +def _fill_defaults(cfg): + p = cfg.get("provider", "openai") + if not cfg.get("base_url"): + cfg["base_url"] = DEFAULT_URLS.get(p, "") + if cfg.get("temperature") is None: + cfg["temperature"] = 0.7 + return cfg + + +# ───────────────────────── 提供商配置 ───────────────────────── + +@app.route("/api/configs", methods=["GET"]) +def list_configs(): + return jsonify(db.list_configs()) + + +@app.route("/api/configs", methods=["POST"]) +def add_config(): + cfg = request.get_json(force=True) or {} + if not cfg.get("name"): + return jsonify({"ok": False, "error": "请填写配置名称"}), 400 + cid = db.add_config(cfg) + return jsonify({"ok": True, "id": cid}) + + +@app.route("/api/configs/", methods=["GET"]) +def get_one_config(cid): + c = db.get_config(cid) + if not c: + return jsonify({"ok": False, "error": "配置不存在"}), 404 + return jsonify(c) + + +@app.route("/api/configs/", methods=["DELETE"]) +def del_config(cid): + db.delete_config(cid) + return jsonify({"ok": True}) + + +@app.route("/api/configs/test", methods=["POST"]) +def test_config(): + cfg = _fill_defaults(request.get_json(force=True) or {}) + if not cfg.get("api_key"): + return jsonify({"ok": False, "error": "请填写 API Key"}), 400 + try: + m = call_stream(cfg, "你好，请只回复：OK", {"max_tokens": 16, "avoid_cache": False}) + return jsonify({"ok": True, "total_ms": m["total_ms"], "metrics": m}) + except ProviderError as e: + return jsonify({"ok": False, "error": str(e)}) + except Exception as e: + return jsonify({"ok": False, "error": str(e)}) + + +# ───────────────────────── 测试 ───────────────────────── + +@app.route("/api/tests", methods=["POST"]) +def start_test(): + body = request.get_json(force=True) or {} + cfg = _fill_defaults(body.get("config") or {}) + gen = body.get("gen") or {} + if not cfg.get("api_key"): + return jsonify({"ok": False, "error": "请填写 API Key"}), 400 + if not cfg.get("model"): + return jsonify({"ok": False, "error": "请填写模型名称"}), 400 + tid = db.create_test(cfg, gen) + runner = TestRunner(tid, cfg, gen) + RUNNERS[tid] = runner + runner.start() + return jsonify({"ok": True, "id": tid}) + + +@app.route("/api/tests", methods=["GET"]) +def list_tests(): + return jsonify(db.list_tests()) + + +@app.route("/api/tests/", methods=["GET"]) +def get_test(tid): + t = db.get_test(tid) + if not t: + return jsonify({"ok": False, "error": "测试不存在"}), 404 + t["runs"] = db.get_runs(tid) + t["logs"] = db.get_logs(tid) + return jsonify(t) + + +@app.route("/api/tests//logs", methods=["GET"]) +def get_logs(tid): + after = int(request.args.get("after", 0)) + data = db.get_logs_after(tid, after) + if data is None: + return jsonify({"ok": False, "error": "测试不存在"}), 404 + return jsonify(data) + + +@app.route("/api/tests//cancel", methods=["POST"]) +def cancel_test(tid): + r = RUNNERS.get(tid) + if r and r.is_alive(): + r.request_cancel() + return jsonify({"ok": True, "msg": "正在停止..."}) + return jsonify({"ok": False, "msg": "测试未在运行"}) + + +@app.route("/api/tests/", methods=["DELETE"]) +def del_test(tid): + db.delete_test(tid) + RUNNERS.pop(tid, None) + return jsonify({"ok": True}) + + +if __name__ == "__main__": + app.run(host=config.HOST, port=config.PORT, threaded=True, debug=False) diff --git a/config.py b/config.py new file mode 100644 index 0000000..30a27cb --- /dev/null +++ b/config.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +"""全局配置""" +import os + +PORT = 16097 +HOST = "0.0.0.0" + +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) +DATA_DIR = os.path.join(BASE_DIR, "data") +LOG_DIR = os.path.join(BASE_DIR, "logs") +DB_PATH = os.path.join(DATA_DIR, "llm_speed_tester.db") + +# 流式请求超时：连接 30s，两次数据包间隔最长 120s（慢模型也够用） +CONNECT_TIMEOUT = 30 +STREAM_READ_TIMEOUT = 120 diff --git a/database.py b/database.py new file mode 100644 index 0000000..966c87f --- /dev/null +++ b/database.py @@ -0,0 +1,294 @@ +# -*- coding: utf-8 -*- +"""SQLite 存储：提供商配置 / 测试记录 / 每次采样指标 / 全量日志""" +import os +import json +import sqlite3 +import threading +import time + +import config + +_lock = threading.RLock() # RLock：允许 get_logs_after 内嵌套调用 get_last_run + +SCHEMA = """ +CREATE TABLE IF NOT EXISTS configs( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT DEFAULT '', + provider TEXT DEFAULT 'openai', + base_url TEXT DEFAULT '', + api_key TEXT DEFAULT '', + model TEXT DEFAULT '', + temperature REAL DEFAULT 0.7, + created_at TEXT DEFAULT (datetime('now','localtime')) +); + +CREATE TABLE IF NOT EXISTS tests( + id INTEGER PRIMARY KEY AUTOINCREMENT, + created_at TEXT DEFAULT (datetime('now','localtime')), + status TEXT DEFAULT 'running', + provider TEXT DEFAULT '', + model TEXT DEFAULT '', + config_json TEXT DEFAULT '{}', + gen_cfg_json TEXT DEFAULT '{}', + summary_json TEXT DEFAULT '{}', + error TEXT DEFAULT '', + started_at REAL DEFAULT 0 +); + +CREATE TABLE IF NOT EXISTS test_runs( + id INTEGER PRIMARY KEY AUTOINCREMENT, + test_id INTEGER NOT NULL, + run_index INTEGER DEFAULT 0, + metrics_json TEXT DEFAULT '{}', + error TEXT DEFAULT '' +); + +CREATE TABLE IF NOT EXISTS logs( + id INTEGER PRIMARY KEY AUTOINCREMENT, + test_id INTEGER NOT NULL, + level TEXT DEFAULT 'INFO', + msg TEXT DEFAULT '', + rel REAL DEFAULT 0, + ts TEXT DEFAULT (datetime('now','localtime')) +); + +CREATE INDEX IF NOT EXISTS idx_runs_test ON test_runs(test_id); +CREATE INDEX IF NOT EXISTS idx_logs_test ON logs(test_id); +""" + + +def _connect(): + os.makedirs(config.DATA_DIR, exist_ok=True) + conn = sqlite3.connect(config.DB_PATH, check_same_thread=False, timeout=30) + conn.row_factory = sqlite3.Row + conn.execute("PRAGMA journal_mode=WAL") + return conn + + +def init_db(): + with _lock: + conn = _connect() + try: + conn.executescript(SCHEMA) + conn.commit() + finally: + conn.close() + + +# ───────────────────────── 提供商配置 ───────────────────────── + +def add_config(cfg: dict) -> int: + with _lock: + conn = _connect() + try: + cur = conn.execute( + "INSERT INTO configs(name,provider,base_url,api_key,model,temperature) VALUES(?,?,?,?,?,?)", + (cfg.get("name", ""), cfg.get("provider", "openai"), + cfg.get("base_url", ""), cfg.get("api_key", ""), + cfg.get("model", ""), float(cfg.get("temperature", 0.7)))) + conn.commit() + return cur.lastrowid + finally: + conn.close() + + +def list_configs(): + with _lock: + conn = _connect() + try: + rows = conn.execute("SELECT id,name,provider,base_url,model,temperature," + "CASE WHEN api_key<>'' THEN 1 ELSE 0 END AS has_key " + "FROM configs ORDER BY id DESC").fetchall() + return [dict(r) for r in rows] + finally: + conn.close() + + +def get_config(cid: int): + with _lock: + conn = _connect() + try: + r = conn.execute("SELECT * FROM configs WHERE id=?", (cid,)).fetchone() + return dict(r) if r else None + finally: + conn.close() + + +def delete_config(cid: int): + with _lock: + conn = _connect() + try: + conn.execute("DELETE FROM configs WHERE id=?", (cid,)) + conn.commit() + finally: + conn.close() + + +# ───────────────────────── 测试记录 ───────────────────────── + +def create_test(cfg: dict, gen: dict) -> int: + with _lock: + conn = _connect() + try: + cur = conn.execute( + "INSERT INTO tests(status,provider,model,config_json,gen_cfg_json,started_at) " + "VALUES('running',?,?,?,?,?)", + (cfg.get("provider", "openai"), cfg.get("model", ""), + json.dumps(cfg, ensure_ascii=False), json.dumps(gen, ensure_ascii=False), + time.time())) + conn.commit() + return cur.lastrowid + finally: + conn.close() + + +def update_status(tid: int, status: str, summary: dict = None, error: str = ""): + with _lock: + conn = _connect() + try: + conn.execute("UPDATE tests SET status=?, summary_json=?, error=? WHERE id=?", + (status, + json.dumps(summary or {}, ensure_ascii=False), + error or "", tid)) + conn.commit() + finally: + conn.close() + + +def get_test(tid: int): + with _lock: + conn = _connect() + try: + r = conn.execute("SELECT * FROM tests WHERE id=?", (tid,)).fetchone() + if not r: + return None + d = dict(r) + d["config"] = json.loads(d.pop("config_json") or "{}") + d["gen"] = json.loads(d.pop("gen_cfg_json") or "{}") + d["summary"] = json.loads(d.pop("summary_json") or "{}") + return d + finally: + conn.close() + + +def list_tests(limit=100): + with _lock: + conn = _connect() + try: + rows = conn.execute( + "SELECT id,created_at,status,provider,model,summary_json,error " + "FROM tests ORDER BY id DESC LIMIT ?", (limit,)).fetchall() + out = [] + for r in rows: + d = dict(r) + d["summary"] = json.loads(d.pop("summary_json") or "{}") + out.append(d) + return out + finally: + conn.close() + + +def delete_test(tid: int): + with _lock: + conn = _connect() + try: + conn.execute("DELETE FROM tests WHERE id=?", (tid,)) + conn.execute("DELETE FROM test_runs WHERE test_id=?", (tid,)) + conn.execute("DELETE FROM logs WHERE test_id=?", (tid,)) + conn.commit() + finally: + conn.close() + + +# ───────────────────────── 采样指标 ───────────────────────── + +def add_run(tid: int, run_index: int, metrics: dict, error: str = ""): + with _lock: + conn = _connect() + try: + conn.execute( + "INSERT INTO test_runs(test_id,run_index,metrics_json,error) VALUES(?,?,?,?)", + (tid, run_index, json.dumps(metrics, ensure_ascii=False), error)) + conn.commit() + finally: + conn.close() + + +def get_runs(tid: int): + with _lock: + conn = _connect() + try: + rows = conn.execute( + "SELECT run_index,metrics_json,error FROM test_runs " + "WHERE test_id=? ORDER BY run_index", (tid,)).fetchall() + out = [] + for r in rows: + d = dict(r) + d["metrics"] = json.loads(d.pop("metrics_json") or "{}") + out.append(d) + return out + finally: + conn.close() + + +def get_last_run(tid: int): + with _lock: + conn = _connect() + try: + r = conn.execute( + "SELECT metrics_json FROM test_runs WHERE test_id=? " + "ORDER BY run_index DESC LIMIT 1", (tid,)).fetchone() + return json.loads(r["metrics_json"] or "{}") if r else None + finally: + conn.close() + + +# ───────────────────────── 日志 ───────────────────────── + +def add_log(tid: int, level: str, msg: str, rel: float = None): + with _lock: + conn = _connect() + try: + if rel is None: + r = conn.execute("SELECT started_at FROM tests WHERE id=?", (tid,)).fetchone() + rel = (time.time() - (r["started_at"] or time.time())) if r else 0.0 + conn.execute("INSERT INTO logs(test_id,level,msg,rel) VALUES(?,?,?,?)", + (tid, level, msg, round(rel, 3))) + conn.commit() + finally: + conn.close() + + +def get_logs_after(tid: int, after_id: int = 0): + with _lock: + conn = _connect() + try: + t = conn.execute("SELECT status,summary_json,error FROM tests WHERE id=?", (tid,)).fetchone() + if not t: + return None + rows = conn.execute( + "SELECT id,level,msg,rel FROM logs WHERE test_id=? AND id>? ORDER BY id", + (tid, after_id)).fetchall() + logs = [dict(r) for r in rows] + last = logs[-1]["id"] if logs else after_id + return { + "status": t["status"], + "error": t["error"], + "summary": json.loads(t["summary_json"] or "{}"), + "last_run": get_last_run(tid), + "logs": logs, + "after": last, + } + finally: + conn.close() + + +def get_logs(tid: int): + with _lock: + conn = _connect() + try: + rows = conn.execute( + "SELECT id,level,msg,rel,ts FROM logs WHERE test_id=? ORDER BY id", (tid,)).fetchall() + return [dict(r) for r in rows] + finally: + conn.close() diff --git a/llm_providers.py b/llm_providers.py new file mode 100644 index 0000000..2954e2d --- /dev/null +++ b/llm_providers.py @@ -0,0 +1,305 @@ +# -*- coding: utf-8 -*- +"""大模型提供商适配器：OpenAI 兼容 / Anthropic / Google Gemini + +统一接口: call_stream(cfg, prompt, gen, log, should_stop) -> metrics(dict) +度量指标: + prompt_tokens 上文/提示词 token 数（来自 provider usage） + output_tokens 输出 token 数 + cached_tokens 命中缓存的 token 数（provider 返回时才有） + ttft_ms 首字延迟（time to first token） + decode_ms 解码阶段耗时（首字 -> 结束） + total_ms 总耗时（连接开始 -> 结束） + prefill_speed 预填充速度 = prompt_tokens / ttft（tok/s） + decode_speed 解码速度 = output_tokens / decode_time（tok/s） +""" +import json +import time + +import requests + +import config + + +class ProviderError(Exception): + """API 调用失败""" + + +class StopRequested(Exception): + """用户请求停止""" + + +DEFAULT_URLS = { + "openai": "https://api.openai.com/v1", + "anthropic": "https://api.anthropic.com", + "google": "https://generativelanguage.googleapis.com", +} + +PROVIDER_LABELS = { + "openai": "OpenAI 兼容", + "anthropic": "Anthropic", + "google": "Google Gemini", +} + + +def _parse_sse_line(line): + line = (line or "").strip() + if not line.startswith("data:"): + return None + data = line[5:].strip() + if not data or data == "[DONE]": + return None + try: + return json.loads(data) + except Exception: + return None + + +def _metrics(start, first_token_at, end, prompt_tokens, output_tokens, + cached_tokens, output_chars, prompt_chars): + ttft_ms = (first_token_at - start) * 1000 + decode_ms = (end - first_token_at) * 1000 + total_ms = (end - start) * 1000 + prefill = (prompt_tokens / (ttft_ms / 1000)) if prompt_tokens and ttft_ms > 0 else None + decode = (output_tokens / (decode_ms / 1000)) if output_tokens and decode_ms > 0 else None + return { + "prompt_tokens": int(prompt_tokens or 0), + "output_tokens": int(output_tokens or 0), + "cached_tokens": int(cached_tokens or 0), + "prompt_chars": int(prompt_chars or 0), + "output_chars": int(output_chars or 0), + "ttft_ms": round(ttft_ms, 1), + "decode_ms": round(decode_ms, 1), + "total_ms": round(total_ms, 1), + "prefill_speed": round(prefill, 1) if prefill else None, + "decode_speed": round(decode, 1) if decode else None, + } + + +# ───────────────────────── OpenAI 兼容 ───────────────────────── + +def stream_openai(cfg, prompt, gen, log, should_stop=None): + base = (cfg.get("base_url") or DEFAULT_URLS["openai"]).rstrip("/") + url = base + "/chat/completions" + headers = { + "Authorization": "Bearer " + (cfg.get("api_key") or ""), + "Content-Type": "application/json", + } + payload = { + "model": cfg["model"], + "messages": [{"role": "user", "content": prompt}], + "stream": True, + "max_tokens": int(gen.get("max_tokens", 256)), + "temperature": float(cfg.get("temperature", 0.7)), + } + use_usage = True # stream_options include_usage（部分网关不支持则自动去掉重试） + + def build_payload(): + p = dict(payload) + if use_usage: + p["stream_options"] = {"include_usage": True} + return p + + start = time.time() + first_token_at = None + output_chars = 0 + prompt_tokens = output_tokens = cached_tokens = 0 + resp = None + try: + while True: + if should_stop and should_stop(): + raise StopRequested() + resp = requests.post(url, json=build_payload(), headers=headers, stream=True, + timeout=(config.CONNECT_TIMEOUT, config.STREAM_READ_TIMEOUT)) + if resp.status_code == 200: + break + err = resp.text[:400] + code = resp.status_code + resp.close() + resp = None + if use_usage and _bad_stream_options(err): + use_usage = False + if log: + log("WARN", "提供商不支持 stream_options=include_usage，已去掉参数重试") + continue + raise ProviderError("HTTP %s: %s" % (code, err)) + + for obj in _iter_json(resp): + if should_stop and should_stop(): + raise StopRequested() + if obj.get("choices"): + delta = obj["choices"][0].get("delta") or {} + text = delta.get("content") or "" + if text: + if first_token_at is None: + first_token_at = time.time() + output_chars += len(text) + usage = obj.get("usage") + if usage: + prompt_tokens = usage.get("prompt_tokens") or 0 + output_tokens = usage.get("completion_tokens") or 0 + details = usage.get("prompt_tokens_details") or {} + if isinstance(details, dict): + cached_tokens = details.get("cached_tokens") or 0 + except StopRequested: + raise + except Exception as e: + raise ProviderError("流式请求异常: %s" % e) + finally: + if resp is not None: + resp.close() + + if first_token_at is None: + raise ProviderError("未收到任何输出内容") + end = time.time() + return _metrics(start, first_token_at, end, prompt_tokens, output_tokens, + cached_tokens, output_chars, len(prompt)) + + +def _bad_stream_options(err: str): + err = (err or "").lower() + return ("stream_options" in err or "unknown parameter" in err or "unknown field" in err + or "additional properties" in err) + + +def _iter_json(resp): + """解析 SSE data: 行，逐个返回 JSON 对象""" + for raw in resp.iter_lines(decode_unicode=True): + obj = _parse_sse_line(raw) + if obj is not None: + yield obj + + +# ───────────────────────── Anthropic ───────────────────────── + +def stream_anthropic(cfg, prompt, gen, log, should_stop=None): + base = (cfg.get("base_url") or DEFAULT_URLS["anthropic"]).rstrip("/") + url = base + "/v1/messages" + headers = { + "x-api-key": cfg.get("api_key") or "", + "anthropic-version": "2023-06-01", + "Content-Type": "application/json", + } + payload = { + "model": cfg["model"], + "messages": [{"role": "user", "content": prompt}], + "max_tokens": int(gen.get("max_tokens", 256)), + "temperature": float(cfg.get("temperature", 0.7)), + "stream": True, + } + start = time.time() + first_token_at = None + output_chars = 0 + prompt_tokens = output_tokens = 0 + resp = None + try: + if should_stop and should_stop(): + raise StopRequested() + resp = requests.post(url, json=payload, headers=headers, stream=True, + timeout=(config.CONNECT_TIMEOUT, config.STREAM_READ_TIMEOUT)) + if resp.status_code != 200: + err = resp.text[:400] + resp.close() + resp = None + raise ProviderError("HTTP %s: %s" % (resp.status_code, err)) + for obj in _iter_json(resp): + if should_stop and should_stop(): + raise StopRequested() + etype = obj.get("type") + if etype == "message_start": + usage = (obj.get("message") or {}).get("usage") or {} + prompt_tokens = usage.get("input_tokens") or 0 + elif etype == "content_block_delta": + text = (obj.get("delta") or {}).get("text") or "" + if text: + if first_token_at is None: + first_token_at = time.time() + output_chars += len(text) + elif etype == "message_delta": + usage = obj.get("usage") or {} + output_tokens = usage.get("output_tokens") or output_tokens + except StopRequested: + raise + except Exception as e: + raise ProviderError("流式请求异常: %s" % e) + finally: + if resp is not None: + resp.close() + + if first_token_at is None: + raise ProviderError("未收到任何输出内容") + end = time.time() + return _metrics(start, first_token_at, end, prompt_tokens, output_tokens, + 0, output_chars, len(prompt)) + + +# ───────────────────────── Google Gemini ───────────────────────── + +def stream_google(cfg, prompt, gen, log, should_stop=None): + base = (cfg.get("base_url") or DEFAULT_URLS["google"]).rstrip("/") + model = cfg["model"] + url = "%s/v1beta/models/%s:streamGenerateContent" % (base, model) + params = {"alt": "sse", "key": cfg.get("api_key") or ""} + headers = {"Content-Type": "application/json"} + payload = { + "contents": [{"parts": [{"text": prompt}]}], + "generationConfig": { + "temperature": float(cfg.get("temperature", 0.7)), + "maxOutputTokens": int(gen.get("max_tokens", 256)), + "candidateCount": 1, + }, + } + start = time.time() + first_token_at = None + output_chars = 0 + prompt_tokens = output_tokens = cached_tokens = 0 + resp = None + try: + if should_stop and should_stop(): + raise StopRequested() + resp = requests.post(url, params=params, json=payload, headers=headers, stream=True, + timeout=(config.CONNECT_TIMEOUT, config.STREAM_READ_TIMEOUT)) + if resp.status_code != 200: + err = resp.text[:400] + resp.close() + resp = None + raise ProviderError("HTTP %s: %s" % (resp.status_code, err)) + for obj in _iter_json(resp): + if should_stop and should_stop(): + raise StopRequested() + cands = obj.get("candidates") or [] + if cands: + parts = (cands[0].get("content") or {}).get("parts") or [] + for part in parts: + text = part.get("text") or "" + if text: + if first_token_at is None: + first_token_at = time.time() + output_chars += len(text) + um = obj.get("usageMetadata") or {} + if um: + prompt_tokens = um.get("promptTokenCount") or 0 + output_tokens = um.get("candidatesTokenCount") or 0 + cached_tokens = um.get("cachedContentTokenCount") or 0 + except StopRequested: + raise + except Exception as e: + raise ProviderError("流式请求异常: %s" % e) + finally: + if resp is not None: + resp.close() + + if first_token_at is None: + raise ProviderError("未收到任何输出内容") + end = time.time() + return _metrics(start, first_token_at, end, prompt_tokens, output_tokens, + cached_tokens, output_chars, len(prompt)) + + +# ───────────────────────── 统一入口 ───────────────────────── + +def call_stream(cfg, prompt, gen, log=None, should_stop=None): + provider = cfg.get("provider", "openai") + fn = {"openai": stream_openai, "anthropic": stream_anthropic, "google": stream_google}.get(provider) + if fn is None: + raise ProviderError("不支持的提供商类型: %s" % provider) + return fn(cfg, prompt, gen, log, should_stop) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..a0d407c --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +flask>=3.0 +requests>=2.31 diff --git a/start.sh b/start.sh new file mode 100755 index 0000000..fc6e902 --- /dev/null +++ b/start.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# LLM 速度测试台 启动/停止脚本 +cd "$(dirname "$0")" +PY=/home/hz1/miniconda3/envs/openclaw/bin/python3 +PORT=16097 +PID_FILE=logs/app.pid +mkdir -p logs data + +start() { + if [ -f "$PID_FILE" ] && kill -0 "$(cat "$PID_FILE")" 2>/dev/null; then + echo "已在运行 (PID $(cat "$PID_FILE"))" + return + fi + nohup "$PY" app.py >> logs/app.log 2>&1 & + echo $! > "$PID_FILE" + sleep 1 + if kill -0 "$(cat "$PID_FILE")" 2>/dev/null; then + echo "启动成功 → http://:$PORT/" + else + echo "启动失败，请查看 logs/app.log" + fi +} + +stop() { + if [ -f "$PID_FILE" ]; then + kill "$(cat "$PID_FILE")" 2>/dev/null + rm -f "$PID_FILE" + echo "已停止" + else + echo "未在运行" + fi +} + +case "${1:-start}" in + start) start ;; + stop) stop ;; + restart) stop; sleep 1; start ;; + *) echo "用法: $0 [start|stop|restart]" ;; +esac diff --git a/static/css/style.css b/static/css/style.css new file mode 100644 index 0000000..0a60b52 --- /dev/null +++ b/static/css/style.css @@ -0,0 +1,194 @@ +:root { + --bg: #0f1420; + --panel: #171e2e; + --panel2: #1d2538; + --border: #2a3550; + --text: #e6ebf5; + --muted: #8b96ad; + --accent: #4f8cff; + --accent2: #22c58b; + --warn: #f5a623; + --danger: #ff5c6c; + --mono: "JetBrains Mono", "SF Mono", Consolas, Menlo, monospace; +} + +* { box-sizing: border-box; margin: 0; padding: 0; } + +body { + background: var(--bg); + color: var(--text); + font: 14px/1.55 -apple-system, "Segoe UI", "PingFang SC", "Microsoft YaHei", sans-serif; +} + +/* ── 顶栏 ── */ +.topbar { + display: flex; align-items: center; justify-content: space-between; + padding: 14px 24px; background: var(--panel); + border-bottom: 1px solid var(--border); position: sticky; top: 0; z-index: 10; +} +.brand { display: flex; align-items: baseline; gap: 12px; } +.logo { font-size: 22px; } +.brand h1 { font-size: 19px; font-weight: 700; } +.brand .sub { color: var(--muted); font-size: 12px; } + +.status-badge { + display: flex; align-items: center; gap: 8px; padding: 6px 14px; + border-radius: 20px; background: var(--panel2); border: 1px solid var(--border); font-size: 13px; +} +.status-badge .dot { width: 9px; height: 9px; border-radius: 50%; background: var(--muted); } +.status-badge.running .dot { background: var(--accent); animation: pulse 1s infinite; } +.status-badge.done .dot { background: var(--accent2); } +.status-badge.error .dot { background: var(--danger); } +@keyframes pulse { 50% { opacity: .3; } } + +/* ── 布局 ── */ +.layout { display: grid; grid-template-columns: 340px 1fr; gap: 16px; padding: 16px 24px; align-items: start; } +.config-panel { display: flex; flex-direction: column; gap: 16px; position: sticky; top: 72px; } +.main { display: flex; flex-direction: column; gap: 16px; min-width: 0; } + +.card { + background: var(--panel); border: 1px solid var(--border); + border-radius: 12px; padding: 16px; +} +.card h2 { font-size: 15px; margin-bottom: 14px; } +.card-head { display: flex; align-items: center; justify-content: space-between; margin-bottom: 14px; } +.card-head h2 { margin-bottom: 0; } + +/* ── 表单 ── */ +.field { margin-bottom: 12px; } +.field label { display: block; font-size: 12px; color: var(--muted); margin-bottom: 5px; } +.field input[type="text"], .field input[type="password"], .field input[type="number"], +.field select { + width: 100%; background: var(--panel2); color: var(--text); + border: 1px solid var(--border); border-radius: 8px; + padding: 8px 10px; font-size: 13px; outline: none; +} +.field input:focus, .field select:focus { border-color: var(--accent); } +.field input[type="range"] { width: 100%; accent-color: var(--accent); } +.hint { color: var(--muted); font-size: 11px; margin-top: 4px; } +.hint-inline { color: var(--muted); font-weight: 400; margin-left: 6px; } +.row { display: flex; gap: 6px; } +.row > select { flex: 1; } +.row > input { flex: 1; } + +/* 开关 */ +.switch-field { display: flex; align-items: center; justify-content: space-between; } +.switch { position: relative; display: inline-block; width: 46px; height: 26px; flex-shrink: 0; } +.switch input { opacity: 0; width: 0; height: 0; } +.slider { + position: absolute; inset: 0; cursor: pointer; border-radius: 26px; + background: #39415a; transition: .2s; +} +.slider::before { + content: ""; position: absolute; width: 20px; height: 20px; left: 3px; top: 3px; + border-radius: 50%; background: #fff; transition: .2s; +} +.switch input:checked + .slider { background: var(--accent); } +.switch input:checked + .slider::before { transform: translateX(20px); } + +/* ── 按钮 ── */ +.btn { + background: var(--panel2); color: var(--text); border: 1px solid var(--border); + border-radius: 8px; padding: 8px 14px; cursor: pointer; font-size: 13px; transition: .15s; +} +.btn:hover { border-color: var(--accent); } +.btn.primary { background: var(--accent); border-color: var(--accent); color: #fff; font-weight: 600; } +.btn.primary:hover { filter: brightness(1.1); } +.btn.danger { background: transparent; border-color: var(--danger); color: var(--danger); } +.btn.danger:hover { background: var(--danger); color: #fff; } +.btn.danger:disabled { opacity: .4; cursor: not-allowed; } +.btn.small { padding: 5px 10px; font-size: 12px; } +.btn.block { width: 100%; } +.btn-group { display: flex; gap: 8px; margin-top: 6px; } +.btn-group .btn { flex: 1; } + +/* ── 连接测试结果 ── */ +.conn-result { margin-top: 10px; padding: 8px 10px; border-radius: 8px; font-size: 12px; } +.conn-result.ok { background: rgba(34,197,139,.12); color: var(--accent2); border: 1px solid rgba(34,197,139,.3); } +.conn-result.fail { background: rgba(255,92,108,.1); color: var(--danger); border: 1px solid rgba(255,92,108,.3); } + +/* ── 指标卡 ── */ +.metrics { display: grid; grid-template-columns: repeat(6, 1fr); gap: 12px; } +.metric-card { + background: var(--panel); border: 1px solid var(--border); border-radius: 12px; + padding: 14px 12px; text-align: center; +} +.m-label { font-size: 11px; color: var(--muted); } +.m-value { font-size: 22px; font-weight: 700; margin: 4px 0 2px; font-family: var(--mono); } +.m-unit { font-size: 11px; color: var(--muted); } +.metric-card .m-value.accent { color: var(--accent); } +.metric-card .m-value.good { color: var(--accent2); } + +/* ── 日志终端 ── */ +.console { + background: #0a0e17; border: 1px solid var(--border); border-radius: 10px; + height: 340px; overflow-y: auto; padding: 12px 14px; + font: 12.5px/1.7 var(--mono); +} +.console .ln { white-space: pre-wrap; word-break: break-all; } +.console .ln .ts { color: #5b6b8c; } +.console .ln.INFO { color: #c9d4ea; } +.console .ln.METRIC { color: var(--accent2); } +.console .ln.WARN { color: var(--warn); } +.console .ln.ERROR { color: var(--danger); } +.console .ln.sys { color: #5b6b8c; font-style: italic; } + +/* ── 历史表 ── */ +.table-wrap { overflow-x: auto; } +table.history { width: 100%; border-collapse: collapse; font-size: 13px; } +.history th, .history td { padding: 8px 10px; text-align: left; border-bottom: 1px solid var(--border); white-space: nowrap; } +.history th { color: var(--muted); font-weight: 600; font-size: 12px; } +.history tbody tr:hover { background: var(--panel2); } +.history td.num { font-family: var(--mono); } +.status-pill { padding: 2px 10px; border-radius: 12px; font-size: 11px; } +.status-pill.running { background: rgba(79,140,255,.15); color: var(--accent); } +.status-pill.done { background: rgba(34,197,139,.15); color: var(--accent2); } +.status-pill.error { background: rgba(255,92,108,.15); color: var(--danger); } +.status-pill.canceled { background: rgba(245,166,35,.15); color: var(--warn); } + +/* ── 弹窗 ── */ +.modal-mask { + position: fixed; inset: 0; background: rgba(4,8,16,.7); z-index: 100; + display: flex; align-items: flex-start; justify-content: center; padding: 40px 20px; +} +.modal-mask[hidden] { display: none; } +.modal { + background: var(--panel); border: 1px solid var(--border); border-radius: 14px; + width: min(920px, 96vw); max-height: 86vh; display: flex; flex-direction: column; +} +.modal-head { + display: flex; align-items: center; justify-content: space-between; + padding: 14px 18px; border-bottom: 1px solid var(--border); +} +.modal-head h2 { font-size: 16px; } +.modal-body { padding: 16px 18px; overflow-y: auto; } +.modal-body h3 { font-size: 14px; margin: 16px 0 8px; color: var(--muted); } +.modal-body h3:first-child { margin-top: 0; } + +.kv { display: grid; grid-template-columns: repeat(auto-fill, minmax(180px, 1fr)); gap: 8px; } +.kv .kv-item { background: var(--panel2); border: 1px solid var(--border); border-radius: 8px; padding: 8px 10px; } +.kv .kv-k { font-size: 11px; color: var(--muted); } +.kv .kv-v { font-family: var(--mono); font-size: 13px; margin-top: 2px; } + +table.mini { width: 100%; border-collapse: collapse; font-size: 12.5px; } +.mini th, .mini td { padding: 6px 8px; border-bottom: 1px solid var(--border); text-align: left; } +.mini th { color: var(--muted); font-size: 11px; } +.mini td.num { font-family: var(--mono); } +.mini tr.err td { color: var(--danger); } + +.detail-log { background: #0a0e17; border: 1px solid var(--border); border-radius: 8px; padding: 10px 12px; font: 12px/1.7 var(--mono); max-height: 240px; overflow-y: auto; } +.detail-log .ln { white-space: pre-wrap; } +.detail-log .ln.METRIC { color: var(--accent2); } +.detail-log .ln.ERROR { color: var(--danger); } +.detail-log .ln.WARN { color: var(--warn); } + +pre.json-box { + background: #0a0e17; border: 1px solid var(--border); border-radius: 8px; + padding: 10px; font: 11.5px/1.6 var(--mono); overflow: auto; max-height: 260px; color: #a8f0d0; +} + +@media (max-width: 1000px) { + .layout { grid-template-columns: 1fr; } + .config-panel { position: static; } + .metrics { grid-template-columns: repeat(3, 1fr); } +} diff --git a/static/index.html b/static/index.html new file mode 100644 index 0000000..8b8967d --- /dev/null +++ b/static/index.html @@ -0,0 +1,154 @@ + + + + + + LLM 速度测试台 + + + +
+
+ +

LLM 速度测试台

+ 大模型推理性能基准 · 预填充 / 首字 / 解码 +
+
空闲
+
+ +
+ + + + +
+
+
首字延迟
—
ms
+
预填充速度
—
tok/s
+
解码速度
—
tok/s
+
上文 Tokens
—
tok
+
输出 Tokens
—
tok
+
总耗时
—
ms
+
+ +
+
+

📋 测试日志

+
+ + + +
+
+
+
+ +
+
+

🗂 测试历史

+ +
+
+ + + + + + +
#时间提供商模型采样首字 ms预填充 tok/s解码 tok/s状态操作
+
+
+
+
+ + + + + + + diff --git a/static/js/app.js b/static/js/app.js new file mode 100644 index 0000000..bd390c8 --- /dev/null +++ b/static/js/app.js @@ -0,0 +1,459 @@ +/* LLM 速度测试台 前端逻辑 */ +"use strict"; + +const $ = (s) => document.querySelector(s); +const $$ = (s) => Array.from(document.querySelectorAll(s)); + +const PROVIDER_DEFAULT_URL = { + openai: "https://api.openai.com/v1", + anthropic: "https://api.anthropic.com", + google: "https://generativelanguage.googleapis.com", +}; +const PROVIDER_LABEL = { + openai: "OpenAI 兼容", anthropic: "Anthropic", google: "Google Gemini", +}; +const STATUS_LABEL = { + running: "测试中", done: "完成", error: "出错", canceled: "已取消", +}; + +let currentTestId = null; // 正在跑的测试 id +let pollTimer = null; +let lastLogId = 0; +let consoleLogs = []; // 当前测试已加载日志 [{id,level,msg,rel}] + +const esc = (s) => String(s ?? "").replace(/[&<>"']/g, + (c) => ({ "&": "&", "<": "<", ">": ">", '"': """, "'": "'" }[c])); +const fmt = (v, d = "—") => (v === null || v === undefined || isNaN(v) ? d : v); +const pad = (n) => String(n).padStart(2, "0"); +const clock = (ts) => `${pad(ts.getHours())}:${pad(ts.getMinutes())}:${pad(ts.getSeconds())}`; + +/* ───────────────────────── 通用请求 ───────────────────────── */ + +async function api(path, method = "GET", body) { + const opt = { method, headers: {} }; + if (body !== undefined) { opt.headers["Content-Type"] = "application/json"; opt.body = JSON.stringify(body); } + const resp = await fetch(path, opt); + return resp.json(); +} + +/* ───────────────────────── 配置管理 ───────────────────────── */ + +function currentConfig() { + return { + name: $("#cfg-name").value.trim(), + provider: $("#cfg-provider").value, + base_url: $("#cfg-baseurl").value.trim(), + api_key: $("#cfg-apikey").value.trim(), + model: $("#cfg-model").value.trim(), + temperature: parseFloat($("#cfg-temp").value) || 0.7, + }; +} + +function currentGen() { + return { + prompt_tokens: parseInt($("#gen-prompt-tokens").value) || 2048, + max_tokens: parseInt($("#gen-max-tokens").value) || 256, + samples: parseInt($("#gen-samples").value) || 3, + avoid_cache: $("#gen-avoid-cache").checked, + }; +} + +function updateDefaultUrlHint() { + const p = $("#cfg-provider").value; + $("#cfg-default-url").textContent = "默认地址：" + PROVIDER_DEFAULT_URL[p]; + if (!$("#cfg-baseurl").value.trim()) { + $("#cfg-baseurl").placeholder = "留空使用默认：" + PROVIDER_DEFAULT_URL[p]; + } +} + +async function loadConfigs() { + const list = await api("/api/configs"); + const sel = $("#cfg-list"); + const keep = sel.value; + sel.innerHTML = ''; + for (const c of list) { + const o = document.createElement("option"); + o.value = c.id; + o.textContent = `${c.name}（${PROVIDER_LABEL[c.provider] || c.provider} / ${c.model}${c.has_key ? "" : " · 无Key"}）`; + sel.appendChild(o); + } + if (keep && [...sel.options].some((o) => o.value === keep)) sel.value = keep; +} + +function loadSelectedConfig() { + const id = $("#cfg-list").value; + if (!id) return; + api(`/api/configs/${id}`).then((c) => { + $("#cfg-name").value = c.name || ""; + $("#cfg-provider").value = c.provider || "openai"; + $("#cfg-baseurl").value = c.base_url || ""; + $("#cfg-apikey").value = c.api_key || ""; + $("#cfg-model").value = c.model || ""; + $("#cfg-temp").value = c.temperature ?? 0.7; + $("#tmp-val").textContent = (c.temperature ?? 0.7).toFixed(1); + updateDefaultUrlHint(); + flash("已加载配置：" + c.name); + }).catch(() => toast("加载失败")); +} + +/* ───────────────────────── 连接测试 ───────────────────────── */ + +async function testConnection() { + const cfg = currentConfig(); + if (!cfg.api_key) { showConn(false, "请先填写 API Key"); return; } + const box = $("#conn-result"); + box.hidden = false; box.className = "conn-result"; + box.textContent = "⏳ 正在测试连接..."; + try { + const r = await api("/api/configs/test", "POST", cfg); + if (r.ok) { + const m = r.metrics || {}; + showConn(true, + `✅ 连接成功（${r.total_ms}ms）| 首字 ${fmt(m.ttft_ms)}ms | 提示词 ${fmt(m.prompt_tokens)} tok | 输出 ${fmt(m.output_tokens)} tok`); + } else { + showConn(false, "❌ " + (r.error || "连接失败")); + } + } catch (e) { + showConn(false, "❌ " + e.message); + } +} +function showConn(ok, text) { + const box = $("#conn-result"); + box.hidden = false; + box.className = "conn-result " + (ok ? "ok" : "fail"); + box.textContent = text; +} + +/* ───────────────────────── 测试运行 ───────────────────────── */ + +function setStatus(state, label) { + const badge = $("#run-status"); + badge.className = "status-badge " + state; + badge.innerHTML = `${label}`; +} + +function clearConsole() { + consoleLogs = []; + $("#console").innerHTML = ""; + $("#log-count").textContent = ""; + resetMetrics(); +} + +function appendLogs(logs) { + const box = $("#console"); + const atBottom = box.scrollHeight - box.scrollTop - box.clientHeight < 40; + let html = ""; + for (const l of logs) { + html += `
[${l.rel.toFixed(3)}s] ${esc(l.msg)}
`; + } + box.insertAdjacentHTML("beforeend", html); + $("#log-count").textContent = `${consoleLogs.length} 条`; + if (atBottom) box.scrollTop = box.scrollHeight; +} + +function resetMetrics() { + ["m-ttft", "m-prefill", "m-decode", "m-prompts", "m-outputs", "m-total"].forEach((id) => { + const el = $(`#${id}`); + el.textContent = "—"; + el.className = "m-value"; + }); +} + +function updateMetrics(m, isSummary = false) { + if (!m) return; + const val = (k, ak) => (m[k] !== null && m[k] !== undefined && !isNaN(m[k])) ? m[k] : m[ak]; + const set = (id, v, accent = false, good = false) => { + const el = $(`#${id}`); + el.textContent = fmt(v); + el.className = "m-value" + (accent ? " accent" : "") + (good ? " good" : ""); + }; + set("m-ttft", val("ttft_ms", "avg_ttft_ms"), false, true); + set("m-prefill", val("prefill_speed", "avg_prefill_speed"), true, false); + set("m-decode", val("decode_speed", "avg_decode_speed"), false, true); + set("m-prompts", val("prompt_tokens", "avg_prompt_tokens")); + set("m-outputs", val("output_tokens", "avg_output_tokens")); + set("m-total", val("total_ms", "avg_total_ms")); + const u = isSummary ? "均 ms" : "ms"; + $("#m-ttft-u").textContent = u; +} + +function startTest() { + const cfg = currentConfig(); + const gen = currentGen(); + if (!cfg.api_key) { toast("请填写 API Key"); return; } + if (!cfg.model) { toast("请填写模型名称"); return; } + + clearConsole(); + currentTestId = null; + lastLogId = 0; + setStatus("running", "正在测试…"); + $("#btn-start").disabled = true; + $("#btn-cancel").disabled = false; + + api("/api/tests", "POST", { config: cfg, gen }).then((r) => { + if (!r.ok) { + throw new Error(r.error || "启动失败"); + } + currentTestId = r.id; + appendLogs([{ id: 0, level: "sys", msg: `测试 #${r.id} 已启动（${cfg.name || cfg.model}）`, rel: 0 }]); + pollTimer = setInterval(pollLogs, 800); + pollLogs(); + }).catch((e) => { + setStatus("error", "启动失败"); + toast(e.message); + $("#btn-start").disabled = false; + $("#btn-cancel").disabled = true; + }); +} + +async function pollLogs() { + if (currentTestId == null) return; + try { + const d = await api(`/api/tests/${currentTestId}/logs?after=${lastLogId}`); + if (d.logs && d.logs.length) { + appendLogs(d.logs); + lastLogId = d.after || lastLogId; + } + if (d.last_run) updateMetrics(d.last_run); + if (d.status === "done" || d.status === "error" || d.status === "canceled") { + finishTest(d); + } + } catch (e) { + /* 网络抖动忽略 */ + } +} + +function finishTest(d) { + clearInterval(pollTimer); + pollTimer = null; + $("#btn-start").disabled = false; + $("#btn-cancel").disabled = true; + if (d.summary && d.summary.samples_ok > 0) { + updateMetrics(d.summary, true); + } + const map = { done: ["done", "✅ 测试完成"], error: ["error", "⚠️ 测试出错"], canceled: ["canceled", "⏹ 已取消"] }; + const [state, label] = map[d.status] || ["done", "完成"]; + setStatus(state, label); + if (d.status === "error" && d.error) toast("测试出错：" + d.error); + loadHistory(); +} + +function stopTest() { + if (currentTestId == null) return; + api(`/api/tests/${currentTestId}/cancel`, "POST").then(() => toast("正在停止…")); + $("#btn-cancel").disabled = true; +} + +/* ───────────────────────── 测试历史 ───────────────────────── */ + +async function loadHistory() { + const list = await api("/api/tests"); + const tb = $("#history tbody"); + tb.innerHTML = ""; + if (!list.length) { + tb.innerHTML = '暂无测试记录'; + return; + } + for (const t of list) { + const s = t.summary || {}; + const tr = document.createElement("tr"); + tr.innerHTML = ` + #${t.id} + ${esc(t.created_at)} + ${esc(PROVIDER_LABEL[t.provider] || t.provider)} + ${esc(t.model)} + ${fmt(s.samples_ok)}/${fmt(s.samples_total)} + ${fmt(s.avg_ttft_ms)} + ${fmt(s.avg_prefill_speed)} + ${fmt(s.avg_decode_speed)} + ${STATUS_LABEL[t.status] || t.status} + + + + `; + tb.appendChild(tr); + } +} + +/* ───────────────────────── 详情弹窗 ───────────────────────── */ + +async function viewDetail(id) { + const t = await api(`/api/tests/${id}`); + const s = t.summary || {}; + const g = t.gen || {}; + const cfg = t.config || {}; + const runs = t.runs || []; + const logs = t.logs || []; + + let runsHtml; + if (runs.length) { + runsHtml = ` + + + ` + + runs.map((r, i) => { + const m = r.metrics || {}; + return ` + + + + + + + + + + `; + }).join("") + `
采样提示词tok缓存tok首字ms预填充tok/s输出tok解码tok/s总耗时ms备注
${i + 1}${fmt(m.prompt_tokens)}${fmt(m.cached_tokens, 0)}${fmt(m.ttft_ms)}${fmt(m.prefill_speed)}${fmt(m.output_tokens)}${fmt(m.decode_speed)}${fmt(m.total_ms)}${r.error ? "❌ " + esc(r.error.slice(0, 60)) : "✅"}
`; + } else { + runsHtml = '
无采样数据
'; + } + + const logHtml = `
` + logs.map((l) => + `
[${l.rel.toFixed(3)}s] ${esc(l.msg)}
` + ).join("") + `
`; + + $("#dt-id").textContent = id; + $("#dt-body").innerHTML = ` +

📌 汇总指标

+
+
状态
${STATUS_LABEL[t.status] || t.status}
+
创建时间
${esc(t.created_at)}
+
提供商 / 模型
${esc(PROVIDER_LABEL[t.provider] || t.provider)} / ${esc(t.model)}
+
采样（成功/总数）
${fmt(s.samples_ok)} / ${fmt(s.samples_total)}
+
平均首字延迟
${fmt(s.avg_ttft_ms)} ms
+
最佳首字延迟
${fmt(s.best_ttft_ms)} ms
+
平均预填充速度
${fmt(s.avg_prefill_speed)} tok/s
+
平均解码速度
${fmt(s.avg_decode_speed)} tok/s
+
平均提示词
${fmt(s.avg_prompt_tokens)} tok
+
平均输出
${fmt(s.avg_output_tokens)} tok
+
平均缓存命中
${fmt(s.avg_cached_tokens, 0)} tok
+
平均总耗时
${fmt(s.avg_total_ms)} ms
+
+ +

⚙️ 测试参数

+
+
上文长度
${g.prompt_tokens ?? "—"} tok
+
生成长度
${g.max_tokens ?? "—"} tok
+
采样次数
${g.samples ?? "—"}
+
避免缓存
${g.avoid_cache ? "开" : "关"}
+
温度
${fmt(cfg.temperature)}
+
Base URL
${esc(cfg.base_url || "（默认）")}
+
校准字符/token
${fmt(s.calibration_chars_per_token)}
+
+ +

📊 每次采样明细

+ ${runsHtml} + +

📋 完整日志

+ ${logHtml} + ${t.error ? `
错误信息：${esc(t.error)}
` : ""} + `; + + $("#detail-mask").hidden = false; + $("#dt-body").scrollTop = 0; + window.__detail = t; +} + +function closeDetail() { $("#detail-mask").hidden = true; } + +function exportDetail() { + if (!window.__detail) return; + download(JSON.stringify(window.__detail, null, 2), `test_${window.__detail.id}.json`, "application/json"); +} + +/* ───────────────────────── 导出 / 下载 ───────────────────────── */ + +function download(text, filename, type = "text/plain") { + const blob = new Blob([text], { type }); + const a = document.createElement("a"); + a.href = URL.createObjectURL(blob); + a.download = filename; + document.body.appendChild(a); + a.click(); + setTimeout(() => { URL.revokeObjectURL(a.href); a.remove(); }, 100); +} + +function exportCurrentLog() { + if (!currentTestId) { toast("当前没有进行中的测试"); return; } + api(`/api/tests/${currentTestId}`).then((t) => { + const lines = t.logs.map((l) => `[${l.rel.toFixed(3)}s] [${l.level}] ${l.msg}`).join("\n"); + const head = `# LLM 速度测试日志 test#${t.id} ${t.created_at}\n# 状态: ${t.status} 提供商: ${t.provider} 模型: ${t.model}\n`; + download(head + lines + "\n", `llm_test_${t.id}.log`, "text/plain"); + }); +} + +/* ───────────────────────── 提示 ───────────────────────── */ + +function toast(msg) { + const el = document.createElement("div"); + el.textContent = msg; + el.style.cssText = "position:fixed;bottom:24px;left:50%;transform:translateX(-50%);background:#1d2538;border:1px solid var(--accent);color:#fff;padding:10px 18px;border-radius:10px;z-index:999;font-size:13px;box-shadow:0 6px 20px rgba(0,0,0,.4)"; + document.body.appendChild(el); + setTimeout(() => el.remove(), 3200); +} + +function flash(msg) { + toast(msg); +} + +/* ───────────────────────── 事件绑定 ───────────────────────── */ + +function bind() { + $("#cfg-provider").addEventListener("change", updateDefaultUrlHint); + $("#cfg-temp").addEventListener("input", () => $("#tmp-val").textContent = parseFloat($("#cfg-temp").value).toFixed(1)); + + $("#btn-toggle-key").addEventListener("click", () => { + const inp = $("#cfg-apikey"); + inp.type = inp.type === "password" ? "text" : "password"; + }); + $("#btn-save-config").addEventListener("click", async () => { + const r = await api("/api/configs", "POST", currentConfig()); + if (r.ok) { toast("配置已保存"); loadConfigs(); } + else toast(r.error || "保存失败"); + }); + $("#btn-load-config").addEventListener("click", loadSelectedConfig); + $("#btn-del-config").addEventListener("click", async () => { + const id = $("#cfg-list").value; + if (!id) { toast("请先选择要删除的配置"); return; } + if (!confirm("确定删除该配置？")) return; + await api(`/api/configs/${id}`, "DELETE"); + toast("已删除"); loadConfigs(); + }); + + $("#btn-test-conn").addEventListener("click", testConnection); + $("#btn-start").addEventListener("click", startTest); + $("#btn-cancel").addEventListener("click", stopTest); + + $("#btn-clear-console").addEventListener("click", clearConsole); + $("#btn-export-log").addEventListener("click", exportCurrentLog); + $("#btn-refresh-history").addEventListener("click", loadHistory); + + $("#history tbody").addEventListener("click", (e) => { + const v = e.target.closest("[data-view]"); + const d = e.target.closest("[data-del]"); + if (v) viewDetail(Number(v.dataset.view)); + if (d) { + const id = Number(d.dataset.del); + if (confirm(`确定删除测试 #${id} 及其全部日志？`)) { + api(`/api/tests/${id}`, "DELETE").then(() => loadHistory()); + } + } + }); + + $("#dt-close").addEventListener("click", closeDetail); + $("#detail-mask").addEventListener("click", (e) => { if (e.target === $("#detail-mask")) closeDetail(); }); + $("#dt-export").addEventListener("click", exportDetail); + + document.addEventListener("keydown", (e) => { if (e.key === "Escape") closeDetail(); }); +} + +/* ───────────────────────── 初始化 ───────────────────────── */ + +(async function init() { + bind(); + updateDefaultUrlHint(); + loadConfigs(); + loadHistory(); + setInterval(() => { if (!currentTestId) loadHistory(); }, 30000); // 空闲时定期刷新历史 +})(); diff --git a/tester.py b/tester.py new file mode 100644 index 0000000..ba5247a --- /dev/null +++ b/tester.py @@ -0,0 +1,167 @@ +# -*- coding: utf-8 -*- +"""速度测试执行器：校准 -> 采样 -> 汇总，全程写日志与指标入库""" +import json +import statistics +import threading +import time +import uuid + +import database as db +import llm_providers as lp +from llm_providers import ProviderError, StopRequested + + +class TestRunner(threading.Thread): + def __init__(self, test_id, cfg, gen): + super().__init__(daemon=True) + self.test_id = test_id + self.cfg = cfg + self.gen = gen + self.cancel_flag = False + self.start_wall = time.time() + self.ratio = None + self.samples = [] + + def request_cancel(self): + self.cancel_flag = True + + def should_stop(self): + return self.cancel_flag + + def log(self, level, msg): + db.add_log(self.test_id, level, msg) + + # ───────────────────────── 主流程 ───────────────────────── + + def run(self): + try: + self._run() + except StopRequested: + self.log("WARN", "用户请求停止测试") + db.update_status(self.test_id, "canceled", + summary=self._make_summary(), error="用户取消") + except Exception as e: + self.log("ERROR", "测试异常终止: %s" % e) + db.update_status(self.test_id, "error", + summary=self._make_summary(), error=str(e)) + + def _run(self): + provider = self.cfg.get("provider", "openai") + model = self.cfg.get("model", "") + n = max(1, int(self.gen.get("samples", 3))) + target_tokens = max(16, int(self.gen.get("prompt_tokens", 2048))) + max_tokens = max(1, int(self.gen.get("max_tokens", 256))) + avoid_cache = bool(self.gen.get("avoid_cache")) + + self.log("INFO", "═══ 开始速度测试 ═══") + self.log("INFO", "提供商: %s | 模型: %s" % (lp.PROVIDER_LABELS.get(provider, provider), model)) + self.log("INFO", "目标上文: %d tokens | 生成长度: %d tokens | 采样: %d 次 | 避免缓存: %s" + % (target_tokens, max_tokens, n, "开" if avoid_cache else "关")) + + ratio = self._calibrate() + self.ratio = ratio + base_prompt = self._build_prompt(target_tokens, ratio) + self.log("INFO", "构造基准提示词完成，目标约 %d tokens" % target_tokens) + + for i in range(1, n + 1): + if self.should_stop(): + raise StopRequested() + prompt = self._finalize_prompt(base_prompt) + self.log("INFO", "── 采样 %d/%d 开始 ──" % (i, n)) + try: + m = lp.call_stream( + self.cfg, prompt, + {"max_tokens": max_tokens, "avoid_cache": avoid_cache}, + log=lambda lv, msg: self.log(lv, msg), + should_stop=self.should_stop) + m["run_index"] = i + self.samples.append({"run_index": i, "ok": True, "metrics": m}) + db.add_run(self.test_id, i, m) + self.log("METRIC", self._fmt_metric(i, n, m)) + except StopRequested: + raise + except ProviderError as e: + self.log("ERROR", "采样 %d/%d 失败: %s" % (i, n, e)) + self.samples.append({"run_index": i, "ok": False, "error": str(e)}) + db.add_run(self.test_id, i, {}, str(e)) + raise e + + summary = self._make_summary() + db.update_status(self.test_id, "done", summary=summary) + self.log("INFO", "═══ 测试完成 ═══") + self.log("INFO", "汇总: 平均首字 %.1f ms | 平均预填充 %.1f tok/s | 平均解码 %.1f tok/s" + % (summary.get("avg_ttft_ms") or 0, + summary.get("avg_prefill_speed") or 0, + summary.get("avg_decode_speed") or 0)) + + # ───────────────────────── 工具方法 ───────────────────────── + + def _calibrate(self): + probe = ("The quick brown fox jumps over the lazy dog. 人工智能大模型推理速度基准语料，" + "用于测量提示词预填充与流式解码性能。\n") * 40 + self.log("INFO", "正在校准 token/字符 比例（发送小探测请求）...") + try: + m = lp.call_stream(self.cfg, probe, + {"max_tokens": 8, "avoid_cache": False}, + log=lambda lv, msg: self.log(lv, msg), + should_stop=self.should_stop) + pt = m.get("prompt_tokens") or 0 + if pt and len(probe): + ratio = pt / len(probe) + self.log("INFO", "探测提示词 %d tokens / %d 字符 = %.3f tok/字符" + % (pt, len(probe), ratio)) + return max(ratio, 0.001) + except StopRequested: + raise + except Exception as e: + self.log("WARN", "校准失败（%s），使用默认估算 0.55 tok/字符" % e) + return 0.55 + + def _build_prompt(self, target_tokens, ratio): + seg = ("基准语料：The quick brown fox jumps over the lazy dog. " + "人工智能大模型推理性能测试文本，用于测量提示词预填充速度、首字延迟与流式解码吞吐。\n") + target_chars = max(64, int(target_tokens / ratio)) + repeats = max(1, target_chars // len(seg)) + return seg * repeats + + def _finalize_prompt(self, base): + if self.gen.get("avoid_cache"): + return "[cache-bust %s]\n%s" % (uuid.uuid4().hex, base) + return base + + def _fmt_metric(self, i, n, m): + return ("采样 %d/%d 完成 | 提示词 %d tok | 缓存 %d tok | 首字 %s ms | 预填充 %s tok/s" + " | 输出 %d tok | 解码 %s tok/s | 总耗时 %s ms" + % (i, n, m.get("prompt_tokens") or 0, m.get("cached_tokens") or 0, + m.get("ttft_ms"), m.get("prefill_speed"), m.get("output_tokens") or 0, + m.get("decode_speed"), m.get("total_ms"))) + + def _make_summary(self): + ok = [s["metrics"] for s in self.samples if s.get("ok")] + base = { + "provider": self.cfg.get("provider"), + "model": self.cfg.get("model"), + "gen": self.gen, + "samples_total": len(self.samples), + "samples_ok": len(ok), + "calibration_chars_per_token": round(1 / self.ratio, 2) if self.ratio else None, + } + if not ok: + return base + + def avg(k): + vals = [m[k] for m in ok if m.get(k) is not None] + return round(statistics.mean(vals), 1) if vals else None + + summary = dict(base) + summary.update({ + "avg_ttft_ms": avg("ttft_ms"), + "avg_prefill_speed": avg("prefill_speed"), + "avg_decode_speed": avg("decode_speed"), + "avg_prompt_tokens": avg("prompt_tokens"), + "avg_output_tokens": avg("output_tokens"), + "avg_cached_tokens": avg("cached_tokens"), + "avg_total_ms": avg("total_ms"), + "best_ttft_ms": min([m["ttft_ms"] for m in ok if m.get("ttft_ms") is not None], default=None), + }) + return summary