hz4th_coder
77fa757959
1. 版本管理增加执行程序(binaries)管理: 每个版本可管理llama-server/llama-cli/llama-bench等程序 - 参数管理中增加执行程序选择器, 可按程序筛选参数 - 参数可绑定到特定执行程序或适用所有程序 2. 模型管理重构: 一个模型统一参数(区分dense/MoE), 量化版本作为子表管理 - models表增加model_type(dense/moe)和num_experts字段 - 新增model_quants表, 每个模型可有多个量化版本 - 前端模型选择改为: 选模型 -> 选量化版本 3. 修复系统设置中默认GPU下拉为空: 异步加载GPU后再渲染设置
528 lines
34 KiB
Python
528 lines
34 KiB
Python
#!/usr/bin/env python3
|
|
"""Database management for llama.cpp command generator."""
|
|
|
|
import sqlite3
|
|
import json
|
|
import os
|
|
|
|
DB_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data.db')
|
|
|
|
|
|
def get_db():
|
|
conn = sqlite3.connect(DB_PATH)
|
|
conn.row_factory = sqlite3.Row
|
|
conn.execute("PRAGMA foreign_keys = ON")
|
|
return conn
|
|
|
|
|
|
def init_db():
|
|
conn = get_db()
|
|
c = conn.cursor()
|
|
|
|
# GPU table
|
|
c.execute('''
|
|
CREATE TABLE IF NOT EXISTS gpus (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
name TEXT NOT NULL UNIQUE,
|
|
vram_mb INTEGER NOT NULL,
|
|
compute_capability TEXT,
|
|
description TEXT,
|
|
sort_order INTEGER DEFAULT 0
|
|
)
|
|
''')
|
|
|
|
# llama.cpp versions table
|
|
c.execute('''
|
|
CREATE TABLE IF NOT EXISTS llama_versions (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
version_tag TEXT NOT NULL UNIQUE,
|
|
description TEXT,
|
|
release_date TEXT,
|
|
is_active INTEGER DEFAULT 1,
|
|
sort_order INTEGER DEFAULT 0
|
|
)
|
|
''')
|
|
|
|
# Version binaries table (llama-server, llama-cli, etc.)
|
|
c.execute('''
|
|
CREATE TABLE IF NOT EXISTS version_binaries (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
version_id INTEGER NOT NULL,
|
|
name TEXT NOT NULL,
|
|
description TEXT,
|
|
sort_order INTEGER DEFAULT 0,
|
|
FOREIGN KEY (version_id) REFERENCES llama_versions(id) ON DELETE CASCADE,
|
|
UNIQUE(version_id, name)
|
|
)
|
|
''')
|
|
|
|
# Parameters table (per version, optionally per binary)
|
|
# binary_id = NULL means the param applies to all binaries
|
|
c.execute('''
|
|
CREATE TABLE IF NOT EXISTS params (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
version_id INTEGER NOT NULL,
|
|
binary_id INTEGER DEFAULT NULL,
|
|
param_key TEXT NOT NULL,
|
|
short_flag TEXT,
|
|
long_flag TEXT NOT NULL,
|
|
description TEXT,
|
|
category TEXT NOT NULL DEFAULT 'common',
|
|
param_type TEXT NOT NULL DEFAULT 'string',
|
|
default_value TEXT,
|
|
options TEXT,
|
|
min_value REAL,
|
|
max_value REAL,
|
|
step REAL,
|
|
unit TEXT,
|
|
is_important INTEGER DEFAULT 0,
|
|
affects_vram INTEGER DEFAULT 0,
|
|
sort_order INTEGER DEFAULT 0,
|
|
FOREIGN KEY (version_id) REFERENCES llama_versions(id) ON DELETE CASCADE
|
|
)
|
|
''')
|
|
|
|
# Settings table (key-value for app config)
|
|
c.execute('''
|
|
CREATE TABLE IF NOT EXISTS settings (
|
|
key TEXT PRIMARY KEY,
|
|
value TEXT
|
|
)
|
|
''')
|
|
|
|
# Models table (base model info, one row per base model)
|
|
c.execute('''
|
|
CREATE TABLE IF NOT EXISTS models (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
base_model TEXT NOT NULL,
|
|
name TEXT NOT NULL,
|
|
model_type TEXT DEFAULT 'dense',
|
|
num_experts INTEGER DEFAULT 0,
|
|
size_gb REAL DEFAULT 0,
|
|
quant TEXT DEFAULT '',
|
|
layers INTEGER NOT NULL,
|
|
embd INTEGER NOT NULL,
|
|
kv_heads INTEGER NOT NULL,
|
|
head_dim INTEGER NOT NULL,
|
|
attention_heads INTEGER NOT NULL,
|
|
default_ctx INTEGER DEFAULT 0,
|
|
description TEXT DEFAULT '',
|
|
sort_order INTEGER DEFAULT 0
|
|
)
|
|
''')
|
|
|
|
# Model quantization variants table
|
|
c.execute('''
|
|
CREATE TABLE IF NOT EXISTS model_quants (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
model_id INTEGER NOT NULL,
|
|
quant_type TEXT NOT NULL,
|
|
size_gb REAL NOT NULL,
|
|
sort_order INTEGER DEFAULT 0,
|
|
FOREIGN KEY (model_id) REFERENCES models(id) ON DELETE CASCADE
|
|
)
|
|
''')
|
|
|
|
conn.commit()
|
|
|
|
# Run migrations for existing databases
|
|
migrate_db(conn)
|
|
|
|
# Insert default data if empty
|
|
insert_default_data(conn)
|
|
|
|
conn.close()
|
|
|
|
|
|
def migrate_db(conn):
|
|
"""Migrate existing database to new schema."""
|
|
c = conn.cursor()
|
|
|
|
# 1. Migrate params table: add binary_id column
|
|
try:
|
|
c.execute("SELECT binary_id FROM params LIMIT 1")
|
|
except sqlite3.OperationalError:
|
|
# Recreate params table with binary_id
|
|
c.execute("ALTER TABLE params RENAME TO params_old")
|
|
c.execute('''
|
|
CREATE TABLE params (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
version_id INTEGER NOT NULL,
|
|
binary_id INTEGER DEFAULT NULL,
|
|
param_key TEXT NOT NULL,
|
|
short_flag TEXT,
|
|
long_flag TEXT NOT NULL,
|
|
description TEXT,
|
|
category TEXT NOT NULL DEFAULT 'common',
|
|
param_type TEXT NOT NULL DEFAULT 'string',
|
|
default_value TEXT,
|
|
options TEXT,
|
|
min_value REAL,
|
|
max_value REAL,
|
|
step REAL,
|
|
unit TEXT,
|
|
is_important INTEGER DEFAULT 0,
|
|
affects_vram INTEGER DEFAULT 0,
|
|
sort_order INTEGER DEFAULT 0,
|
|
FOREIGN KEY (version_id) REFERENCES llama_versions(id) ON DELETE CASCADE
|
|
)
|
|
''')
|
|
c.execute('''INSERT INTO params (id, version_id, binary_id, param_key, short_flag, long_flag,
|
|
description, category, param_type, default_value, options, min_value, max_value,
|
|
step, unit, is_important, affects_vram, sort_order)
|
|
SELECT id, version_id, NULL, param_key, short_flag, long_flag,
|
|
description, category, param_type, default_value, options, min_value, max_value,
|
|
step, unit, is_important, affects_vram, sort_order FROM params_old''')
|
|
c.execute("DROP TABLE params_old")
|
|
|
|
# 2. Add model_type to models table
|
|
try:
|
|
c.execute("SELECT model_type FROM models LIMIT 1")
|
|
except sqlite3.OperationalError:
|
|
c.execute("ALTER TABLE models ADD COLUMN model_type TEXT DEFAULT 'dense'")
|
|
|
|
# 3. Add num_experts to models table
|
|
try:
|
|
c.execute("SELECT num_experts FROM models LIMIT 1")
|
|
except sqlite3.OperationalError:
|
|
c.execute("ALTER TABLE models ADD COLUMN num_experts INTEGER DEFAULT 0")
|
|
|
|
conn.commit()
|
|
|
|
# 4. Insert default binaries for existing versions
|
|
c.execute("SELECT COUNT(*) as cnt FROM version_binaries")
|
|
if c.fetchone()['cnt'] == 0:
|
|
c.execute("SELECT id FROM llama_versions")
|
|
for v in c.fetchall():
|
|
vid = v['id']
|
|
c.execute("INSERT OR IGNORE INTO version_binaries (version_id, name, description, sort_order) VALUES (?, ?, ?, ?)",
|
|
(vid, 'llama-server', 'HTTP API 服务器', 1))
|
|
c.execute("INSERT OR IGNORE INTO version_binaries (version_id, name, description, sort_order) VALUES (?, ?, ?, ?)",
|
|
(vid, 'llama-cli', '命令行交互', 2))
|
|
c.execute("INSERT OR IGNORE INTO version_binaries (version_id, name, description, sort_order) VALUES (?, ?, ?, ?)",
|
|
(vid, 'llama-bench', '性能基准测试', 3))
|
|
conn.commit()
|
|
|
|
# 5. Migrate existing model data to model_quants table
|
|
c.execute("SELECT COUNT(*) as cnt FROM model_quants")
|
|
if c.fetchone()['cnt'] == 0:
|
|
c.execute("PRAGMA table_info(models)")
|
|
columns = [col['name'] for col in c.fetchall()]
|
|
|
|
if 'quant' in columns:
|
|
c.execute("SELECT DISTINCT base_model FROM models")
|
|
base_models = [r['base_model'] for r in c.fetchall()]
|
|
|
|
moe_models = {
|
|
'Mixtral-8x7B-Instruct': 8,
|
|
'DeepSeek-V2-Chat': 160,
|
|
'DeepSeek-V2.5-Chat': 160,
|
|
}
|
|
|
|
for base in base_models:
|
|
c.execute("SELECT * FROM models WHERE base_model = ? ORDER BY sort_order", (base,))
|
|
variants = c.fetchall()
|
|
|
|
if not variants:
|
|
continue
|
|
|
|
first = variants[0]
|
|
model_id = first['id']
|
|
|
|
# Insert all quant variants into model_quants
|
|
for v in variants:
|
|
quant = v['quant'] if v['quant'] else 'FP16'
|
|
c.execute(
|
|
"INSERT INTO model_quants (model_id, quant_type, size_gb, sort_order) VALUES (?, ?, ?, ?)",
|
|
(model_id, quant, v['size_gb'], v['sort_order'])
|
|
)
|
|
|
|
# Set model_type and num_experts
|
|
model_type = 'dense'
|
|
num_experts = 0
|
|
if base in moe_models:
|
|
model_type = 'moe'
|
|
num_experts = moe_models[base]
|
|
|
|
c.execute("UPDATE models SET model_type = ?, num_experts = ?, name = ?, size_gb = 0, quant = '' WHERE id = ?",
|
|
(model_type, num_experts, base, model_id))
|
|
|
|
# Delete duplicate variants
|
|
for v in variants[1:]:
|
|
c.execute("DELETE FROM models WHERE id = ?", (v['id'],))
|
|
|
|
conn.commit()
|
|
|
|
|
|
def insert_default_data(conn):
|
|
c = conn.cursor()
|
|
|
|
# Check if data already exists
|
|
c.execute("SELECT COUNT(*) as cnt FROM gpus")
|
|
if c.fetchone()['cnt'] > 0:
|
|
return
|
|
|
|
# ===== Default GPUs =====
|
|
default_gpus = [
|
|
("RTX 3090", 24576, "8.6", "NVIDIA GeForce RTX 3090, 24GB VRAM", 1),
|
|
("RTX 4090", 24576, "8.9", "NVIDIA GeForce RTX 4090, 24GB VRAM", 2),
|
|
("RTX 4080", 16384, "8.9", "NVIDIA GeForce RTX 4080, 16GB VRAM", 3),
|
|
("RTX 3080", 10240, "8.6", "NVIDIA GeForce RTX 3080, 10GB VRAM", 4),
|
|
("RTX 3090 Ti", 24576, "8.6", "NVIDIA GeForce RTX 3090 Ti, 24GB VRAM", 5),
|
|
("RTX 5090", 32768, "12.0", "NVIDIA GeForce RTX 5090, 32GB VRAM", 6),
|
|
("RTX 4070 Ti", 12288, "8.9", "NVIDIA GeForce RTX 4070 Ti, 12GB VRAM", 7),
|
|
("RTX 4060", 8192, "8.9", "NVIDIA GeForce RTX 4060, 8GB VRAM", 8),
|
|
("A100 80GB", 81920, "8.0", "NVIDIA A100 80GB", 9),
|
|
("A100 40GB", 40960, "8.0", "NVIDIA A100 40GB", 10),
|
|
("H100 80GB", 81920, "9.0", "NVIDIA H100 80GB", 11),
|
|
("V100 32GB", 32768, "7.0", "NVIDIA V100 32GB", 12),
|
|
("RTX A6000", 49152, "8.6", "NVIDIA RTX A6000, 48GB VRAM", 13),
|
|
("RTX A5000", 24576, "8.6", "NVIDIA RTX A5000, 24GB VRAM", 14),
|
|
("RX 7900 XTX", 24576, "N/A", "AMD Radeon RX 7900 XTX, 24GB VRAM", 15),
|
|
]
|
|
for g in default_gpus:
|
|
c.execute('INSERT INTO gpus (name, vram_mb, compute_capability, description, sort_order) VALUES (?, ?, ?, ?, ?)', g)
|
|
|
|
# ===== Default llama.cpp versions =====
|
|
c.execute('INSERT INTO llama_versions (version_tag, description, release_date, is_active, sort_order) VALUES (?, ?, ?, ?, ?)',
|
|
('b6310', 'llama.cpp build 6310 (2025年中)', '2025-06-01', 1, 1))
|
|
v1_id = c.lastrowid
|
|
|
|
c.execute('INSERT INTO llama_versions (version_tag, description, release_date, is_active, sort_order) VALUES (?, ?, ?, ?, ?)',
|
|
('b10068', 'llama.cpp build 10068 (2026年最新)', '2026-07-01', 1, 2))
|
|
v2_id = c.lastrowid
|
|
|
|
# ===== Default binaries for each version =====
|
|
for vid in [v1_id, v2_id]:
|
|
c.execute('INSERT OR IGNORE INTO version_binaries (version_id, name, description, sort_order) VALUES (?, ?, ?, ?)',
|
|
(vid, 'llama-server', 'HTTP API 服务器', 1))
|
|
c.execute('INSERT OR IGNORE INTO version_binaries (version_id, name, description, sort_order) VALUES (?, ?, ?, ?)',
|
|
(vid, 'llama-cli', '命令行交互', 2))
|
|
c.execute('INSERT OR IGNORE INTO version_binaries (version_id, name, description, sort_order) VALUES (?, ?, ?, ?)',
|
|
(vid, 'llama-bench', '性能基准测试', 3))
|
|
|
|
# ===== Parameters for b6310 =====
|
|
params_b6310 = [
|
|
# === Common params ===
|
|
("model", "-m", "--model", "模型文件路径", "common", "string", "", None, None, None, None, None, 1, 0, 1),
|
|
("ctx_size", "-c", "--ctx-size", "上下文窗口大小 (0=从模型加载)", "common", "number", "0", None, 0, 131072, 512, "tokens", 1, 1, 2),
|
|
("n_gpu_layers", "-ngl", "--n-gpu-layers", "存储在VRAM中的最大层数 (0=不卸载, -1=全部)", "common", "number", "0", None, -1, 999, 1, "layers", 1, 1, 3),
|
|
("threads", "-t", "--threads", "生成期间使用的CPU线程数", "common", "number", "-1", None, -1, 128, 1, "threads", 1, 0, 4),
|
|
("batch_size", "-b", "--batch-size", "逻辑最大批处理大小", "common", "number", "2048", None, 1, 8192, 128, "tokens", 1, 1, 5),
|
|
("ubatch_size", "-ub", "--ubatch-size", "物理最大批处理大小", "common", "number", "512", None, 1, 4096, 64, "tokens", 0, 1, 6),
|
|
("flash_attn", "-fa", "--flash-attn", "Flash Attention (on/off)", "common", "select", "off", json.dumps(["on", "off"]), None, None, None, None, 1, 1, 7),
|
|
("predict", "-n", "--predict", "要预测的token数量 (-1=无限)", "common", "number", "-1", None, -1, 999999, 1, "tokens", 0, 0, 10),
|
|
("keep", "", "--keep", "从初始提示中保留的token数", "common", "number", "0", None, -1, 999999, 1, "tokens", 0, 0, 11),
|
|
("cache_type_k", "-ctk", "--cache-type-k", "KV缓存K的数据类型", "common", "select", "f16", json.dumps(["f32", "f16", "bf16", "q8_0", "q4_0", "q4_1", "iq4_nl", "q5_0", "q5_1"]), None, None, None, None, 0, 1, 12),
|
|
("cache_type_v", "-ctv", "--cache-type-v", "KV缓存V的数据类型", "common", "select", "f16", json.dumps(["f32", "f16", "bf16", "q8_0", "q4_0", "q4_1", "iq4_nl", "q5_0", "q5_1"]), None, None, None, None, 0, 1, 13),
|
|
("split_mode", "-sm", "--split-mode", "跨多个GPU的分割模式", "common", "select", "layer", json.dumps(["none", "layer", "row"]), None, None, None, None, 0, 0, 14),
|
|
("tensor_split", "-ts", "--tensor-split", "每个GPU卸载的模型比例, 逗号分隔", "common", "string", "", None, None, None, None, None, 0, 0, 15),
|
|
("main_gpu", "-mg", "--main-gpu", "用于模型的主GPU索引", "common", "number", "0", None, 0, 7, 1, None, 0, 0, 16),
|
|
("mlock", "", "--mlock", "强制系统将模型保留在RAM中", "common", "boolean", "false", None, None, None, None, None, 0, 0, 17),
|
|
("mmap", "", "--mmap", "内存映射模型文件", "common", "boolean", "true", None, None, None, None, None, 0, 0, 18),
|
|
("numa", "", "--numa", "NUMA优化", "common", "select", "", json.dumps(["", "distribute", "isolate", "numactl"]), None, None, None, None, 0, 0, 19),
|
|
("cpu_moe", "-cmoe", "--cpu-moe", "将所有MoE权重保留在CPU", "common", "boolean", "false", None, None, None, None, None, 0, 0, 20),
|
|
("rope_scaling", "", "--rope-scaling", "RoPE频率缩放方法", "common", "select", "linear", json.dumps(["none", "linear", "yarn"]), None, None, None, None, 0, 0, 21),
|
|
("rope_freq_base", "", "--rope-freq-base", "RoPE基础频率", "common", "number", "10000", None, 1, 1000000, 1, None, 0, 0, 22),
|
|
("rope_freq_scale", "", "--rope-freq-scale", "RoPE频率缩放因子", "common", "number", "1.0", None, 0.01, 10, 0.01, None, 0, 0, 23),
|
|
("yarn_orig_ctx", "", "--yarn-orig-ctx", "YaRN原始上下文大小", "common", "number", "0", None, 0, 131072, 512, "tokens", 0, 0, 24),
|
|
("yarn_ext_factor", "", "--yarn-ext-factor", "YaRN外推混合因子", "common", "number", "-1.0", None, -1, 1, 0.1, None, 0, 0, 25),
|
|
("yarn_attn_factor", "", "--yarn-attn-factor", "YaRN注意力缩放因子", "common", "number", "-1.0", None, -1, 10, 0.1, None, 0, 0, 26),
|
|
("yarn_beta_slow", "", "--yarn-beta-slow", "YaRN高修正维度", "common", "number", "-1.0", None, -1, 10, 0.1, None, 0, 0, 27),
|
|
("yarn_beta_fast", "", "--yarn-beta-fast", "YaRN低修正维度", "common", "number", "-1.0", None, -1, 10, 0.1, None, 0, 0, 28),
|
|
("kv_offload", "-kvo", "--kv-offload", "启用KV缓存卸载", "common", "boolean", "true", None, None, None, None, None, 0, 0, 29),
|
|
("device", "-dev", "--device", "用于卸载的设备列表", "common", "string", "", None, None, None, None, None, 0, 0, 30),
|
|
("override_tensor", "-ot", "--override-tensor", "覆盖张量缓冲区类型", "common", "string", "", None, None, None, None, None, 0, 0, 31),
|
|
# === Sampling params ===
|
|
("temperature", "", "--temp", "温度 (创造力)", "sampling", "number", "0.8", None, 0.01, 2.0, 0.05, None, 1, 0, 1),
|
|
("seed", "-s", "--seed", "随机种子 (-1=随机)", "sampling", "number", "-1", None, -1, 999999, 1, None, 0, 0, 2),
|
|
("top_k", "", "--top-k", "Top-K采样", "sampling", "number", "40", None, 0, 200, 1, None, 1, 0, 3),
|
|
("top_p", "", "--top-p", "Top-P (核) 采样", "sampling", "number", "0.95", None, 0.0, 1.0, 0.05, None, 1, 0, 4),
|
|
("min_p", "", "--min-p", "Min-P采样", "sampling", "number", "0.05", None, 0.0, 1.0, 0.01, None, 0, 0, 5),
|
|
("typical", "", "--typical", "局部典型采样", "sampling", "number", "1.0", None, 0.0, 1.0, 0.05, None, 0, 0, 6),
|
|
("repeat_penalty", "", "--repeat-penalty", "重复惩罚", "sampling", "number", "1.0", None, 0.5, 2.0, 0.05, None, 0, 0, 7),
|
|
("repeat_last_n", "", "--repeat-last-n", "惩罚考虑的最后n个token", "sampling", "number", "64", None, 0, 999999, 1, "tokens", 0, 0, 8),
|
|
("presence_penalty", "", "--presence-penalty", "存在惩罚", "sampling", "number", "0.0", None, -2.0, 2.0, 0.1, None, 0, 0, 9),
|
|
("frequency_penalty", "", "--frequency-penalty", "频率惩罚", "sampling", "number", "0.0", None, -2.0, 2.0, 0.1, None, 0, 0, 10),
|
|
("mirostat", "", "--mirostat", "Mirostat采样模式", "sampling", "select", "0", json.dumps(["0", "1", "2"]), None, None, None, None, 0, 0, 11),
|
|
("mirostat_lr", "", "--mirostat-lr", "Mirostat学习率", "sampling", "number", "0.1", None, 0.01, 1.0, 0.01, None, 0, 0, 12),
|
|
("mirostat_ent", "", "--mirostat-ent", "Mirostat目标熵", "sampling", "number", "5.0", None, 1.0, 10.0, 0.1, None, 0, 0, 13),
|
|
("ignore_eos", "", "--ignore-eos", "忽略结束流token", "sampling", "boolean", "false", None, None, None, None, None, 0, 0, 14),
|
|
("samplers", "", "--samplers", "采样器序列", "sampling", "string", "penalties;dry;top_n_sigma;top_k;typ_p;top_p;min_p;xtc;temperature", None, None, None, None, None, 0, 0, 15),
|
|
# === Server params ===
|
|
("port", "", "--port", "服务器监听端口", "server", "number", "8080", None, 1, 65535, 1, None, 1, 0, 1),
|
|
("host", "", "--host", "服务器监听地址", "server", "string", "0.0.0.0", None, None, None, None, None, 1, 0, 2),
|
|
("parallel", "-np", "--parallel", "服务器槽位数", "server", "number", "-1", None, -1, 64, 1, "slots", 0, 1, 3),
|
|
("cont_batching", "-cb", "--cont-batching", "连续批处理", "server", "boolean", "true", None, None, None, None, None, 0, 0, 4),
|
|
("context_shift", "", "--context-shift", "上下文移位", "server", "boolean", "false", None, None, None, None, None, 0, 0, 5),
|
|
("special", "-sp", "--special", "特殊token输出", "server", "boolean", "false", None, None, None, None, None, 0, 0, 6),
|
|
("warmup", "", "--warmup", "预热运行", "server", "boolean", "true", None, None, None, None, None, 0, 0, 7),
|
|
("pooling", "", "--pooling", "嵌入池化类型", "server", "select", "", json.dumps(["", "none", "mean", "cls", "last", "rank"]), None, None, None, None, 0, 0, 8),
|
|
("mmproj", "-mm", "--mmproj", "多模态投影文件路径", "server", "string", "", None, None, None, None, None, 0, 0, 9),
|
|
# === Model download ===
|
|
("hf_repo", "-hf", "--hf-repo", "Hugging Face模型仓库", "model_source", "string", "", None, None, None, None, None, 0, 0, 1),
|
|
("hf_file", "-hff", "--hf-file", "Hugging Face模型文件", "model_source", "string", "", None, None, None, None, None, 0, 0, 2),
|
|
("hf_token", "-hft", "--hf-token", "Hugging Face访问令牌", "model_source", "string", "", None, None, None, None, None, 0, 0, 3),
|
|
("model_url", "-mu", "--model-url", "模型下载URL", "model_source", "string", "", None, None, None, None, None, 0, 0, 4),
|
|
("docker_repo", "-dr", "--docker-repo", "Docker Hub模型仓库", "model_source", "string", "", None, None, None, None, None, 0, 0, 5),
|
|
# === Logging ===
|
|
("verbose", "-v", "--verbose", "详细日志输出", "logging", "boolean", "false", None, None, None, None, None, 0, 0, 1),
|
|
("log_file", "", "--log-file", "日志文件路径", "logging", "string", "", None, None, None, None, None, 0, 0, 2),
|
|
("log_colors", "", "--log-colors", "彩色日志", "logging", "select", "auto", json.dumps(["on", "off", "auto"]), None, None, None, None, 0, 0, 3),
|
|
("log_verbosity", "-lv", "--verbosity", "日志详细级别", "logging", "select", "3", json.dumps(["0", "1", "2", "3", "4", "5"]), None, None, None, None, 0, 0, 4),
|
|
# === LoRA ===
|
|
("lora", "", "--lora", "LoRA适配器路径", "lora", "string", "", None, None, None, None, None, 0, 0, 1),
|
|
("lora_scaled", "", "--lora-scaled", "带缩放的LoRA适配器", "lora", "string", "", None, None, None, None, None, 0, 0, 2),
|
|
# === Advanced ===
|
|
("cpu_mask", "-C", "--cpu-mask", "CPU亲和性掩码", "advanced", "string", "", None, None, None, None, None, 0, 0, 1),
|
|
("cpu_range", "-Cr", "--cpu-range", "CPU亲和性范围", "advanced", "string", "", None, None, None, None, None, 0, 0, 2),
|
|
("cpu_strict", "", "--cpu-strict", "严格CPU放置", "advanced", "boolean", "false", None, None, None, None, None, 0, 0, 3),
|
|
("prio", "", "--prio", "进程/线程优先级", "advanced", "number", "0", None, -1, 3, 1, None, 0, 0, 4),
|
|
("poll", "", "--poll", "轮询级别", "advanced", "number", "50", None, 0, 100, 1, None, 0, 0, 5),
|
|
("threads_batch", "-tb", "--threads-batch", "批处理和提示处理线程数", "advanced", "number", "-1", None, -1, 128, 1, None, 0, 0, 6),
|
|
("rope_scale", "", "--rope-scale", "RoPE上下文缩放因子", "advanced", "number", "1.0", None, 0.1, 10, 0.1, None, 0, 0, 7),
|
|
("op_offload", "", "--op-offload", "卸载主机张量操作到设备", "advanced", "boolean", "true", None, None, None, None, None, 0, 0, 8),
|
|
("check_tensors", "", "--check-tensors", "检查模型张量数据", "advanced", "boolean", "false", None, None, None, None, None, 0, 0, 9),
|
|
("override_kv", "", "--override-kv", "覆盖模型元数据", "advanced", "string", "", None, None, None, None, None, 0, 0, 10),
|
|
]
|
|
|
|
for p in params_b6310:
|
|
c.execute('''INSERT INTO params
|
|
(version_id, binary_id, param_key, short_flag, long_flag, description, category, param_type,
|
|
default_value, options, min_value, max_value, step, unit, is_important, affects_vram, sort_order)
|
|
VALUES (?, NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''',
|
|
(v1_id,) + p)
|
|
|
|
# ===== Parameters for b10068 (latest) =====
|
|
params_b10068 = list(params_b6310)
|
|
params_b10068 = [(p[0], p[1], p[2], p[3], p[4], p[5],
|
|
"auto" if p[0] == "n_gpu_layers" else p[6],
|
|
p[7], p[8], p[9], p[10], p[11], p[12], p[13], p[14])
|
|
if p[0] == "n_gpu_layers" else p for p in params_b10068]
|
|
params_b10068 = [(p[0], p[1], p[2], p[3], p[4], p[5],
|
|
"auto" if p[0] == "flash_attn" else p[6],
|
|
'["on", "off", "auto"]' if p[0] == "flash_attn" else p[7],
|
|
p[8], p[9], p[10], p[11], p[12], p[13], p[14])
|
|
if p[0] == "flash_attn" else p for p in params_b10068]
|
|
|
|
new_params_b10068 = [
|
|
("fit", "-fit", "--fit", "自动调整参数以适应设备内存", "common", "select", "on", json.dumps(["on", "off"]), None, None, None, None, 1, 1, 8),
|
|
("fit_target", "-fitt", "--fit-target", "每个设备的目标余量(MiB)", "common", "string", "1024", None, None, None, None, "MiB", 0, 1, 9),
|
|
("fit_ctx", "-fitc", "--fit-ctx", "fit选项可设置的最小ctx大小", "common", "number", "4096", None, 1024, 131072, 512, "tokens", 0, 1, 10),
|
|
("cache_ram", "-cram", "--cache-ram", "最大缓存大小(MiB)", "server", "number", "8192", None, -1, 999999, 512, "MiB", 0, 1, 10),
|
|
("kv_unified", "-kvu", "--kv-unified", "使用统一KV缓冲区", "server", "boolean", "false", None, None, None, None, None, 0, 1, 11),
|
|
("cache_idle_slots", "", "--cache-idle-slots", "缓存空闲槽位", "server", "boolean", "true", None, None, None, None, None, 0, 0, 12),
|
|
("ctx_checkpoints", "-ctxcp", "--ctx-checkpoints", "每槽最大上下文检查点数", "server", "number", "32", None, 1, 999, 1, None, 0, 0, 13),
|
|
("checkpoint_min_step", "-cms", "--checkpoint-min-step", "检查点最小间距", "server", "number", "8192", None, 0, 999999, 512, "tokens", 0, 0, 14),
|
|
("swa_full", "", "--swa-full", "使用全尺寸SWA缓存", "common", "boolean", "false", None, None, None, None, None, 0, 1, 32),
|
|
("perf", "", "--perf", "启用性能计时", "common", "boolean", "false", None, None, None, None, None, 0, 0, 33),
|
|
("repack", "", "--repack", "权重重打包", "common", "boolean", "true", None, None, None, None, None, 0, 0, 34),
|
|
("no_host", "", "--no-host", "绕过主机缓冲区", "common", "boolean", "false", None, None, None, None, None, 0, 0, 35),
|
|
("n_cpu_moe", "-ncmoe", "--n-cpu-moe", "前N层MoE权重保留在CPU", "common", "number", "0", None, 0, 999, 1, "layers", 0, 0, 36),
|
|
("direct_io", "-dio", "--direct-io", "使用DirectIO", "common", "boolean", "false", None, None, None, None, None, 0, 0, 37),
|
|
("offline", "", "--offline", "离线模式", "common", "boolean", "false", None, None, None, None, None, 0, 0, 38),
|
|
("spec_draft_cache_type_k", "-ctkd", "--cache-type-k-draft", "草稿模型KV缓存K类型", "advanced", "select", "f16", json.dumps(["f32", "f16", "bf16", "q8_0", "q4_0", "q4_1", "iq4_nl", "q5_0", "q5_1"]), None, None, None, None, 0, 0, 11),
|
|
("spec_draft_cache_type_v", "-ctvd", "--cache-type-v-draft", "草稿模型KV缓存V类型", "advanced", "select", "f16", json.dumps(["f32", "f16", "bf16", "q8_0", "q4_0", "q4_1", "iq4_nl", "q5_0", "q5_1"]), None, None, None, None, 0, 0, 12),
|
|
("adaptive_target", "", "--adaptive-target", "adaptive-p目标概率", "sampling", "number", "-1.0", None, -1.0, 1.0, 0.05, None, 0, 0, 16),
|
|
("adaptive_decay", "", "--adaptive-decay", "adaptive-p衰减率", "sampling", "number", "0.9", None, 0.0, 0.99, 0.01, None, 0, 0, 17),
|
|
("dynatemp_range", "", "--dynatemp-range", "动态温度范围", "sampling", "number", "0.0", None, 0.0, 2.0, 0.05, None, 0, 0, 18),
|
|
("dynatemp_exp", "", "--dynatemp-exp", "动态温度指数", "sampling", "number", "1.0", None, 0.1, 5.0, 0.1, None, 0, 0, 19),
|
|
("top_n_sigma", "--top-nsigma", "--top-n-sigma", "Top-n-sigma采样", "sampling", "number", "-1.0", None, -1.0, 10.0, 0.1, None, 0, 0, 20),
|
|
("xtc_probability", "", "--xtc-probability", "XTC概率", "sampling", "number", "0.0", None, 0.0, 1.0, 0.01, None, 0, 0, 21),
|
|
("xtc_threshold", "", "--xtc-threshold", "XTC阈值", "sampling", "number", "0.1", None, 0.0, 1.0, 0.01, None, 0, 0, 22),
|
|
("dry_multiplier", "", "--dry-multiplier", "DRY采样乘数", "sampling", "number", "0.0", None, 0.0, 5.0, 0.1, None, 0, 0, 23),
|
|
("dry_base", "", "--dry-base", "DRY采样基础值", "sampling", "number", "1.75", None, 1.0, 3.0, 0.05, None, 0, 0, 24),
|
|
("dry_allowed_length", "", "--dry-allowed-length", "DRY允许长度", "sampling", "number", "2", None, 1, 20, 1, None, 0, 0, 25),
|
|
("dry_penalty_last_n", "", "--dry-penalty-last-n", "DRY惩罚最后n个token", "sampling", "number", "-1", None, -1, 999999, 1, "tokens", 0, 0, 26),
|
|
("sampler_seq", "", "--sampler-seq", "简化采样器序列", "sampling", "string", "edskypmxt", None, None, None, None, None, 0, 0, 27),
|
|
("backend_sampling", "-bs", "--backend-sampling", "后端采样(实验性)", "sampling", "boolean", "false", None, None, None, None, None, 0, 0, 28),
|
|
("hf_repo_v", "-hfv", "--hf-repo-v", "vocoder模型HF仓库", "model_source", "string", "", None, None, None, None, None, 0, 0, 6),
|
|
("hf_file_v", "-hffv", "--hf-file-v", "vocoder模型HF文件", "model_source", "string", "", None, None, None, None, None, 0, 0, 7),
|
|
]
|
|
|
|
all_params_b10068 = list(params_b10068) + list(new_params_b10068)
|
|
for p in all_params_b10068:
|
|
c.execute('''INSERT INTO params
|
|
(version_id, binary_id, param_key, short_flag, long_flag, description, category, param_type,
|
|
default_value, options, min_value, max_value, step, unit, is_important, affects_vram, sort_order)
|
|
VALUES (?, NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''',
|
|
(v2_id,) + p)
|
|
|
|
# ===== Default models (one entry per base_model) =====
|
|
# (base_model, name, model_type, num_experts, layers, embd, kv_heads, head_dim, attention_heads, default_ctx, description, sort_order)
|
|
default_models = [
|
|
("Llama-3-8B-Instruct", "Llama-3-8B-Instruct", "dense", 0, 32, 4096, 8, 128, 32, 8192, "Meta Llama 3 8B Instruct", 1),
|
|
("Llama-3-70B-Instruct", "Llama-3-70B-Instruct", "dense", 0, 80, 8192, 8, 128, 64, 8192, "Meta Llama 3 70B Instruct", 2),
|
|
("Llama-3.1-8B-Instruct", "Llama-3.1-8B-Instruct", "dense", 0, 32, 4096, 8, 128, 32, 131072, "Meta Llama 3.1 8B Instruct", 3),
|
|
("Llama-3.1-70B-Instruct", "Llama-3.1-70B-Instruct", "dense", 0, 80, 8192, 8, 128, 64, 131072, "Meta Llama 3.1 70B Instruct", 4),
|
|
("Llama-3.1-405B-Instruct", "Llama-3.1-405B-Instruct", "dense", 0, 126, 16384, 8, 128, 128, 131072, "Meta Llama 3.1 405B Instruct", 5),
|
|
("Qwen2.5-7B-Instruct", "Qwen2.5-7B-Instruct", "dense", 0, 28, 3584, 4, 128, 28, 32768, "Qwen2.5 7B Instruct", 6),
|
|
("Qwen2.5-14B-Instruct", "Qwen2.5-14B-Instruct", "dense", 0, 40, 5120, 8, 128, 40, 32768, "Qwen2.5 14B Instruct", 7),
|
|
("Qwen2.5-32B-Instruct", "Qwen2.5-32B-Instruct", "dense", 0, 64, 5120, 8, 128, 64, 32768, "Qwen2.5 32B Instruct", 8),
|
|
("Qwen2.5-72B-Instruct", "Qwen2.5-72B-Instruct", "dense", 0, 80, 8192, 8, 128, 64, 32768, "Qwen2.5 72B Instruct", 9),
|
|
("DeepSeek-V2-Chat", "DeepSeek-V2-Chat", "moe", 160, 60, 5120, 8, 128, 60, 4096, "DeepSeek V2 Chat (MoE, 160 experts)", 10),
|
|
("DeepSeek-V2.5-Chat", "DeepSeek-V2.5-Chat", "moe", 160, 60, 5120, 8, 128, 60, 4096, "DeepSeek V2.5 Chat (MoE, 160 experts)", 11),
|
|
("DeepSeek-R1-Distill-Qwen-32B", "DeepSeek-R1-Distill-Qwen-32B", "dense", 0, 64, 5120, 8, 128, 64, 131072, "DeepSeek R1 Distill Qwen 32B", 12),
|
|
("DeepSeek-R1-Distill-Llama-70B", "DeepSeek-R1-Distill-Llama-70B", "dense", 0, 80, 8192, 8, 128, 64, 131072, "DeepSeek R1 Distill Llama 70B", 13),
|
|
("Mistral-7B-Instruct-v0.3", "Mistral-7B-Instruct-v0.3", "dense", 0, 32, 4096, 8, 128, 32, 32768, "Mistral 7B Instruct v0.3", 14),
|
|
("Mixtral-8x7B-Instruct", "Mixtral-8x7B-Instruct", "moe", 8, 32, 4096, 8, 128, 32, 32768, "Mixtral 8x7B Instruct (MoE, 8 experts)", 15),
|
|
("Gemma-2-9B-It", "Gemma-2-9B-It", "dense", 0, 42, 3584, 4, 256, 14, 8192, "Google Gemma 2 9B It", 16),
|
|
("Gemma-2-27B-It", "Gemma-2-27B-It", "dense", 0, 46, 4608, 4, 128, 36, 8192, "Google Gemma 2 27B It", 17),
|
|
("Phi-3-Mini-4K-Instruct", "Phi-3-Mini-4K-Instruct", "dense", 0, 32, 3072, 32, 96, 32, 4096, "Microsoft Phi-3 Mini 4K Instruct", 18),
|
|
("Phi-3-Medium-14B-Instruct", "Phi-3-Medium-14B-Instruct", "dense", 0, 40, 5120, 10, 128, 40, 14336, "Microsoft Phi-3 Medium 14B Instruct", 19),
|
|
("GLM-4-9B-Chat", "GLM-4-9B-Chat", "dense", 0, 40, 4096, 4, 128, 40, 131072, "Zhipu GLM-4 9B Chat", 20),
|
|
]
|
|
|
|
# Quant variants: { base_model: [(quant_type, size_gb, sort_order), ...] }
|
|
default_model_quants = {
|
|
"Llama-3-8B-Instruct": [("Q4_K_M", 4.9, 1), ("Q8_0", 8.5, 2), ("FP16", 15.5, 3)],
|
|
"Llama-3-70B-Instruct": [("Q4_K_M", 38.5, 1), ("Q8_0", 74.0, 2), ("FP16", 138.0, 3)],
|
|
"Llama-3.1-8B-Instruct": [("Q4_K_M", 4.9, 1), ("Q8_0", 8.5, 2)],
|
|
"Llama-3.1-70B-Instruct": [("Q4_K_M", 38.5, 1), ("Q8_0", 74.0, 2)],
|
|
"Llama-3.1-405B-Instruct": [("Q4_K_M", 226.0, 1)],
|
|
"Qwen2.5-7B-Instruct": [("Q4_K_M", 4.7, 1), ("Q8_0", 7.6, 2)],
|
|
"Qwen2.5-14B-Instruct": [("Q4_K_M", 8.7, 1)],
|
|
"Qwen2.5-32B-Instruct": [("Q4_K_M", 19.5, 1), ("Q8_0", 32.0, 2)],
|
|
"Qwen2.5-72B-Instruct": [("Q4_K_M", 42.0, 1), ("Q8_0", 75.0, 2)],
|
|
"DeepSeek-V2-Chat": [("Q4_K_M", 23.0, 1)],
|
|
"DeepSeek-V2.5-Chat": [("Q4_K_M", 23.0, 1)],
|
|
"DeepSeek-R1-Distill-Qwen-32B": [("Q4_K_M", 19.5, 1), ("Q8_0", 32.0, 2)],
|
|
"DeepSeek-R1-Distill-Llama-70B": [("Q4_K_M", 42.0, 1), ("Q8_0", 75.0, 2)],
|
|
"Mistral-7B-Instruct-v0.3": [("Q4_K_M", 4.4, 1), ("Q8_0", 7.5, 2)],
|
|
"Mixtral-8x7B-Instruct": [("Q4_K_M", 26.0, 1)],
|
|
"Gemma-2-9B-It": [("Q4_K_M", 5.4, 1)],
|
|
"Gemma-2-27B-It": [("Q4_K_M", 16.5, 1)],
|
|
"Phi-3-Mini-4K-Instruct": [("Q4_K_M", 2.5, 1)],
|
|
"Phi-3-Medium-14B-Instruct": [("Q4_K_M", 8.4, 1)],
|
|
"GLM-4-9B-Chat": [("Q4_K_M", 5.5, 1), ("Q8_0", 9.0, 2)],
|
|
}
|
|
|
|
for m in default_models:
|
|
c.execute('''INSERT INTO models (base_model, name, model_type, num_experts, size_gb, quant, layers, embd, kv_heads, head_dim, attention_heads, default_ctx, description, sort_order)
|
|
VALUES (?, ?, ?, ?, 0, '', ?, ?, ?, ?, ?, ?, ?, ?)''', m)
|
|
model_id = c.lastrowid
|
|
base = m[0]
|
|
for q in default_model_quants.get(base, []):
|
|
c.execute('INSERT INTO model_quants (model_id, quant_type, size_gb, sort_order) VALUES (?, ?, ?, ?)',
|
|
(model_id, q[0], q[1], q[2]))
|
|
|
|
# ===== Default settings =====
|
|
c.execute("INSERT OR IGNORE INTO settings (key, value) VALUES (?, ?)", ("admin_password", "admin123"))
|
|
c.execute("INSERT OR IGNORE INTO settings (key, value) VALUES (?, ?)", ("default_gpu", "RTX 3090"))
|
|
c.execute("INSERT OR IGNORE INTO settings (key, value) VALUES (?, ?)", ("default_version", "b10068"))
|
|
c.execute("INSERT OR IGNORE INTO settings (key, value) VALUES (?, ?)", ("default_mode", "gpu"))
|
|
c.execute("INSERT OR IGNORE INTO settings (key, value) VALUES (?, ?)", ("default_quant", "Q4_K_M"))
|
|
c.execute("INSERT OR IGNORE INTO settings (key, value) VALUES (?, ?)", ("llm_enabled", "false"))
|
|
c.execute("INSERT OR IGNORE INTO settings (key, value) VALUES (?, ?)", ("llm_api_url", ""))
|
|
c.execute("INSERT OR IGNORE INTO settings (key, value) VALUES (?, ?)", ("llm_api_key", ""))
|
|
c.execute("INSERT OR IGNORE INTO settings (key, value) VALUES (?, ?)", ("llm_api_model", ""))
|
|
c.execute("INSERT OR IGNORE INTO settings (key, value) VALUES (?, ?)", ("nl_default_text", "用自然语言描述你想要的配置，支持多行输入。\n例如：\n用RTX 4090跑Llama-3-70B，上下文8192\n温度0.7，开启flash attention\n端口设为8080"))
|
|
c.execute("INSERT OR IGNORE INTO settings (key, value) VALUES (?, ?)", ("show_nl_section", "true"))
|
|
|
|
conn.commit()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
init_db()
|
|
print(f"Database initialized at {DB_PATH}")
|