v2.0.0: 重构模型管理(dense/MoE+量化版本分离), 版本管理增加执行程序管理, 修复设置页GPU下拉为空
1. 版本管理增加执行程序(binaries)管理: 每个版本可管理llama-server/llama-cli/llama-bench等程序 - 参数管理中增加执行程序选择器, 可按程序筛选参数 - 参数可绑定到特定执行程序或适用所有程序 2. 模型管理重构: 一个模型统一参数(区分dense/MoE), 量化版本作为子表管理 - models表增加model_type(dense/moe)和num_experts字段 - 新增model_quants表, 每个模型可有多个量化版本 - 前端模型选择改为: 选模型 -> 选量化版本 3. 修复系统设置中默认GPU下拉为空: 异步加载GPU后再渲染设置
This commit is contained in:
@@ -43,11 +43,26 @@ def init_db():
|
||||
)
|
||||
''')
|
||||
|
||||
# Parameters table (per version)
|
||||
# Version binaries table (llama-server, llama-cli, etc.)
|
||||
c.execute('''
|
||||
CREATE TABLE IF NOT EXISTS version_binaries (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
version_id INTEGER NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
description TEXT,
|
||||
sort_order INTEGER DEFAULT 0,
|
||||
FOREIGN KEY (version_id) REFERENCES llama_versions(id) ON DELETE CASCADE,
|
||||
UNIQUE(version_id, name)
|
||||
)
|
||||
''')
|
||||
|
||||
# Parameters table (per version, optionally per binary)
|
||||
# binary_id = NULL means the param applies to all binaries
|
||||
c.execute('''
|
||||
CREATE TABLE IF NOT EXISTS params (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
version_id INTEGER NOT NULL,
|
||||
binary_id INTEGER DEFAULT NULL,
|
||||
param_key TEXT NOT NULL,
|
||||
short_flag TEXT,
|
||||
long_flag TEXT NOT NULL,
|
||||
@@ -63,8 +78,7 @@ def init_db():
|
||||
is_important INTEGER DEFAULT 0,
|
||||
affects_vram INTEGER DEFAULT 0,
|
||||
sort_order INTEGER DEFAULT 0,
|
||||
FOREIGN KEY (version_id) REFERENCES llama_versions(id) ON DELETE CASCADE,
|
||||
UNIQUE(version_id, param_key)
|
||||
FOREIGN KEY (version_id) REFERENCES llama_versions(id) ON DELETE CASCADE
|
||||
)
|
||||
''')
|
||||
|
||||
@@ -76,33 +90,170 @@ def init_db():
|
||||
)
|
||||
''')
|
||||
|
||||
# Models table
|
||||
# Models table (base model info, one row per base model)
|
||||
c.execute('''
|
||||
CREATE TABLE IF NOT EXISTS models (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
base_model TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
size_gb REAL NOT NULL,
|
||||
model_type TEXT DEFAULT 'dense',
|
||||
num_experts INTEGER DEFAULT 0,
|
||||
size_gb REAL DEFAULT 0,
|
||||
quant TEXT DEFAULT '',
|
||||
layers INTEGER NOT NULL,
|
||||
embd INTEGER NOT NULL,
|
||||
kv_heads INTEGER NOT NULL,
|
||||
head_dim INTEGER NOT NULL,
|
||||
attention_heads INTEGER NOT NULL,
|
||||
default_ctx INTEGER DEFAULT 0,
|
||||
quant TEXT DEFAULT '',
|
||||
description TEXT DEFAULT '',
|
||||
sort_order INTEGER DEFAULT 0
|
||||
)
|
||||
''')
|
||||
|
||||
# Model quantization variants table
|
||||
c.execute('''
|
||||
CREATE TABLE IF NOT EXISTS model_quants (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
model_id INTEGER NOT NULL,
|
||||
quant_type TEXT NOT NULL,
|
||||
size_gb REAL NOT NULL,
|
||||
sort_order INTEGER DEFAULT 0,
|
||||
FOREIGN KEY (model_id) REFERENCES models(id) ON DELETE CASCADE
|
||||
)
|
||||
''')
|
||||
|
||||
conn.commit()
|
||||
|
||||
# Insert default data
|
||||
# Run migrations for existing databases
|
||||
migrate_db(conn)
|
||||
|
||||
# Insert default data if empty
|
||||
insert_default_data(conn)
|
||||
|
||||
conn.close()
|
||||
|
||||
|
||||
def migrate_db(conn):
|
||||
"""Migrate existing database to new schema."""
|
||||
c = conn.cursor()
|
||||
|
||||
# 1. Migrate params table: add binary_id column
|
||||
try:
|
||||
c.execute("SELECT binary_id FROM params LIMIT 1")
|
||||
except sqlite3.OperationalError:
|
||||
# Recreate params table with binary_id
|
||||
c.execute("ALTER TABLE params RENAME TO params_old")
|
||||
c.execute('''
|
||||
CREATE TABLE params (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
version_id INTEGER NOT NULL,
|
||||
binary_id INTEGER DEFAULT NULL,
|
||||
param_key TEXT NOT NULL,
|
||||
short_flag TEXT,
|
||||
long_flag TEXT NOT NULL,
|
||||
description TEXT,
|
||||
category TEXT NOT NULL DEFAULT 'common',
|
||||
param_type TEXT NOT NULL DEFAULT 'string',
|
||||
default_value TEXT,
|
||||
options TEXT,
|
||||
min_value REAL,
|
||||
max_value REAL,
|
||||
step REAL,
|
||||
unit TEXT,
|
||||
is_important INTEGER DEFAULT 0,
|
||||
affects_vram INTEGER DEFAULT 0,
|
||||
sort_order INTEGER DEFAULT 0,
|
||||
FOREIGN KEY (version_id) REFERENCES llama_versions(id) ON DELETE CASCADE
|
||||
)
|
||||
''')
|
||||
c.execute('''INSERT INTO params (id, version_id, binary_id, param_key, short_flag, long_flag,
|
||||
description, category, param_type, default_value, options, min_value, max_value,
|
||||
step, unit, is_important, affects_vram, sort_order)
|
||||
SELECT id, version_id, NULL, param_key, short_flag, long_flag,
|
||||
description, category, param_type, default_value, options, min_value, max_value,
|
||||
step, unit, is_important, affects_vram, sort_order FROM params_old''')
|
||||
c.execute("DROP TABLE params_old")
|
||||
|
||||
# 2. Add model_type to models table
|
||||
try:
|
||||
c.execute("SELECT model_type FROM models LIMIT 1")
|
||||
except sqlite3.OperationalError:
|
||||
c.execute("ALTER TABLE models ADD COLUMN model_type TEXT DEFAULT 'dense'")
|
||||
|
||||
# 3. Add num_experts to models table
|
||||
try:
|
||||
c.execute("SELECT num_experts FROM models LIMIT 1")
|
||||
except sqlite3.OperationalError:
|
||||
c.execute("ALTER TABLE models ADD COLUMN num_experts INTEGER DEFAULT 0")
|
||||
|
||||
conn.commit()
|
||||
|
||||
# 4. Insert default binaries for existing versions
|
||||
c.execute("SELECT COUNT(*) as cnt FROM version_binaries")
|
||||
if c.fetchone()['cnt'] == 0:
|
||||
c.execute("SELECT id FROM llama_versions")
|
||||
for v in c.fetchall():
|
||||
vid = v['id']
|
||||
c.execute("INSERT OR IGNORE INTO version_binaries (version_id, name, description, sort_order) VALUES (?, ?, ?, ?)",
|
||||
(vid, 'llama-server', 'HTTP API 服务器', 1))
|
||||
c.execute("INSERT OR IGNORE INTO version_binaries (version_id, name, description, sort_order) VALUES (?, ?, ?, ?)",
|
||||
(vid, 'llama-cli', '命令行交互', 2))
|
||||
c.execute("INSERT OR IGNORE INTO version_binaries (version_id, name, description, sort_order) VALUES (?, ?, ?, ?)",
|
||||
(vid, 'llama-bench', '性能基准测试', 3))
|
||||
conn.commit()
|
||||
|
||||
# 5. Migrate existing model data to model_quants table
|
||||
c.execute("SELECT COUNT(*) as cnt FROM model_quants")
|
||||
if c.fetchone()['cnt'] == 0:
|
||||
c.execute("PRAGMA table_info(models)")
|
||||
columns = [col['name'] for col in c.fetchall()]
|
||||
|
||||
if 'quant' in columns:
|
||||
c.execute("SELECT DISTINCT base_model FROM models")
|
||||
base_models = [r['base_model'] for r in c.fetchall()]
|
||||
|
||||
moe_models = {
|
||||
'Mixtral-8x7B-Instruct': 8,
|
||||
'DeepSeek-V2-Chat': 160,
|
||||
'DeepSeek-V2.5-Chat': 160,
|
||||
}
|
||||
|
||||
for base in base_models:
|
||||
c.execute("SELECT * FROM models WHERE base_model = ? ORDER BY sort_order", (base,))
|
||||
variants = c.fetchall()
|
||||
|
||||
if not variants:
|
||||
continue
|
||||
|
||||
first = variants[0]
|
||||
model_id = first['id']
|
||||
|
||||
# Insert all quant variants into model_quants
|
||||
for v in variants:
|
||||
quant = v['quant'] if v['quant'] else 'FP16'
|
||||
c.execute(
|
||||
"INSERT INTO model_quants (model_id, quant_type, size_gb, sort_order) VALUES (?, ?, ?, ?)",
|
||||
(model_id, quant, v['size_gb'], v['sort_order'])
|
||||
)
|
||||
|
||||
# Set model_type and num_experts
|
||||
model_type = 'dense'
|
||||
num_experts = 0
|
||||
if base in moe_models:
|
||||
model_type = 'moe'
|
||||
num_experts = moe_models[base]
|
||||
|
||||
c.execute("UPDATE models SET model_type = ?, num_experts = ?, name = ?, size_gb = 0, quant = '' WHERE id = ?",
|
||||
(model_type, num_experts, base, model_id))
|
||||
|
||||
# Delete duplicate variants
|
||||
for v in variants[1:]:
|
||||
c.execute("DELETE FROM models WHERE id = ?", (v['id'],))
|
||||
|
||||
conn.commit()
|
||||
|
||||
|
||||
def insert_default_data(conn):
|
||||
c = conn.cursor()
|
||||
|
||||
@@ -129,28 +280,30 @@ def insert_default_data(conn):
|
||||
("RTX A5000", 24576, "8.6", "NVIDIA RTX A5000, 24GB VRAM", 14),
|
||||
("RX 7900 XTX", 24576, "N/A", "AMD Radeon RX 7900 XTX, 24GB VRAM", 15),
|
||||
]
|
||||
|
||||
for g in default_gpus:
|
||||
c.execute('''INSERT INTO gpus (name, vram_mb, compute_capability, description, sort_order)
|
||||
VALUES (?, ?, ?, ?, ?)''', g)
|
||||
c.execute('INSERT INTO gpus (name, vram_mb, compute_capability, description, sort_order) VALUES (?, ?, ?, ?, ?)', g)
|
||||
|
||||
# ===== Default llama.cpp versions =====
|
||||
# Version 1: b6310 (mid 2025)
|
||||
c.execute('''INSERT INTO llama_versions (version_tag, description, release_date, is_active, sort_order)
|
||||
VALUES (?, ?, ?, ?, ?)''',
|
||||
c.execute('INSERT INTO llama_versions (version_tag, description, release_date, is_active, sort_order) VALUES (?, ?, ?, ?, ?)',
|
||||
('b6310', 'llama.cpp build 6310 (2025年中)', '2025-06-01', 1, 1))
|
||||
v1_id = c.lastrowid
|
||||
|
||||
# Version 2: b10068 (latest 2026)
|
||||
c.execute('''INSERT INTO llama_versions (version_tag, description, release_date, is_active, sort_order)
|
||||
VALUES (?, ?, ?, ?, ?)''',
|
||||
c.execute('INSERT INTO llama_versions (version_tag, description, release_date, is_active, sort_order) VALUES (?, ?, ?, ?, ?)',
|
||||
('b10068', 'llama.cpp build 10068 (2026年最新)', '2026-07-01', 1, 2))
|
||||
v2_id = c.lastrowid
|
||||
|
||||
# ===== Default binaries for each version =====
|
||||
for vid in [v1_id, v2_id]:
|
||||
c.execute('INSERT OR IGNORE INTO version_binaries (version_id, name, description, sort_order) VALUES (?, ?, ?, ?)',
|
||||
(vid, 'llama-server', 'HTTP API 服务器', 1))
|
||||
c.execute('INSERT OR IGNORE INTO version_binaries (version_id, name, description, sort_order) VALUES (?, ?, ?, ?)',
|
||||
(vid, 'llama-cli', '命令行交互', 2))
|
||||
c.execute('INSERT OR IGNORE INTO version_binaries (version_id, name, description, sort_order) VALUES (?, ?, ?, ?)',
|
||||
(vid, 'llama-bench', '性能基准测试', 3))
|
||||
|
||||
# ===== Parameters for b6310 =====
|
||||
params_b6310 = [
|
||||
# === Common params ===
|
||||
# Important
|
||||
("model", "-m", "--model", "模型文件路径", "common", "string", "", None, None, None, None, None, 1, 0, 1),
|
||||
("ctx_size", "-c", "--ctx-size", "上下文窗口大小 (0=从模型加载)", "common", "number", "0", None, 0, 131072, 512, "tokens", 1, 1, 2),
|
||||
("n_gpu_layers", "-ngl", "--n-gpu-layers", "存储在VRAM中的最大层数 (0=不卸载, -1=全部)", "common", "number", "0", None, -1, 999, 1, "layers", 1, 1, 3),
|
||||
@@ -158,8 +311,6 @@ def insert_default_data(conn):
|
||||
("batch_size", "-b", "--batch-size", "逻辑最大批处理大小", "common", "number", "2048", None, 1, 8192, 128, "tokens", 1, 1, 5),
|
||||
("ubatch_size", "-ub", "--ubatch-size", "物理最大批处理大小", "common", "number", "512", None, 1, 4096, 64, "tokens", 0, 1, 6),
|
||||
("flash_attn", "-fa", "--flash-attn", "Flash Attention (on/off)", "common", "select", "off", json.dumps(["on", "off"]), None, None, None, None, 1, 1, 7),
|
||||
|
||||
# Less important
|
||||
("predict", "-n", "--predict", "要预测的token数量 (-1=无限)", "common", "number", "-1", None, -1, 999999, 1, "tokens", 0, 0, 10),
|
||||
("keep", "", "--keep", "从初始提示中保留的token数", "common", "number", "0", None, -1, 999999, 1, "tokens", 0, 0, 11),
|
||||
("cache_type_k", "-ctk", "--cache-type-k", "KV缓存K的数据类型", "common", "select", "f16", json.dumps(["f32", "f16", "bf16", "q8_0", "q4_0", "q4_1", "iq4_nl", "q5_0", "q5_1"]), None, None, None, None, 0, 1, 12),
|
||||
@@ -182,7 +333,6 @@ def insert_default_data(conn):
|
||||
("kv_offload", "-kvo", "--kv-offload", "启用KV缓存卸载", "common", "boolean", "true", None, None, None, None, None, 0, 0, 29),
|
||||
("device", "-dev", "--device", "用于卸载的设备列表", "common", "string", "", None, None, None, None, None, 0, 0, 30),
|
||||
("override_tensor", "-ot", "--override-tensor", "覆盖张量缓冲区类型", "common", "string", "", None, None, None, None, None, 0, 0, 31),
|
||||
|
||||
# === Sampling params ===
|
||||
("temperature", "", "--temp", "温度 (创造力)", "sampling", "number", "0.8", None, 0.01, 2.0, 0.05, None, 1, 0, 1),
|
||||
("seed", "-s", "--seed", "随机种子 (-1=随机)", "sampling", "number", "-1", None, -1, 999999, 1, None, 0, 0, 2),
|
||||
@@ -199,7 +349,6 @@ def insert_default_data(conn):
|
||||
("mirostat_ent", "", "--mirostat-ent", "Mirostat目标熵", "sampling", "number", "5.0", None, 1.0, 10.0, 0.1, None, 0, 0, 13),
|
||||
("ignore_eos", "", "--ignore-eos", "忽略结束流token", "sampling", "boolean", "false", None, None, None, None, None, 0, 0, 14),
|
||||
("samplers", "", "--samplers", "采样器序列", "sampling", "string", "penalties;dry;top_n_sigma;top_k;typ_p;top_p;min_p;xtc;temperature", None, None, None, None, None, 0, 0, 15),
|
||||
|
||||
# === Server params ===
|
||||
("port", "", "--port", "服务器监听端口", "server", "number", "8080", None, 1, 65535, 1, None, 1, 0, 1),
|
||||
("host", "", "--host", "服务器监听地址", "server", "string", "0.0.0.0", None, None, None, None, None, 1, 0, 2),
|
||||
@@ -210,24 +359,20 @@ def insert_default_data(conn):
|
||||
("warmup", "", "--warmup", "预热运行", "server", "boolean", "true", None, None, None, None, None, 0, 0, 7),
|
||||
("pooling", "", "--pooling", "嵌入池化类型", "server", "select", "", json.dumps(["", "none", "mean", "cls", "last", "rank"]), None, None, None, None, 0, 0, 8),
|
||||
("mmproj", "-mm", "--mmproj", "多模态投影文件路径", "server", "string", "", None, None, None, None, None, 0, 0, 9),
|
||||
|
||||
# === Model download ===
|
||||
("hf_repo", "-hf", "--hf-repo", "Hugging Face模型仓库", "model_source", "string", "", None, None, None, None, None, 0, 0, 1),
|
||||
("hf_file", "-hff", "--hf-file", "Hugging Face模型文件", "model_source", "string", "", None, None, None, None, None, 0, 0, 2),
|
||||
("hf_token", "-hft", "--hf-token", "Hugging Face访问令牌", "model_source", "string", "", None, None, None, None, None, 0, 0, 3),
|
||||
("model_url", "-mu", "--model-url", "模型下载URL", "model_source", "string", "", None, None, None, None, None, 0, 0, 4),
|
||||
("docker_repo", "-dr", "--docker-repo", "Docker Hub模型仓库", "model_source", "string", "", None, None, None, None, None, 0, 0, 5),
|
||||
|
||||
# === Logging ===
|
||||
("verbose", "-v", "--verbose", "详细日志输出", "logging", "boolean", "false", None, None, None, None, None, 0, 0, 1),
|
||||
("log_file", "", "--log-file", "日志文件路径", "logging", "string", "", None, None, None, None, None, 0, 0, 2),
|
||||
("log_colors", "", "--log-colors", "彩色日志", "logging", "select", "auto", json.dumps(["on", "off", "auto"]), None, None, None, None, 0, 0, 3),
|
||||
("log_verbosity", "-lv", "--verbosity", "日志详细级别", "logging", "select", "3", json.dumps(["0", "1", "2", "3", "4", "5"]), None, None, None, None, 0, 0, 4),
|
||||
|
||||
# === LoRA ===
|
||||
("lora", "", "--lora", "LoRA适配器路径", "lora", "string", "", None, None, None, None, None, 0, 0, 1),
|
||||
("lora_scaled", "", "--lora-scaled", "带缩放的LoRA适配器", "lora", "string", "", None, None, None, None, None, 0, 0, 2),
|
||||
|
||||
# === Advanced ===
|
||||
("cpu_mask", "-C", "--cpu-mask", "CPU亲和性掩码", "advanced", "string", "", None, None, None, None, None, 0, 0, 1),
|
||||
("cpu_range", "-Cr", "--cpu-range", "CPU亲和性范围", "advanced", "string", "", None, None, None, None, None, 0, 0, 2),
|
||||
@@ -243,32 +388,24 @@ def insert_default_data(conn):
|
||||
|
||||
for p in params_b6310:
|
||||
c.execute('''INSERT INTO params
|
||||
(version_id, param_key, short_flag, long_flag, description, category, param_type,
|
||||
default_value, options, min_value, max_value, step, unit, is_important,
|
||||
affects_vram, sort_order)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''',
|
||||
(version_id, binary_id, param_key, short_flag, long_flag, description, category, param_type,
|
||||
default_value, options, min_value, max_value, step, unit, is_important, affects_vram, sort_order)
|
||||
VALUES (?, NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''',
|
||||
(v1_id,) + p)
|
||||
|
||||
# ===== Parameters for b10068 (latest) =====
|
||||
params_b10068 = list(params_b6310) # Copy all from b6310
|
||||
|
||||
# Modify defaults that changed
|
||||
# n_gpu_layers default changed from "0" to "auto"
|
||||
params_b10068 = list(params_b6310)
|
||||
params_b10068 = [(p[0], p[1], p[2], p[3], p[4], p[5],
|
||||
"auto" if p[0] == "n_gpu_layers" else p[6],
|
||||
p[7], p[8], p[9], p[10], p[11], p[12], p[13], p[14])
|
||||
if p[0] == "n_gpu_layers" else p for p in params_b10068]
|
||||
|
||||
# flash_attn: default changed from "off" to "auto", and options include "auto"
|
||||
params_b10068 = [(p[0], p[1], p[2], p[3], p[4], p[5],
|
||||
"auto" if p[0] == "flash_attn" else p[6],
|
||||
'["on", "off", "auto"]' if p[0] == "flash_attn" else p[7],
|
||||
p[8], p[9], p[10], p[11], p[12], p[13], p[14])
|
||||
if p[0] == "flash_attn" else p for p in params_b10068]
|
||||
|
||||
# Add new parameters for b10068
|
||||
new_params_b10068 = [
|
||||
# New in b10068
|
||||
("fit", "-fit", "--fit", "自动调整参数以适应设备内存", "common", "select", "on", json.dumps(["on", "off"]), None, None, None, None, 1, 1, 8),
|
||||
("fit_target", "-fitt", "--fit-target", "每个设备的目标余量(MiB)", "common", "string", "1024", None, None, None, None, "MiB", 0, 1, 9),
|
||||
("fit_ctx", "-fitc", "--fit-ctx", "fit选项可设置的最小ctx大小", "common", "number", "4096", None, 1024, 131072, 512, "tokens", 0, 1, 10),
|
||||
@@ -304,76 +441,70 @@ def insert_default_data(conn):
|
||||
]
|
||||
|
||||
all_params_b10068 = list(params_b10068) + list(new_params_b10068)
|
||||
|
||||
for p in all_params_b10068:
|
||||
c.execute('''INSERT INTO params
|
||||
(version_id, param_key, short_flag, long_flag, description, category, param_type,
|
||||
default_value, options, min_value, max_value, step, unit, is_important,
|
||||
affects_vram, sort_order)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''',
|
||||
(version_id, binary_id, param_key, short_flag, long_flag, description, category, param_type,
|
||||
default_value, options, min_value, max_value, step, unit, is_important, affects_vram, sort_order)
|
||||
VALUES (?, NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''',
|
||||
(v2_id,) + p)
|
||||
|
||||
# ===== Default models =====
|
||||
# (base_model, name, size_gb, layers, embd, kv_heads, head_dim, attention_heads, default_ctx, quant, description, sort_order)
|
||||
# ===== Default models (one entry per base_model) =====
|
||||
# (base_model, name, model_type, num_experts, layers, embd, kv_heads, head_dim, attention_heads, default_ctx, description, sort_order)
|
||||
default_models = [
|
||||
# Llama-3-8B-Instruct (ctx 8192)
|
||||
("Llama-3-8B-Instruct", "Llama-3-8B-Instruct (Q4_K_M)", 4.9, 32, 4096, 8, 128, 32, 8192, "Q4_K_M", "Meta Llama 3 8B Instruct, Q4_K_M 量化", 1),
|
||||
("Llama-3-8B-Instruct", "Llama-3-8B-Instruct (Q8_0)", 8.5, 32, 4096, 8, 128, 32, 8192, "Q8_0", "Meta Llama 3 8B Instruct, Q8_0 量化", 2),
|
||||
("Llama-3-8B-Instruct", "Llama-3-8B-Instruct (FP16)", 15.5, 32, 4096, 8, 128, 32, 8192, "FP16", "Meta Llama 3 8B Instruct, FP16", 3),
|
||||
# Llama-3-70B-Instruct (ctx 8192)
|
||||
("Llama-3-70B-Instruct", "Llama-3-70B-Instruct (Q4_K_M)", 38.5, 80, 8192, 8, 128, 64, 8192, "Q4_K_M", "Meta Llama 3 70B Instruct, Q4_K_M 量化", 4),
|
||||
("Llama-3-70B-Instruct", "Llama-3-70B-Instruct (Q8_0)", 74.0, 80, 8192, 8, 128, 64, 8192, "Q8_0", "Meta Llama 3 70B Instruct, Q8_0 量化", 5),
|
||||
("Llama-3-70B-Instruct", "Llama-3-70B-Instruct (FP16)", 138.0, 80, 8192, 8, 128, 64, 8192, "FP16", "Meta Llama 3 70B Instruct, FP16", 6),
|
||||
# Llama-3.1-8B-Instruct (ctx 131072)
|
||||
("Llama-3.1-8B-Instruct", "Llama-3.1-8B-Instruct (Q4_K_M)", 4.9, 32, 4096, 8, 128, 32, 131072, "Q4_K_M", "Meta Llama 3.1 8B Instruct, Q4_K_M 量化", 7),
|
||||
("Llama-3.1-8B-Instruct", "Llama-3.1-8B-Instruct (Q8_0)", 8.5, 32, 4096, 8, 128, 32, 131072, "Q8_0", "Meta Llama 3.1 8B Instruct, Q8_0 量化", 8),
|
||||
# Llama-3.1-70B-Instruct (ctx 131072)
|
||||
("Llama-3.1-70B-Instruct", "Llama-3.1-70B-Instruct (Q4_K_M)", 38.5, 80, 8192, 8, 128, 64, 131072, "Q4_K_M", "Meta Llama 3.1 70B Instruct, Q4_K_M 量化", 9),
|
||||
("Llama-3.1-70B-Instruct", "Llama-3.1-70B-Instruct (Q8_0)", 74.0, 80, 8192, 8, 128, 64, 131072, "Q8_0", "Meta Llama 3.1 70B Instruct, Q8_0 量化", 10),
|
||||
# Llama-3.1-405B-Instruct (ctx 131072)
|
||||
("Llama-3.1-405B-Instruct", "Llama-3.1-405B-Instruct (Q4_K_M)", 226.0, 126, 16384, 8, 128, 128, 131072, "Q4_K_M", "Meta Llama 3.1 405B Instruct, Q4_K_M 量化", 11),
|
||||
# Qwen2.5-7B-Instruct (ctx 32768)
|
||||
("Qwen2.5-7B-Instruct", "Qwen2.5-7B-Instruct (Q4_K_M)", 4.7, 28, 3584, 4, 128, 28, 32768, "Q4_K_M", "Qwen2.5 7B Instruct, Q4_K_M 量化", 12),
|
||||
("Qwen2.5-7B-Instruct", "Qwen2.5-7B-Instruct (Q8_0)", 7.6, 28, 3584, 4, 128, 28, 32768, "Q8_0", "Qwen2.5 7B Instruct, Q8_0 量化", 13),
|
||||
# Qwen2.5-14B-Instruct (ctx 32768)
|
||||
("Qwen2.5-14B-Instruct", "Qwen2.5-14B-Instruct (Q4_K_M)", 8.7, 40, 5120, 8, 128, 40, 32768, "Q4_K_M", "Qwen2.5 14B Instruct, Q4_K_M 量化", 14),
|
||||
# Qwen2.5-32B-Instruct (ctx 32768)
|
||||
("Qwen2.5-32B-Instruct", "Qwen2.5-32B-Instruct (Q4_K_M)", 19.5, 64, 5120, 8, 128, 64, 32768, "Q4_K_M", "Qwen2.5 32B Instruct, Q4_K_M 量化", 15),
|
||||
("Qwen2.5-32B-Instruct", "Qwen2.5-32B-Instruct (Q8_0)", 32.0, 64, 5120, 8, 128, 64, 32768, "Q8_0", "Qwen2.5 32B Instruct, Q8_0 量化", 16),
|
||||
# Qwen2.5-72B-Instruct (ctx 32768)
|
||||
("Qwen2.5-72B-Instruct", "Qwen2.5-72B-Instruct (Q4_K_M)", 42.0, 80, 8192, 8, 128, 64, 32768, "Q4_K_M", "Qwen2.5 72B Instruct, Q4_K_M 量化", 17),
|
||||
("Qwen2.5-72B-Instruct", "Qwen2.5-72B-Instruct (Q8_0)", 75.0, 80, 8192, 8, 128, 64, 32768, "Q8_0", "Qwen2.5 72B Instruct, Q8_0 量化", 18),
|
||||
# DeepSeek-V2-Chat (ctx 4096)
|
||||
("DeepSeek-V2-Chat", "DeepSeek-V2-Chat (Q4_K_M)", 23.0, 60, 5120, 8, 128, 60, 4096, "Q4_K_M", "DeepSeek V2 Chat, Q4_K_M 量化", 19),
|
||||
# DeepSeek-V2.5-Chat (ctx 4096)
|
||||
("DeepSeek-V2.5-Chat", "DeepSeek-V2.5-Chat (Q4_K_M)", 23.0, 60, 5120, 8, 128, 60, 4096, "Q4_K_M", "DeepSeek V2.5 Chat, Q4_K_M 量化", 20),
|
||||
# DeepSeek-R1-Distill-Qwen-32B (ctx 131072)
|
||||
("DeepSeek-R1-Distill-Qwen-32B", "DeepSeek-R1-Distill-Qwen-32B (Q4_K_M)", 19.5, 64, 5120, 8, 128, 64, 131072, "Q4_K_M", "DeepSeek R1 Distill Qwen 32B, Q4_K_M 量化", 21),
|
||||
("DeepSeek-R1-Distill-Qwen-32B", "DeepSeek-R1-Distill-Qwen-32B (Q8_0)", 32.0, 64, 5120, 8, 128, 64, 131072, "Q8_0", "DeepSeek R1 Distill Qwen 32B, Q8_0 量化", 22),
|
||||
# DeepSeek-R1-Distill-Llama-70B (ctx 131072)
|
||||
("DeepSeek-R1-Distill-Llama-70B", "DeepSeek-R1-Distill-Llama-70B (Q4_K_M)", 42.0, 80, 8192, 8, 128, 64, 131072, "Q4_K_M", "DeepSeek R1 Distill Llama 70B, Q4_K_M 量化", 23),
|
||||
("DeepSeek-R1-Distill-Llama-70B", "DeepSeek-R1-Distill-Llama-70B (Q8_0)", 75.0, 80, 8192, 8, 128, 64, 131072, "Q8_0", "DeepSeek R1 Distill Llama 70B, Q8_0 量化", 24),
|
||||
# Mistral-7B-Instruct-v0.3 (ctx 32768)
|
||||
("Mistral-7B-Instruct-v0.3", "Mistral-7B-Instruct-v0.3 (Q4_K_M)", 4.4, 32, 4096, 8, 128, 32, 32768, "Q4_K_M", "Mistral 7B Instruct v0.3, Q4_K_M 量化", 25),
|
||||
("Mistral-7B-Instruct-v0.3", "Mistral-7B-Instruct-v0.3 (Q8_0)", 7.5, 32, 4096, 8, 128, 32, 32768, "Q8_0", "Mistral 7B Instruct v0.3, Q8_0 量化", 26),
|
||||
# Mixtral-8x7B-Instruct (ctx 32768)
|
||||
("Mixtral-8x7B-Instruct", "Mixtral-8x7B-Instruct (Q4_K_M)", 26.0, 32, 4096, 8, 128, 32, 32768, "Q4_K_M", "Mixtral 8x7B Instruct, Q4_K_M 量化", 27),
|
||||
# Gemma-2-9B-It (ctx 8192)
|
||||
("Gemma-2-9B-It", "Gemma-2-9B-It (Q4_K_M)", 5.4, 42, 3584, 4, 256, 14, 8192, "Q4_K_M", "Google Gemma 2 9B It, Q4_K_M 量化", 28),
|
||||
# Gemma-2-27B-It (ctx 8192)
|
||||
("Gemma-2-27B-It", "Gemma-2-27B-It (Q4_K_M)", 16.5, 46, 4608, 4, 128, 36, 8192, "Q4_K_M", "Google Gemma 2 27B It, Q4_K_M 量化", 29),
|
||||
# Phi-3-Mini-4K-Instruct (ctx 4096)
|
||||
("Phi-3-Mini-4K-Instruct", "Phi-3-Mini-4K-Instruct (Q4_K_M)", 2.5, 32, 3072, 32, 96, 32, 4096, "Q4_K_M", "Microsoft Phi-3 Mini 4K Instruct, Q4_K_M 量化", 30),
|
||||
# Phi-3-Medium-14B-Instruct (ctx 14336)
|
||||
("Phi-3-Medium-14B-Instruct", "Phi-3-Medium-14B-Instruct (Q4_K_M)", 8.4, 40, 5120, 10, 128, 40, 14336, "Q4_K_M", "Microsoft Phi-3 Medium 14B Instruct, Q4_K_M 量化", 31),
|
||||
# GLM-4-9B-Chat (ctx 131072)
|
||||
("GLM-4-9B-Chat", "GLM-4-9B-Chat (Q4_K_M)", 5.5, 40, 4096, 4, 128, 40, 131072, "Q4_K_M", "Zhipu GLM-4 9B Chat, Q4_K_M 量化", 32),
|
||||
("GLM-4-9B-Chat", "GLM-4-9B-Chat (Q8_0)", 9.0, 40, 4096, 4, 128, 40, 131072, "Q8_0", "Zhipu GLM-4 9B Chat, Q8_0 量化", 33),
|
||||
("Llama-3-8B-Instruct", "Llama-3-8B-Instruct", "dense", 0, 32, 4096, 8, 128, 32, 8192, "Meta Llama 3 8B Instruct", 1),
|
||||
("Llama-3-70B-Instruct", "Llama-3-70B-Instruct", "dense", 0, 80, 8192, 8, 128, 64, 8192, "Meta Llama 3 70B Instruct", 2),
|
||||
("Llama-3.1-8B-Instruct", "Llama-3.1-8B-Instruct", "dense", 0, 32, 4096, 8, 128, 32, 131072, "Meta Llama 3.1 8B Instruct", 3),
|
||||
("Llama-3.1-70B-Instruct", "Llama-3.1-70B-Instruct", "dense", 0, 80, 8192, 8, 128, 64, 131072, "Meta Llama 3.1 70B Instruct", 4),
|
||||
("Llama-3.1-405B-Instruct", "Llama-3.1-405B-Instruct", "dense", 0, 126, 16384, 8, 128, 128, 131072, "Meta Llama 3.1 405B Instruct", 5),
|
||||
("Qwen2.5-7B-Instruct", "Qwen2.5-7B-Instruct", "dense", 0, 28, 3584, 4, 128, 28, 32768, "Qwen2.5 7B Instruct", 6),
|
||||
("Qwen2.5-14B-Instruct", "Qwen2.5-14B-Instruct", "dense", 0, 40, 5120, 8, 128, 40, 32768, "Qwen2.5 14B Instruct", 7),
|
||||
("Qwen2.5-32B-Instruct", "Qwen2.5-32B-Instruct", "dense", 0, 64, 5120, 8, 128, 64, 32768, "Qwen2.5 32B Instruct", 8),
|
||||
("Qwen2.5-72B-Instruct", "Qwen2.5-72B-Instruct", "dense", 0, 80, 8192, 8, 128, 64, 32768, "Qwen2.5 72B Instruct", 9),
|
||||
("DeepSeek-V2-Chat", "DeepSeek-V2-Chat", "moe", 160, 60, 5120, 8, 128, 60, 4096, "DeepSeek V2 Chat (MoE, 160 experts)", 10),
|
||||
("DeepSeek-V2.5-Chat", "DeepSeek-V2.5-Chat", "moe", 160, 60, 5120, 8, 128, 60, 4096, "DeepSeek V2.5 Chat (MoE, 160 experts)", 11),
|
||||
("DeepSeek-R1-Distill-Qwen-32B", "DeepSeek-R1-Distill-Qwen-32B", "dense", 0, 64, 5120, 8, 128, 64, 131072, "DeepSeek R1 Distill Qwen 32B", 12),
|
||||
("DeepSeek-R1-Distill-Llama-70B", "DeepSeek-R1-Distill-Llama-70B", "dense", 0, 80, 8192, 8, 128, 64, 131072, "DeepSeek R1 Distill Llama 70B", 13),
|
||||
("Mistral-7B-Instruct-v0.3", "Mistral-7B-Instruct-v0.3", "dense", 0, 32, 4096, 8, 128, 32, 32768, "Mistral 7B Instruct v0.3", 14),
|
||||
("Mixtral-8x7B-Instruct", "Mixtral-8x7B-Instruct", "moe", 8, 32, 4096, 8, 128, 32, 32768, "Mixtral 8x7B Instruct (MoE, 8 experts)", 15),
|
||||
("Gemma-2-9B-It", "Gemma-2-9B-It", "dense", 0, 42, 3584, 4, 256, 14, 8192, "Google Gemma 2 9B It", 16),
|
||||
("Gemma-2-27B-It", "Gemma-2-27B-It", "dense", 0, 46, 4608, 4, 128, 36, 8192, "Google Gemma 2 27B It", 17),
|
||||
("Phi-3-Mini-4K-Instruct", "Phi-3-Mini-4K-Instruct", "dense", 0, 32, 3072, 32, 96, 32, 4096, "Microsoft Phi-3 Mini 4K Instruct", 18),
|
||||
("Phi-3-Medium-14B-Instruct", "Phi-3-Medium-14B-Instruct", "dense", 0, 40, 5120, 10, 128, 40, 14336, "Microsoft Phi-3 Medium 14B Instruct", 19),
|
||||
("GLM-4-9B-Chat", "GLM-4-9B-Chat", "dense", 0, 40, 4096, 4, 128, 40, 131072, "Zhipu GLM-4 9B Chat", 20),
|
||||
]
|
||||
|
||||
# Quant variants: { base_model: [(quant_type, size_gb, sort_order), ...] }
|
||||
default_model_quants = {
|
||||
"Llama-3-8B-Instruct": [("Q4_K_M", 4.9, 1), ("Q8_0", 8.5, 2), ("FP16", 15.5, 3)],
|
||||
"Llama-3-70B-Instruct": [("Q4_K_M", 38.5, 1), ("Q8_0", 74.0, 2), ("FP16", 138.0, 3)],
|
||||
"Llama-3.1-8B-Instruct": [("Q4_K_M", 4.9, 1), ("Q8_0", 8.5, 2)],
|
||||
"Llama-3.1-70B-Instruct": [("Q4_K_M", 38.5, 1), ("Q8_0", 74.0, 2)],
|
||||
"Llama-3.1-405B-Instruct": [("Q4_K_M", 226.0, 1)],
|
||||
"Qwen2.5-7B-Instruct": [("Q4_K_M", 4.7, 1), ("Q8_0", 7.6, 2)],
|
||||
"Qwen2.5-14B-Instruct": [("Q4_K_M", 8.7, 1)],
|
||||
"Qwen2.5-32B-Instruct": [("Q4_K_M", 19.5, 1), ("Q8_0", 32.0, 2)],
|
||||
"Qwen2.5-72B-Instruct": [("Q4_K_M", 42.0, 1), ("Q8_0", 75.0, 2)],
|
||||
"DeepSeek-V2-Chat": [("Q4_K_M", 23.0, 1)],
|
||||
"DeepSeek-V2.5-Chat": [("Q4_K_M", 23.0, 1)],
|
||||
"DeepSeek-R1-Distill-Qwen-32B": [("Q4_K_M", 19.5, 1), ("Q8_0", 32.0, 2)],
|
||||
"DeepSeek-R1-Distill-Llama-70B": [("Q4_K_M", 42.0, 1), ("Q8_0", 75.0, 2)],
|
||||
"Mistral-7B-Instruct-v0.3": [("Q4_K_M", 4.4, 1), ("Q8_0", 7.5, 2)],
|
||||
"Mixtral-8x7B-Instruct": [("Q4_K_M", 26.0, 1)],
|
||||
"Gemma-2-9B-It": [("Q4_K_M", 5.4, 1)],
|
||||
"Gemma-2-27B-It": [("Q4_K_M", 16.5, 1)],
|
||||
"Phi-3-Mini-4K-Instruct": [("Q4_K_M", 2.5, 1)],
|
||||
"Phi-3-Medium-14B-Instruct": [("Q4_K_M", 8.4, 1)],
|
||||
"GLM-4-9B-Chat": [("Q4_K_M", 5.5, 1), ("Q8_0", 9.0, 2)],
|
||||
}
|
||||
|
||||
for m in default_models:
|
||||
c.execute('''INSERT INTO models (base_model, name, size_gb, layers, embd, kv_heads, head_dim, attention_heads, default_ctx, quant, description, sort_order)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''', m)
|
||||
c.execute('''INSERT INTO models (base_model, name, model_type, num_experts, size_gb, quant, layers, embd, kv_heads, head_dim, attention_heads, default_ctx, description, sort_order)
|
||||
VALUES (?, ?, ?, ?, 0, '', ?, ?, ?, ?, ?, ?, ?, ?)''', m)
|
||||
model_id = c.lastrowid
|
||||
base = m[0]
|
||||
for q in default_model_quants.get(base, []):
|
||||
c.execute('INSERT INTO model_quants (model_id, quant_type, size_gb, sort_order) VALUES (?, ?, ?, ?)',
|
||||
(model_id, q[0], q[1], q[2]))
|
||||
|
||||
# ===== Default settings =====
|
||||
c.execute("INSERT OR IGNORE INTO settings (key, value) VALUES (?, ?)", ("admin_password", "admin123"))
|
||||
@@ -381,7 +512,6 @@ def insert_default_data(conn):
|
||||
c.execute("INSERT OR IGNORE INTO settings (key, value) VALUES (?, ?)", ("default_version", "b10068"))
|
||||
c.execute("INSERT OR IGNORE INTO settings (key, value) VALUES (?, ?)", ("default_mode", "gpu"))
|
||||
c.execute("INSERT OR IGNORE INTO settings (key, value) VALUES (?, ?)", ("default_quant", "Q4_K_M"))
|
||||
# LLM API settings for natural language parsing
|
||||
c.execute("INSERT OR IGNORE INTO settings (key, value) VALUES (?, ?)", ("llm_enabled", "false"))
|
||||
c.execute("INSERT OR IGNORE INTO settings (key, value) VALUES (?, ?)", ("llm_api_url", ""))
|
||||
c.execute("INSERT OR IGNORE INTO settings (key, value) VALUES (?, ?)", ("llm_api_key", ""))
|
||||
|
||||
Reference in New Issue
Block a user