feat: llama.cpp 命令生成器 v1.0.0
- 支持多版本 llama.cpp 参数 (b6310, b10068) - GPU 模式 / GPU+CPU 模式 - 多 GPU 支持 (最多4张) - 实时显存/内存估算 - 自然语言解析生成命令 - 参数分级显示 (重要/隐藏) - 仅输出非默认值参数 - 后台管理 (GPU/版本/参数 CRUD) - 模型预设 (7B-70B)
This commit is contained in:
@@ -0,0 +1,741 @@
|
||||
#!/usr/bin/env python3
|
||||
"""llama.cpp command generator - main Flask application."""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import re
|
||||
import math
|
||||
from flask import Flask, request, jsonify, send_from_directory
|
||||
from flask_cors import CORS
|
||||
from db import get_db, init_db, DB_PATH
|
||||
|
||||
app = Flask(__name__, static_folder='static', static_url_path='')
|
||||
CORS(app)
|
||||
|
||||
|
||||
# ==================== Helper Functions ====================
|
||||
|
||||
def parse_param_value(param_type, value):
|
||||
"""Parse parameter value based on its type."""
|
||||
if value is None or value == '':
|
||||
return None
|
||||
if param_type == 'number':
|
||||
try:
|
||||
if '.' in str(value):
|
||||
return float(value)
|
||||
return int(value)
|
||||
except (ValueError, TypeError):
|
||||
return value
|
||||
elif param_type == 'boolean':
|
||||
return str(value).lower() in ('true', '1', 'yes', 'on')
|
||||
return value
|
||||
|
||||
|
||||
def format_param_value(param_type, value):
|
||||
"""Format parameter value for display."""
|
||||
if param_type == 'number':
|
||||
try:
|
||||
f = float(value)
|
||||
if f == int(f):
|
||||
return str(int(f))
|
||||
return str(f)
|
||||
except (ValueError, TypeError):
|
||||
return str(value)
|
||||
return str(value)
|
||||
|
||||
|
||||
def get_flag_for_param(param):
|
||||
"""Get the flag string for a parameter (prefers short flag)."""
|
||||
if param['short_flag']:
|
||||
return param['short_flag']
|
||||
return param['long_flag']
|
||||
|
||||
|
||||
def get_kv_cache_bytes_per_element(cache_type):
|
||||
"""Get bytes per element for KV cache type."""
|
||||
cache_type_map = {
|
||||
'f32': 4,
|
||||
'f16': 2,
|
||||
'bf16': 2,
|
||||
'q8_0': 1,
|
||||
'q4_0': 0.5,
|
||||
'q4_1': 0.5625,
|
||||
'iq4_nl': 0.5,
|
||||
'q5_0': 0.625,
|
||||
'q5_1': 0.6875,
|
||||
}
|
||||
return cache_type_map.get(cache_type, 2)
|
||||
|
||||
|
||||
def estimate_vram(params_dict, gpus, version_params):
|
||||
"""
|
||||
Estimate VRAM usage based on parameters.
|
||||
Returns dict with breakdown.
|
||||
"""
|
||||
# Model parameters (user configurable hints)
|
||||
model_size_gb = float(params_dict.get('_model_size_gb', 0) or 0)
|
||||
model_layers = int(params_dict.get('_model_layers', 0) or 0)
|
||||
model_embd = int(params_dict.get('_model_embd', 0) or 0)
|
||||
model_heads = int(params_dict.get('_model_heads', 0) or 0)
|
||||
model_kv_heads = int(params_dict.get('_model_kv_heads', 0) or 0)
|
||||
model_head_dim = int(params_dict.get('_model_head_dim', 0) or 0)
|
||||
|
||||
# If model_layers not specified, try to estimate from model size
|
||||
if model_layers == 0 and model_size_gb > 0:
|
||||
# Rough estimate: ~0.3GB per layer for 7B, scale accordingly
|
||||
model_layers = max(1, int(model_size_gb / 0.4))
|
||||
|
||||
# n_gpu_layers
|
||||
ngl_raw = params_dict.get('n_gpu_layers', 'auto')
|
||||
if ngl_raw in ('auto', 'all', '-1'):
|
||||
ngl = model_layers if model_layers > 0 else 32
|
||||
if ngl_raw == '0':
|
||||
ngl = 0
|
||||
else:
|
||||
try:
|
||||
ngl = int(ngl_raw)
|
||||
except (ValueError, TypeError):
|
||||
ngl = 0
|
||||
|
||||
# Context size
|
||||
ctx_size = int(params_dict.get('ctx_size', 0) or 0)
|
||||
if ctx_size == 0:
|
||||
ctx_size = 4096 # default assumption
|
||||
|
||||
# Batch size
|
||||
batch_size = int(params_dict.get('batch_size', 2048) or 2048)
|
||||
ubatch_size = int(params_dict.get('ubatch_size', 512) or 512)
|
||||
|
||||
# KV cache types
|
||||
ctk = params_dict.get('cache_type_k', 'f16')
|
||||
ctv = params_dict.get('cache_type_v', 'f16')
|
||||
k_bytes = get_kv_cache_bytes_per_element(ctk)
|
||||
v_bytes = get_kv_cache_bytes_per_element(ctv)
|
||||
|
||||
# Parallel slots
|
||||
parallel = int(params_dict.get('parallel', 1) or 1)
|
||||
if parallel < 1:
|
||||
parallel = 1
|
||||
|
||||
# Flash attention
|
||||
fa = params_dict.get('flash_attn', 'auto')
|
||||
fa_enabled = fa in ('on', 'auto')
|
||||
|
||||
# === Calculate VRAM components ===
|
||||
total_vram = sum(g['vram_mb'] for g in gpus) if gpus else 0
|
||||
max_single_vram = max((g['vram_mb'] for g in gpus), default=0)
|
||||
|
||||
# 1. Model weights in VRAM
|
||||
if model_size_gb > 0 and model_layers > 0:
|
||||
ratio = min(1.0, ngl / model_layers) if ngl > 0 else 0
|
||||
weights_vram_mb = model_size_gb * 1024 * ratio
|
||||
else:
|
||||
# Fallback estimate: ~1GB per 1B params at Q4
|
||||
weights_vram_mb = 0
|
||||
|
||||
# 2. KV cache
|
||||
# KV cache per layer = 2 (K and V) * n_kv_heads * head_dim * ctx_size * bytes_per_element
|
||||
if model_kv_heads > 0 and model_head_dim > 0:
|
||||
kv_per_layer_bytes = 2 * model_kv_heads * model_head_dim * ctx_size
|
||||
kv_total_bytes = kv_per_layer_bytes * model_layers * parallel
|
||||
elif model_embd > 0:
|
||||
# Fallback: use embedding dim
|
||||
kv_per_layer_bytes = 2 * model_embd * ctx_size
|
||||
kv_total_bytes = kv_per_layer_bytes * model_layers * parallel
|
||||
else:
|
||||
# Very rough estimate: ~0.5MB per layer per 1K context at f16
|
||||
kv_total_bytes = 0
|
||||
if model_layers > 0:
|
||||
kv_total_bytes = int(0.5 * 1024 * 1024 * model_layers * (ctx_size / 1024) * parallel)
|
||||
|
||||
kv_vram_mb = (kv_total_bytes * (k_bytes + v_bytes) / 2) / (1024 * 1024)
|
||||
|
||||
# If ngl < model_layers, only ngl layers' KV is on GPU
|
||||
if model_layers > 0 and ngl < model_layers:
|
||||
kv_vram_mb = kv_vram_mb * (ngl / model_layers)
|
||||
|
||||
# 3. Compute buffer / overhead
|
||||
# Rough: batch_size * model_embd * 4 bytes * some factor
|
||||
compute_mb = 0
|
||||
if model_embd > 0:
|
||||
compute_mb = (batch_size * model_embd * 4 * 2) / (1024 * 1024) # logits buffer
|
||||
compute_mb = max(compute_mb, 100) # minimum overhead
|
||||
|
||||
# CUDA context overhead (~300-500MB per GPU)
|
||||
cuda_overhead_mb = len(gpus) * 400 if gpus else 0
|
||||
|
||||
# Total
|
||||
total_estimate = weights_vram_mb + kv_vram_mb + compute_mb + cuda_overhead_mb
|
||||
|
||||
return {
|
||||
'total_mb': round(total_estimate, 1),
|
||||
'total_gb': round(total_estimate / 1024, 2),
|
||||
'weights_mb': round(weights_vram_mb, 1),
|
||||
'weights_gb': round(weights_vram_mb / 1024, 2),
|
||||
'kv_cache_mb': round(kv_vram_mb, 1),
|
||||
'kv_cache_gb': round(kv_vram_mb / 1024, 2),
|
||||
'compute_mb': round(compute_mb, 1),
|
||||
'cuda_overhead_mb': round(cuda_overhead_mb, 1),
|
||||
'total_vram_available_mb': total_vram,
|
||||
'total_vram_available_gb': round(total_vram / 1024, 2) if total_vram else 0,
|
||||
'max_single_vram_mb': max_single_vram,
|
||||
'fits': total_estimate <= total_vram if total_vram > 0 else None,
|
||||
'usage_percent': round(total_estimate / total_vram * 100, 1) if total_vram > 0 else None,
|
||||
# For CPU+GPU mode
|
||||
'cpu_weights_mb': round(model_size_gb * 1024 * max(0, 1 - (ngl / model_layers if model_layers > 0 else 0)), 1) if model_size_gb > 0 else 0,
|
||||
}
|
||||
|
||||
|
||||
def parse_natural_language(text):
|
||||
"""
|
||||
Parse natural language description into parameter values.
|
||||
Supports Chinese and English keywords.
|
||||
"""
|
||||
result = {}
|
||||
text_lower = text.lower()
|
||||
|
||||
# GPU selection
|
||||
gpu_patterns = [
|
||||
(r'(?:rtx\s*)?3090', 'RTX 3090'),
|
||||
(r'(?:rtx\s*)?4090', 'RTX 4090'),
|
||||
(r'(?:rtx\s*)?4080', 'RTX 4080'),
|
||||
(r'(?:rtx\s*)?3080', 'RTX 3080'),
|
||||
(r'(?:rtx\s*)?5090', 'RTX 5090'),
|
||||
(r'(?:rtx\s*)?4070\s*ti', 'RTX 4070 Ti'),
|
||||
(r'(?:rtx\s*)?4060', 'RTX 4060'),
|
||||
(r'a100\s*80', 'A100 80GB'),
|
||||
(r'a100\s*40', 'A100 40GB'),
|
||||
(r'h100', 'H100 80GB'),
|
||||
(r'v100', 'V100 32GB'),
|
||||
(r'a6000', 'RTX A6000'),
|
||||
(r'a5000', 'RTX A5000'),
|
||||
(r'7900\s*xtx', 'RX 7900 XTX'),
|
||||
]
|
||||
for pattern, gpu_name in gpu_patterns:
|
||||
if re.search(pattern, text_lower):
|
||||
result['_gpu_name'] = gpu_name
|
||||
break
|
||||
|
||||
# GPU count
|
||||
gpu_count_match = re.search(r'(\d+)\s*(?:张|块|个)?\s*(?:gpu|显卡|卡)', text_lower)
|
||||
if gpu_count_match:
|
||||
result['_gpu_count'] = int(gpu_count_match.group(1))
|
||||
|
||||
# Context size
|
||||
ctx_patterns = [
|
||||
r'(?:上下文|context|ctx)[\s大小为]*[:：\s]*(\d+)',
|
||||
r'(\d+)\s*(?:k|K)\s*(?:上下文|context|ctx)',
|
||||
]
|
||||
for pattern in ctx_patterns:
|
||||
m = re.search(pattern, text_lower)
|
||||
if m:
|
||||
val = int(m.group(1))
|
||||
if val < 100: # like "8k context"
|
||||
val = val * 1024
|
||||
result['ctx_size'] = str(val)
|
||||
break
|
||||
|
||||
# GPU layers
|
||||
ngl_patterns = [
|
||||
r'(?:gpu\s*层|gpu\s*layers?|ngl|offload)[\s:：]*(\d+)',
|
||||
r'(\d+)\s*(?:层|layers?)\s*(?:gpu|offload)',
|
||||
r'全部(?:offload|卸载|gpu)|all\s*(?:gpu|layers?)',
|
||||
]
|
||||
for pattern in ngl_patterns:
|
||||
m = re.search(pattern, text_lower)
|
||||
if m:
|
||||
val = m.group(1) if m.lastindex else '-1'
|
||||
result['n_gpu_layers'] = val
|
||||
break
|
||||
|
||||
# Temperature
|
||||
temp_match = re.search(r'(?:温度|temp|temperature)[\s:：]*(\d+(?:\.\d+)?)', text_lower)
|
||||
if temp_match:
|
||||
result['temperature'] = temp_match.group(1)
|
||||
|
||||
# Top-k
|
||||
topk_match = re.search(r'(?:top[\s-]*k)[\s:：]*(\d+)', text_lower)
|
||||
if topk_match:
|
||||
result['top_k'] = topk_match.group(1)
|
||||
|
||||
# Top-p
|
||||
topp_match = re.search(r'(?:top[\s-]*p)[\s:：]*(\d+(?:\.\d+)?)', text_lower)
|
||||
if topp_match:
|
||||
result['top_p'] = topp_match.group(1)
|
||||
|
||||
# Threads
|
||||
threads_match = re.search(r'(?:线程|threads?|cpu\s*线程)[\s:：]*(\d+)', text_lower)
|
||||
if threads_match:
|
||||
result['threads'] = threads_match.group(1)
|
||||
|
||||
# Batch size
|
||||
batch_match = re.search(r'(?:batch|批处理|批次)[\s大小:：]*(\d+)', text_lower)
|
||||
if batch_match:
|
||||
result['batch_size'] = batch_match.group(1)
|
||||
|
||||
# Port
|
||||
port_match = re.search(r'(?:端口|port)[\s:：]*(\d+)', text_lower)
|
||||
if port_match:
|
||||
result['port'] = port_match.group(1)
|
||||
|
||||
# Model path
|
||||
model_match = re.search(r'(?:模型|model)[\s路径]*[:：\s]+([^\s,，]+\.gguf)', text_lower)
|
||||
if model_match:
|
||||
result['model'] = model_match.group(1)
|
||||
|
||||
# HF repo
|
||||
hf_match = re.search(r'(?:hf|hugging\s*face|仓库)[\s:：]+([^\s,，]+)', text_lower)
|
||||
if hf_match and not model_match:
|
||||
result['hf_repo'] = hf_match.group(1)
|
||||
|
||||
# Flash attention
|
||||
if re.search(r'flash\s*atten|flash\s*attn|fa', text_lower):
|
||||
result['flash_attn'] = 'on'
|
||||
|
||||
# Mode
|
||||
if re.search(r'cpu\s*\+\s*gpu|gpu\s*\+\s*cpu|混合', text_lower):
|
||||
result['_mode'] = 'gpu_cpu'
|
||||
elif re.search(r'纯\s*gpu|gpu\s*only|仅\s*gpu', text_lower):
|
||||
result['_mode'] = 'gpu'
|
||||
|
||||
# Parallel
|
||||
parallel_match = re.search(r'(?:并行|parallel|slots?)[\s:：]*(\d+)', text_lower)
|
||||
if parallel_match:
|
||||
result['parallel'] = parallel_match.group(1)
|
||||
|
||||
# Split mode
|
||||
if re.search(r'row\s*split|行分割|按行分割', text_lower):
|
||||
result['split_mode'] = 'row'
|
||||
elif re.search(r'tensor\s*split|张量分割|按张量分割', text_lower):
|
||||
result['split_mode'] = 'tensor'
|
||||
|
||||
# mlock
|
||||
if re.search(r'mlock|锁[\s定]*内存|内存锁', text_lower):
|
||||
result['mlock'] = 'true'
|
||||
|
||||
# numa
|
||||
numa_match = re.search(r'numa[\s:：]*(distribute|isolate|numactl)', text_lower)
|
||||
if numa_match:
|
||||
result['numa'] = numa_match.group(1)
|
||||
|
||||
# Reasoning budget
|
||||
reasoning_match = re.search(r'(?:reasoning|推理|thinking)[\s预算budget]*[:：\s]*(\d+)', text_lower)
|
||||
if reasoning_match:
|
||||
result['reasoning_budget'] = reasoning_match.group(1)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# ==================== API Routes ====================
|
||||
|
||||
@app.route('/')
|
||||
def index():
|
||||
return send_from_directory('static', 'index.html')
|
||||
|
||||
|
||||
@app.route('/admin')
|
||||
def admin():
|
||||
return send_from_directory('static', 'admin.html')
|
||||
|
||||
|
||||
# ----- Versions -----
|
||||
@app.route('/api/versions')
|
||||
def get_versions():
|
||||
db = get_db()
|
||||
versions = db.execute('SELECT * FROM llama_versions WHERE is_active = 1 ORDER BY sort_order').fetchall()
|
||||
result = [dict(v) for v in versions]
|
||||
db.close()
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
@app.route('/api/versions/<int:vid>/params')
|
||||
def get_version_params(vid):
|
||||
db = get_db()
|
||||
params = db.execute(
|
||||
'SELECT * FROM params WHERE version_id = ? ORDER BY is_important DESC, sort_order',
|
||||
(vid,)
|
||||
).fetchall()
|
||||
result = []
|
||||
for p in params:
|
||||
d = dict(p)
|
||||
if d.get('options'):
|
||||
try:
|
||||
d['options'] = json.loads(d['options'])
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
pass
|
||||
result.append(d)
|
||||
db.close()
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
# ----- GPUs -----
|
||||
@app.route('/api/gpus')
|
||||
def get_gpus():
|
||||
db = get_db()
|
||||
gpus = db.execute('SELECT * FROM gpus ORDER BY sort_order, name').fetchall()
|
||||
result = [dict(g) for g in gpus]
|
||||
db.close()
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
# ----- Generate Command -----
|
||||
@app.route('/api/generate', methods=['POST'])
|
||||
def generate_command():
|
||||
data = request.json
|
||||
version_id = data.get('version_id')
|
||||
params = data.get('params', {})
|
||||
mode = data.get('mode', 'gpu')
|
||||
gpu_selections = data.get('gpu_selections', [])
|
||||
binary = data.get('binary', 'llama-server')
|
||||
system_memory_gb = data.get('system_memory_gb', 0)
|
||||
|
||||
db = get_db()
|
||||
|
||||
# Get param definitions
|
||||
param_defs = db.execute(
|
||||
'SELECT * FROM params WHERE version_id = ?',
|
||||
(version_id,)
|
||||
).fetchall()
|
||||
param_def_map = {p['param_key']: dict(p) for p in param_defs}
|
||||
|
||||
db.close()
|
||||
|
||||
# Build command
|
||||
cmd_parts = [binary]
|
||||
|
||||
# GPU layers
|
||||
if mode == 'gpu':
|
||||
# Add GPU-related args
|
||||
if gpu_selections:
|
||||
gpu_names = [g.get('name', '') for g in gpu_selections]
|
||||
device_str = ','.join(str(i) for i in range(len(gpu_selections)))
|
||||
|
||||
# n_gpu_layers
|
||||
ngl = params.get('n_gpu_layers', 'auto')
|
||||
if ngl and ngl != param_def_map.get('n_gpu_layers', {}).get('default_value', 'auto'):
|
||||
flag = param_def_map.get('n_gpu_layers', {}).get('short_flag') or param_def_map.get('n_gpu_layers', {}).get('long_flag', '-ngl')
|
||||
cmd_parts.append(f'{flag} {ngl}')
|
||||
|
||||
# tensor_split for multiple GPUs
|
||||
if len(gpu_selections) > 1:
|
||||
ts = params.get('tensor_split', '')
|
||||
if ts and ts != param_def_map.get('tensor_split', {}).get('default_value', ''):
|
||||
flag = param_def_map.get('tensor_split', {}).get('short_flag', '-ts')
|
||||
cmd_parts.append(f'{flag} {ts}')
|
||||
elif not ts:
|
||||
# Auto-generate tensor split based on VRAM ratio
|
||||
total_vram = sum(g.get('vram_mb', 0) for g in gpu_selections)
|
||||
if total_vram > 0:
|
||||
ratios = [str(round(g.get('vram_mb', 0) / total_vram, 2)) for g in gpu_selections]
|
||||
cmd_parts.append(f'-ts {",".join(ratios)}')
|
||||
|
||||
elif mode == 'gpu_cpu':
|
||||
# GPU+CPU mode
|
||||
if gpu_selections:
|
||||
ngl = params.get('n_gpu_layers', 'auto')
|
||||
if ngl and ngl != param_def_map.get('n_gpu_layers', {}).get('default_value', 'auto'):
|
||||
flag = param_def_map.get('n_gpu_layers', {}).get('short_flag') or param_def_map.get('n_gpu_layers', {}).get('long_flag', '-ngl')
|
||||
cmd_parts.append(f'{flag} {ngl}')
|
||||
|
||||
# Iterate through params and add non-default ones
|
||||
skip_params = {'n_gpu_layers', 'tensor_split'}
|
||||
for key, value in params.items():
|
||||
if key.startswith('_'):
|
||||
continue
|
||||
if key in skip_params:
|
||||
continue
|
||||
|
||||
p_def = param_def_map.get(key)
|
||||
if not p_def:
|
||||
continue
|
||||
|
||||
default_val = p_def['default_value']
|
||||
# Skip if value equals default
|
||||
if str(value) == str(default_val):
|
||||
continue
|
||||
|
||||
# Skip empty values
|
||||
if value is None or value == '' or value == 'false':
|
||||
if str(default_val).lower() == 'false' and str(value).lower() == 'false':
|
||||
continue
|
||||
if value == '' or value is None:
|
||||
continue
|
||||
|
||||
# Boolean params: only add if true (and default is false)
|
||||
if p_def['param_type'] == 'boolean':
|
||||
if str(value).lower() == 'true' and str(default_val).lower() != 'true':
|
||||
flag = p_def['short_flag'] or p_def['long_flag']
|
||||
cmd_parts.append(flag)
|
||||
elif str(value).lower() == 'false' and str(default_val).lower() == 'true':
|
||||
# Add --no- variant
|
||||
flag = p_def['long_flag']
|
||||
cmd_parts.append(f'--no-{flag.lstrip("--")}')
|
||||
continue
|
||||
|
||||
flag = p_def['short_flag'] or p_def['long_flag']
|
||||
cmd_parts.append(f'{flag} {value}')
|
||||
|
||||
command = ' '.join(cmd_parts)
|
||||
return jsonify({'command': command, 'mode': mode, 'binary': binary})
|
||||
|
||||
|
||||
# ----- Estimate VRAM -----
|
||||
@app.route('/api/estimate', methods=['POST'])
|
||||
def estimate():
|
||||
data = request.json
|
||||
params = data.get('params', {})
|
||||
gpu_selections = data.get('gpu_selections', [])
|
||||
mode = data.get('mode', 'gpu')
|
||||
system_memory_gb = data.get('system_memory_gb', 0)
|
||||
|
||||
# Prepare GPU list with VRAM
|
||||
gpus = []
|
||||
for gs in gpu_selections:
|
||||
gpus.append({'vram_mb': gs.get('vram_mb', 0), 'name': gs.get('name', '')})
|
||||
|
||||
result = estimate_vram(params, gpus, [])
|
||||
|
||||
# Add system memory estimation for GPU+CPU mode
|
||||
if mode == 'gpu_cpu':
|
||||
model_size_gb = float(params.get('_model_size_gb', 0) or 0)
|
||||
model_layers = int(params.get('_model_layers', 0) or 0)
|
||||
ngl_raw = params.get('n_gpu_layers', 'auto')
|
||||
|
||||
if ngl_raw in ('auto', 'all', '-1'):
|
||||
ngl = model_layers if model_layers > 0 else 32
|
||||
else:
|
||||
try:
|
||||
ngl = int(ngl_raw)
|
||||
except (ValueError, TypeError):
|
||||
ngl = 0
|
||||
|
||||
# CPU portion of weights
|
||||
if model_layers > 0 and ngl < model_layers:
|
||||
cpu_weights_gb = model_size_gb * (1 - ngl / model_layers)
|
||||
else:
|
||||
cpu_weights_gb = 0
|
||||
|
||||
# KV cache on CPU
|
||||
ctx_size = int(params.get('ctx_size', 0) or 0)
|
||||
if ctx_size == 0:
|
||||
ctx_size = 4096
|
||||
parallel = int(params.get('parallel', 1) or 1)
|
||||
if parallel < 1:
|
||||
parallel = 1
|
||||
|
||||
# Estimate KV cache on CPU (layers not on GPU)
|
||||
model_kv_heads = int(params.get('_model_kv_heads', 0) or 0)
|
||||
model_head_dim = int(params.get('_model_head_dim', 0) or 0)
|
||||
model_embd = int(params.get('_model_embd', 0) or 0)
|
||||
|
||||
if model_layers > 0 and ngl < model_layers:
|
||||
remaining_layers = model_layers - ngl
|
||||
if model_kv_heads > 0 and model_head_dim > 0:
|
||||
kv_cpu_bytes = 2 * model_kv_heads * model_head_dim * ctx_size * remaining_layers * parallel
|
||||
elif model_embd > 0:
|
||||
kv_cpu_bytes = 2 * model_embd * ctx_size * remaining_layers * parallel
|
||||
else:
|
||||
kv_cpu_bytes = int(0.5 * 1024 * 1024 * remaining_layers * (ctx_size / 1024) * parallel) * 2
|
||||
kv_cpu_mb = kv_cpu_bytes * 2 / (1024 * 1024) # Assuming f16
|
||||
else:
|
||||
kv_cpu_mb = 0
|
||||
|
||||
total_cpu_mb = cpu_weights_gb * 1024 + kv_cpu_mb + 500 # overhead
|
||||
result['cpu_total_mb'] = round(total_cpu_mb, 1)
|
||||
result['cpu_total_gb'] = round(total_cpu_mb / 1024, 2)
|
||||
result['cpu_weights_gb'] = round(cpu_weights_gb, 2)
|
||||
result['cpu_kv_cache_mb'] = round(kv_cpu_mb, 1)
|
||||
result['system_memory_gb'] = system_memory_gb
|
||||
if system_memory_gb > 0:
|
||||
result['cpu_usage_percent'] = round(total_cpu_mb / (system_memory_gb * 1024) * 100, 1)
|
||||
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
# ----- Parse Natural Language -----
|
||||
@app.route('/api/parse-nl', methods=['POST'])
|
||||
def parse_nl():
|
||||
data = request.json
|
||||
text = data.get('text', '')
|
||||
result = parse_natural_language(text)
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
# ==================== Admin API ====================
|
||||
|
||||
@app.route('/api/admin/gpus', methods=['GET', 'POST'])
|
||||
def admin_gpus():
|
||||
db = get_db()
|
||||
if request.method == 'GET':
|
||||
gpus = db.execute('SELECT * FROM gpus ORDER BY sort_order, name').fetchall()
|
||||
result = [dict(g) for g in gpus]
|
||||
db.close()
|
||||
return jsonify(result)
|
||||
|
||||
elif request.method == 'POST':
|
||||
data = request.json
|
||||
db.execute(
|
||||
'INSERT INTO gpus (name, vram_mb, compute_capability, description, sort_order) VALUES (?, ?, ?, ?, ?)',
|
||||
(data['name'], data['vram_mb'], data.get('compute_capability', ''),
|
||||
data.get('description', ''), data.get('sort_order', 0))
|
||||
)
|
||||
db.commit()
|
||||
db.close()
|
||||
return jsonify({'status': 'ok'})
|
||||
|
||||
|
||||
@app.route('/api/admin/gpus/<int:gid>', methods=['PUT', 'DELETE'])
|
||||
def admin_gpu_edit(gid):
|
||||
db = get_db()
|
||||
if request.method == 'PUT':
|
||||
data = request.json
|
||||
db.execute(
|
||||
'UPDATE gpus SET name=?, vram_mb=?, compute_capability=?, description=?, sort_order=? WHERE id=?',
|
||||
(data['name'], data['vram_mb'], data.get('compute_capability', ''),
|
||||
data.get('description', ''), data.get('sort_order', 0), gid)
|
||||
)
|
||||
db.commit()
|
||||
db.close()
|
||||
return jsonify({'status': 'ok'})
|
||||
elif request.method == 'DELETE':
|
||||
db.execute('DELETE FROM gpus WHERE id=?', (gid,))
|
||||
db.commit()
|
||||
db.close()
|
||||
return jsonify({'status': 'ok'})
|
||||
|
||||
|
||||
@app.route('/api/admin/versions', methods=['GET', 'POST'])
|
||||
def admin_versions():
|
||||
db = get_db()
|
||||
if request.method == 'GET':
|
||||
versions = db.execute('SELECT * FROM llama_versions ORDER BY sort_order').fetchall()
|
||||
result = [dict(v) for v in versions]
|
||||
db.close()
|
||||
return jsonify(result)
|
||||
elif request.method == 'POST':
|
||||
data = request.json
|
||||
db.execute(
|
||||
'INSERT INTO llama_versions (version_tag, description, release_date, is_active, sort_order) VALUES (?, ?, ?, ?, ?)',
|
||||
(data['version_tag'], data.get('description', ''), data.get('release_date', ''),
|
||||
data.get('is_active', 1), data.get('sort_order', 0))
|
||||
)
|
||||
db.commit()
|
||||
db.close()
|
||||
return jsonify({'status': 'ok'})
|
||||
|
||||
|
||||
@app.route('/api/admin/versions/<int:vid>', methods=['PUT', 'DELETE'])
|
||||
def admin_version_edit(vid):
|
||||
db = get_db()
|
||||
if request.method == 'PUT':
|
||||
data = request.json
|
||||
db.execute(
|
||||
'UPDATE llama_versions SET version_tag=?, description=?, release_date=?, is_active=?, sort_order=? WHERE id=?',
|
||||
(data['version_tag'], data.get('description', ''), data.get('release_date', ''),
|
||||
data.get('is_active', 1), data.get('sort_order', 0), vid)
|
||||
)
|
||||
db.commit()
|
||||
db.close()
|
||||
return jsonify({'status': 'ok'})
|
||||
elif request.method == 'DELETE':
|
||||
db.execute('DELETE FROM llama_versions WHERE id=?', (vid,))
|
||||
db.commit()
|
||||
db.close()
|
||||
return jsonify({'status': 'ok'})
|
||||
|
||||
|
||||
@app.route('/api/admin/versions/<int:vid>/params', methods=['GET', 'POST'])
|
||||
def admin_params(vid):
|
||||
db = get_db()
|
||||
if request.method == 'GET':
|
||||
params = db.execute(
|
||||
'SELECT * FROM params WHERE version_id = ? ORDER BY is_important DESC, sort_order',
|
||||
(vid,)
|
||||
).fetchall()
|
||||
result = []
|
||||
for p in params:
|
||||
d = dict(p)
|
||||
if d.get('options'):
|
||||
try:
|
||||
d['options'] = json.loads(d['options'])
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
pass
|
||||
result.append(d)
|
||||
db.close()
|
||||
return jsonify(result)
|
||||
elif request.method == 'POST':
|
||||
data = request.json
|
||||
options = data.get('options')
|
||||
if isinstance(options, list):
|
||||
options = json.dumps(options)
|
||||
db.execute(
|
||||
'''INSERT INTO params
|
||||
(version_id, param_key, short_flag, long_flag, description, category, param_type,
|
||||
default_value, options, min_value, max_value, step, unit, is_important, affects_vram, sort_order)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''',
|
||||
(vid, data['param_key'], data.get('short_flag', ''), data['long_flag'],
|
||||
data.get('description', ''), data.get('category', 'common'),
|
||||
data.get('param_type', 'string'), data.get('default_value', ''),
|
||||
options, data.get('min_value'), data.get('max_value'),
|
||||
data.get('step'), data.get('unit'), data.get('is_important', 0),
|
||||
data.get('affects_vram', 0), data.get('sort_order', 0))
|
||||
)
|
||||
db.commit()
|
||||
db.close()
|
||||
return jsonify({'status': 'ok'})
|
||||
|
||||
|
||||
@app.route('/api/admin/params/<int:pid>', methods=['PUT', 'DELETE'])
|
||||
def admin_param_edit(pid):
|
||||
db = get_db()
|
||||
if request.method == 'PUT':
|
||||
data = request.json
|
||||
options = data.get('options')
|
||||
if isinstance(options, list):
|
||||
options = json.dumps(options)
|
||||
db.execute(
|
||||
'''UPDATE params SET
|
||||
param_key=?, short_flag=?, long_flag=?, description=?, category=?, param_type=?,
|
||||
default_value=?, options=?, min_value=?, max_value=?, step=?, unit=?,
|
||||
is_important=?, affects_vram=?, sort_order=? WHERE id=?''',
|
||||
(data['param_key'], data.get('short_flag', ''), data['long_flag'],
|
||||
data.get('description', ''), data.get('category', 'common'),
|
||||
data.get('param_type', 'string'), data.get('default_value', ''),
|
||||
options, data.get('min_value'), data.get('max_value'),
|
||||
data.get('step'), data.get('unit'), data.get('is_important', 0),
|
||||
data.get('affects_vram', 0), data.get('sort_order', 0), pid)
|
||||
)
|
||||
db.commit()
|
||||
db.close()
|
||||
return jsonify({'status': 'ok'})
|
||||
elif request.method == 'DELETE':
|
||||
db.execute('DELETE FROM params WHERE id=?', (pid,))
|
||||
db.commit()
|
||||
db.close()
|
||||
return jsonify({'status': 'ok'})
|
||||
|
||||
|
||||
@app.route('/api/admin/settings', methods=['GET', 'PUT'])
|
||||
def admin_settings():
|
||||
db = get_db()
|
||||
if request.method == 'GET':
|
||||
settings = db.execute('SELECT * FROM settings').fetchall()
|
||||
result = {s['key']: s['value'] for s in settings}
|
||||
db.close()
|
||||
return jsonify(result)
|
||||
elif request.method == 'PUT':
|
||||
data = request.json
|
||||
for key, value in data.items():
|
||||
db.execute(
|
||||
'INSERT OR REPLACE INTO settings (key, value) VALUES (?, ?)',
|
||||
(key, str(value))
|
||||
)
|
||||
db.commit()
|
||||
db.close()
|
||||
return jsonify({'status': 'ok'})
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
init_db()
|
||||
app.run(host='0.0.0.0', port=16052, debug=False)
|
||||
Reference in New Issue
Block a user