← Qwen3.8 27B Q4_K_M · 32k ctx, f16 KV
Run dd42310b
| Kind | chat benchmark |
| Started | 2026-09-17 17:34 UTC |
| Duration | 782 s |
| Engine | llama.cpp@91f6a6cf3 (build 10883) |
| Driver | 595.99.02 |
| Throttled | no |
| Command | lab bench qwen3.8-27b-q4_k_m-gguf/32k-f16kv --http |
| Lab version | 0.1.0+795a2d1 |
Metrics
| Metric | Method | Prompt | Gen | Depth | Value | ± sd | Unit |
|---|
decode_tps | greedy | — | 1024 | 0 | 30.8 | 0.03 | t/s |
decode_tps | default sampling | — | 1024 | 0 | 30.8 | 0.06 | t/s |
e2e_tps | greedy | — | 1024 | 0 | 30.6 | — | t/s |
e2e_tps | default sampling | — | 1024 | 0 | 30.6 | — | t/s |
gpu_w_avg | greedy | — | 1024 | 0 | 250 | — | W |
gpu_w_avg | default sampling | — | 1024 | 0 | 249 | — | W |
load_s | chat benchmark | — | — | 0 | 6.20 | — | s |
oom_kills | chat benchmark | — | — | 0 | 0.00 | — | count |
out_tokens_mean | greedy | — | 1024 | 0 | 946 | — | tokens |
out_tokens_mean | default sampling | — | 1024 | 0 | 948 | — | tokens |
prefill_tps | greedy | — | 1024 | 0 | 460 | — | t/s |
prefill_tps | default sampling | — | 1024 | 0 | 459 | — | t/s |
ram_peak_mb | chat benchmark | — | — | 0 | 24,448 | — | MiB |
tokens_per_joule | greedy | — | 1024 | 0 | 0.12 | — | tok/J |
tokens_per_joule | default sampling | — | 1024 | 0 | 0.12 | — | tok/J |
tpot_ms | greedy | — | 1024 | 0 | 32.5 | 0.04 | ms |
tpot_ms | default sampling | — | 1024 | 0 | 32.4 | 0.06 | ms |
ttft_ms | greedy | — | 1024 | 0 | 220 | 40.6 | ms |
ttft_ms | default sampling | — | 1024 | 0 | 220 | 41.1 | ms |
vram_peak_mb | chat benchmark | — | — | 0 | 18,406 | — | MiB |
Raw
{
"bench": {
"protocol": "chat-c1-v1",
"prompts": 8,
"max_tokens": 1024,
"concurrency": 1,
"cohorts": {
"http-sampled": {
"temperature": 1,
"top_p": 0.95,
"top_k": 20,
"min_p": 0,
"presence_penalty": 0
},
"http-greedy": {
"temperature": 0
}
},
"thinking": false,
"warmup": "all prompts once, discarded"
},
"telemetry_summary": {
"vram_baseline_mb": 0.9375,
"vram_peak_mb": 18407.125,
"ram_baseline_mb": 23629.37109375,
"ram_peak_mb": 24448.46484375,
"proc_rss_peak_mb": 17065.44140625,
"temp_max_c": 65,
"power_max_w": 251.185,
"clock_max_mhz": 1620,
"throttled": false,
"power_capped_frac": 0.9883922615076718,
"phases": {
"boot": {
"duration_s": 6.216221084003337,
"energy_j": 626.443,
"power_avg_w": 100.77553412829415
},
"warmup": {
"duration_s": 250.2421405549394,
"energy_j": 62429.96,
"power_avg_w": 249.47820483614277
},
"http-sampled": {
"duration_s": 247.43428473907989,
"energy_j": 61730.746,
"power_avg_w": 249.48339744063858
},
"http-greedy": {
"duration_s": 247.1656156849349,
"energy_j": 61685.715,
"power_avg_w": 249.57239634266747
}
}
},
"protocol": "chat-c1-v1",
"requests": [
{
"phase": "warmup",
"prompt": 0,
"prompt_tokens": 60,
"completion_tokens": 1024,
"ttft_ms": 284.02,
"tpot_ms": 31.769,
"decode_tps": 31.48,
"e2e_s": 32.784,
"finish_reason": "length",
"server_decode_tps": 31.48
},
{
"phase": "warmup",
"prompt": 1,
"prompt_tokens": 49,
"completion_tokens": 1024,
"ttft_ms": 190.69,
"tpot_ms": 32.451,
"decode_tps": 30.82,
"e2e_s": 33.389,
"finish_reason": "length",
"server_decode_tps": 30.82
},
{
"phase": "warmup",
"prompt": 2,
"prompt_tokens": 68,
"completion_tokens": 1024,
"ttft_ms": 196.78,
"tpot_ms": 33.171,
"decode_tps": 30.15,
"e2e_s": 34.131,
"finish_reason": "length",
"server_decode_tps": 30.15
},
{
"phase": "warmup",
"prompt": 3,
"prompt_tokens": 196,
"completion_tokens": 1024,
"ttft_ms": 268.26,
"tpot_ms": 33.294,
"decode_tps": 30.04,
"e2e_s": 34.328,
"finish_reason": "length",
"server_decode_tps": 30.04
},
{
"phase": "warmup",
"prompt": 4,
"prompt_tokens": 92,
"completion_tokens": 1024,
"ttft_ms": 212.95,
"tpot_ms": 33.01,
"decode_tps": 30.29,
"e2e_s": 33.982,
"finish_reason": "length",
"server_decode_tps": 30.29
},
{
"phase": "warmup",
"prompt": 5,
"prompt_tokens": 55,
"completion_tokens": 1024,
"ttft_ms": 195.77,
"tpot_ms": 32.829,
"decode_tps": 30.46,
"e2e_s": 33.781,
"finish_reason": "length",
"server_decode_tps": 30.46
},
{
"phase": "warmup",
"prompt": 6,
"prompt_tokens": 50,
"completion_tokens": 1024,
"ttft_ms": 191.74,
"tpot_ms": 32.659,
"decode_tps": 30.62,
"e2e_s": 33.602,
"finish_reason": "length",
"server_decode_tps": 30.62
},
{
"phase": "warmup",
"prompt": 7,
"prompt_tokens": 240,
"completion_tokens": 429,
"ttft_ms": 305.6,
"tpot_ms": 32.484,
"decode_tps": 30.78,
"e2e_s": 14.242,
"finish_reason": "stop",
"server_decode_tps": 30.71
},
{
"phase": "http-sampled",
"prompt": 0,
"prompt_tokens": 60,
"completion_tokens": 1024,
"ttft_ms": 195.28,
"tpot_ms": 32.51,
"decode_tps": 30.76,
"e2e_s": 33.453,
"finish_reason": "length",
"server_decode_tps": 30.76
},
{
"phase": "http-sampled",
"prompt": 1,
"prompt_tokens": 49,
"completion_tokens": 1024,
"ttft_ms": 192.46,
"tpot_ms": 32.369,
"decode_tps": 30.89,
"e2e_s": 33.306,
"finish_reason": "length",
"server_decode_tps": 30.89
},
{
"phase": "http-sampled",
"prompt": 2,
"prompt_tokens": 68,
"completion_tokens": 1024,
"ttft_ms": 198.46,
"tpot_ms": 32.345,
"decode_tps": 30.92,
"e2e_s": 33.288,
"finish_reason": "length",
"server_decode_tps": 30.92
},
{
"phase": "http-sampled",
"prompt": 3,
"prompt_tokens": 196,
"completion_tokens": 1024,
"ttft_ms": 264.01,
"tpot_ms": 32.454,
"decode_tps": 30.81,
"e2e_s": 33.465,
"finish_reason": "length",
"server_decode_tps": 30.81
},
{
"phase": "http-sampled",
"prompt": 4,
"prompt_tokens": 92,
"completion_tokens": 1024,
"ttft_ms": 215.9,
"tpot_ms": 32.481,
"decode_tps": 30.79,
"e2e_s": 33.444,
"finish_reason": "length",
"server_decode_tps": 30.79
},
{
"phase": "http-sampled",
"prompt": 5,
"prompt_tokens": 55,
"completion_tokens": 1024,
"ttft_ms": 200.97,
"tpot_ms": 32.469,
"decode_tps": 30.8,
"e2e_s": 33.417,
"finish_reason": "length",
"server_decode_tps": 30.8
},
{
"phase": "http-sampled",
"prompt": 6,
"prompt_tokens": 50,
"completion_tokens": 1024,
"ttft_ms": 193.15,
"tpot_ms": 32.478,
"decode_tps": 30.79,
"e2e_s": 33.418,
"finish_reason": "length",
"server_decode_tps": 30.79
},
{
"phase": "http-sampled",
"prompt": 7,
"prompt_tokens": 240,
"completion_tokens": 412,
"ttft_ms": 303.47,
"tpot_ms": 32.361,
"decode_tps": 30.9,
"e2e_s": 13.637,
"finish_reason": "stop",
"server_decode_tps": 30.83
},
{
"phase": "http-greedy",
"prompt": 0,
"prompt_tokens": 60,
"completion_tokens": 1024,
"ttft_ms": 195.67,
"tpot_ms": 32.453,
"decode_tps": 30.81,
"e2e_s": 33.396,
"finish_reason": "length",
"server_decode_tps": 30.81
},
{
"phase": "http-greedy",
"prompt": 1,
"prompt_tokens": 49,
"completion_tokens": 1024,
"ttft_ms": 191.68,
"tpot_ms": 32.463,
"decode_tps": 30.8,
"e2e_s": 33.402,
"finish_reason": "length",
"server_decode_tps": 30.8
},
{
"phase": "http-greedy",
"prompt": 2,
"prompt_tokens": 68,
"completion_tokens": 1024,
"ttft_ms": 198.58,
"tpot_ms": 32.442,
"decode_tps": 30.82,
"e2e_s": 33.387,
"finish_reason": "length",
"server_decode_tps": 30.82
},
{
"phase": "http-greedy",
"prompt": 3,
"prompt_tokens": 196,
"completion_tokens": 1024,
"ttft_ms": 262.74,
"tpot_ms": 32.462,
"decode_tps": 30.81,
"e2e_s": 33.472,
"finish_reason": "length",
"server_decode_tps": 30.8
},
{
"phase": "http-greedy",
"prompt": 4,
"prompt_tokens": 92,
"completion_tokens": 1024,
"ttft_ms": 214.53,
"tpot_ms": 32.48,
"decode_tps": 30.79,
"e2e_s": 33.442,
"finish_reason": "length",
"server_decode_tps": 30.79
},
{
"phase": "http-greedy",
"prompt": 5,
"prompt_tokens": 55,
"completion_tokens": 1024,
"ttft_ms": 200.12,
"tpot_ms": 32.523,
"decode_tps": 30.75,
"e2e_s": 33.472,
"finish_reason": "length",
"server_decode_tps": 30.75
},
{
"phase": "http-greedy",
"prompt": 6,
"prompt_tokens": 50,
"completion_tokens": 1024,
"ttft_ms": 195.24,
"tpot_ms": 32.51,
"decode_tps": 30.76,
"e2e_s": 33.454,
"finish_reason": "length",
"server_decode_tps": 30.76
},
{
"phase": "http-greedy",
"prompt": 7,
"prompt_tokens": 240,
"completion_tokens": 396,
"ttft_ms": 302.48,
"tpot_ms": 32.408,
"decode_tps": 30.86,
"e2e_s": 13.136,
"finish_reason": "stop",
"server_decode_tps": 30.78
}
],
"server": {
"cmdline": [
"llama-server",
"-m",
"Qwen3.8-27B-Q4_K_M.gguf",
"--host",
"127.0.0.1",
"--port",
"8081",
"-a",
"qwen3.8-27b-q4_k_m-gguf/32k-f16kv",
"--fit",
"off",
"-c",
"32768",
"-np",
"1",
"--cache-ram",
"0",
"-ngl",
"999",
"-fa",
"on",
"-ctk",
"f16",
"-ctv",
"f16",
"-b",
"2048",
"-ub",
"512",
"-t",
"6",
"-ncmoe",
"0",
"-lm",
"mmap",
"--metrics",
"--temp",
"1.0",
"--top-p",
"0.95",
"--top-k",
"20",
"--min-p",
"0",
"--presence-penalty",
"0"
],
"env": {}
}
}