← Qwen3.8 27B Q4_K_M · 64k ctx, q8_0 KV
Run e75bd3f1
| Kind | chat benchmark |
| Started | 2026-09-17 17:21 UTC |
| Duration | 766 s |
| Engine | llama.cpp@91f6a6cf3 (build 10883) |
| Driver | 595.99.02 |
| Throttled | no |
| Command | lab bench qwen3.8-27b-q4_k_m-gguf/64k-q8kv --http |
| Lab version | 0.1.0+795a2d1 |
Metrics
| Metric | Method | Prompt | Gen | Depth | Value | ± sd | Unit |
|---|
decode_tps | greedy | — | 1024 | 0 | 30.5 | 0.03 | t/s |
decode_tps | default sampling | — | 1024 | 0 | 30.6 | 0.05 | t/s |
e2e_tps | greedy | — | 1024 | 0 | 30.3 | — | t/s |
e2e_tps | default sampling | — | 1024 | 0 | 30.4 | — | t/s |
gpu_w_avg | greedy | — | 1024 | 0 | 250 | — | W |
gpu_w_avg | default sampling | — | 1024 | 0 | 249 | — | W |
load_s | chat benchmark | — | — | 0 | 18.1 | — | s |
oom_kills | chat benchmark | — | — | 0 | 0.00 | — | count |
out_tokens_mean | greedy | — | 1024 | 0 | 948 | — | tokens |
out_tokens_mean | default sampling | — | 1024 | 0 | 947 | — | tokens |
prefill_tps | greedy | — | 1024 | 0 | 449 | — | t/s |
prefill_tps | default sampling | — | 1024 | 0 | 450 | — | t/s |
ram_peak_mb | chat benchmark | — | — | 0 | 24,448 | — | MiB |
tokens_per_joule | greedy | — | 1024 | 0 | 0.12 | — | tok/J |
tokens_per_joule | default sampling | — | 1024 | 0 | 0.12 | — | tok/J |
tpot_ms | greedy | — | 1024 | 0 | 32.8 | 0.04 | ms |
tpot_ms | default sampling | — | 1024 | 0 | 32.7 | 0.05 | ms |
ttft_ms | greedy | — | 1024 | 0 | 225 | 40.9 | ms |
ttft_ms | default sampling | — | 1024 | 0 | 225 | 39.9 | ms |
vram_peak_mb | chat benchmark | — | — | 0 | 18,780 | — | MiB |
Raw
{
"bench": {
"protocol": "chat-c1-v1",
"prompts": 8,
"max_tokens": 1024,
"concurrency": 1,
"cohorts": {
"http-sampled": {
"temperature": 1,
"top_p": 0.95,
"top_k": 20,
"min_p": 0,
"presence_penalty": 0
},
"http-greedy": {
"temperature": 0
}
},
"thinking": false,
"warmup": "all prompts once, discarded"
},
"telemetry_summary": {
"vram_baseline_mb": 0.9375,
"vram_peak_mb": 18781.125,
"ram_baseline_mb": 24125.5390625,
"ram_peak_mb": 24448.44921875,
"proc_rss_peak_mb": 15609.2265625,
"temp_max_c": 65,
"power_max_w": 252.307,
"clock_max_mhz": 1620,
"throttled": false,
"power_capped_frac": 0.9732283464566929,
"phases": {
"boot": {
"duration_s": 18.103008003905416,
"energy_j": 1840.731,
"power_avg_w": 101.68094714441344
},
"warmup": {
"duration_s": 246.06503996800166,
"energy_j": 61381.046,
"power_avg_w": 249.45049490972795
},
"http-sampled": {
"duration_s": 249.32226216094568,
"energy_j": 62198.215,
"power_avg_w": 249.4691587542592
},
"http-greedy": {
"duration_s": 250.10854888509493,
"energy_j": 62404.232,
"power_avg_w": 249.50859248185796
}
}
},
"protocol": "chat-c1-v1",
"requests": [
{
"phase": "warmup",
"prompt": 0,
"prompt_tokens": 60,
"completion_tokens": 1024,
"ttft_ms": 330.72,
"tpot_ms": 31.386,
"decode_tps": 31.86,
"e2e_s": 32.441,
"finish_reason": "length",
"server_decode_tps": 31.86
},
{
"phase": "warmup",
"prompt": 1,
"prompt_tokens": 49,
"completion_tokens": 1024,
"ttft_ms": 193.81,
"tpot_ms": 32.16,
"decode_tps": 31.09,
"e2e_s": 33.094,
"finish_reason": "length",
"server_decode_tps": 31.09
},
{
"phase": "warmup",
"prompt": 2,
"prompt_tokens": 68,
"completion_tokens": 1024,
"ttft_ms": 200.86,
"tpot_ms": 32.963,
"decode_tps": 30.34,
"e2e_s": 33.922,
"finish_reason": "length",
"server_decode_tps": 30.34
},
{
"phase": "warmup",
"prompt": 3,
"prompt_tokens": 196,
"completion_tokens": 921,
"ttft_ms": 270.76,
"tpot_ms": 33.453,
"decode_tps": 29.89,
"e2e_s": 31.081,
"finish_reason": "stop",
"server_decode_tps": 29.86
},
{
"phase": "warmup",
"prompt": 4,
"prompt_tokens": 92,
"completion_tokens": 1024,
"ttft_ms": 219.51,
"tpot_ms": 33.533,
"decode_tps": 29.82,
"e2e_s": 34.524,
"finish_reason": "length",
"server_decode_tps": 29.82
},
{
"phase": "warmup",
"prompt": 5,
"prompt_tokens": 55,
"completion_tokens": 1024,
"ttft_ms": 199.2,
"tpot_ms": 33.177,
"decode_tps": 30.14,
"e2e_s": 34.139,
"finish_reason": "length",
"server_decode_tps": 30.14
},
{
"phase": "warmup",
"prompt": 6,
"prompt_tokens": 50,
"completion_tokens": 1024,
"ttft_ms": 195.99,
"tpot_ms": 32.979,
"decode_tps": 30.32,
"e2e_s": 33.934,
"finish_reason": "length",
"server_decode_tps": 30.32
},
{
"phase": "warmup",
"prompt": 7,
"prompt_tokens": 240,
"completion_tokens": 385,
"ttft_ms": 304.72,
"tpot_ms": 32.782,
"decode_tps": 30.5,
"e2e_s": 12.926,
"finish_reason": "stop",
"server_decode_tps": 30.43
},
{
"phase": "http-sampled",
"prompt": 0,
"prompt_tokens": 60,
"completion_tokens": 1024,
"ttft_ms": 202.22,
"tpot_ms": 32.731,
"decode_tps": 30.55,
"e2e_s": 33.686,
"finish_reason": "length",
"server_decode_tps": 30.55
},
{
"phase": "http-sampled",
"prompt": 1,
"prompt_tokens": 49,
"completion_tokens": 1024,
"ttft_ms": 199.1,
"tpot_ms": 32.657,
"decode_tps": 30.62,
"e2e_s": 33.607,
"finish_reason": "length",
"server_decode_tps": 30.62
},
{
"phase": "http-sampled",
"prompt": 2,
"prompt_tokens": 68,
"completion_tokens": 1024,
"ttft_ms": 204.16,
"tpot_ms": 32.601,
"decode_tps": 30.67,
"e2e_s": 33.555,
"finish_reason": "length",
"server_decode_tps": 30.67
},
{
"phase": "http-sampled",
"prompt": 3,
"prompt_tokens": 196,
"completion_tokens": 1024,
"ttft_ms": 270.74,
"tpot_ms": 32.701,
"decode_tps": 30.58,
"e2e_s": 33.724,
"finish_reason": "length",
"server_decode_tps": 30.58
},
{
"phase": "http-sampled",
"prompt": 4,
"prompt_tokens": 92,
"completion_tokens": 1024,
"ttft_ms": 220.9,
"tpot_ms": 32.723,
"decode_tps": 30.56,
"e2e_s": 33.697,
"finish_reason": "length",
"server_decode_tps": 30.56
},
{
"phase": "http-sampled",
"prompt": 5,
"prompt_tokens": 55,
"completion_tokens": 1024,
"ttft_ms": 201.84,
"tpot_ms": 32.691,
"decode_tps": 30.59,
"e2e_s": 33.645,
"finish_reason": "length",
"server_decode_tps": 30.59
},
{
"phase": "http-sampled",
"prompt": 6,
"prompt_tokens": 50,
"completion_tokens": 1024,
"ttft_ms": 198.5,
"tpot_ms": 32.735,
"decode_tps": 30.55,
"e2e_s": 33.687,
"finish_reason": "length",
"server_decode_tps": 30.55
},
{
"phase": "http-sampled",
"prompt": 7,
"prompt_tokens": 240,
"completion_tokens": 411,
"ttft_ms": 303.68,
"tpot_ms": 32.634,
"decode_tps": 30.64,
"e2e_s": 13.717,
"finish_reason": "stop",
"server_decode_tps": 30.57
},
{
"phase": "http-greedy",
"prompt": 0,
"prompt_tokens": 60,
"completion_tokens": 1024,
"ttft_ms": 201.39,
"tpot_ms": 32.791,
"decode_tps": 30.5,
"e2e_s": 33.747,
"finish_reason": "length",
"server_decode_tps": 30.5
},
{
"phase": "http-greedy",
"prompt": 1,
"prompt_tokens": 49,
"completion_tokens": 1024,
"ttft_ms": 195.6,
"tpot_ms": 32.785,
"decode_tps": 30.5,
"e2e_s": 33.735,
"finish_reason": "length",
"server_decode_tps": 30.5
},
{
"phase": "http-greedy",
"prompt": 2,
"prompt_tokens": 68,
"completion_tokens": 1024,
"ttft_ms": 204.84,
"tpot_ms": 32.734,
"decode_tps": 30.55,
"e2e_s": 33.692,
"finish_reason": "length",
"server_decode_tps": 30.55
},
{
"phase": "http-greedy",
"prompt": 3,
"prompt_tokens": 196,
"completion_tokens": 1024,
"ttft_ms": 270.66,
"tpot_ms": 32.775,
"decode_tps": 30.51,
"e2e_s": 33.8,
"finish_reason": "length",
"server_decode_tps": 30.51
},
{
"phase": "http-greedy",
"prompt": 4,
"prompt_tokens": 92,
"completion_tokens": 1024,
"ttft_ms": 218.66,
"tpot_ms": 32.743,
"decode_tps": 30.54,
"e2e_s": 33.715,
"finish_reason": "length",
"server_decode_tps": 30.54
},
{
"phase": "http-greedy",
"prompt": 5,
"prompt_tokens": 55,
"completion_tokens": 1024,
"ttft_ms": 204.7,
"tpot_ms": 32.769,
"decode_tps": 30.52,
"e2e_s": 33.729,
"finish_reason": "length",
"server_decode_tps": 30.52
},
{
"phase": "http-greedy",
"prompt": 6,
"prompt_tokens": 50,
"completion_tokens": 1024,
"ttft_ms": 199.66,
"tpot_ms": 32.731,
"decode_tps": 30.55,
"e2e_s": 33.684,
"finish_reason": "length",
"server_decode_tps": 30.55
},
{
"phase": "http-greedy",
"prompt": 7,
"prompt_tokens": 240,
"completion_tokens": 419,
"ttft_ms": 306.78,
"tpot_ms": 32.686,
"decode_tps": 30.59,
"e2e_s": 14.003,
"finish_reason": "stop",
"server_decode_tps": 30.52
}
],
"server": {
"cmdline": [
"llama-server",
"-m",
"Qwen3.8-27B-Q4_K_M.gguf",
"--host",
"127.0.0.1",
"--port",
"8081",
"-a",
"qwen3.8-27b-q4_k_m-gguf/64k-q8kv",
"--fit",
"off",
"-c",
"65536",
"-np",
"1",
"--cache-ram",
"0",
"-ngl",
"999",
"-fa",
"on",
"-ctk",
"q8_0",
"-ctv",
"q8_0",
"-b",
"2048",
"-ub",
"512",
"-t",
"6",
"-ncmoe",
"0",
"-lm",
"mmap",
"--metrics",
"--temp",
"1.0",
"--top-p",
"0.95",
"--top-k",
"20",
"--min-p",
"0",
"--presence-penalty",
"0"
],
"env": {}
}
}