← Qwen3.8 27B W4A16 AutoRound · 64k ctx, DFlash2 speculative decoding

Run 372dd3de

Kind	chat benchmark
Started	2026-09-17 17:16 UTC
Duration	229 s
Engine	`vllm@bae2023ff` (build 0.28.0)
Driver	595.99.02
Throttled	no
Command	`lab bench qwen3.8-27b-w4a16-autoround-fast/64k-dflash2`
Lab version	`0.1.0+795a2d1`

Metrics

Metric	Method	Prompt	Gen	Value	± sd	Unit
`decode_tps`	greedy	—	1024	152	50.9	t/s
`decode_tps`	default sampling	—	1024	146	46.8	t/s
`e2e_tps`	greedy	—	1024	147	—	t/s
`e2e_tps`	default sampling	—	1024	141	—	t/s
`gpu_w_avg`	greedy	—	1024	250	—	W
`gpu_w_avg`	default sampling	—	1024	249	—	W
`load_s`	chat benchmark	—	—	62.3	—	s
`oom_kills`	chat benchmark	—	—	0.00	—	count
`out_tokens_mean`	greedy	—	1024	951	—	tokens
`out_tokens_mean`	default sampling	—	1024	938	—	tokens
`prefill_tps`	greedy	—	1024	723	—	t/s
`prefill_tps`	default sampling	—	1024	741	—	t/s
`ram_peak_mb`	chat benchmark	—	—	24,448	—	MiB
`spec_accept_len`	greedy	—	1024	3.84	—	tok/step
`spec_accept_len`	default sampling	—	1024	3.71	—	tok/step
`tokens_per_joule`	greedy	—	1024	0.59	—	tok/J
`tokens_per_joule`	default sampling	—	1024	0.57	—	tok/J
`tpot_ms`	greedy	—	1024	6.58	3.15	ms
`tpot_ms`	default sampling	—	1024	6.86	3.10	ms
`ttft_ms`	greedy	—	1024	140	65.7	ms
`ttft_ms`	default sampling	—	1024	137	66.3	ms
`vram_peak_mb`	chat benchmark	—	—	24,052	—	MiB

Raw

{
  "bench": {
    "protocol": "chat-c1-v1",
    "prompts": 8,
    "max_tokens": 1024,
    "concurrency": 1,
    "cohorts": {
      "http-sampled": {
        "temperature": 1,
        "top_p": 0.95,
        "top_k": 20,
        "min_p": 0,
        "presence_penalty": 0
      },
      "http-greedy": {
        "temperature": 0
      }
    },
    "thinking": false,
    "warmup": "all prompts once, discarded"
  },
  "telemetry_summary": {
    "vram_baseline_mb": 0.9375,
    "vram_peak_mb": 24053.125,
    "ram_baseline_mb": 23303.23046875,
    "ram_peak_mb": 24447.8671875,
    "proc_rss_peak_mb": 1806.640625,
    "temp_max_c": 64,
    "power_max_w": 250.545,
    "clock_max_mhz": 1620,
    "throttled": false,
    "power_capped_frac": 0.7133610725843735,
    "phases": {
      "boot": {
        "duration_s": 62.32641507091466,
        "energy_j": 4892.548,
        "power_avg_w": 78.49878730283596
      },
      "warmup": {
        "duration_s": 49.73119433305692,
        "energy_j": 12379.744,
        "power_avg_w": 248.9331729515902
      },
      "http-sampled": {
        "duration_s": 53.092629292048514,
        "energy_j": 13241.847,
        "power_avg_w": 249.41026987305716
      },
      "http-greedy": {
        "duration_s": 51.65955868200399,
        "energy_j": 12896.076,
        "power_avg_w": 249.63581433947573
      }
    }
  },
  "protocol": "chat-c1-v1",
  "requests": [
    {
      "phase": "warmup",
      "prompt": 0,
      "prompt_tokens": 60,
      "completion_tokens": 1024,
      "ttft_ms": 110.92,
      "tpot_ms": 5.034,
      "decode_tps": 198.64,
      "e2e_s": 5.261,
      "finish_reason": "length"
    },
    {
      "phase": "warmup",
      "prompt": 1,
      "prompt_tokens": 49,
      "completion_tokens": 1024,
      "ttft_ms": 74.97,
      "tpot_ms": 7.558,
      "decode_tps": 132.3,
      "e2e_s": 7.807,
      "finish_reason": "length"
    },
    {
      "phase": "warmup",
      "prompt": 2,
      "prompt_tokens": 68,
      "completion_tokens": 1024,
      "ttft_ms": 116.33,
      "tpot_ms": 12.389,
      "decode_tps": 80.72,
      "e2e_s": 12.791,
      "finish_reason": "length"
    },
    {
      "phase": "warmup",
      "prompt": 3,
      "prompt_tokens": 196,
      "completion_tokens": 579,
      "ttft_ms": 223.35,
      "tpot_ms": 6.254,
      "decode_tps": 159.9,
      "e2e_s": 3.838,
      "finish_reason": "stop"
    },
    {
      "phase": "warmup",
      "prompt": 4,
      "prompt_tokens": 92,
      "completion_tokens": 1024,
      "ttft_ms": 134.13,
      "tpot_ms": 4.712,
      "decode_tps": 212.25,
      "e2e_s": 4.954,
      "finish_reason": "length"
    },
    {
      "phase": "warmup",
      "prompt": 5,
      "prompt_tokens": 55,
      "completion_tokens": 1024,
      "ttft_ms": 100.32,
      "tpot_ms": 4.786,
      "decode_tps": 208.95,
      "e2e_s": 4.997,
      "finish_reason": "length"
    },
    {
      "phase": "warmup",
      "prompt": 6,
      "prompt_tokens": 50,
      "completion_tokens": 1024,
      "ttft_ms": 77.4,
      "tpot_ms": 6.662,
      "decode_tps": 150.1,
      "e2e_s": 6.893,
      "finish_reason": "length"
    },
    {
      "phase": "warmup",
      "prompt": 7,
      "prompt_tokens": 240,
      "completion_tokens": 446,
      "ttft_ms": 252.8,
      "tpot_ms": 6.483,
      "decode_tps": 154.24,
      "e2e_s": 3.164,
      "finish_reason": "stop"
    },
    {
      "phase": "http-sampled",
      "prompt": 0,
      "prompt_tokens": 60,
      "completion_tokens": 1024,
      "ttft_ms": 76.1,
      "tpot_ms": 5.15,
      "decode_tps": 194.16,
      "e2e_s": 5.345,
      "finish_reason": "length"
    },
    {
      "phase": "http-sampled",
      "prompt": 1,
      "prompt_tokens": 49,
      "completion_tokens": 1024,
      "ttft_ms": 100.11,
      "tpot_ms": 7.731,
      "decode_tps": 129.35,
      "e2e_s": 8.009,
      "finish_reason": "length"
    },
    {
      "phase": "http-sampled",
      "prompt": 2,
      "prompt_tokens": 68,
      "completion_tokens": 1024,
      "ttft_ms": 97.64,
      "tpot_ms": 14.133,
      "decode_tps": 70.76,
      "e2e_s": 14.556,
      "finish_reason": "length"
    },
    {
      "phase": "http-sampled",
      "prompt": 3,
      "prompt_tokens": 196,
      "completion_tokens": 945,
      "ttft_ms": 225.76,
      "tpot_ms": 5.933,
      "decode_tps": 168.55,
      "e2e_s": 5.827,
      "finish_reason": "stop"
    },
    {
      "phase": "http-sampled",
      "prompt": 4,
      "prompt_tokens": 92,
      "completion_tokens": 1024,
      "ttft_ms": 135.93,
      "tpot_ms": 4.89,
      "decode_tps": 204.51,
      "e2e_s": 5.138,
      "finish_reason": "length"
    },
    {
      "phase": "http-sampled",
      "prompt": 5,
      "prompt_tokens": 55,
      "completion_tokens": 1024,
      "ttft_ms": 101.04,
      "tpot_ms": 4.612,
      "decode_tps": 216.84,
      "e2e_s": 4.819,
      "finish_reason": "length"
    },
    {
      "phase": "http-sampled",
      "prompt": 6,
      "prompt_tokens": 50,
      "completion_tokens": 1024,
      "ttft_ms": 101.69,
      "tpot_ms": 6.4,
      "decode_tps": 156.26,
      "e2e_s": 6.649,
      "finish_reason": "length"
    },
    {
      "phase": "http-sampled",
      "prompt": 7,
      "prompt_tokens": 240,
      "completion_tokens": 416,
      "ttft_ms": 254.31,
      "tpot_ms": 5.996,
      "decode_tps": 166.77,
      "e2e_s": 2.743,
      "finish_reason": "stop"
    },
    {
      "phase": "http-greedy",
      "prompt": 0,
      "prompt_tokens": 60,
      "completion_tokens": 1024,
      "ttft_ms": 76.01,
      "tpot_ms": 5.024,
      "decode_tps": 199.04,
      "e2e_s": 5.216,
      "finish_reason": "length"
    },
    {
      "phase": "http-greedy",
      "prompt": 1,
      "prompt_tokens": 49,
      "completion_tokens": 1024,
      "ttft_ms": 100.74,
      "tpot_ms": 7.114,
      "decode_tps": 140.57,
      "e2e_s": 7.379,
      "finish_reason": "length"
    },
    {
      "phase": "http-greedy",
      "prompt": 2,
      "prompt_tokens": 68,
      "completion_tokens": 1024,
      "ttft_ms": 120.6,
      "tpot_ms": 14,
      "decode_tps": 71.43,
      "e2e_s": 14.442,
      "finish_reason": "length"
    },
    {
      "phase": "http-greedy",
      "prompt": 3,
      "prompt_tokens": 196,
      "completion_tokens": 1024,
      "ttft_ms": 228.72,
      "tpot_ms": 5.113,
      "decode_tps": 195.58,
      "e2e_s": 5.46,
      "finish_reason": "length"
    },
    {
      "phase": "http-greedy",
      "prompt": 4,
      "prompt_tokens": 92,
      "completion_tokens": 1024,
      "ttft_ms": 136.5,
      "tpot_ms": 4.626,
      "decode_tps": 216.15,
      "e2e_s": 4.87,
      "finish_reason": "length"
    },
    {
      "phase": "http-greedy",
      "prompt": 5,
      "prompt_tokens": 55,
      "completion_tokens": 1024,
      "ttft_ms": 101.4,
      "tpot_ms": 4.372,
      "decode_tps": 228.74,
      "e2e_s": 4.574,
      "finish_reason": "length"
    },
    {
      "phase": "http-greedy",
      "prompt": 6,
      "prompt_tokens": 50,
      "completion_tokens": 1024,
      "ttft_ms": 100.79,
      "tpot_ms": 6.729,
      "decode_tps": 148.6,
      "e2e_s": 6.985,
      "finish_reason": "length"
    },
    {
      "phase": "http-greedy",
      "prompt": 7,
      "prompt_tokens": 240,
      "completion_tokens": 439,
      "ttft_ms": 255.31,
      "tpot_ms": 5.647,
      "decode_tps": 177.09,
      "e2e_s": 2.729,
      "finish_reason": "stop"
    }
  ],
  "compile_cache_before": {
    "entries": 4,
    "bytes": 428620966
  },
  "server": {
    "cmdline": [
      "python",
      "vllm",
      "serve",
      "Qwen3.8-27B-W4A16-AutoRound-fast",
      "--served-model-name",
      "qwen3.8-27b",
      "--host",
      "127.0.0.1",
      "--port",
      "8081",
      "--gpu-memory-utilization",
      "0.93",
      "--max-model-len",
      "65536",
      "--max-num-seqs",
      "8",
      "--api-server-count",
      "1",
      "--language-model-only",
      "--attention-backend",
      "FLASH_ATTN",
      "--kv-cache-dtype",
      "bfloat16",
      "--mamba-ssm-cache-dtype",
      "float16",
      "--async-scheduling",
      "--max-num-batched-tokens",
      "2048",
      "--speculative-config",
      "{\"method\":\"dflash\",\"model\":\"Qwen3.8-27B-DFlash2-W4A16\",\"num_speculative_tokens\":7,\"draft_sample_method\":\"probabilistic\"}",
      "--compilation-config",
      "{\"max_cudagraph_capture_size\":64,\"custom_ops\":[\"+rms_norm\",\"+silu_and_mul\"]}",
      "--reasoning-parser",
      "qwen3",
      "--enable-prompt-tokens-details",
      "--enable-auto-tool-choice",
      "--tool-call-parser",
      "qwen3_coder",
      "--enable-prefix-caching",
      "--mamba-cache-mode",
      "align",
      "--kv-cache-memory=5583457484",
      "--served-model-name",
      "qwen3.8-27b-w4a16-autoround-fast/64k-dflash2",
      "--load-format",
      "auto",
      "--sse-keep-alive-interval",
      "30"
    ],
    "env": {
      "FLASHINFER_DISABLE_VERSION_CHECK": "1",
      "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True",
      "VLLM_DFLASH2_LOOKUP": "1",
      "VLLM_SPEC_DECODE_ATTN": "1",
      "VLLM_SPEC_DECODE_ATTN_QMAX": "8",
      "VLLM_USE_FLASHINFER_SAMPLER": "0",
      "VLLM_V2_CUDAGRAPH_MEM_MIB": "1400"
    }
  },
  "counters": {
    "http-sampled": {
      "vllm:spec_decode_num_drafts_total": 2028,
      "vllm:spec_decode_num_accepted_tokens_total": 5488,
      "vllm:prefix_cache_hits_total": 0,
      "vllm:prefix_cache_queries_total": 810,
      "vllm:num_preemptions_total": 0
    },
    "http-greedy": {
      "vllm:spec_decode_num_drafts_total": 1981,
      "vllm:spec_decode_num_accepted_tokens_total": 5627,
      "vllm:prefix_cache_hits_total": 0,
      "vllm:prefix_cache_queries_total": 810,
      "vllm:num_preemptions_total": 0
    }
  },
  "boot": {
    "weights_load_s": 12.27732,
    "init_engine_s": 22.41,
    "compile_s": 12.7,
    "kv_cache_tokens": 68605
  }
}