vllm-recipe/device/nvidia-agx-thor/cyankiwi-gemma-4-26B-A4B-it-AWQ-4bit.md

• 20260622

serve

vllm serve "/workspace/thor-wm/cyankiwi-gemma-4-26B-A4B-it-AWQ-4bit" \
--port "8001" \
--host "0.0.0.0" \
--served-model-name "cyankiwi/gemma-4-26B-A4B-it-AWQ-4bit" \
--max-model-len 262144 \
--tensor-parallel-size 1 \
--max-num-seqs 8 \
--gpu-memory-utilization 0.01 \
--kv-cache-memory-bytes 9g \
--attention-backend TRITON_ATTN \
--enable-chunked-prefil \
--max-num-batched-tokens 16384 \
--enable-prefix-caching \
--reasoning-parser gemma4 \
--enable-auto-tool-choice \
--tool-call-parser gemma4 \
--mm-processor-kwargs '{"max_soft_tokens": 1120}'

bench

vllm bench serve \
  --model "/workspace/thor-wm/cyankiwi-gemma-4-26B-A4B-it-AWQ-4bit" \
  --served-model-name "cyankiwi/gemma-4-26B-A4B-it-AWQ-4bit" \
  --host localhost \
  --port 8001 \
  --dataset-name random \
  --random-input-len 1024 \
  --random-output-len 1024 \
  --num-prompts 5 \
  --max-concurrency 1

vllm bench serve \
  --model "/workspace/thor-wm/cyankiwi-gemma-4-26B-A4B-it-AWQ-4bit" \
  --served-model-name "cyankiwi/gemma-4-26B-A4B-it-AWQ-4bit" \
  --host localhost \
  --port 8001 \
  --dataset-name random \
  --random-input-len 1024 \
  --random-output-len 1024 \
  --num-prompts 32 \
  --max-concurrency 8