# vllm locally on 5060Ti 16GB x 2

> Source: <https://gist.github.com/bouroo/71d5dfe6131880956ff8e0cb5cc4105c>
> Published: 2026-07-12 05:51:10+00:00

|
services: |
|
vllm-server: |
|
image: vllm/vllm-openai:latest |
|
container_name: vllm-server |
|
restart: unless-stopped |
|
ports: |
|
- "${VLLM_PORT}:8000" |
|
environment: |
|
- VLLM_FP4_GEMM_BACKEND=cutlass |
|
- NVIDIA_VISIBLE_DEVICES=all |
|
- NVIDIA_DRIVER_CAPABILITIES=compute,utility |
|
volumes: |
|
- ${HF_CACHE_DIR}:/root/.cache/huggingface |
|
ipc: host |
|
shm_size: '16gb' |
|
deploy: |
|
resources: |
|
reservations: |
|
devices: |
|
- driver: nvidia |
|
count: all |
|
capabilities: [gpu] |
|
command: > |
|
--model ${VLLM_MODEL} |
|
--served-model-name ${VLLM_SERVED_NAME} |
|
--tensor-parallel-size ${VLLM_TP_SIZE} |
|
--gpu-memory-utilization ${VLLM_MEMORY_UTIL} |
|
--max-model-len ${VLLM_MAX_MODEL_LEN} |
|
--max-num-seqs ${VLLM_MAX_NUM_SEQS} |
|
--api-key ${API_KEY} |
|
--language-model-only |
|
--enable-auto-tool-choice |
|
--generation-config vllm |
|
--trust-remote-code |
|
--kv-cache-dtype fp8 |
|
--reasoning-parser ${VLLM_REASON_PARSER} |
|
--tool-call-parser ${VLLM_TOOL_PARSER} |
|
--speculative-config '{"method": "mtp", "num_speculative_tokens": 2}' |
|
--default-chat-template-kwargs '{"preserve_thinking": true}' |
|
--host 0.0.0.0 |