vllm locally on 5060Ti 16GB x 2

| services: | | vllm-server: | | image: vllm/vllm-openai:latest | | container_name: vllm-server | | restart: unless-stopped | | ports: | |

• "${VLLM_PORT}:8000" | | environment: | |
• VLLM_FP4_GEMM_BACKEND=cutlass | |
• NVIDIA_VISIBLE_DEVICES=all | |
• NVIDIA_DRIVER_CAPABILITIES=compute,utility | | volumes: | |
• ${HF_CACHE_DIR}:/root/.cache/huggingface | | ipc: host | | shm_size: '16gb' | | deploy: | | resources: | | reservations: | | devices: | |
• driver: nvidia | | count: all | | capabilities: [gpu] | | command: > | | --model ${VLLM_MODEL} | | --served-model-name ${VLLM_SERVED_NAME} | | --tensor-parallel-size ${VLLM_TP_SIZE} | | --gpu-memory-utilization ${VLLM_MEMORY_UTIL} | | --max-model-len ${VLLM_MAX_MODEL_LEN} | | --max-num-seqs ${VLLM_MAX_NUM_SEQS} | | --api-key ${API_KEY} | | --language-model-only | | --enable-auto-tool-choice | | --generation-config vllm | | --trust-remote-code | | --kv-cache-dtype fp8 | | --reasoning-parser ${VLLM_REASON_PARSER} | | --tool-call-parser ${VLLM_TOOL_PARSER} | | --speculative-config '{"method": "mtp", "num_speculative_tokens": 2}' | | --default-chat-template-kwargs '{"preserve_thinking": true}' | | --host 0.0.0.0 |