Qwen 3.6 config example

A developer shared a configuration example for running Qwen 3.6 with llama-turboquant, a Docker-based CUDA inference container. The setup includes GPU offloading, flash attention, speculative decoding with MTP and ngram-mod, and memory limits of 30GB RAM with 46GB swap.

| /usr/bin/env bash | | | set -euo pipefail | | | cd "$ dirname "$0" " | | | export CUDA MALLOC ASYNC SUPPORTED=1 | | | export GGML CUDA FORCE MMQ=1 | | | IMAGE="${IMAGE:-llama-turboquant:cuda}" | | | HOST PORT="${HOST PORT:-8080}" | | | NETWORK="${NETWORK:-runner-network}" | | | STATIC IP="${STATIC IP:-172.18.0.10}" | | | 65 layers | | | MODEL FILE="${MODEL FILE:-Qwen3.6-27B-NVFP4-MTP.gguf}" | | | MMPROJ FILE="${MMPROJ FILE:-mmproj-Qwen3.6-27B-F16.gguf}" | | | N GPU LAYERS="${N GPU LAYERS:-999}" | | | NAME="${NAME:-llama-turboquant}" | | | if docker inspect "${NAME}" /dev/null 2 &1; then | | | docker rm -f "${NAME}" /dev/null | | | fi | | | docker create \ | | | --name "${NAME}" \ | | | --restart=unless-stopped \ | | | --gpus all \ | | | -e CUDA DEVICE ORDER=PCI BUS ID \ | | | -e CUDA VISIBLE DEVICES=1,0 \ | | | --memory=30g \ | | | --memory-swap=46g \ | | | --cap-add=IPC LOCK \ | | | --ulimit memlock=-1:-1 \ | | | --ulimit core=0 \ | | | -e TURBO AUTO ASYMMETRIC=0 \ | | | -p "${HOST PORT}:8080" \ | | | --network "${NETWORK}" \ | | | --ip "${STATIC IP}" \ | | | -v "$ pwd /models:/models:ro" \ | | | -v "$ pwd /scripts:/scripts:ro" \ | | | --entrypoint /scripts/entrypoint.sh \ | | | "${IMAGE}" \ | | | --model "/models/${MODEL FILE}" \ | | | --mmproj "/models/${MMPROJ FILE}" \ | | | --mmproj-offload \ | | | --host 0.0.0.0 \ | | | --port 8080 \ | | | --metrics \ | | | --n-gpu-layers "${N GPU LAYERS}" \ | | | --main-gpu 0 \ | | | --split-mode layer \ | | | --tensor-split 40,25 \ | | | -fit off \ | | | --flash-attn on \ | | | -c 120000 \ | | | -n -1 \ | | | --parallel 1 \ | | | -ctk q8 0 \ | | | -ctv turbo3 \ | | | -ctkd q8 0 \ | | | -ctvd turbo3 \ | | | --kv-unified \ | | | --no-mmap \ | | | --mlock \ | | | --jinja \ | | | --reasoning off \ | | | --spec-type draft-mtp,ngram-mod \ | | | --spec-draft-n-max 3 \ | | | --spec-ngram-mod-n-match 24 \ | | | --spec-ngram-mod-n-min 4 \ | | | --spec-ngram-mod-n-max 48 \ | | | --temp 0.7 \ | | | --top-p 0.8 \ | | | --top-k 20 \ | | | --min-p 0.0 \ | | | --presence-penalty 1.5 \ | | | --repeat-penalty 1.0 \ | | | -b 1024 \ | | | -ub 512 \ | | | --cache-idle-slots \ | | | --cache-ram 16384 \ | | | --cache-reuse 256 \ | | | --threads 8 \ | | | --cpu-range 0-7 \ | | | --timeout 360 \ | | | "$@" /dev/null | | | docker start -a "${NAME}" |