# llamacpp.sh

> Source: <https://gist.github.com/0chroma/db735f95bf47e4b822463935e6c997ec>
> Published: 2026-06-24 08:43:19+00:00

| #!/usr/bin/env bash | |
| set -euo pipefail | |
| # tuning info: https://carteakey.dev/blog/local-inference/local-llm-optimization/ | |
| # dependencies: podman | |
| # Probe VRAM from sysfs (zero dependencies) | |
| get_max_vram_gb() { | |
| local max_bytes=0 | |
| for vram_file in /sys/class/drm/card*/device/mem_info_vram_total; do | |
| [[ -f "$vram_file" ]] || continue | |
| local bytes | |
| bytes=$(<"$vram_file") | |
| (( bytes > max_bytes )) && max_bytes=$bytes | |
| done | |
| echo $(( max_bytes / 1024 / 1024 / 1024 )) | |
| } | |
| VRAM_GB=$(get_max_vram_gb) | |
| # Select model and configuration based on available VRAM | |
| if (( VRAM_GB >= 23 )); then | |
| MODEL_ORG="unsloth" | |
| MODEL_NAME="Qwen3.6-27B" | |
| MODEL_QUANT="UD-Q4_K_XL" | |
| MODEL_REPO="${MODEL_NAME}-MTP-GGUF" | |
| CUSTOM_TEMPLATE_URL="https://gist.githubusercontent.com/jscott3201/e4b155885cc68c038d6ac8909a3bd9fe/raw/08b4cce5971d4075591204995fdd7a17e29826fc/custom_pub_chat_template_qwen36.jinja" | |
| NEEDS_MTP_DRAFTER=false | |
| NEEDS_MMPROJ=true | |
| TEMP=0.6 TOP_P=0.95 TOP_K=20 MIN_P=0.0 | |
| CTX_SIZE=98304 | |
| elif (( VRAM_GB >= 15 )); then | |
| # MTP Q8_0 drafter + mmproj needs ~9GB; leaves headroom for desktop env on 16-17GB cards | |
| MODEL_ORG="unsloth" | |
| MODEL_NAME="gemma-4-12b-it" | |
| MODEL_QUANT="UD-Q5_K_XL" | |
| MODEL_REPO="gemma-4-12b-it-GGUF" | |
| CUSTOM_TEMPLATE_URL="" | |
| NEEDS_MTP_DRAFTER=true | |
| NEEDS_MMPROJ=true | |
| TEMP=0.6 TOP_P=0.95 TOP_K=64 MIN_P=0.0 | |
| CTX_SIZE=98304 | |
| else | |
| # Fallback for low VRAM — may require CPU offload | |
| echo "WARN: Low VRAM (${VRAM_GB}GB) — falling back to Ornith 9B (may require CPU offload)." >&2 | |
| MODEL_ORG="protoLabsAI" | |
| MODEL_NAME="ornith-9b-mtp-kl" | |
| MODEL_QUANT="Q4_K_M" | |
| MODEL_REPO="Ornith-1.0-9B-MTP-GGUF" | |
| CUSTOM_TEMPLATE_URL="" | |
| NEEDS_MTP_DRAFTER=false | |
| NEEDS_MMPROJ=false | |
| TEMP=0.6 TOP_P=0.95 TOP_K=20 MIN_P=0.0 | |
| CTX_SIZE=32768 | |
| fi | |
| MODEL_NAME_QUANT="$MODEL_NAME-$MODEL_QUANT" | |
| MODEL_URL="https://huggingface.co/${MODEL_ORG}/${MODEL_REPO}/resolve/main/$MODEL_NAME_QUANT.gguf" | |
| VOLUME_NAME="llm-models" | |
| # last working version was -b9570 | |
| LLAMA_IMAGE="ghcr.io/ggml-org/llama.cpp:server-vulkan" | |
| # Check if the volume exists; create it if it doesn't | |
| if ! podman volume inspect "$VOLUME_NAME" &>/dev/null; then | |
| echo "Volume '$VOLUME_NAME' not found. Creating..." | |
| podman volume create "$VOLUME_NAME" | |
| fi | |
| # Get the volume's mount path on the host | |
| MOUNT_PATH="$(podman volume inspect --format '{{.Mountpoint}}' "$VOLUME_NAME")" | |
| MODEL_DIR="$MOUNT_PATH/$MODEL_NAME_QUANT" | |
| MODEL_FILE="$MODEL_DIR/$MODEL_NAME_QUANT.gguf" | |
| if $NEEDS_MMPROJ; then | |
| MMPROJ_FILE="$MODEL_DIR/mmproj-BF16.gguf" | |
| MMPROJ_URL="https://huggingface.co/${MODEL_ORG}/${MODEL_REPO}/resolve/main/mmproj-BF16.gguf" | |
| fi | |
| if $NEEDS_MTP_DRAFTER; then | |
| MTP_FILE="mtp-$MODEL_NAME.gguf" | |
| MTP_URL="https://huggingface.co/${MODEL_ORG}/${MODEL_REPO}/resolve/main/$MTP_FILE" | |
| MTP_MODEL_FILE="$MODEL_DIR/$MTP_FILE" | |
| fi | |
| PRESET_FILE="$MOUNT_PATH/models-preset.ini" | |
| if [[ ! -f "$MODEL_FILE" ]]; then | |
| echo "Model file '$MODEL_NAME_QUANT.gguf' not found. Downloading..." | |
| mkdir -p "$MODEL_DIR" | |
| curl -# -L -o "$MODEL_FILE" "$MODEL_URL" | |
| if $NEEDS_MMPROJ; then | |
| curl -# -L -o "$MMPROJ_FILE" "$MMPROJ_URL" | |
| fi | |
| if $NEEDS_MTP_DRAFTER; then | |
| curl -# -L -o "$MTP_MODEL_FILE" "$MTP_URL" | |
| fi | |
| echo "Model downloaded successfully." | |
| else | |
| echo "Model file '$MODEL_NAME_QUANT.gguf' already exists." | |
| fi | |
| if [[ -n "$CUSTOM_TEMPLATE_URL" ]]; then | |
| CUSTOM_TEMPLATE_PATH="$MODEL_DIR/chat_template.jinja" | |
| if [[ ! -f "$CUSTOM_TEMPLATE_PATH" ]]; then | |
| echo "Custom chat template not found. Downloading..." | |
| curl -# -L -o "$CUSTOM_TEMPLATE_PATH" "$CUSTOM_TEMPLATE_URL" | |
| echo "Chat template downloaded." | |
| fi | |
| fi | |
| # Generate models-preset.ini for per-model configuration | |
| if [[ -n "$CUSTOM_TEMPLATE_URL" ]]; then | |
| cat > "$PRESET_FILE" << INI | |
| [*] | |
| chat-template-kwargs = {"preserve_thinking_enabled": true} | |
| [$MODEL_NAME_QUANT] | |
| chat-template-file = /models/$MODEL_NAME_QUANT/chat_template.jinja | |
| temp = $TEMP | |
| top-p = $TOP_P | |
| top-k = $TOP_K | |
| min-p = $MIN_P | |
| INI | |
| else | |
| cat > "$PRESET_FILE" << INI | |
| [$MODEL_NAME_QUANT] | |
| temp = $TEMP | |
| top-p = $TOP_P | |
| top-k = $TOP_K | |
| min-p = $MIN_P | |
| INI | |
| fi | |
| # Build MTP speculative decoding args | |
| if $NEEDS_MTP_DRAFTER; then | |
| MTP_DRAFT_ARGS="--model-draft /models/$MODEL_NAME_QUANT/$MTP_FILE --spec-draft-n-max 2" | |
| else | |
| MTP_DRAFT_ARGS="--spec-draft-n-max 2" | |
| fi | |
| podman run \ | |
| --name llama.cpp \ | |
| --pull newer \ | |
| --detach \ | |
| --replace \ | |
| --group-add keep-groups \ | |
| --shm-size 16G \ | |
| --ulimit memlock=-1:-1 \ | |
| --security-opt label=type:container_runtime_t \ | |
| --device /dev/kfd \ | |
| --device /dev/dri \ | |
| -v "$VOLUME_NAME":/models \ | |
| -p 8080:8080 \ | |
| "$LLAMA_IMAGE" \ | |
| --models-dir /models \ | |
| --models-preset /models/models-preset.ini \ | |
| --parallel 1 \ | |
| --gpu-layers all \ | |
| --mlock \ | |
| --direct-io \ | |
| --cache-reuse 128 \ | |
| --sleep-idle-seconds 1200 \ | |
| --ctx-size "$CTX_SIZE" \ | |
| --cache-type-k q8_0 \ | |
| --cache-type-v q8_0 \ | |
| --fit off \ | |
| --spec-type draft-mtp \ | |
| $MTP_DRAFT_ARGS | |
| # Set up systemd service for auto-starting the container on boot | |
| SYSTEMD_DIR="$HOME/.config/systemd/user" | |
| SERVICE_FILE="$SYSTEMD_DIR/llamacpp.service" | |
| if [[ ! -f "$SERVICE_FILE" ]]; then | |
| echo "Systemd service not found. Creating..." | |
| mkdir -p "$SYSTEMD_DIR" | |
| cat > "$SERVICE_FILE" << 'SERVICE_EOF' | |
| [Unit] | |
| Description=llama.cpp Inference Server | |
| Documentation=https://github.com/ggml-org/llama.cpp | |
| After=network-online.target | |
| Wants=network-online.target | |
| [Service] | |
| Type=forking | |
| RemainAfterExit=yes | |
| ExecStart=/usr/bin/podman start llama.cpp | |
| ExecStop=/usr/bin/podman stop llama.cpp | |
| Restart=on-failure | |
| RestartSec=10 | |
| [Install] | |
| WantedBy=default.target | |
| SERVICE_EOF | |
| systemctl --user daemon-reload | |
| systemctl --user enable llamacpp.service | |
| echo "Systemd service created and enabled." | |
| else | |
| echo "Systemd service already exists." | |
| fi |