llamacpp.sh

A developer created llamacpp.sh, a Bash script that automates local LLM inference using llama.cpp and Podman. The script probes available VRAM from sysfs and selects an appropriate model and configuration, supporting models like Qwen3.6-27B and Gemma-4-12b-it with automatic download and setup.

| /usr/bin/env bash | | | set -euo pipefail | | | tuning info: https://carteakey.dev/blog/local-inference/local-llm-optimization/ | | | dependencies: podman | | | Probe VRAM from sysfs zero dependencies | | | get max vram gb { | | | local max bytes=0 | | | for vram file in /sys/class/drm/card /device/mem info vram total; do | | | -f "$vram file" || continue | | | local bytes | | | bytes=$ <"$vram file" | | | bytes max bytes && max bytes=$bytes | | | done | | | echo $ max bytes / 1024 / 1024 / 1024 | | | } | | | VRAM GB=$ get max vram gb | | | Select model and configuration based on available VRAM | | | if VRAM GB = 23 ; then | | | MODEL ORG="unsloth" | | | MODEL NAME="Qwen3.6-27B" | | | MODEL QUANT="UD-Q4 K XL" | | | MODEL REPO="${MODEL NAME}-MTP-GGUF" | | | CUSTOM TEMPLATE URL="https://gist.githubusercontent.com/jscott3201/e4b155885cc68c038d6ac8909a3bd9fe/raw/08b4cce5971d4075591204995fdd7a17e29826fc/custom pub chat template qwen36.jinja" | | | NEEDS MTP DRAFTER=false | | | NEEDS MMPROJ=true | | | TEMP=0.6 TOP P=0.95 TOP K=20 MIN P=0.0 | | | CTX SIZE=98304 | | | elif VRAM GB = 15 ; then | | | MTP Q8 0 drafter + mmproj needs ~9GB; leaves headroom for desktop env on 16-17GB cards | | | MODEL ORG="unsloth" | | | MODEL NAME="gemma-4-12b-it" | | | MODEL QUANT="UD-Q5 K XL" | | | MODEL REPO="gemma-4-12b-it-GGUF" | | | CUSTOM TEMPLATE URL="" | | | NEEDS MTP DRAFTER=true | | | NEEDS MMPROJ=true | | | TEMP=0.6 TOP P=0.95 TOP K=64 MIN P=0.0 | | | CTX SIZE=98304 | | | else | | | Fallback for low VRAM — may require CPU offload | | | echo "WARN: Low VRAM ${VRAM GB}GB — falling back to Ornith 9B may require CPU offload ." &2 | | | MODEL ORG="protoLabsAI" | | | MODEL NAME="ornith-9b-mtp-kl" | | | MODEL QUANT="Q4 K M" | | | MODEL REPO="Ornith-1.0-9B-MTP-GGUF" | | | CUSTOM TEMPLATE URL="" | | | NEEDS MTP DRAFTER=false | | | NEEDS MMPROJ=false | | | TEMP=0.6 TOP P=0.95 TOP K=20 MIN P=0.0 | | | CTX SIZE=32768 | | | fi | | | MODEL NAME QUANT="$MODEL NAME-$MODEL QUANT" | | | MODEL URL="https://huggingface.co/${MODEL ORG}/${MODEL REPO}/resolve/main/$MODEL NAME QUANT.gguf" | | | VOLUME NAME="llm-models" | | | last working version was -b9570 | | | LLAMA IMAGE="ghcr.io/ggml-org/llama.cpp:server-vulkan" | | | Check if the volume exists; create it if it doesn't | | | if podman volume inspect "$VOLUME NAME" & /dev/null; then | | | echo "Volume '$VOLUME NAME' not found. Creating..." | | | podman volume create "$VOLUME NAME" | | | fi | | | Get the volume's mount path on the host | | | MOUNT PATH="$ podman volume inspect --format '{{.Mountpoint}}' "$VOLUME NAME" " | | | MODEL DIR="$MOUNT PATH/$MODEL NAME QUANT" | | | MODEL FILE="$MODEL DIR/$MODEL NAME QUANT.gguf" | | | if $NEEDS MMPROJ; then | | | MMPROJ FILE="$MODEL DIR/mmproj-BF16.gguf" | | | MMPROJ URL="https://huggingface.co/${MODEL ORG}/${MODEL REPO}/resolve/main/mmproj-BF16.gguf" | | | fi | | | if $NEEDS MTP DRAFTER; then | | | MTP FILE="mtp-$MODEL NAME.gguf" | | | MTP URL="https://huggingface.co/${MODEL ORG}/${MODEL REPO}/resolve/main/$MTP FILE" | | | MTP MODEL FILE="$MODEL DIR/$MTP FILE" | | | fi | | | PRESET FILE="$MOUNT PATH/models-preset.ini" | | | if -f "$MODEL FILE" ; then | | | echo "Model file '$MODEL NAME QUANT.gguf' not found. Downloading..." | | | mkdir -p "$MODEL DIR" | | | curl - -L -o "$MODEL FILE" "$MODEL URL" | | | if $NEEDS MMPROJ; then | | | curl - -L -o "$MMPROJ FILE" "$MMPROJ URL" | | | fi | | | if $NEEDS MTP DRAFTER; then | | | curl - -L -o "$MTP MODEL FILE" "$MTP URL" | | | fi | | | echo "Model downloaded successfully." | | | else | | | echo "Model file '$MODEL NAME QUANT.gguf' already exists." | | | fi | | | if -n "$CUSTOM TEMPLATE URL" ; then | | | CUSTOM TEMPLATE PATH="$MODEL DIR/chat template.jinja" | | | if -f "$CUSTOM TEMPLATE PATH" ; then | | | echo "Custom chat template not found. Downloading..." | | | curl - -L -o "$CUSTOM TEMPLATE PATH" "$CUSTOM TEMPLATE URL" | | | echo "Chat template downloaded." | | | fi | | | fi | | | Generate models-preset.ini for per-model configuration | | | if -n "$CUSTOM TEMPLATE URL" ; then | | | cat "$PRESET FILE" << INI | | | | | | chat-template-kwargs = {"preserve thinking enabled": true} | | | $MODEL NAME QUANT | | | chat-template-file = /models/$MODEL NAME QUANT/chat template.jinja | | | temp = $TEMP | | | top-p = $TOP P | | | top-k = $TOP K | | | min-p = $MIN P | | | INI | | | else | | | cat "$PRESET FILE" << INI | | | $MODEL NAME QUANT | | | temp = $TEMP | | | top-p = $TOP P | | | top-k = $TOP K | | | min-p = $MIN P | | | INI | | | fi | | | Build MTP speculative decoding args | | | if $NEEDS MTP DRAFTER; then | | | MTP DRAFT ARGS="--model-draft /models/$MODEL NAME QUANT/$MTP FILE --spec-draft-n-max 2" | | | else | | | MTP DRAFT ARGS="--spec-draft-n-max 2" | | | fi | | | podman run \ | | | --name llama.cpp \ | | | --pull newer \ | | | --detach \ | | | --replace \ | | | --group-add keep-groups \ | | | --shm-size 16G \ | | | --ulimit memlock=-1:-1 \ | | | --security-opt label=type:container runtime t \ | | | --device /dev/kfd \ | | | --device /dev/dri \ | | | -v "$VOLUME NAME":/models \ | | | -p 8080:8080 \ | | | "$LLAMA IMAGE" \ | | | --models-dir /models \ | | | --models-preset /models/models-preset.ini \ | | | --parallel 1 \ | | | --gpu-layers all \ | | | --mlock \ | | | --direct-io \ | | | --cache-reuse 128 \ | | | --sleep-idle-seconds 1200 \ | | | --ctx-size "$CTX SIZE" \ | | | --cache-type-k q8 0 \ | | | --cache-type-v q8 0 \ | | | --fit off \ | | | --spec-type draft-mtp \ | | | $MTP DRAFT ARGS | | | Set up systemd service for auto-starting the container on boot | | | SYSTEMD DIR="$HOME/.config/systemd/user" | | | SERVICE FILE="$SYSTEMD DIR/llamacpp.service" | | | if -f "$SERVICE FILE" ; then | | | echo "Systemd service not found. Creating..." | | | mkdir -p "$SYSTEMD DIR" | | | cat "$SERVICE FILE" << 'SERVICE EOF' | | | Unit | | | Description=llama.cpp Inference Server | | | Documentation=https://github.com/ggml-org/llama.cpp | | | After=network-online.target | | | Wants=network-online.target | | | Service | | | Type=forking | | | RemainAfterExit=yes | | | ExecStart=/usr/bin/podman start llama.cpp | | | ExecStop=/usr/bin/podman stop llama.cpp | | | Restart=on-failure | | | RestartSec=10 | | | Install | | | WantedBy=default.target | | | SERVICE EOF | | | systemctl --user daemon-reload | | | systemctl --user enable llamacpp.service | | | echo "Systemd service created and enabled." | | | else | | | echo "Systemd service already exists." | | | fi |