{"slug": "llamacpp-sh", "title": "llamacpp.sh", "summary": "A developer created llamacpp.sh, a Bash script that automates local LLM inference using llama.cpp and Podman. The script probes available VRAM from sysfs and selects an appropriate model and configuration, supporting models like Qwen3.6-27B and Gemma-4-12b-it with automatic download and setup.", "body_md": "| #!/usr/bin/env bash | |\n| set -euo pipefail | |\n| # tuning info: https://carteakey.dev/blog/local-inference/local-llm-optimization/ | |\n| # dependencies: podman | |\n| # Probe VRAM from sysfs (zero dependencies) | |\n| get_max_vram_gb() { | |\n| local max_bytes=0 | |\n| for vram_file in /sys/class/drm/card*/device/mem_info_vram_total; do | |\n| [[ -f \"$vram_file\" ]] || continue | |\n| local bytes | |\n| bytes=$(<\"$vram_file\") | |\n| (( bytes > max_bytes )) && max_bytes=$bytes | |\n| done | |\n| echo $(( max_bytes / 1024 / 1024 / 1024 )) | |\n| } | |\n| VRAM_GB=$(get_max_vram_gb) | |\n| # Select model and configuration based on available VRAM | |\n| if (( VRAM_GB >= 23 )); then | |\n| MODEL_ORG=\"unsloth\" | |\n| MODEL_NAME=\"Qwen3.6-27B\" | |\n| MODEL_QUANT=\"UD-Q4_K_XL\" | |\n| MODEL_REPO=\"${MODEL_NAME}-MTP-GGUF\" | |\n| CUSTOM_TEMPLATE_URL=\"https://gist.githubusercontent.com/jscott3201/e4b155885cc68c038d6ac8909a3bd9fe/raw/08b4cce5971d4075591204995fdd7a17e29826fc/custom_pub_chat_template_qwen36.jinja\" | |\n| NEEDS_MTP_DRAFTER=false | |\n| NEEDS_MMPROJ=true | |\n| TEMP=0.6 TOP_P=0.95 TOP_K=20 MIN_P=0.0 | |\n| CTX_SIZE=98304 | |\n| elif (( VRAM_GB >= 15 )); then | |\n| # MTP Q8_0 drafter + mmproj needs ~9GB; leaves headroom for desktop env on 16-17GB cards | |\n| MODEL_ORG=\"unsloth\" | |\n| MODEL_NAME=\"gemma-4-12b-it\" | |\n| MODEL_QUANT=\"UD-Q5_K_XL\" | |\n| MODEL_REPO=\"gemma-4-12b-it-GGUF\" | |\n| CUSTOM_TEMPLATE_URL=\"\" | |\n| NEEDS_MTP_DRAFTER=true | |\n| NEEDS_MMPROJ=true | |\n| TEMP=0.6 TOP_P=0.95 TOP_K=64 MIN_P=0.0 | |\n| CTX_SIZE=98304 | |\n| else | |\n| # Fallback for low VRAM — may require CPU offload | |\n| echo \"WARN: Low VRAM (${VRAM_GB}GB) — falling back to Ornith 9B (may require CPU offload).\" >&2 | |\n| MODEL_ORG=\"protoLabsAI\" | |\n| MODEL_NAME=\"ornith-9b-mtp-kl\" | |\n| MODEL_QUANT=\"Q4_K_M\" | |\n| MODEL_REPO=\"Ornith-1.0-9B-MTP-GGUF\" | |\n| CUSTOM_TEMPLATE_URL=\"\" | |\n| NEEDS_MTP_DRAFTER=false | |\n| NEEDS_MMPROJ=false | |\n| TEMP=0.6 TOP_P=0.95 TOP_K=20 MIN_P=0.0 | |\n| CTX_SIZE=32768 | |\n| fi | |\n| MODEL_NAME_QUANT=\"$MODEL_NAME-$MODEL_QUANT\" | |\n| MODEL_URL=\"https://huggingface.co/${MODEL_ORG}/${MODEL_REPO}/resolve/main/$MODEL_NAME_QUANT.gguf\" | |\n| VOLUME_NAME=\"llm-models\" | |\n| # last working version was -b9570 | |\n| LLAMA_IMAGE=\"ghcr.io/ggml-org/llama.cpp:server-vulkan\" | |\n| # Check if the volume exists; create it if it doesn't | |\n| if ! podman volume inspect \"$VOLUME_NAME\" &>/dev/null; then | |\n| echo \"Volume '$VOLUME_NAME' not found. Creating...\" | |\n| podman volume create \"$VOLUME_NAME\" | |\n| fi | |\n| # Get the volume's mount path on the host | |\n| MOUNT_PATH=\"$(podman volume inspect --format '{{.Mountpoint}}' \"$VOLUME_NAME\")\" | |\n| MODEL_DIR=\"$MOUNT_PATH/$MODEL_NAME_QUANT\" | |\n| MODEL_FILE=\"$MODEL_DIR/$MODEL_NAME_QUANT.gguf\" | |\n| if $NEEDS_MMPROJ; then | |\n| MMPROJ_FILE=\"$MODEL_DIR/mmproj-BF16.gguf\" | |\n| MMPROJ_URL=\"https://huggingface.co/${MODEL_ORG}/${MODEL_REPO}/resolve/main/mmproj-BF16.gguf\" | |\n| fi | |\n| if $NEEDS_MTP_DRAFTER; then | |\n| MTP_FILE=\"mtp-$MODEL_NAME.gguf\" | |\n| MTP_URL=\"https://huggingface.co/${MODEL_ORG}/${MODEL_REPO}/resolve/main/$MTP_FILE\" | |\n| MTP_MODEL_FILE=\"$MODEL_DIR/$MTP_FILE\" | |\n| fi | |\n| PRESET_FILE=\"$MOUNT_PATH/models-preset.ini\" | |\n| if [[ ! -f \"$MODEL_FILE\" ]]; then | |\n| echo \"Model file '$MODEL_NAME_QUANT.gguf' not found. Downloading...\" | |\n| mkdir -p \"$MODEL_DIR\" | |\n| curl -# -L -o \"$MODEL_FILE\" \"$MODEL_URL\" | |\n| if $NEEDS_MMPROJ; then | |\n| curl -# -L -o \"$MMPROJ_FILE\" \"$MMPROJ_URL\" | |\n| fi | |\n| if $NEEDS_MTP_DRAFTER; then | |\n| curl -# -L -o \"$MTP_MODEL_FILE\" \"$MTP_URL\" | |\n| fi | |\n| echo \"Model downloaded successfully.\" | |\n| else | |\n| echo \"Model file '$MODEL_NAME_QUANT.gguf' already exists.\" | |\n| fi | |\n| if [[ -n \"$CUSTOM_TEMPLATE_URL\" ]]; then | |\n| CUSTOM_TEMPLATE_PATH=\"$MODEL_DIR/chat_template.jinja\" | |\n| if [[ ! -f \"$CUSTOM_TEMPLATE_PATH\" ]]; then | |\n| echo \"Custom chat template not found. Downloading...\" | |\n| curl -# -L -o \"$CUSTOM_TEMPLATE_PATH\" \"$CUSTOM_TEMPLATE_URL\" | |\n| echo \"Chat template downloaded.\" | |\n| fi | |\n| fi | |\n| # Generate models-preset.ini for per-model configuration | |\n| if [[ -n \"$CUSTOM_TEMPLATE_URL\" ]]; then | |\n| cat > \"$PRESET_FILE\" << INI | |\n| [*] | |\n| chat-template-kwargs = {\"preserve_thinking_enabled\": true} | |\n| [$MODEL_NAME_QUANT] | |\n| chat-template-file = /models/$MODEL_NAME_QUANT/chat_template.jinja | |\n| temp = $TEMP | |\n| top-p = $TOP_P | |\n| top-k = $TOP_K | |\n| min-p = $MIN_P | |\n| INI | |\n| else | |\n| cat > \"$PRESET_FILE\" << INI | |\n| [$MODEL_NAME_QUANT] | |\n| temp = $TEMP | |\n| top-p = $TOP_P | |\n| top-k = $TOP_K | |\n| min-p = $MIN_P | |\n| INI | |\n| fi | |\n| # Build MTP speculative decoding args | |\n| if $NEEDS_MTP_DRAFTER; then | |\n| MTP_DRAFT_ARGS=\"--model-draft /models/$MODEL_NAME_QUANT/$MTP_FILE --spec-draft-n-max 2\" | |\n| else | |\n| MTP_DRAFT_ARGS=\"--spec-draft-n-max 2\" | |\n| fi | |\n| podman run \\ | |\n| --name llama.cpp \\ | |\n| --pull newer \\ | |\n| --detach \\ | |\n| --replace \\ | |\n| --group-add keep-groups \\ | |\n| --shm-size 16G \\ | |\n| --ulimit memlock=-1:-1 \\ | |\n| --security-opt label=type:container_runtime_t \\ | |\n| --device /dev/kfd \\ | |\n| --device /dev/dri \\ | |\n| -v \"$VOLUME_NAME\":/models \\ | |\n| -p 8080:8080 \\ | |\n| \"$LLAMA_IMAGE\" \\ | |\n| --models-dir /models \\ | |\n| --models-preset /models/models-preset.ini \\ | |\n| --parallel 1 \\ | |\n| --gpu-layers all \\ | |\n| --mlock \\ | |\n| --direct-io \\ | |\n| --cache-reuse 128 \\ | |\n| --sleep-idle-seconds 1200 \\ | |\n| --ctx-size \"$CTX_SIZE\" \\ | |\n| --cache-type-k q8_0 \\ | |\n| --cache-type-v q8_0 \\ | |\n| --fit off \\ | |\n| --spec-type draft-mtp \\ | |\n| $MTP_DRAFT_ARGS | |\n| # Set up systemd service for auto-starting the container on boot | |\n| SYSTEMD_DIR=\"$HOME/.config/systemd/user\" | |\n| SERVICE_FILE=\"$SYSTEMD_DIR/llamacpp.service\" | |\n| if [[ ! -f \"$SERVICE_FILE\" ]]; then | |\n| echo \"Systemd service not found. Creating...\" | |\n| mkdir -p \"$SYSTEMD_DIR\" | |\n| cat > \"$SERVICE_FILE\" << 'SERVICE_EOF' | |\n| [Unit] | |\n| Description=llama.cpp Inference Server | |\n| Documentation=https://github.com/ggml-org/llama.cpp | |\n| After=network-online.target | |\n| Wants=network-online.target | |\n| [Service] | |\n| Type=forking | |\n| RemainAfterExit=yes | |\n| ExecStart=/usr/bin/podman start llama.cpp | |\n| ExecStop=/usr/bin/podman stop llama.cpp | |\n| Restart=on-failure | |\n| RestartSec=10 | |\n| [Install] | |\n| WantedBy=default.target | |\n| SERVICE_EOF | |\n| systemctl --user daemon-reload | |\n| systemctl --user enable llamacpp.service | |\n| echo \"Systemd service created and enabled.\" | |\n| else | |\n| echo \"Systemd service already exists.\" | |\n| fi |", "url": "https://wpnews.pro/news/llamacpp-sh", "canonical_source": "https://gist.github.com/0chroma/db735f95bf47e4b822463935e6c997ec", "published_at": "2026-06-24 08:43:19+00:00", "updated_at": "2026-06-30 11:48:50.074141+00:00", "lang": "en", "topics": ["large-language-models", "developer-tools", "ai-infrastructure"], "entities": ["llamacpp.sh", "Podman", "llama.cpp", "Hugging Face", "unsloth", "Qwen3.6-27B", "Gemma-4-12b-it", "protoLabsAI"], "alternates": {"html": "https://wpnews.pro/news/llamacpp-sh", "markdown": "https://wpnews.pro/news/llamacpp-sh.md", "text": "https://wpnews.pro/news/llamacpp-sh.txt", "jsonld": "https://wpnews.pro/news/llamacpp-sh.jsonld"}}