{"slug": "minimal-llm-watermarking-from-scratch", "title": "Minimal LLM Watermarking from scratch", "summary": "A developer has released a minimal implementation of LLM watermarking, inspired by Anthropic's upcoming 'is it AI' API and Google DeepMind's SynthID-Text. The code, written in Python using PyTorch and Hugging Face Transformers, demonstrates Tournament Sampling for watermarking and a statistical detection method based on z-scores and p-values. The implementation is designed to be run with 'uv run synthid.py' and uses a small Qwen model for demonstration.", "body_md": "| #!/usr/bin/env -S uv run --script | |\n| # /// script | |\n| # requires-python = \">=3.11\" | |\n| # dependencies = [ | |\n| # \"torch\", | |\n| # \"transformers>=5.15\", | |\n| # ] | |\n| # /// | |\n| \"\"\"LLM Watermarks. | |\n| Anthropic will roll out their \"is it AI\" API soon. | |\n| This is ~roughly how it works. | |\n| SynthID-Text from scratch. | |\n| A tiny implementation of Tournament Sampling + watermark detection. | |\n| uv run synthid.py | |\n| \"\"\" | |\n| from __future__ import annotations | |\n| import hashlib | |\n| import hmac | |\n| import math | |\n| import torch | |\n| from transformers import AutoModelForCausalLM, AutoTokenizer | |\n| MODEL = \"Qwen/Qwen2.5-0.5B-Instruct\" | |\n| KEY = b\"player-piano\" | |\n| H = 4 # previous tokens used as watermark context | |\n| M = 30 # tournament layers | |\n| TOP_K = 100 | |\n| TEMP = 0.7 | |\n| MAX_NEW = 120 | |\n| def bits(ctx: tuple[int, ...], tok: int, key: bytes = KEY) -> list[int]: | |\n| \"\"\"Return M deterministic keyed random bits for (context, token).\"\"\" | |\n| msg = b\"\".join(int(x).to_bytes(4, \"little\") for x in (*ctx, tok)) | |\n| digest = hmac.new(key, msg, hashlib.sha256).digest() | |\n| n = int.from_bytes(digest, \"little\") | |\n| return [(n >> i) & 1 for i in range(M)] | |\n| def watermark(p: torch.Tensor, toks: torch.Tensor, ctx: tuple[int, ...]) -> torch.Tensor: | |\n| \"\"\"Apply M layers of N=2 Tournament Sampling.\"\"\" | |\n| ids = toks.detach().cpu().tolist() | |\n| g = torch.tensor([bits(ctx, t) for t in ids], dtype=torch.float32, device=p.device) | |\n| p = p.float() | |\n| for i in range(M): | |\n| q = (p * g[:, i]).sum().clamp(0, 1) | |\n| # The whole trick: | |\n| # | |\n| # p'(x) = p(x) [1 + g(x) - q] | |\n| # | |\n| p = p * (1 + g[:, i] - q) | |\n| p = p.clamp_min(0) | |\n| return p / p.sum() | |\n| @torch.inference_mode() | |\n| def generate(model, tok, prompt: str, *, wm: bool, seed: int = 42) -> tuple[str, list[int]]: | |\n| \"\"\"Generate text with or without the watermark.\"\"\" | |\n| device = next(model.parameters()).device | |\n| torch.manual_seed(seed) | |\n| chat = tok.apply_chat_template( | |\n| [{\"role\": \"user\", \"content\": prompt}], | |\n| tokenize=False, | |\n| add_generation_prompt=True, | |\n| ) | |\n| prompt_ids = tok(chat, return_tensors=\"pt\", add_special_tokens=False).input_ids.to(device) | |\n| out: list[int] = [] | |\n| seen: set[tuple[int, ...]] = set() | |\n| eos = model.generation_config.eos_token_id | |\n| if eos is None: | |\n| eos = tok.eos_token_id | |\n| eos_ids = set(eos if isinstance(eos, (list, tuple)) else [eos]) | |\n| for _ in range(MAX_NEW): | |\n| ids = prompt_ids | |\n| if out: | |\n| ids = torch.cat( | |\n| [prompt_ids, torch.tensor([out], dtype=torch.long, device=device)], | |\n| dim=1, | |\n| ) | |\n| logits = model(ids, use_cache=False).logits[0, -1].float() / TEMP | |\n| top_logits, top_ids = torch.topk(logits, min(TOP_K, logits.numel())) | |\n| p = torch.softmax(top_logits, dim=-1) | |\n| # Normal decoding first, watermark second. | |\n| if wm and len(out) >= H: | |\n| ctx = tuple(out[-H:]) | |\n| # Don't watermark the same context twice. | |\n| if ctx not in seen: | |\n| seen.add(ctx) | |\n| p = watermark(p, top_ids, ctx) | |\n| sample = torch.multinomial(p, 1).item() | |\n| token = int(top_ids[sample].item()) | |\n| if token in eos_ids: | |\n| break | |\n| out.append(token) | |\n| return tok.decode(out, skip_special_tokens=True), out | |\n| def detect(ids: list[int], key: bytes = KEY) -> tuple[float, float, float]: | |\n| \"\"\"Return watermark score, z-score and one-sided p-value.\"\"\" | |\n| seen: set[tuple[int, ...]] = set() | |\n| ones = 0 | |\n| n = 0 | |\n| for i in range(H, len(ids)): | |\n| ctx = tuple(ids[i - H:i]) | |\n| if ctx in seen: | |\n| continue | |\n| seen.add(ctx) | |\n| ones += sum(bits(ctx, ids[i], key)) | |\n| n += M | |\n| if n == 0: | |\n| return float(\"nan\"), 0.0, 1.0 | |\n| # Under the null, g ~ Bernoulli(0.5). | |\n| score = ones / n | |\n| z = (ones - n / 2) / math.sqrt(n / 4) | |\n| p = 0.5 * math.erfc(z / math.sqrt(2)) | |\n| return score, z, p | |\n| def detect_text(tok, text: str, key: bytes = KEY) -> tuple[float, float, float]: | |\n| \"\"\"Detect our watermark in arbitrary visible text.\"\"\" | |\n| ids = tok.encode(text, add_special_tokens=False) | |\n| return detect([int(x) for x in ids], key) | |\n| def show_detection(tok, text: str, label: str, key: bytes = KEY) -> None: | |\n| \"\"\"Print an interpretable detection result.\"\"\" | |\n| score, z, p = detect_text(tok, text, key) | |\n| if z >= 4: | |\n| verdict = \"strong watermark evidence\" | |\n| elif z >= 2: | |\n| verdict = \"some watermark evidence\" | |\n| else: | |\n| verdict = \"no watermark evidence\" | |\n| print(f\"\\n{label}\") | |\n| print(f\"score = {score:.4f} z = {z:.2f} p ≈ {p:.2e}\") | |\n| print(f\"result: {verdict}\") | |\n| def show_prefixes(ids: list[int]) -> None: | |\n| \"\"\"Show watermark evidence accumulating with text length.\"\"\" | |\n| print(\"\\nEvidence as tokens accumulate:\") | |\n| for n in [20, 40, 60, 80, 120]: | |\n| if len(ids) < n: | |\n| continue | |\n| score, z, p = detect(ids[:n]) | |\n| print(f\"{n:>3} tokens score={score:.3f} z={z:>6.2f} p≈{p:.1e}\") | |\n| def main() -> None: | |\n| \"\"\"Generate watermarked/plain text and demonstrate detection.\"\"\" | |\n| device = \"cuda\" if torch.cuda.is_available() else \"cpu\" | |\n| dtype = torch.float16 if device == \"cuda\" else torch.float32 | |\n| print(f\"device: {device}\") | |\n| if device == \"cuda\": | |\n| print(f\"gpu: {torch.cuda.get_device_name(0)}\") | |\n| tok = AutoTokenizer.from_pretrained(MODEL) | |\n| model = AutoModelForCausalLM.from_pretrained(MODEL, dtype=dtype).to(device).eval() | |\n| prompt = ( | |\n| \"To the engineers and managers of tomorrow: \" | |\n| \"when machines can do much of the work we now consider skilled, \" | |\n| \"what should humans get exceptionally good at?\" | |\n| ) | |\n| # 1. Generate with our watermark. | |\n| wm_text, wm_ids = generate(model, tok, prompt, wm=True, seed=42) | |\n| print(\"\\n\" + \"=\" * 70) | |\n| print(\"WATERMARKED\") | |\n| print(\"=\" * 70) | |\n| print(wm_text) | |\n| show_detection(tok, wm_text, \"Correct secret key\") | |\n| show_prefixes(wm_ids) | |\n| # Without the correct key, the hidden pattern disappears into noise. | |\n| show_detection(tok, wm_text, \"Wrong secret key\", b\"wrong-key\") | |\n| # 2. Same model, same prompt, ordinary sampling. | |\n| plain_text, _ = generate(model, tok, prompt, wm=False, seed=42) | |\n| print(\"\\n\" + \"=\" * 70) | |\n| print(\"UNWATERMARKED\") | |\n| print(\"=\" * 70) | |\n| print(plain_text) | |\n| show_detection(tok, plain_text, \"Detection\") | |\n| # 3. Treat arbitrary text as an unknown sample. | |\n| mystery = \"\"\" | |\n| Good judgement starts with understanding what matters, what can change, | |\n| and what cannot. Tools may make execution cheaper, but choosing the | |\n| right problem and deciding what trade-offs are acceptable still matter. | |\n| \"\"\" | |\n| print(\"\\n\" + \"=\" * 70) | |\n| print(\"MYSTERY TEXT\") | |\n| print(\"=\" * 70) | |\n| print(mystery.strip()) | |\n| show_detection(tok, mystery, \"Detection\") | |\n| if __name__ == \"__main__\": | |\n| main() |", "url": "https://wpnews.pro/news/minimal-llm-watermarking-from-scratch", "canonical_source": "https://gist.github.com/jSwords91/2732ff6017213526da73e8dc2bd54770", "published_at": "2026-08-23 15:47:38+00:00", "updated_at": "2026-08-24 04:13:55.781141+00:00", "lang": "en", "topics": ["artificial-intelligence", "large-language-models", "ai-safety", "ai-tools", "developer-tools"], "entities": ["Anthropic", "Google DeepMind", "SynthID-Text", "Qwen", "PyTorch", "Hugging Face Transformers"], "alternates": {"html": "https://wpnews.pro/news/minimal-llm-watermarking-from-scratch", "markdown": "https://wpnews.pro/news/minimal-llm-watermarking-from-scratch.md", "text": "https://wpnews.pro/news/minimal-llm-watermarking-from-scratch.txt", "jsonld": "https://wpnews.pro/news/minimal-llm-watermarking-from-scratch.jsonld"}}