Qwen3.8-27B NVFP4-MTP benchmark on RTX 5090 (tier comparison, 64K long-context, spec-draft-n-max sweep) + mtp-bench.py tool

| #!/usr/bin/env python3 | | import argparse, json, os, sys, time | | from urllib import request | | | | PROMPTS = [ | | {"name": "code_python", "prompt": "Write a Python function that returns the n-th Fibonacci number using memoization. Include a docstring."}, | | {"name": "code_cpp", "prompt": "Write a C++ template function clamp(x, lo, hi) that returns x clamped to [lo, hi]. No std::clamp."}, | | {"name": "explain_concept", "prompt": "Explain how speculative decoding works in large language model inference, in three short paragraphs."}, | | {"name": "summarize", "prompt": "Summarize in two sentences: The Industrial Revolution began in Britain in the late 18th century, transforming manufacturing through mechanization, steam power, and the factory system. It spread to continental Europe and North America during the 19th century."}, | | {"name": "qa_factual", "prompt": "Q: What are the four fundamental forces of physics?\nA:"}, | | {"name": "translation", "prompt": "Translate to French: 'The quick brown fox jumps over the lazy dog.'"}, | | {"name": "creative_short", "prompt": "Write a four-line poem about an old lighthouse."}, | | {"name": "stepwise_math", "prompt": "Solve step by step: A train leaves station A at 60 km/h. Two hours later, a second train leaves the same station on the same track at 90 km/h. How long until the second train catches the first?"}, | | {"name": "long_code_review", "prompt": ( | | "You are reviewing a backend service that has been suffering intermittent latency spikes " | | "in production. Below is the relevant code and a description of the system. After reading " | | "carefully, produce a structured review with three sections: (1) likely root causes ranked " | | "by probability, (2) concrete code or configuration changes you would make first, " | | "(3) what telemetry you would add to confirm the diagnosis.\n\n" | | "System description: a Python FastAPI service in front of a Postgres 15 database, deployed " | | "as four replicas behind an nginx load balancer. Each request reads a user record, fetches " | | "their last 50 events from a partitioned events table, computes an aggregate score, writes " | | "the score back to the user row, and returns a JSON response. Average payload is 4 KB. " | | "p50 latency is 35 ms; p99 spikes to 1.8 seconds approximately every 90 seconds in a " | | "regular pattern. The spikes correlate with elevated Postgres CPU but not with elevated " | | "Postgres connection count. The application pool is sized at 20 connections per replica. " | | "PgBouncer is in front of Postgres in transaction pooling mode with a pool size of 50.\n\n" | | "Code excerpt — the hot endpoint:\n" | | " python\n@app.post('/score/{user_id}')\nasync def score(user_id: int, payload: ScoreRequest):\n" | | " async with db.transaction() as tx:\n user = await tx.fetchrow(\n" | | " 'SELECT id, tier, last_score FROM users WHERE id = $1 FOR UPDATE',\n user_id,\n )\n" | | " if user is None:\n raise HTTPException(404)\n events = await tx.fetch(\n" | | " 'SELECT type, weight, ts FROM events '\n 'WHERE user_id = $1 ORDER BY ts DESC LIMIT 50',\n user_id,\n )\n" | | " new_score = compute_score(user['tier'], events, payload.signals)\n" | | " await tx.execute(\n 'UPDATE users SET last_score = $1, updated_at = now() WHERE id = $2',\n new_score, user_id,\n )\n" | | " await tx.execute(\n 'INSERT INTO score_history (user_id, score, ts) VALUES ($1, $2, now())',\n user_id, new_score,\n )\n" | | " await cache.set(f'score:{user_id}', new_score, ex=300)\n" | | " metrics.histogram('score.latency_ms').observe((time.time() - start) * 1000)\n" | | " return {'user_id': user_id, 'score': new_score}\n\n\n" | | "Schema notes: users is ~50M rows, events is partitioned by month with ~2B rows total " | | "and a btree index on (user_id, ts DESC). score_history is unpartitioned, ~800M rows, " | | "with a single index on user_id. Postgres autovacuum is at default settings. There is " | | "a nightly batch job that rebuilds materialized views starting at 02:00 UTC; spikes occur " | | "throughout the day, not just during the batch window. Connection pooling metrics show " | | "PgBouncer waiting connections occasionally hit 8-12 during spikes but never saturate. " | | "CPU on the FastAPI replicas stays below 30% even during spikes. Network round-trip time " | | "between the application and Postgres is consistently 0.4 ms.\n\nBegin your review now." | | )}, | | ] | | | | def estimate_tokens(text): | | """Very rough token estimate: ~4 chars per English token.""" | | return len(text) // 4 | | | | def generate_long_prompt(target_tokens=64000): | | """Generate a long context prompt by repeating a passage until we hit target_tokens.""" | | passage = ( | | "The Industrial Revolution began in Britain in the late 18th century, transforming manufacturing " | | "through mechanization, steam power, and the factory system. It spread to continental Europe and " | | "North America during the 19th century. Key innovations included the spinning jenny, the water frame, " | | "and James Watt's improved steam engine. These inventions dramatically increased production capacity " | | "and led to urbanization as workers moved from rural areas to factory towns. Social changes included " | | "the rise of the middle class, labor movements, and new political ideologies such as liberalism, " | | "conservatism, and socialism. Economic shifts included the decline of feudalism, the growth of global " | | "trade, and the establishment of modern banking systems. The transportation revolution brought " | | "canals, turnpikes, railroads, and steamships, reducing costs and connecting markets across vast distances. " | | "These transformations laid the foundation for the modern world economy.\n" | | ) | | passage_tokens = estimate_tokens(passage) | | if passage_tokens == 0: | | return "Empty prompt" | | repeats_needed = max(1, target_tokens // passage_tokens) | | long_text = passage * repeats_needed | | | approx_tokens = estimate_tokens(long_text) | | if approx_tokens > target_tokens * 1.1: | | word_limit = int(target_tokens * 2) # ~2 chars per word | | long_text = " ".join(long_text.split()[:word_limit]) | | return ( | | f"You will be given a very long document about the Industrial Revolution. " | | f"Read it carefully and answer questions at the end.\n" | | f"Estimated tokens: ~{estimate_tokens(long_text)}\n\n" | | f"{long_text}\n\n" | | f"Q1: What were the key technological innovations mentioned? " | | f"Q2: What social changes occurred during this period? " | | f"Q3: How did transportation evolve?" | | ) | | | | def load_long_prompt(file_path, target_tokens=64000): | | """Load a text file and use it as context, trimming to approximately target_tokens.""" | | if not os.path.exists(file_path): | | print(f"ERROR: File not found: {file_path}"); sys.exit(1) | | with open(file_path, "r", encoding="utf-8") as f: | | text = f.read() | | file_tokens = estimate_tokens(text) | | if file_tokens > target_tokens * 2: | | word_limit = int(target_tokens * 2) | | text = " ".join(text.split()[:word_limit]) | | return ( | | f"You will be given a long document. Read it carefully and answer the questions below.\n" | | f"File: {file_path}\nEstimated tokens: ~{estimate_tokens(text)}\n\n" | | f"{text}\n\n" | | f"Q1: What were the main topics discussed? " | | f"Q2: Summarize the key points in three sentences." | | ) | | | | def post(url, payload): | | req = request.Request(url, data=json.dumps(payload).encode(), headers={"Content-Type":"application/json"}, method="POST") | | with request.urlopen(req, timeout=300) as r: | | return json.loads(r.read()) | | | | def run(args): | | out = {"results": []} | | | | | prompts_to_run = list(PROMPTS) | | if args.long_context: | | if args.context_file: | | content = load_long_prompt(args.context_file, args.context_size) | | else: | | content = generate_long_prompt(args.context_size) | | token_est = estimate_tokens(content) | | prompts_to_run.append({ | | "name": f"long_ctx_{args.context_size // 1000}k", | | "prompt": content, | | "_token_estimate": token_est, | | }) | | print(f"\n[+] Long context prompt: ~{token_est:,} tokens (target: {args.context_size:,})") | | | | for p in prompts_to_run: | | t0 = time.time() | | r = post(f"{args.url}/v1/chat/completions", { | | "model": "qwen-3.8-reasoning", | | "messages": [{"role": "user", "content": p["prompt"]}], | | "max_tokens": 256 if "long_ctx" in (p.get("name", "")) else 192, | | "seed": 42, | | }) | | wall = time.time() - t0 | | | usage = r.get("usage", {}) or {} | | t = r.get("timings", {}) or {} | | predicted_n = usage.get("completion_tokens") or t.get("predicted_n") | | predicted_per_second = t.get("predicted_per_second") or (predicted_n / wall if wall > 0 else 0) | | rec = {"name": p["name"], "wall_s": round(wall,3), | | "predicted_n": predicted_n, "predicted_per_second": round(predicted_per_second, 2), | | "draft_n": t.get("draft_n",0), "draft_n_accepted": t.get("draft_n_accepted",0)} | | rec["accept_rate"] = round(rec["draft_n_accepted"]/rec["draft_n"],4) if rec["draft_n"] else None | | out["results"].append(rec) | | ar = f"{rec['accept_rate']:.3f}" if rec["accept_rate"] is not None else "n/a" | | tok_est = p.get("_token_estimate", 0) | | est_str = f" ctx={tok_est:,}t" if tok_est else "" | | print(f" {rec['name']:<18} pred={rec['predicted_n']:>4} draft={rec['draft_n']:>4} acc={rec['draft_n_accepted']:>4} rate={ar} tok/s={rec['predicted_per_second']:.1f}{est_str}") | | td = sum(x["draft_n"] or 0 for x in out["results"]) | | ta = sum(x["draft_n_accepted"] or 0 for x in out["results"]) | | tp = sum(x["predicted_n"] or 0 for x in out["results"]) | | tw = sum(x["wall_s"] for x in out["results"]) | | out["aggregate"] = {"n_requests": len(out["results"]), "total_predicted": tp, "total_draft": td, "total_draft_accepted": ta, | | "aggregate_accept_rate": round(ta/td,4) if td else None, "wall_s_total": round(tw,2)} | | print("\nAggregate:", json.dumps(out["aggregate"], indent=2)) | | if args.out: | | json.dump(out, open(args.out,"w"), indent=2); print("Wrote", args.out) | | | | def diff(a, b): | | A, B = json.load(open(a)), json.load(open(b)) | | print(f"{'metric':<24} {'A':>14} {'B':>14} {'delta':>10}") | | for k in ("aggregate_accept_rate","total_predicted","total_draft","total_draft_accepted","wall_s_total"): | | va, vb = A["aggregate"].get(k), B["aggregate"].get(k) | | if va is None or vb is None: print(f"{k:<24} {str(va):>14} {str(vb):>14}"); continue | | d = vb - va | | s = f"{d:>+10.4f}" if isinstance(d,float) else f"{d:>+10}" | | print(f"{k:<24} {va:>14} {vb:>14} {s}") | | by_a = {x["name"]: x for x in A["results"]} | | print("\n{:<20} {:>8} {:>8} {:>8}".format("prompt","A","B","delta")) | | for rb in B["results"]: | | ra = by_a.get(rb["name"]) or {} | | ar = ra.get("accept_rate") or 0; br = rb.get("accept_rate") or 0 | | print(f"{rb['name']:<20} {ar:>8.3f} {br:>8.3f} {br-ar:>+8.3f}") | | | | ap = argparse.ArgumentParser(description="MTP Bench — test speculative decoding performance") | | ap.add_argument("--url", default="http://127.0.0.1:8080") | | ap.add_argument("--out") | | ap.add_argument("--diff", nargs=2) | | | ap.add_argument("--long-context", action="store_true", help="Add a large-context prompt to the benchmark") | | ap.add_argument("--context-size", type=int, default=64000, help="Target context size in tokens (default: 64000)") | | ap.add_argument("--context-file", type=str, default=None, help="Path to a text file to use as long context") | | a = ap.parse_args() | | if a.diff: diff(*a.diff) | | else: run(a) |