A simple CLI tool for testing OpenAI-compatible APIs and measuring streaming performance. It calculates TTFT, generation time, completion tokens, and real-time TPS with optional proxy support.

A developer has released a simple command-line tool for testing OpenAI-compatible APIs and measuring streaming performance. The tool calculates time-to-first-token (TTFT), generation time, completion tokens, and real-time tokens per second (TPS), with optional proxy support.

| | /usr/bin/env python3 | | | import json | | | import time | | | import urllib.request | | | import urllib.error | | | def main : | | | base url = input "Base URL: " .strip .rstrip "/" | | | model = input "Model: " .strip | | | api key = input "API Key: " .strip | | | proxy = input "Proxy optional : " .strip | | | prompt = input "Prompt: " .strip | | | url = f"{base url}/chat/completions" | | | payload = { | | | "model": model, | | | "messages": | | | { | | | "role": "user", | | | "content": prompt, | | | } | | | , | | | "stream": True, | | | "stream options": { | | | "include usage": True, | | | }, | | | } | | | data = json.dumps payload .encode | | | request = urllib.request.Request | | | url, | | | data=data, | | | headers={ | | | "Authorization": f"Bearer {api key}", | | | "Content-Type": "application/json", | | | "Accept": "text/event-stream", | | | }, | | | method="POST", | | | | | | Proxy | | | if proxy: | | | proxy handler = urllib.request.ProxyHandler { | | | "http": proxy, | | | "https": proxy, | | | } | | | opener = urllib.request.build opener proxy handler | | | else: | | | opener = urllib.request.build opener | | | print | | | print "Connecting..." | | | print | | | request start = time.perf counter | | | first token time = None | | | last token time = None | | | completion tokens = None | | | try: | | | with opener.open request, timeout=600 as response: | | | for raw line in response: | | | line = raw line.decode "utf-8", errors="replace" .strip | | | if not line.startswith "data:" : | | | continue | | | data str = line 5: .strip | | | if data str == " DONE ": | | | break | | | try: | | | chunk = json.loads data str | | | except json.JSONDecodeError: | | | continue | | | Usage | | | usage = chunk.get "usage" | | | if usage: | | | completion tokens = usage.get "completion tokens" | | | choices = chunk.get "choices", | | | if not choices: | | | continue | | | delta = choices 0 .get "delta", {} | | | content = delta.get "content" | | | if content: | | | now = time.perf counter | | | if first token time is None: | | | first token time = now | | | ttft = first token time - request start | | | print | | | f"\n TTFT: {ttft:.3f}s \n" | | | | | | last token time = now | | | print content, end="", flush=True | | | except urllib.error.HTTPError as e: | | | print f"\nHTTP Error {e.code}:" | | | print e.read .decode errors="replace" | | | return | | | except Exception as e: | | | print f"\nError: {e}" | | | return | | | print "\n" | | | print "=" 50 | | | if first token time is None: | | | print "No tokens received." | | | return | | | if last token time is None: | | | last token time = first token time | | | generation time = last token time - first token time | | | total time = last token time - request start | | | print f"Model: {model}" | | | print f"Completion tokens: {completion tokens}" | | | print f"TTFT: {first token time - request start:.3f}s" | | | print f"Generation time: {generation time:.3f}s" | | | print f"Total time: {total time:.3f}s" | | | if completion tokens and generation time 0: | | | tps = completion tokens / generation time | | | print f"TPS: {tps:.2f} tokens/s" | | | else: | | | print "TPS: N/A" | | | if name == " main ": | | | main |