Four small tools that turn "words ≠ tokens" into something you can measure: count tokens and cost, compare τ across text types, check a prompt against a context window, and size an embedding matrix in memory.
cl100k_base) counts measured with tiktoken. Every tool has the exact Python version underneath it — run that for production numbers.
Paste any text. Get word count, estimated token count, τ = tokens ÷ words, and the API cost at a price per million tokens.
"""token_counter.py — python token_counter.py "your text" --price 30 --requests 1000"""
import argparse
import tiktoken
parser = argparse.ArgumentParser()
parser.add_argument("text")
parser.add_argument("--model", default="gpt-4")
parser.add_argument("--price", type=float, default=30.0, help="USD per 1M tokens")
parser.add_argument("--requests", type=int, default=1)
args = parser.parse_args()
enc = tiktoken.encoding_for_model(args.model)
tokens = len(enc.encode(args.text))
words = len(args.text.split())
tau = tokens / words if words else 0.0
cost = tokens / 1_000_000 * args.price
print(f"Words: {words}")
print(f"Tokens: {tokens}")
print(f"τ: {tau:.2f}")
print(f"Cost/request: ${cost:.6f}")
print(f"Cost x {args.requests}: ${cost * args.requests:.4f}")Ten samples run through the real GPT-4 tokenizer (cl100k_base). Switch the metric to see why whitespace-based τ breaks down for scripts without spaces.
"""tau_visualizer.py — pip install tiktoken matplotlib"""
import tiktoken
import matplotlib.pyplot as plt
SAMPLES = {
"English prose": "The quick brown fox jumps over the lazy dog while the sun sets slowly behind the hills.",
"Technical jargon": "Hyperparameter optimization of transformer architectures requires backpropagation through heterogeneous embeddings.",
"Python code": "def add(a, b):\n return a + b\n\nprint(add(2, 3))",
"JSON": '{"id": 42, "name": "Ada", "tags": ["ai", "ml"], "active": true}',
"Spanish": "El rápido zorro marrón salta sobre el perro perezoso mientras el sol se pone detrás de las colinas.",
"German": "Die Donaudampfschifffahrtsgesellschaft veröffentlichte gestern ihre Jahresbilanz.",
"Chinese": "今天天气很好，我们一起去公园散步吧。",
"Hindi": "आज मौसम बहुत अच्छा है, चलो पार्क में टहलने चलते हैं।",
"Emoji": "😀🎉🚀🔥👍🌍💡🙌",
"Numbers": "3.14159265358979 2718281828 1234567890 0.000123",
}
enc = tiktoken.encoding_for_model("gpt-4")
taus = {}
for name, text in SAMPLES.items():
tokens = len(enc.encode(text))
words = len(text.split())
taus[name] = tokens / words
print(f"{name:18} tokens={tokens:3} words={words:3} chars={len(text):3} tau={taus[name]:.2f}")
print("highest:", max(taus, key=taus.get), "| lowest:", min(taus, key=taus.get))
plt.barh(list(taus), list(taus.values()))
plt.xlabel("τ = tokens / word")
plt.tight_layout()
plt.show()Will the system prompt + user message + reserved reply fit in the model's context window? Measured in tokens, not words.
import tiktoken
CONTEXT_WINDOWS = {
"gpt-4": 8_192,
"gpt-4-turbo": 128_000,
"gpt-4o": 128_000,
}
def fits_in_context(system_prompt: str, user_message: str,
model: str = "gpt-4", reserve_for_reply: int = 512) -> dict:
enc = tiktoken.encoding_for_model(model)
used = len(enc.encode(system_prompt)) + len(enc.encode(user_message))
window = CONTEXT_WINDOWS[model]
left = window - reserve_for_reply - used
return {"used": used, "window": window, "left": left, "fits": left >= 0}
print(fits_in_context("You are a helpful assistant.", "Hello!"))The embedding matrix has V × dmodel parameters. Memory = parameters × bytes per parameter.
"""vram_calc.py — python vram_calc.py 50257 768"""
import sys
BYTES = {"FP32": 4, "FP16": 2, "INT8": 1}
vocab, d_model = int(sys.argv[1]), int(sys.argv[2])
params = vocab * d_model
print(f"Embedding parameters: {params:,}")
for name, size in BYTES.items():
mib = params * size / 1024**2
print(f"{name}: {mib:,.1f} MiB")