Performance, Latency & Cost Benchmarking

Why Performance Benchmarking Matters

A model that gives correct answers but takes 30 seconds to respond is not usable in a customer-facing product. Performance benchmarking measures the operational characteristics of your AI system — not just whether it’s correct, but how fast and how expensive.

The three dimensions:

• Latency — time to first token (TTFT) and total response time
• Throughput — requests per second your system can handle
• Cost — price per request, per 1M tokens, per day

---

Latency Benchmarking

Time to First Token (TTFT)

For streaming APIs, TTFT is the most important latency metric — the time the user waits before anything appears on screen. Even if a response takes 10 seconds total, a 200ms TTFT makes it feel fast.

import time
import anthropic
import openai
import statistics


def measure_ttft_anthropic(prompt: str, model: str) -> float:
    """Measure time to first token for Anthropic streaming."""
    client = anthropic.Anthropic()
    start = time.perf_counter()
    ttft = None

    with client.messages.stream(
        model=model,
        max_tokens=500,
        messages=[{"role": "user", "content": prompt}]
    ) as stream:
        for text in stream.text_stream:
            if ttft is None:
                ttft = time.perf_counter() - start
            break  # We only need the first token

    return ttft


def measure_ttft_openai(prompt: str, model: str) -> float:
    """Measure time to first token for OpenAI streaming."""
    client = openai.OpenAI()
    start = time.perf_counter()
    ttft = None

    stream = client.chat.completions.create(
        model=model,
        max_tokens=500,
        stream=True,
        messages=[{"role": "user", "content": prompt}]
    )

    for chunk in stream:
        if chunk.choices[0].delta.content and ttft is None:
            ttft = time.perf_counter() - start
            break

    return ttft


def benchmark_latency(prompt: str, n_runs: int = 20) -> dict:
    """Run multiple samples and compute statistics."""
    ttfts = [measure_ttft_anthropic(prompt, "claude-sonnet-4-6") for _ in range(n_runs)]
    return {
        "model": "claude-sonnet-4-6",
        "p50_ttft_ms": statistics.median(ttfts) * 1000,
        "p95_ttft_ms": sorted(ttfts)[int(0.95 * n_runs)] * 1000,
        "p99_ttft_ms": sorted(ttfts)[int(0.99 * n_runs)] * 1000,
        "mean_ttft_ms": statistics.mean(ttfts) * 1000,
    }

Total Response Time

def measure_total_latency(prompt: str, model: str = "claude-sonnet-4-6") -> dict:
    """Measure total response time and tokens generated."""
    client = anthropic.Anthropic()
    start = time.perf_counter()

    response = client.messages.create(
        model=model,
        max_tokens=500,
        messages=[{"role": "user", "content": prompt}]
    )

    elapsed = time.perf_counter() - start
    output_tokens = response.usage.output_tokens

    return {
        "total_ms": elapsed * 1000,
        "output_tokens": output_tokens,
        "tokens_per_second": output_tokens / elapsed,
    }

---

Cost Benchmarking

Price Per Request Calculator

# Current pricing (May 2026 — always verify at provider pricing pages)
PRICING = {
    "claude-sonnet-4-6": {
        "input_per_1m": 3.00,
        "output_per_1m": 15.00,
    },
    "claude-haiku-4-5": {
        "input_per_1m": 0.80,
        "output_per_1m": 4.00,
    },
    "gpt-4o": {
        "input_per_1m": 2.50,
        "output_per_1m": 10.00,
    },
    "gpt-4o-mini": {
        "input_per_1m": 0.15,
        "output_per_1m": 0.60,
    },
}


def estimate_cost(input_tokens: int, output_tokens: int, model: str) -> float:
    """Calculate cost in USD for a single request."""
    pricing = PRICING[model]
    input_cost = (input_tokens / 1_000_000) * pricing["input_per_1m"]
    output_cost = (output_tokens / 1_000_000) * pricing["output_per_1m"]
    return input_cost + output_cost


def project_monthly_cost(
    daily_requests: int,
    avg_input_tokens: int,
    avg_output_tokens: int,
    model: str
) -> dict:
    """Project monthly cost given usage patterns."""
    cost_per_request = estimate_cost(avg_input_tokens, avg_output_tokens, model)
    monthly_requests = daily_requests * 30
    monthly_cost = cost_per_request * monthly_requests

    return {
        "model": model,
        "cost_per_request_usd": round(cost_per_request, 6),
        "monthly_requests": monthly_requests,
        "monthly_cost_usd": round(monthly_cost, 2),
    }


# Example: 10,000 daily requests, 500 input + 200 output tokens average
for model in PRICING:
    print(project_monthly_cost(10_000, 500, 200, model))

---

Multi-Model Comparison Benchmark

Compare models across accuracy, speed, and cost simultaneously:

import asyncio
import json
from dataclasses import dataclass, asdict

@dataclass
class ModelBenchmarkResult:
    model: str
    accuracy: float
    p95_ttft_ms: float
    cost_per_1000_requests_usd: float
    tokens_per_second: float


async def benchmark_model(model_config: dict, test_cases: list[dict]) -> ModelBenchmarkResult:
    correct = 0
    latencies = []
    costs = []

    for case in test_cases:
        result = await run_model_async(model_config, case["input"])

        if result["answer"].strip().lower() == case["expected"].strip().lower():
            correct += 1

        latencies.append(result["ttft_ms"])
        costs.append(estimate_cost(
            result["input_tokens"],
            result["output_tokens"],
            model_config["name"]
        ))

    return ModelBenchmarkResult(
        model=model_config["name"],
        accuracy=correct / len(test_cases),
        p95_ttft_ms=sorted(latencies)[int(0.95 * len(latencies))],
        cost_per_1000_requests_usd=sum(costs) / len(costs) * 1000,
        tokens_per_second=sum(r["tokens_per_second"] for r in results) / len(results)
    )


# Run benchmarks across models in parallel
models = [
    {"name": "claude-sonnet-4-6", "provider": "anthropic"},
    {"name": "claude-haiku-4-5", "provider": "anthropic"},
    {"name": "gpt-4o", "provider": "openai"},
    {"name": "gpt-4o-mini", "provider": "openai"},
]

results = asyncio.run(asyncio.gather(*[
    benchmark_model(m, test_cases) for m in models
]))

# Print comparison table
print(f"{'Model':<25} {'Accuracy':>8} {'P95 TTFT':>10} {'Cost/1K':>10} {'Tok/s':>8}")
print("-" * 65)
for r in sorted(results, key=lambda x: -x.accuracy):
    print(f"{r.model:<25} {r.accuracy:>8.1%} {r.p95_ttft_ms:>9.0f}ms ${r.cost_per_1000_requests_usd:>8.3f} {r.tokens_per_second:>7.0f}")

Sample output:

Model                     Accuracy   P95 TTFT    Cost/1K    Tok/s
-----------------------------------------------------------------
claude-sonnet-4-6            91.2%       820ms    $ 2.890      87
gpt-4o                       88.7%       710ms    $ 2.120      95
claude-haiku-4-5             79.3%       280ms    $ 0.340     210
gpt-4o-mini                  77.1%       240ms    $ 0.054     220

---

Load Testing

Measure how your system performs under concurrent load:

import asyncio
import time
from dataclasses import dataclass


@dataclass
class LoadTestResult:
    concurrency: int
    total_requests: int
    success_rate: float
    p50_latency_ms: float
    p95_latency_ms: float
    requests_per_second: float
    errors: list[str]


async def run_load_test(
    prompt: str,
    concurrency: int,
    total_requests: int
) -> LoadTestResult:
    semaphore = asyncio.Semaphore(concurrency)
    latencies = []
    errors = []

    async def single_request():
        async with semaphore:
            try:
                start = time.perf_counter()
                await run_model_async({"name": "gpt-4o"}, {"input": prompt})
                latencies.append((time.perf_counter() - start) * 1000)
            except Exception as e:
                errors.append(str(e))

    start_time = time.perf_counter()
    await asyncio.gather(*[single_request() for _ in range(total_requests)])
    total_time = time.perf_counter() - start_time

    sorted_latencies = sorted(latencies)
    return LoadTestResult(
        concurrency=concurrency,
        total_requests=total_requests,
        success_rate=len(latencies) / total_requests,
        p50_latency_ms=sorted_latencies[len(latencies) // 2],
        p95_latency_ms=sorted_latencies[int(0.95 * len(latencies))],
        requests_per_second=len(latencies) / total_time,
        errors=errors[:5],  # First 5 errors
    )


# Test at increasing concurrency levels
for concurrency in [1, 5, 10, 20, 50]:
    result = asyncio.run(run_load_test("Summarise the benefits of microservices", concurrency, 100))
    print(f"Concurrency {concurrency:2d}: {result.requests_per_second:.1f} req/s, p95={result.p95_latency_ms:.0f}ms, errors={len(result.errors)}")

---

Setting SLAs

Based on your benchmarking data, define SLAs before production launch:

PRODUCTION_SLAS = {
    "ttft_p95_ms": 1000,          # First token within 1 second 95% of the time
    "total_latency_p99_ms": 15000, # Full response within 15 seconds 99% of the time
    "error_rate_max": 0.01,        # Less than 1% errors
    "cost_per_day_usd_max": 500,   # Daily cost cap
    "accuracy_min": 0.85,          # At least 85% correct on golden set
}


def check_sla_compliance(benchmark_results: dict) -> dict:
    violations = []
    for metric, threshold in PRODUCTION_SLAS.items():
        actual = benchmark_results.get(metric)
        if actual is None:
            continue

        if metric.endswith("_max") and actual > threshold:
            violations.append(f"{metric}: {actual} > {threshold}")
        elif metric.endswith("_min") and actual < threshold:
            violations.append(f"{metric}: {actual} < {threshold}")

    return {
        "compliant": len(violations) == 0,
        "violations": violations
    }