Regression Testing for AI Models

What is AI Regression Testing?

In traditional software, a regression is when a code change breaks something that previously worked. AI systems have the same problem, but the cause is harder to spot:

• A model provider silently updates their model (e.g., GPT-4o “1106” vs “0125”)
• A prompt template change improves one task while degrading another
• A retrieval change shifts which documents are fetched, affecting downstream answer quality
• A new tool or system prompt message changes model behaviour unpredictably

AI regression testing is the discipline of detecting these changes before users do.

---

The Regression Testing Lifecycle

Establish Baseline
    │  (run evaluation against current system, store scores)
    ▼
Make a Change
    │  (model update, prompt edit, retrieval config change)
    ▼
Run Regression Suite
    │  (same evaluation on same dataset)
    ▼
Compare to Baseline
    ├── Within tolerance? ──► ✅ Proceed
    └── Outside tolerance? ──► ❌ Investigate, revert, or accept and update baseline

---

Establishing and Storing a Baseline

baseline.py

import json
import hashlib
from datetime import datetime
from pathlib import Path
from typing import Any

BASELINE_FILE = Path("eval_baselines.json")


def compute_dataset_hash(dataset_path: str) -> str:
    """Hash the dataset so we know if it changed between runs."""
    content = Path(dataset_path).read_bytes()
    return hashlib.sha256(content).hexdigest()[:12]


def save_baseline(
    scores: dict[str, float],
    model: str,
    prompt_version: str,
    dataset_path: str,
    notes: str = ""
) -> str:
    """Save evaluation scores as the new baseline."""
    baselines = json.loads(BASELINE_FILE.read_text()) if BASELINE_FILE.exists() else []

    baseline_id = f"{model}_{prompt_version}_{datetime.utcnow().strftime('%Y%m%d_%H%M%S')}"
    entry = {
        "id": baseline_id,
        "timestamp": datetime.utcnow().isoformat(),
        "model": model,
        "prompt_version": prompt_version,
        "dataset_hash": compute_dataset_hash(dataset_path),
        "scores": scores,
        "notes": notes,
    }

    baselines.append(entry)
    BASELINE_FILE.write_text(json.dumps(baselines, indent=2))
    print(f"Baseline saved: {baseline_id}")
    return baseline_id


def load_latest_baseline(model: str, prompt_version: str) -> dict | None:
    """Load the most recent baseline for a given model and prompt version."""
    if not BASELINE_FILE.exists():
        return None

    baselines = json.loads(BASELINE_FILE.read_text())
    matches = [
        b for b in baselines
        if b["model"] == model and b["prompt_version"] == prompt_version
    ]
    return matches[-1] if matches else None

---

Regression Detection

regression_check.py

from typing import Optional


REGRESSION_THRESHOLDS = {
    "answer_relevancy": 0.03,    # Fail if score drops more than 3 percentage points
    "faithfulness": 0.05,        # Fail if faithfulness drops more than 5pp
    "context_recall": 0.05,
    "accuracy": 0.02,            # Fail if accuracy drops more than 2pp
}


def detect_regression(
    baseline_scores: dict[str, float],
    current_scores: dict[str, float],
    thresholds: Optional[dict[str, float]] = None,
) -> dict:
    """Compare current scores to baseline and identify regressions."""
    if thresholds is None:
        thresholds = REGRESSION_THRESHOLDS

    regressions = []
    improvements = []
    unchanged = []

    for metric, current in current_scores.items():
        baseline = baseline_scores.get(metric)
        if baseline is None:
            continue

        delta = current - baseline
        threshold = thresholds.get(metric, 0.03)

        if delta < -threshold:
            regressions.append({
                "metric": metric,
                "baseline": baseline,
                "current": current,
                "delta": delta,
                "threshold": -threshold,
            })
        elif delta > threshold:
            improvements.append({
                "metric": metric,
                "baseline": baseline,
                "current": current,
                "delta": delta,
            })
        else:
            unchanged.append(metric)

    return {
        "passed": len(regressions) == 0,
        "regressions": regressions,
        "improvements": improvements,
        "unchanged": unchanged,
    }


def format_regression_report(result: dict) -> str:
    lines = ["## Regression Report\n"]

    if result["passed"]:
        lines.append("✅ No regressions detected.\n")
    else:
        lines.append(f"❌ {len(result['regressions'])} regression(s) detected.\n")
        for r in result["regressions"]:
            lines.append(
                f"  - **{r['metric']}**: {r['baseline']:.3f} → {r['current']:.3f} "
                f"(Δ {r['delta']:+.3f}, threshold {r['threshold']:.3f})"
            )

    if result["improvements"]:
        lines.append(f"\n✅ {len(result['improvements'])} improvement(s):\n")
        for i in result["improvements"]:
            lines.append(f"  + **{i['metric']}**: {i['baseline']:.3f} → {i['current']:.3f} (Δ {i['delta']:+.3f})")

    return "\n".join(lines)

---

A/B Evaluation for Model Changes

When comparing two models or two prompt versions, A/B evaluation uses a judge LLM to pick the winner on each test case:

import openai

client = openai.OpenAI()


def ab_evaluate(
    question: str,
    response_a: str,
    response_b: str,
    expected: str,
) -> str:
    """Use GPT-4o as judge to pick the better response. Returns 'A', 'B', or 'tie'."""

    prompt = f"""You are evaluating two AI responses to a question.

Question: {question}

Expected answer: {expected}

Response A:
{response_a}

Response B:
{response_b}

Which response is better? Consider: accuracy, completeness, and clarity.
Reply with only: A, B, or TIE"""

    result = client.chat.completions.create(
        model="gpt-4o",
        messages=[{"role": "user", "content": prompt}],
        max_tokens=5,
        temperature=0,
    )

    verdict = result.choices[0].message.content.strip().upper()
    return verdict if verdict in ("A", "B", "TIE") else "TIE"


def run_ab_test(
    test_cases: list[dict],
    model_a: str,
    model_b: str,
    n_runs: int = 3,  # Run each test case multiple times to reduce variance
) -> dict:
    """Run a full A/B evaluation between two models."""
    wins_a = 0
    wins_b = 0
    ties = 0

    for case in test_cases:
        case_verdicts = []
        for _ in range(n_runs):
            response_a = call_model(model_a, case["input"])
            response_b = call_model(model_b, case["input"])
            verdict = ab_evaluate(case["input"], response_a, response_b, case["expected"])
            case_verdicts.append(verdict)

        # Majority vote across n_runs
        from collections import Counter
        winner = Counter(case_verdicts).most_common(1)[0][0]
        if winner == "A":
            wins_a += 1
        elif winner == "B":
            wins_b += 1
        else:
            ties += 1

    total = wins_a + wins_b + ties
    return {
        "model_a": model_a,
        "model_b": model_b,
        "wins_a": wins_a,
        "wins_b": wins_b,
        "ties": ties,
        "win_rate_a": wins_a / total,
        "win_rate_b": wins_b / total,
        "recommendation": model_a if wins_a > wins_b else model_b if wins_b > wins_a else "no_change",
    }

---

Tracking Score History

Visualise quality trends over time:

import json
from pathlib import Path
from datetime import datetime


SCORE_HISTORY_FILE = Path("score_history.jsonl")


def record_eval_run(
    run_id: str,
    model: str,
    prompt_version: str,
    scores: dict[str, float],
    git_sha: str = "",
):
    entry = {
        "timestamp": datetime.utcnow().isoformat(),
        "run_id": run_id,
        "model": model,
        "prompt_version": prompt_version,
        "git_sha": git_sha,
        "scores": scores,
    }
    with SCORE_HISTORY_FILE.open("a") as f:
        f.write(json.dumps(entry) + "\n")


def load_score_history() -> list[dict]:
    if not SCORE_HISTORY_FILE.exists():
        return []
    return [json.loads(line) for line in SCORE_HISTORY_FILE.read_text().strip().splitlines()]


def plot_score_trend(metric: str = "answer_relevancy"):
    """Print a simple text trend chart."""
    history = load_score_history()
    if not history:
        print("No history yet.")
        return

    print(f"\nScore trend for: {metric}")
    print("-" * 50)
    for entry in history[-20:]:  # Last 20 runs
        score = entry["scores"].get(metric, 0)
        bar = "█" * int(score * 40)
        print(f"{entry['timestamp'][:10]} {entry['model']:<25} {score:.3f} {bar}")

---

Prompt Regression Testing

Test every prompt template change against the full golden set automatically:

tests/test_prompt_regression.py

import pytest
from my_app.prompts import SUPPORT_PROMPT_V1, SUPPORT_PROMPT_V2
from my_app.eval import run_prompt_against_dataset

BASELINE_SCORES = {
    "answer_relevancy": 0.85,
    "faithfulness": 0.90,
}

@pytest.fixture(scope="session")
def golden_dataset():
    import json
    return [json.loads(l) for l in open("tests/golden_dataset.jsonl")]


def test_prompt_v2_does_not_regress(golden_dataset):
    """Verify SUPPORT_PROMPT_V2 does not regress on our golden test set."""
    scores = run_prompt_against_dataset(SUPPORT_PROMPT_V2, golden_dataset)

    for metric, baseline in BASELINE_SCORES.items():
        assert scores[metric] >= baseline - 0.03, (
            f"{metric} regressed: {scores[metric]:.3f} < baseline {baseline:.3f} - 0.03"
        )


def test_prompt_v2_improves_or_matches(golden_dataset):
    """Verify V2 is not worse than V1 overall."""
    scores_v1 = run_prompt_against_dataset(SUPPORT_PROMPT_V1, golden_dataset)
    scores_v2 = run_prompt_against_dataset(SUPPORT_PROMPT_V2, golden_dataset)

    # V2 should win or tie on the majority of metrics
    v2_wins = sum(
        1 for m in BASELINE_SCORES
        if scores_v2.get(m, 0) >= scores_v1.get(m, 0) - 0.01
    )
    assert v2_wins >= len(BASELINE_SCORES) * 0.7, "V2 regressed on too many metrics"

---

Accepting a Regression Intentionally

Sometimes a regression is an acceptable tradeoff (e.g., trading accuracy for speed). Document this explicitly:

# Update the baseline after an intentional tradeoff
python scripts/update_baseline.py \
  --model claude-haiku-4-5 \
  --prompt-version v3.1 \
  --notes "Switched to Haiku for 4x speed, accepting 6pp accuracy drop — approved by @product-lead"

Never silently update a baseline. Always require a human decision and a written justification captured in version control.