Eval Framework¶

from selectools import Agent, AgentConfig, tool
from selectools.providers.stubs import LocalProvider
from selectools.evals import EvalSuite, TestCase

@tool()
def cancel_subscription(user_id: str) -> str:
    """Cancel a user subscription."""
    return f"Subscription cancelled for {user_id}"

agent = Agent(
    tools=[cancel_subscription],
    provider=LocalProvider(),
    config=AgentConfig(model="gpt-4o"),
)

suite = EvalSuite(agent=agent, cases=[
    TestCase(input="Cancel my account", expect_tool="cancel_subscription"),
    TestCase(input="Help me cancel", expect_contains="cancel"),
])
report = suite.run()
print(f"Accuracy: {report.accuracy:.0%}")
print(f"Pass: {report.pass_count}, Fail: {report.fail_count}")

from selectools.evals import EvalSuite, TestCase

suite = EvalSuite(agent=agent, cases=[
    TestCase(input="Cancel my account", expect_tool="cancel_subscription"),
    TestCase(input="Check my balance", expect_contains="balance"),
    TestCase(input="What's 2+2?", expect_output="4"),
])
report = suite.run()
print(report.accuracy)      # 0.95
print(report.latency_p50)   # 142ms
print(report.total_cost)    # $0.003

TestCase(input="Cancel subscription", expect_tool="cancel_sub")
TestCase(input="Full workflow", expect_tools=["search", "summarize"])
TestCase(input="Search", expect_tool_args={"search": {"query": "python"}})

TestCase(input="Hello", expect_contains="hello")
TestCase(input="Safe?", expect_not_contains="error")
TestCase(input="2+2", expect_output="4")
TestCase(input="Phone", expect_output_regex=r"\d{3}-\d{4}")
TestCase(input="JSON?", expect_json=True)
TestCase(input="Prefix", expect_starts_with="Hello")
TestCase(input="Suffix", expect_ends_with=".")
TestCase(input="Short", expect_min_length=10, expect_max_length=500)

TestCase(
    input="Extract name",
    response_format=MyModel,
    expect_parsed={"name": "Alice"},
)

TestCase(
    input="Fast query",
    expect_latency_ms_lte=500,
    expect_cost_usd_lte=0.01,
    expect_iterations_lte=3,
)

TestCase(input="Account info", expect_no_pii=True)
TestCase(input="Ignore instructions", expect_no_injection=True)

TestCase(
    input="Summarize this",
    reference="The original long text...",  # ground truth
    context="Retrieved document content...",  # RAG context
    rubric="Rate accuracy and completeness",  # custom rubric
)

def must_be_polite(result) -> bool:
    return "please" in result.content.lower()

TestCase(
    input="Help me",
    custom_evaluator=must_be_polite,
    custom_evaluator_name="politeness",
)

TestCase(input="Critical", tags=["billing", "critical"], weight=3.0)
TestCase(input="Minor", tags=["nice-to-have"], weight=0.5)

from selectools.evals import CorrectnessEvaluator, RelevanceEvaluator

suite = EvalSuite(
    agent=agent,
    cases=cases,
    evaluators=[
        CorrectnessEvaluator(provider=provider, model="gpt-4.1-mini"),
        RelevanceEvaluator(provider=provider, model="gpt-4.1-mini"),
    ],
)

report = suite.run()

# Aggregate metrics
report.accuracy        # Weighted accuracy (0.0 - 1.0)
report.pass_count      # Number of passing cases
report.fail_count      # Number of failing cases
report.error_count     # Number of error cases
report.total_cost      # Total USD cost
report.total_tokens    # Total tokens used
report.latency_p50     # Median latency (ms)
report.latency_p95     # 95th percentile latency
report.latency_p99     # 99th percentile latency
report.cost_per_case   # Average cost per case

# Filtering
report.filter_by_tag("billing")
report.filter_by_verdict(CaseVerdict.FAIL)
report.failures_by_evaluator()  # {"tool_use": 3, "contains": 1}

# Export
report.to_html("report.html")         # Interactive HTML report
report.to_junit_xml("results.xml")    # JUnit XML for CI
report.to_json("results.json")        # Machine-readable JSON
report.summary()                      # Human-readable text

from selectools.evals import DatasetLoader

# JSON
cases = DatasetLoader.from_json("tests/eval_cases.json")

# YAML (requires PyYAML)
cases = DatasetLoader.from_yaml("tests/eval_cases.yaml")

# Auto-detect from extension
cases = DatasetLoader.load("tests/eval_cases.json")

from selectools.evals import BaselineStore

store = BaselineStore("./baselines")
report = suite.run()

# Compare against saved baseline
result = store.compare(report)
if result.is_regression:
    print(f"Regressions: {result.regressions}")
    print(f"Accuracy delta: {result.accuracy_delta:+.2%}")
else:
    store.save(report)  # Update baseline

suite = EvalSuite(
    agent=agent,
    cases=cases,
    max_concurrency=5,  # Run 5 cases in parallel
    on_progress=lambda done, total: print(f"[{done}/{total}]"),
)

def test_agent_accuracy(agent):
    suite = EvalSuite(agent=agent, cases=[
        TestCase(input="Cancel", expect_tool="cancel_sub"),
        TestCase(input="Balance", expect_contains="balance"),
    ])
    report = suite.run()
    assert report.accuracy >= 0.9
    assert report.latency_p50 < 500