fit-boilerplate-wox/evals/runner.py

#!/usr/bin/env python3
"""Eval runner — Capas 2 (behavior) y 3 (final state).

Lee `evals/scenarios/*.yaml`, ejecuta cada scenario contra el agente
correspondiente, y verifica las trazas observables + estado final.

Formato del scenario YAML (ver docs/eval-strategy.md):

    name: reset_password_happy_path
    agent: ad_specialist_cotemar
    input: "Necesito resetear..."
    expect:
      agent_response_contains: ["TKT-"]
      tool_calls_in_order:
        - tool: lookup_user
        - tool: reset_password
        - tool: create_ticket
      no_tool_calls:
        - escalate_to_n2
    final_state:
      # ... asserts contra DB / API
"""
from __future__ import annotations

import json
import os
import sys
import sqlite3
import time
from pathlib import Path

try:
    import yaml
except ImportError:
    print("✗ pyyaml required: pip install pyyaml", file=sys.stderr)
    sys.exit(2)

ROOT = Path(__file__).resolve().parent.parent
SCENARIOS_DIR = ROOT / "evals" / "scenarios"
TRACES_DB = os.environ.get("TRACES_DB_PATH", "/tmp/traces.db")


def run_scenario(spec: dict) -> tuple[bool, list[str]]:
    """Returns (passed, errors)."""
    errors: list[str] = []
    name = spec.get("name", "?")
    agent = spec.get("agent")
    input_text = spec.get("input")
    expect = spec.get("expect", {})

    if not agent or not input_text:
        return False, ["scenario sin 'agent' o 'input'"]

    started = int(time.time() * 1000)

    # 1) Mandar el input al agente (usando orchestrate CLI)
    fixture = {"messages": [{"role": "user", "content": input_text}]}
    fixture_path = Path("/tmp") / f"_eval_{name}.json"
    fixture_path.write_text(json.dumps(fixture))
    import subprocess
    proc = subprocess.run(
        ["orchestrate", "agents", "test", agent, "--input-file", str(fixture_path)],
        capture_output=True, text=True, timeout=120,
    )
    output = proc.stdout + "\n" + proc.stderr

    # 2) Verificar response (Capa 2)
    for needle in expect.get("agent_response_contains", []):
        if needle not in output:
            errors.append(f"agent_response_contains: '{needle}' not in output")

    # 3) Verificar tool calls vía trazas (Capa 2)
    traces = _read_traces_since(started)
    tool_names = [t["tool"] for t in traces]

    expected_order = [t.get("tool") for t in expect.get("tool_calls_in_order", [])]
    if expected_order:
        if not _is_subsequence(expected_order, tool_names):
            errors.append(f"tool_calls_in_order: expected {expected_order}, got {tool_names}")

    for bad_tool in expect.get("no_tool_calls", []):
        if bad_tool in tool_names:
            errors.append(f"no_tool_calls: {bad_tool} should NOT have been called")

    # 4) Final state (Capa 3) — placeholder, depende del backend
    # TODO en v2: parametrizable según backend del proyecto
    final_state = spec.get("final_state")
    if final_state:
        # Stub — implementación específica del proyecto
        pass

    return (len(errors) == 0), errors


def _is_subsequence(needle: list[str], haystack: list[str]) -> bool:
    """¿`needle` es subsequence de `haystack`? (order preserved, gaps OK)"""
    it = iter(haystack)
    return all(x in it for x in needle)


def _read_traces_since(ts_ms: int) -> list[dict]:
    if not Path(TRACES_DB).exists():
        return []
    conn = sqlite3.connect(TRACES_DB)
    rows = conn.execute(
        "SELECT payload FROM traces WHERE started_at >= datetime(?, 'unixepoch')",
        (ts_ms / 1000,)
    ).fetchall()
    conn.close()
    return [json.loads(r[0]) for r in rows]


def main() -> int:
    scenarios = sorted(SCENARIOS_DIR.glob("*.yaml"))
    if not scenarios:
        print("(no scenarios in evals/scenarios/*.yaml — skipping)")
        return 0

    pass_count = 0
    fail_count = 0
    for f in scenarios:
        if f.name.startswith("_"):
            continue
        spec = yaml.safe_load(f.read_text())
        name = spec.get("name", f.stem)
        ok, errors = run_scenario(spec)
        if ok:
            print(f"  ✓ {name}")
            pass_count += 1
        else:
            print(f"  ✗ {name}")
            for e in errors:
                print(f"     - {e}")
            fail_count += 1

    print(f"\n  Total: {pass_count} pass / {fail_count} fail")
    return 0 if fail_count == 0 else 1


if __name__ == "__main__":
    sys.exit(main())