feat: v1 — boilerplate WxO + web
Boilerplate completo para construir soluciones agénticas sobre IBM watsonx Orchestrate (ADK 2.x) con una capa web encima. Destilado de cotemar-poc-n1 y dun-casos-prueba. Incluye: - 11 docs (best practices, architecture patterns, ADK cheatsheet, observability, tool authoring, deployment, RUNBOOK, DEPLOY_TO_NEW_WOX, known-issues con 16 errores reales y fix, eval strategy, INDEX) - 13 templates WxO (orchestrator/specialist/single-meta agents, 3 connections, KB + runbook, observable_tool decorator, coercion helpers, Python tools, OpenAPI spec, backend filter endpoint, webhook validator HMAC, MCP connection) - 6 scripts (deploy idempotente con fallback ADK, undeploy, reset, check-adk-version, new-specialist scaffold, eval-agents runner) - 4 eval pieces (linter de best practices, runner, smoke-test, direct backend probe) + scenario templates - 8 subagentes Claude (.claude/agents/) — wxo-architect, wxo-agent-author, wxo-tool-author, runbook-author, mock-builder, backend-tool-builder, eval-author, web-layer-builder - Skill bundle fit-wxo-bootstrap/ (SKILL.md + 3 templates) listo para copiar a ~/.claude/skills/ - Web layer default FastAPI+HTMX con vista timeline observable + endpoint receptor de trazas - Docker compose Coolify-ready (healthcheck wget -qO-, sin labels Traefik, network split internal:true) - CI Gitea workflow con lint + smoke Best practices enforced por evals/lint_wxo_yaml.py: - A1: máx 10 tools por agente - A3: orchestrator sin tools de remediación - A6: agente sin propósito (react sin tools ni KB) - T1: @observable_tool obligatorio, no @tool directo - T3: _compat shim inline en tools.py - D1: docker-compose sin wget --spider - I-005: OpenAPI con description per-operation - I-009: sin labels Traefik manuales Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
141
evals/runner.py
Normal file
141
evals/runner.py
Normal file
@@ -0,0 +1,141 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Eval runner — Capas 2 (behavior) y 3 (final state).
|
||||
|
||||
Lee `evals/scenarios/*.yaml`, ejecuta cada scenario contra el agente
|
||||
correspondiente, y verifica las trazas observables + estado final.
|
||||
|
||||
Formato del scenario YAML (ver docs/eval-strategy.md):
|
||||
|
||||
name: reset_password_happy_path
|
||||
agent: ad_specialist_cotemar
|
||||
input: "Necesito resetear..."
|
||||
expect:
|
||||
agent_response_contains: ["TKT-"]
|
||||
tool_calls_in_order:
|
||||
- tool: lookup_user
|
||||
- tool: reset_password
|
||||
- tool: create_ticket
|
||||
no_tool_calls:
|
||||
- escalate_to_n2
|
||||
final_state:
|
||||
# ... asserts contra DB / API
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import sqlite3
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError:
|
||||
print("✗ pyyaml required: pip install pyyaml", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
SCENARIOS_DIR = ROOT / "evals" / "scenarios"
|
||||
TRACES_DB = os.environ.get("TRACES_DB_PATH", "/tmp/traces.db")
|
||||
|
||||
|
||||
def run_scenario(spec: dict) -> tuple[bool, list[str]]:
|
||||
"""Returns (passed, errors)."""
|
||||
errors: list[str] = []
|
||||
name = spec.get("name", "?")
|
||||
agent = spec.get("agent")
|
||||
input_text = spec.get("input")
|
||||
expect = spec.get("expect", {})
|
||||
|
||||
if not agent or not input_text:
|
||||
return False, ["scenario sin 'agent' o 'input'"]
|
||||
|
||||
started = int(time.time() * 1000)
|
||||
|
||||
# 1) Mandar el input al agente (usando orchestrate CLI)
|
||||
fixture = {"messages": [{"role": "user", "content": input_text}]}
|
||||
fixture_path = Path("/tmp") / f"_eval_{name}.json"
|
||||
fixture_path.write_text(json.dumps(fixture))
|
||||
import subprocess
|
||||
proc = subprocess.run(
|
||||
["orchestrate", "agents", "test", agent, "--input-file", str(fixture_path)],
|
||||
capture_output=True, text=True, timeout=120,
|
||||
)
|
||||
output = proc.stdout + "\n" + proc.stderr
|
||||
|
||||
# 2) Verificar response (Capa 2)
|
||||
for needle in expect.get("agent_response_contains", []):
|
||||
if needle not in output:
|
||||
errors.append(f"agent_response_contains: '{needle}' not in output")
|
||||
|
||||
# 3) Verificar tool calls vía trazas (Capa 2)
|
||||
traces = _read_traces_since(started)
|
||||
tool_names = [t["tool"] for t in traces]
|
||||
|
||||
expected_order = [t.get("tool") for t in expect.get("tool_calls_in_order", [])]
|
||||
if expected_order:
|
||||
if not _is_subsequence(expected_order, tool_names):
|
||||
errors.append(f"tool_calls_in_order: expected {expected_order}, got {tool_names}")
|
||||
|
||||
for bad_tool in expect.get("no_tool_calls", []):
|
||||
if bad_tool in tool_names:
|
||||
errors.append(f"no_tool_calls: {bad_tool} should NOT have been called")
|
||||
|
||||
# 4) Final state (Capa 3) — placeholder, depende del backend
|
||||
# TODO en v2: parametrizable según backend del proyecto
|
||||
final_state = spec.get("final_state")
|
||||
if final_state:
|
||||
# Stub — implementación específica del proyecto
|
||||
pass
|
||||
|
||||
return (len(errors) == 0), errors
|
||||
|
||||
|
||||
def _is_subsequence(needle: list[str], haystack: list[str]) -> bool:
|
||||
"""¿`needle` es subsequence de `haystack`? (order preserved, gaps OK)"""
|
||||
it = iter(haystack)
|
||||
return all(x in it for x in needle)
|
||||
|
||||
|
||||
def _read_traces_since(ts_ms: int) -> list[dict]:
|
||||
if not Path(TRACES_DB).exists():
|
||||
return []
|
||||
conn = sqlite3.connect(TRACES_DB)
|
||||
rows = conn.execute(
|
||||
"SELECT payload FROM traces WHERE started_at >= datetime(?, 'unixepoch')",
|
||||
(ts_ms / 1000,)
|
||||
).fetchall()
|
||||
conn.close()
|
||||
return [json.loads(r[0]) for r in rows]
|
||||
|
||||
|
||||
def main() -> int:
|
||||
scenarios = sorted(SCENARIOS_DIR.glob("*.yaml"))
|
||||
if not scenarios:
|
||||
print("(no scenarios in evals/scenarios/*.yaml — skipping)")
|
||||
return 0
|
||||
|
||||
pass_count = 0
|
||||
fail_count = 0
|
||||
for f in scenarios:
|
||||
if f.name.startswith("_"):
|
||||
continue
|
||||
spec = yaml.safe_load(f.read_text())
|
||||
name = spec.get("name", f.stem)
|
||||
ok, errors = run_scenario(spec)
|
||||
if ok:
|
||||
print(f" ✓ {name}")
|
||||
pass_count += 1
|
||||
else:
|
||||
print(f" ✗ {name}")
|
||||
for e in errors:
|
||||
print(f" - {e}")
|
||||
fail_count += 1
|
||||
|
||||
print(f"\n Total: {pass_count} pass / {fail_count} fail")
|
||||
return 0 if fail_count == 0 else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user