feat: v1 — boilerplate WxO + web
Boilerplate completo para construir soluciones agénticas sobre IBM watsonx Orchestrate (ADK 2.x) con una capa web encima. Destilado de cotemar-poc-n1 y dun-casos-prueba. Incluye: - 11 docs (best practices, architecture patterns, ADK cheatsheet, observability, tool authoring, deployment, RUNBOOK, DEPLOY_TO_NEW_WOX, known-issues con 16 errores reales y fix, eval strategy, INDEX) - 13 templates WxO (orchestrator/specialist/single-meta agents, 3 connections, KB + runbook, observable_tool decorator, coercion helpers, Python tools, OpenAPI spec, backend filter endpoint, webhook validator HMAC, MCP connection) - 6 scripts (deploy idempotente con fallback ADK, undeploy, reset, check-adk-version, new-specialist scaffold, eval-agents runner) - 4 eval pieces (linter de best practices, runner, smoke-test, direct backend probe) + scenario templates - 8 subagentes Claude (.claude/agents/) — wxo-architect, wxo-agent-author, wxo-tool-author, runbook-author, mock-builder, backend-tool-builder, eval-author, web-layer-builder - Skill bundle fit-wxo-bootstrap/ (SKILL.md + 3 templates) listo para copiar a ~/.claude/skills/ - Web layer default FastAPI+HTMX con vista timeline observable + endpoint receptor de trazas - Docker compose Coolify-ready (healthcheck wget -qO-, sin labels Traefik, network split internal:true) - CI Gitea workflow con lint + smoke Best practices enforced por evals/lint_wxo_yaml.py: - A1: máx 10 tools por agente - A3: orchestrator sin tools de remediación - A6: agente sin propósito (react sin tools ni KB) - T1: @observable_tool obligatorio, no @tool directo - T3: _compat shim inline en tools.py - D1: docker-compose sin wget --spider - I-005: OpenAPI con description per-operation - I-009: sin labels Traefik manuales Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
65
evals/direct-backend-probe.sh
Executable file
65
evals/direct-backend-probe.sh
Executable file
@@ -0,0 +1,65 @@
|
||||
#!/usr/bin/env bash
|
||||
# Diagnóstico: ¿el backend responde correctamente a llamadas estilo WxO?
|
||||
# Aísla problemas de auth/conectividad de problemas de lógica del agente.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
PUBLIC_HOST="${PUBLIC_HOST:-mi-cliente.fitlabs.dev}"
|
||||
BASE="https://${PUBLIC_HOST}"
|
||||
TOKEN="${WXO_BACKEND_TOKEN:-}"
|
||||
|
||||
echo "──────────────────────────────────────────────────────"
|
||||
echo " Direct backend probe — ${BASE}"
|
||||
echo "──────────────────────────────────────────────────────"
|
||||
|
||||
# [1/3] Health
|
||||
echo "[1/3] Healthcheck"
|
||||
CODE=$(curl -s -o /dev/null -w "%{http_code}" "${BASE}/health" || echo "000")
|
||||
if [ "$CODE" = "200" ]; then
|
||||
echo " ✓ 200 OK"
|
||||
else
|
||||
echo " ✗ $CODE — el backend no responde. ¿Container up? ¿Healthcheck OK?"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# [2/3] Direct tool call — con token, expecting controlled failure (case not found)
|
||||
echo "[2/3] Direct tool call (con token, case inexistente)"
|
||||
RESP=$(curl -s -X POST "${BASE}/api/v1/cases/run" \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "X-Orchestrate-Token: ${TOKEN}" \
|
||||
-d '{"case_id":"probe-nonexistent-uuid","target_id":1}')
|
||||
CODE=$(curl -s -o /dev/null -w "%{http_code}" -X POST "${BASE}/api/v1/cases/run" \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "X-Orchestrate-Token: ${TOKEN}" \
|
||||
-d '{"case_id":"probe-nonexistent-uuid","target_id":1}')
|
||||
|
||||
if [ "$CODE" = "200" ]; then
|
||||
echo " ✓ 200 con error de lógica: $RESP"
|
||||
echo " → Backend responde OK. Si el agente falla, es el agente o el LLM."
|
||||
elif [ "$CODE" = "401" ] || [ "$CODE" = "403" ]; then
|
||||
echo " ✗ $CODE — auth fallida. Verificar:"
|
||||
echo " - El token (WXO_BACKEND_TOKEN) es correcto"
|
||||
echo " - La connection en WxO tiene la API key correcta"
|
||||
echo " - El header es X-Orchestrate-Token (no Authorization)"
|
||||
exit 1
|
||||
elif [ "$CODE" = "502" ] || [ "$CODE" = "504" ]; then
|
||||
echo " ✗ $CODE — gateway. Backend caído o muy lento."
|
||||
exit 1
|
||||
else
|
||||
echo " ⚠ $CODE inesperado: $RESP"
|
||||
fi
|
||||
|
||||
# [3/3] Sin token, debe fallar 401
|
||||
echo "[3/3] Sin token (debe rechazar)"
|
||||
CODE_NOAUTH=$(curl -s -o /dev/null -w "%{http_code}" -X POST "${BASE}/api/v1/cases/run" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"case_id":"probe","target_id":1}')
|
||||
|
||||
if [ "$CODE_NOAUTH" = "401" ] || [ "$CODE_NOAUTH" = "403" ]; then
|
||||
echo " ✓ rechaza correctamente sin token ($CODE_NOAUTH)"
|
||||
else
|
||||
echo " ⚠ acepta sin token ($CODE_NOAUTH) — posible problema de seguridad"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "✓ Probe completed. Backend está sano desde el punto de vista de WxO."
|
||||
72
evals/eval-agents.sh
Executable file
72
evals/eval-agents.sh
Executable file
@@ -0,0 +1,72 @@
|
||||
#!/usr/bin/env bash
|
||||
# Runner completo de evals. Corre las 4 capas en orden.
|
||||
# Exit 0 si todo pasa, 1 si alguna falla.
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
cd "$ROOT"
|
||||
|
||||
FAILED=0
|
||||
|
||||
# ─── Capa 0: Lint ───────────────────────────────────────────────────────────
|
||||
echo "[lint] best practices"
|
||||
if python3 evals/lint_wxo_yaml.py; then
|
||||
echo " ✓ lint PASS"
|
||||
else
|
||||
echo " ✗ lint FAIL"
|
||||
FAILED=$((FAILED + 1))
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# ─── Capa 1: Native (ADK) ───────────────────────────────────────────────────
|
||||
echo "[native] orchestrate agents test"
|
||||
NATIVE_PASS=0; NATIVE_FAIL=0
|
||||
for f in evals/scenarios/*.input.json; do
|
||||
[ -f "$f" ] || continue
|
||||
# Derivar nombre del agente (convención: scenario_X_agentName.input.json o leer del JSON)
|
||||
agent=$(python3 -c "import json,sys; d=json.load(open('$f')); print(d.get('_agent',''))" 2>/dev/null)
|
||||
if [ -z "$agent" ]; then
|
||||
echo " ⚠ $f sin '_agent' field, skip"
|
||||
continue
|
||||
fi
|
||||
if orchestrate agents test "$agent" --input-file "$f" > /tmp/eval_native_$$.log 2>&1; then
|
||||
NATIVE_PASS=$((NATIVE_PASS + 1))
|
||||
else
|
||||
NATIVE_FAIL=$((NATIVE_FAIL + 1))
|
||||
echo " ✗ $f failed (see /tmp/eval_native_$$.log)"
|
||||
fi
|
||||
done
|
||||
echo " native: $NATIVE_PASS pass / $NATIVE_FAIL fail"
|
||||
[ "$NATIVE_FAIL" -gt 0 ] && FAILED=$((FAILED + 1))
|
||||
echo ""
|
||||
|
||||
# ─── Capa 2 + 3: Behavior + Final state (runner.py) ─────────────────────────
|
||||
echo "[behavior+final] custom runner"
|
||||
if python3 evals/runner.py; then
|
||||
echo " ✓ runner PASS"
|
||||
else
|
||||
echo " ✗ runner FAIL"
|
||||
FAILED=$((FAILED + 1))
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# ─── Capa 4: Smoke test ─────────────────────────────────────────────────────
|
||||
echo "[smoke] end-to-end"
|
||||
if ./evals/smoke-test.sh > /tmp/eval_smoke_$$.log 2>&1; then
|
||||
echo " ✓ smoke PASS"
|
||||
else
|
||||
echo " ✗ smoke FAIL (see /tmp/eval_smoke_$$.log)"
|
||||
FAILED=$((FAILED + 1))
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# ─── Resumen ────────────────────────────────────────────────────────────────
|
||||
echo "════════════════════════════════════════"
|
||||
if [ "$FAILED" -eq 0 ]; then
|
||||
echo " ✓ ALL EVALS PASS"
|
||||
exit 0
|
||||
else
|
||||
echo " ✗ $FAILED layer(s) FAILED"
|
||||
exit 1
|
||||
fi
|
||||
196
evals/lint_wxo_yaml.py
Normal file
196
evals/lint_wxo_yaml.py
Normal file
@@ -0,0 +1,196 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Linter de buenas prácticas WxO.
|
||||
|
||||
Falla con exit code 1 si encuentra violaciones. Usado en CI antes de
|
||||
deploy. Las reglas están en docs/wxo-best-practices.md.
|
||||
|
||||
Reglas enforced:
|
||||
- A1: máx 10 tools por agente
|
||||
- A3: orchestrator no tiene tools de remediación
|
||||
- A6: agente con style=react debe tener KB o tools
|
||||
- T1: toda función @tool debe estar bajo @observable_tool
|
||||
- T3: tools.py debe inlinear el _compat shim
|
||||
- D1: docker-compose no usa `wget --spider`
|
||||
- I-005: OpenAPI tiene description per-operation
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import sys
|
||||
import textwrap
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError:
|
||||
print("✗ pyyaml required: pip install pyyaml", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
|
||||
# Tools típicas de remediación que un orchestrator NO debe tener.
|
||||
REMEDIATION_TOOLS = {
|
||||
"reset_password", "restart_service", "create_user", "grant_access",
|
||||
"assign_groups", "rotate_logs", "delete_user", "revoke_access",
|
||||
"kill_process", "redeploy", "rollback",
|
||||
}
|
||||
|
||||
errors: list[str] = []
|
||||
warnings: list[str] = []
|
||||
|
||||
|
||||
def err(msg: str) -> None:
|
||||
errors.append(msg)
|
||||
print(f"✗ {msg}")
|
||||
|
||||
|
||||
def warn(msg: str) -> None:
|
||||
warnings.append(msg)
|
||||
print(f"⚠ {msg}")
|
||||
|
||||
|
||||
def ok(msg: str) -> None:
|
||||
print(f"✓ {msg}")
|
||||
|
||||
|
||||
# ─── Agents ──────────────────────────────────────────────────────────────────
|
||||
|
||||
def lint_agents() -> None:
|
||||
agent_dir = ROOT / "wxo" / "agents"
|
||||
for f in sorted(agent_dir.glob("*.agent.yaml")):
|
||||
if f.name.startswith("_"):
|
||||
continue
|
||||
try:
|
||||
data = yaml.safe_load(f.read_text())
|
||||
except yaml.YAMLError as e:
|
||||
err(f"{f}: invalid YAML — {e}")
|
||||
continue
|
||||
|
||||
name = data.get("name", f.stem)
|
||||
tools = data.get("tools") or []
|
||||
kb = data.get("knowledge_base") or []
|
||||
style = data.get("style", "")
|
||||
collaborators = data.get("collaborators") or []
|
||||
instructions = data.get("instructions") or ""
|
||||
|
||||
# A1: máx 10 tools
|
||||
if len(tools) > 10:
|
||||
err(f"{f.name}: agente '{name}' tiene {len(tools)} tools (regla A1: máx 10)")
|
||||
|
||||
# A3: orchestrator sin tools de remediación
|
||||
is_orchestrator = bool(collaborators) or "orchestrator" in name or "n1" in name
|
||||
if is_orchestrator:
|
||||
bad = [t for t in tools if t in REMEDIATION_TOOLS]
|
||||
if bad:
|
||||
err(f"{f.name}: orchestrator '{name}' declara tools de remediación: {bad} (regla A3)")
|
||||
|
||||
# A6: style=react sin KB ni tools = agente sin propósito
|
||||
if style == "react" and not tools and not kb:
|
||||
err(f"{f.name}: agente '{name}' tiene style=react pero ni tools ni KB (regla A6)")
|
||||
|
||||
# P1: LLM pinneado a gpt-oss-120b (warning si no)
|
||||
llm = data.get("llm", "")
|
||||
if "gpt-oss-120b" not in llm and "llama" in llm:
|
||||
warn(f"{f.name}: agente '{name}' usa llama. Issue I-002 — preferible gpt-oss-120b.")
|
||||
|
||||
# P2: REGLA #0 presente en instructions
|
||||
if "REGLA #0" not in instructions and "NUNCA escribas la tool call" not in instructions:
|
||||
warn(f"{f.name}: agente '{name}' no menciona REGLA #0 en instructions (regla P2)")
|
||||
|
||||
# ok message
|
||||
if len(errors) == 0:
|
||||
ok(f"agent {name} (tools={len(tools)}, kb={len(kb)}, style={style})")
|
||||
|
||||
|
||||
# ─── Tools (Python) ──────────────────────────────────────────────────────────
|
||||
|
||||
def lint_python_tools() -> None:
|
||||
tools_dir = ROOT / "wxo" / "tools" / "python"
|
||||
for f in sorted(tools_dir.glob("*.py")):
|
||||
if f.name.startswith("_"):
|
||||
continue # skip helpers
|
||||
src = f.read_text()
|
||||
|
||||
# T3: compat shim presente
|
||||
if "inline compat shim" not in src and "from ibm_watsonx_orchestrate.agent_builder.tools import tool" not in src:
|
||||
err(f"{f.name}: falta compat shim inline (regla T3, issue I-010)")
|
||||
|
||||
# T1: @tool sin @observable_tool
|
||||
# Buscar @tool( no precedido por observable_tool
|
||||
for line_no, line in enumerate(src.splitlines(), start=1):
|
||||
line_stripped = line.strip()
|
||||
if re.match(r"^@tool\b", line_stripped):
|
||||
# ¿el archivo lo usa solo como import alias _adk_tool? OK.
|
||||
# Si no, es violación.
|
||||
err(f"{f.name}:{line_no}: usa @tool directo, debe ser @observable_tool (regla T1)")
|
||||
|
||||
ok(f"tools {f.name}")
|
||||
|
||||
|
||||
# ─── OpenAPI ─────────────────────────────────────────────────────────────────
|
||||
|
||||
def lint_openapi() -> None:
|
||||
openapi_dir = ROOT / "wxo" / "tools" / "openapi"
|
||||
for f in sorted(list(openapi_dir.glob("*.yaml")) + list(openapi_dir.glob("*.json"))):
|
||||
if f.name.startswith("_"):
|
||||
continue
|
||||
try:
|
||||
data = yaml.safe_load(f.read_text())
|
||||
except yaml.YAMLError as e:
|
||||
err(f"{f.name}: invalid YAML/JSON — {e}")
|
||||
continue
|
||||
|
||||
paths = data.get("paths", {})
|
||||
for path, item in paths.items():
|
||||
for method, op in item.items():
|
||||
if not isinstance(op, dict) or "responses" not in op:
|
||||
continue
|
||||
# I-005: description per-op
|
||||
if not op.get("description"):
|
||||
err(f"{f.name}: {method.upper()} {path} sin description (issue I-005)")
|
||||
# I-006: security per-op
|
||||
if op.get("security") is None and data.get("security") is not None:
|
||||
warn(f"{f.name}: {method.upper()} {path} no declara security per-op (issue I-006)")
|
||||
|
||||
ok(f"openapi {f.name}")
|
||||
|
||||
|
||||
# ─── docker-compose ──────────────────────────────────────────────────────────
|
||||
|
||||
def lint_compose() -> None:
|
||||
for f in [ROOT / "docker-compose.yml", ROOT / "docker-compose.local.yml"]:
|
||||
if not f.exists():
|
||||
continue
|
||||
src = f.read_text()
|
||||
# D1: no `wget --spider`
|
||||
if "wget --spider" in src or "wget -S --spider" in src:
|
||||
err(f"{f.name}: usa `wget --spider`, cambiá por `wget -qO-` o `curl -sf` (issue I-008)")
|
||||
# I-009: labels Traefik manuales
|
||||
if re.search(r"traefik\.http\.routers\.", src):
|
||||
warn(f"{f.name}: declara labels Traefik manuales — Coolify v4 los autogenera (issue I-009)")
|
||||
ok(f"compose {f.name}")
|
||||
|
||||
|
||||
# ─── Main ────────────────────────────────────────────────────────────────────
|
||||
|
||||
def main() -> int:
|
||||
print("─── Linting agents ───")
|
||||
lint_agents()
|
||||
print("\n─── Linting Python tools ───")
|
||||
lint_python_tools()
|
||||
print("\n─── Linting OpenAPI ───")
|
||||
lint_openapi()
|
||||
print("\n─── Linting docker-compose ───")
|
||||
lint_compose()
|
||||
|
||||
print("\n" + "─" * 60)
|
||||
if errors:
|
||||
print(f"✗ {len(errors)} ERROR(es), {len(warnings)} warning(s)")
|
||||
return 1
|
||||
print(f"✓ Linting OK ({len(warnings)} warning(s))")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
141
evals/runner.py
Normal file
141
evals/runner.py
Normal file
@@ -0,0 +1,141 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Eval runner — Capas 2 (behavior) y 3 (final state).
|
||||
|
||||
Lee `evals/scenarios/*.yaml`, ejecuta cada scenario contra el agente
|
||||
correspondiente, y verifica las trazas observables + estado final.
|
||||
|
||||
Formato del scenario YAML (ver docs/eval-strategy.md):
|
||||
|
||||
name: reset_password_happy_path
|
||||
agent: ad_specialist_cotemar
|
||||
input: "Necesito resetear..."
|
||||
expect:
|
||||
agent_response_contains: ["TKT-"]
|
||||
tool_calls_in_order:
|
||||
- tool: lookup_user
|
||||
- tool: reset_password
|
||||
- tool: create_ticket
|
||||
no_tool_calls:
|
||||
- escalate_to_n2
|
||||
final_state:
|
||||
# ... asserts contra DB / API
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import sqlite3
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError:
|
||||
print("✗ pyyaml required: pip install pyyaml", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
SCENARIOS_DIR = ROOT / "evals" / "scenarios"
|
||||
TRACES_DB = os.environ.get("TRACES_DB_PATH", "/tmp/traces.db")
|
||||
|
||||
|
||||
def run_scenario(spec: dict) -> tuple[bool, list[str]]:
|
||||
"""Returns (passed, errors)."""
|
||||
errors: list[str] = []
|
||||
name = spec.get("name", "?")
|
||||
agent = spec.get("agent")
|
||||
input_text = spec.get("input")
|
||||
expect = spec.get("expect", {})
|
||||
|
||||
if not agent or not input_text:
|
||||
return False, ["scenario sin 'agent' o 'input'"]
|
||||
|
||||
started = int(time.time() * 1000)
|
||||
|
||||
# 1) Mandar el input al agente (usando orchestrate CLI)
|
||||
fixture = {"messages": [{"role": "user", "content": input_text}]}
|
||||
fixture_path = Path("/tmp") / f"_eval_{name}.json"
|
||||
fixture_path.write_text(json.dumps(fixture))
|
||||
import subprocess
|
||||
proc = subprocess.run(
|
||||
["orchestrate", "agents", "test", agent, "--input-file", str(fixture_path)],
|
||||
capture_output=True, text=True, timeout=120,
|
||||
)
|
||||
output = proc.stdout + "\n" + proc.stderr
|
||||
|
||||
# 2) Verificar response (Capa 2)
|
||||
for needle in expect.get("agent_response_contains", []):
|
||||
if needle not in output:
|
||||
errors.append(f"agent_response_contains: '{needle}' not in output")
|
||||
|
||||
# 3) Verificar tool calls vía trazas (Capa 2)
|
||||
traces = _read_traces_since(started)
|
||||
tool_names = [t["tool"] for t in traces]
|
||||
|
||||
expected_order = [t.get("tool") for t in expect.get("tool_calls_in_order", [])]
|
||||
if expected_order:
|
||||
if not _is_subsequence(expected_order, tool_names):
|
||||
errors.append(f"tool_calls_in_order: expected {expected_order}, got {tool_names}")
|
||||
|
||||
for bad_tool in expect.get("no_tool_calls", []):
|
||||
if bad_tool in tool_names:
|
||||
errors.append(f"no_tool_calls: {bad_tool} should NOT have been called")
|
||||
|
||||
# 4) Final state (Capa 3) — placeholder, depende del backend
|
||||
# TODO en v2: parametrizable según backend del proyecto
|
||||
final_state = spec.get("final_state")
|
||||
if final_state:
|
||||
# Stub — implementación específica del proyecto
|
||||
pass
|
||||
|
||||
return (len(errors) == 0), errors
|
||||
|
||||
|
||||
def _is_subsequence(needle: list[str], haystack: list[str]) -> bool:
|
||||
"""¿`needle` es subsequence de `haystack`? (order preserved, gaps OK)"""
|
||||
it = iter(haystack)
|
||||
return all(x in it for x in needle)
|
||||
|
||||
|
||||
def _read_traces_since(ts_ms: int) -> list[dict]:
|
||||
if not Path(TRACES_DB).exists():
|
||||
return []
|
||||
conn = sqlite3.connect(TRACES_DB)
|
||||
rows = conn.execute(
|
||||
"SELECT payload FROM traces WHERE started_at >= datetime(?, 'unixepoch')",
|
||||
(ts_ms / 1000,)
|
||||
).fetchall()
|
||||
conn.close()
|
||||
return [json.loads(r[0]) for r in rows]
|
||||
|
||||
|
||||
def main() -> int:
|
||||
scenarios = sorted(SCENARIOS_DIR.glob("*.yaml"))
|
||||
if not scenarios:
|
||||
print("(no scenarios in evals/scenarios/*.yaml — skipping)")
|
||||
return 0
|
||||
|
||||
pass_count = 0
|
||||
fail_count = 0
|
||||
for f in scenarios:
|
||||
if f.name.startswith("_"):
|
||||
continue
|
||||
spec = yaml.safe_load(f.read_text())
|
||||
name = spec.get("name", f.stem)
|
||||
ok, errors = run_scenario(spec)
|
||||
if ok:
|
||||
print(f" ✓ {name}")
|
||||
pass_count += 1
|
||||
else:
|
||||
print(f" ✗ {name}")
|
||||
for e in errors:
|
||||
print(f" - {e}")
|
||||
fail_count += 1
|
||||
|
||||
print(f"\n Total: {pass_count} pass / {fail_count} fail")
|
||||
return 0 if fail_count == 0 else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
10
evals/scenarios/_template_scenario.input.json
Normal file
10
evals/scenarios/_template_scenario.input.json
Normal file
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"_agent": "REPLACE_agent_name",
|
||||
"_description": "REPLACE — qué prueba este scenario",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "REPLACE — qué le dice el usuario al agente"
|
||||
}
|
||||
]
|
||||
}
|
||||
48
evals/scenarios/_template_scenario.yaml
Normal file
48
evals/scenarios/_template_scenario.yaml
Normal file
@@ -0,0 +1,48 @@
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Template — Eval scenario
|
||||
#
|
||||
# Cada archivo en este directorio es un test case. El runner los procesa todos.
|
||||
# Asegurate de tener también el archivo input JSON correspondiente:
|
||||
# evals/scenarios/<name>.input.json
|
||||
#
|
||||
# Documentación: docs/eval-strategy.md
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
# Nombre único del scenario (para reportes).
|
||||
name: REPLACE_scenario_name
|
||||
|
||||
# Agente al que mandar el input.
|
||||
agent: REPLACE_agent_name
|
||||
|
||||
# Texto del usuario.
|
||||
input: "REPLACE — qué le dice el usuario al agente"
|
||||
|
||||
# Capa 2 — Behavior eval
|
||||
expect:
|
||||
# El response final del agente debe contener estos strings (case sensitive)
|
||||
agent_response_contains:
|
||||
- "REPLACE_token_1"
|
||||
|
||||
# Las tools deben llamarse en este orden (puede haber tools intermedias)
|
||||
tool_calls_in_order:
|
||||
- tool: REPLACE_tool_1
|
||||
# inputs: opcional, asserts sobre los inputs
|
||||
- tool: REPLACE_tool_2
|
||||
|
||||
# Estas tools NO deben aparecer en las trazas
|
||||
no_tool_calls:
|
||||
- REPLACE_tool_que_no_deberia_llamarse
|
||||
|
||||
# Capa 3 — Final state eval (opcional)
|
||||
final_state:
|
||||
# query: "SELECT * FROM tickets WHERE created_at > $START_TIME"
|
||||
# expect_count: 1
|
||||
# expect_fields:
|
||||
# status: "RESOLVED"
|
||||
# extra.runbook: "01"
|
||||
|
||||
# Capa 4 — UI eval (opcional)
|
||||
# ui:
|
||||
# visit: "/insights"
|
||||
# expect_text:
|
||||
# - "Resueltos: 1"
|
||||
55
evals/smoke-test.sh
Executable file
55
evals/smoke-test.sh
Executable file
@@ -0,0 +1,55 @@
|
||||
#!/usr/bin/env bash
|
||||
# Smoke test mínimo: end-to-end del flujo principal.
|
||||
#
|
||||
# Sirve como:
|
||||
# - Validación post-deploy
|
||||
# - Health check nightly (cron en CI)
|
||||
# - Reproducible bug report
|
||||
#
|
||||
# Pasos:
|
||||
# 1. Healthcheck del backend público
|
||||
# 2. Llamar al endpoint principal con un input válido
|
||||
# 3. Pollear hasta done
|
||||
# 4. Assert sobre el estado final
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
PUBLIC_HOST="${PUBLIC_HOST:-mi-cliente.fitlabs.dev}"
|
||||
BASE="https://${PUBLIC_HOST}"
|
||||
|
||||
echo "→ [1/4] Healthcheck"
|
||||
curl -sf "${BASE}/health" > /dev/null && echo " ✓ 200 OK" || { echo " ✗ FAIL"; exit 1; }
|
||||
|
||||
echo "→ [2/4] Create case (smoke input)"
|
||||
# REPLACE: ajustar al endpoint y payload de tu solución
|
||||
RESULT=$(curl -sf -X POST "${BASE}/api/v1/cases/run" \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "X-Orchestrate-Token: ${WXO_BACKEND_TOKEN:-smoke-test-token}" \
|
||||
-d '{"case_id":"smoke-001","target_id":1}' \
|
||||
2>&1) || { echo " ✗ FAIL: $RESULT"; exit 1; }
|
||||
echo " ✓ accepted: $RESULT"
|
||||
|
||||
echo "→ [3/4] Poll until done (max 60s)"
|
||||
for i in $(seq 1 60); do
|
||||
STATUS=$(curl -sf "${BASE}/api/v1/cases/smoke-001" | python3 -c "import sys,json; print(json.load(sys.stdin).get('status','?'))" 2>/dev/null || echo "?")
|
||||
[ "$STATUS" = "done" ] && { echo " ✓ done after ${i}s"; break; }
|
||||
[ "$STATUS" = "failed" ] && { echo " ✗ failed"; exit 1; }
|
||||
sleep 1
|
||||
done
|
||||
[ "$STATUS" = "done" ] || { echo " ✗ timeout (last status: $STATUS)"; exit 1; }
|
||||
|
||||
echo "→ [4/4] Assert final state"
|
||||
# REPLACE: ajustar al assert de tu solución
|
||||
RESULT=$(curl -sf "${BASE}/api/v1/cases/smoke-001")
|
||||
echo "$RESULT" | python3 -c "
|
||||
import sys, json
|
||||
data = json.load(sys.stdin)
|
||||
assert data['status'] == 'done', f'status={data[\"status\"]}'
|
||||
assert data.get('package_url'), 'no package_url'
|
||||
print(' ✓ assertions OK')
|
||||
"
|
||||
|
||||
echo ""
|
||||
echo "════════════════════════════════════════"
|
||||
echo " ✓ Smoke test passed"
|
||||
echo "════════════════════════════════════════"
|
||||
Reference in New Issue
Block a user