{
  "slug": "evaluate-agents",
  "label": "test and evaluate agents",
  "blurb": "Trace, test and score LLM apps and agents before and after they ship.",
  "keywords": [
    "evals",
    "evaluation",
    "llm observability",
    "tracing",
    "prompt testing"
  ],
  "categories": [
    "infrastructure",
    "code-devtools",
    "orchestration"
  ],
  "access": [
    "api"
  ],
  "count": 29,
  "official": 13,
  "url": "https://agentswitchboard.dev/tasks/evaluate-agents",
  "browseUrl": "https://agentswitchboard.dev/browse?task=evaluate-agents",
  "agentsUrl": "https://agentswitchboard.dev/api/agents?task=evaluate-agents",
  "agents": [
    {
      "name": "Langfuse",
      "slug": "langfuse",
      "url": "https://agentswitchboard.dev/agents/langfuse",
      "iconUrl": "https://agentswitchboard.dev/icons/langfuse.png?v=b2abe0af",
      "description": "Open-source LLM observability and analytics. Trace, evaluate, and debug LLM apps with prompt versioning, score tracking, and a full analytics dashboard.",
      "provider": "Langfuse",
      "categories": [
        "infrastructure"
      ],
      "accessMethods": [
        "api",
        "cli"
      ],
      "verified": true,
      "official": true,
      "featured": false,
      "authType": "apiKey",
      "why": "Score traces with custom evaluation rubrics to track quality over time"
    },
    {
      "name": "Future AGI",
      "slug": "future-agi",
      "url": "https://agentswitchboard.dev/agents/future-agi",
      "iconUrl": "https://agentswitchboard.dev/icons/future-agi.png?v=54a3c3c3",
      "description": "Evaluate, observe, and improve LLM and agent applications from one self-hostable platform — tracing, evals, simulations, datasets, guardrails, and a model gateway under Apache 2.0.",
      "provider": "Future AGI",
      "categories": [
        "code-devtools"
      ],
      "accessMethods": [
        "api",
        "cli"
      ],
      "verified": true,
      "official": false,
      "featured": false,
      "authType": "apiKey",
      "why": "Evaluation Suite: Scores outputs for hallucination, relevance, and task success using configurable evaluators across datasets."
    },
    {
      "name": "Microsoft PromptFlow",
      "slug": "promptflow",
      "url": "https://agentswitchboard.dev/agents/promptflow",
      "iconUrl": "https://agentswitchboard.dev/icons/promptflow.png?v=b809713c",
      "description": "Build, evaluate, and deploy LLM applications as visual DAG flows — chain prompts, Python, and tools, then run batch evaluations with built-in quality and safety metrics.",
      "provider": "Microsoft",
      "categories": [
        "orchestration",
        "code-devtools"
      ],
      "accessMethods": [
        "api",
        "cli"
      ],
      "verified": true,
      "official": true,
      "featured": false,
      "authType": "apiKey",
      "why": "Batch Evaluation: Executes a flow across a dataset and scores results with groundedness, relevance, and safety evaluators."
    },
    {
      "name": "OpenLLMetry",
      "slug": "openllmetry",
      "url": "https://agentswitchboard.dev/agents/openllmetry",
      "iconUrl": "https://agentswitchboard.dev/icons/openllmetry.png?v=f2ed5797",
      "description": "Instruments LLM and agent applications with OpenTelemetry-native tracing, exporting spans for prompts, tool calls, and chains to any OTel-compatible backend.",
      "provider": "Traceloop",
      "categories": [
        "infrastructure",
        "research"
      ],
      "accessMethods": [
        "api"
      ],
      "verified": true,
      "official": true,
      "featured": false,
      "authType": "apiKey",
      "why": "Workflow Tracing: Groups multi-step agent runs into one trace so latency and cost attribute per step."
    },
    {
      "name": "Latitude",
      "slug": "latitude",
      "url": "https://agentswitchboard.dev/agents/latitude",
      "iconUrl": "https://agentswitchboard.dev/icons/latitude.png?v=eec7c13f",
      "description": "Traces, scores, and evaluates production AI agents via OpenTelemetry and SDKs, groups failures into signals, and dispatches coding agents over MCP to fix them.",
      "provider": "Latitude",
      "categories": [
        "code-devtools",
        "data-analytics"
      ],
      "accessMethods": [
        "api",
        "cli",
        "mcp"
      ],
      "verified": true,
      "official": false,
      "featured": false,
      "authType": "apiKey",
      "why": "Automated Evaluations: Scores sessions with condition sets, LLM judges, or custom JavaScript scripts, and replays regression datasets against fixed…"
    },
    {
      "name": "MCPJam",
      "slug": "mcpjam",
      "url": "https://agentswitchboard.dev/agents/mcpjam",
      "iconUrl": "https://agentswitchboard.dev/icons/mcpjam.png?v=53d45d85",
      "description": "Tests and evaluates MCP servers and MCP apps for developers, with a playground, OAuth debugger, cross-client evals, and CI checks, via web app, npx inspector, CLI, and SDK.",
      "provider": "MCPJam",
      "categories": [
        "code-devtools"
      ],
      "accessMethods": [
        "api",
        "cli",
        "mcp"
      ],
      "verified": true,
      "official": false,
      "featured": false,
      "authType": "none",
      "why": "Cross-Model Evals: Runs test cases with expected tool calls across multiple LLMs and tracks accuracy over time to catch MCP server regressions."
    },
    {
      "name": "LLM Space",
      "slug": "llm-space",
      "url": "https://agentswitchboard.dev/agents/llm-space",
      "iconUrl": "https://agentswitchboard.dev/icons/llm-space.png?v=7e2f7933",
      "description": "Prototypes, traces, replays, and evaluates LLM agents in a local macOS desktop app, and exports threads as runnable LangGraph agents, for agent builders.",
      "provider": "DeerFlow",
      "categories": [
        "code-devtools",
        "orchestration"
      ],
      "accessMethods": [],
      "verified": false,
      "official": false,
      "featured": false,
      "authType": "apiKey",
      "why": "Agent Run Tracing: Shows every model call and tool run inside the agent loop as it happens."
    },
    {
      "name": "Opik",
      "slug": "opik",
      "url": "https://agentswitchboard.dev/agents/opik",
      "iconUrl": "https://agentswitchboard.dev/icons/opik.png?v=800b3344",
      "description": "Traces LLM and agent runs, scores them with LLM-as-a-judge and heuristic metrics, and monitors production quality via Python and TypeScript SDKs and 60+ integrations.",
      "provider": "Comet ML",
      "categories": [
        "infrastructure",
        "code-devtools"
      ],
      "accessMethods": [
        "api",
        "mcp",
        "cli"
      ],
      "verified": true,
      "official": false,
      "featured": false,
      "authType": "apiKey",
      "why": "PyTest CI Evaluation: Tests LLM pipelines on every commit through the PyTest integration so evaluation results are recorded per CI run."
    },
    {
      "name": "Datadog LLM Observability",
      "slug": "datadog-llm-observability",
      "url": "https://agentswitchboard.dev/agents/datadog-llm-observability",
      "iconUrl": "https://agentswitchboard.dev/icons/datadog-llm-observability.png?v=cb427b54",
      "description": "End-to-end monitoring for LLM applications — trace prompts, evaluate responses, track costs, and debug AI pipelines in production.",
      "provider": "Datadog",
      "categories": [
        "infrastructure"
      ],
      "accessMethods": [
        "api",
        "mcp",
        "cli"
      ],
      "verified": true,
      "official": true,
      "featured": false,
      "authType": "apiKey",
      "why": null
    },
    {
      "name": "Helicone",
      "slug": "helicone",
      "url": "https://agentswitchboard.dev/agents/helicone",
      "iconUrl": "https://agentswitchboard.dev/icons/helicone.png?v=9bb53a82",
      "description": "Monitors LLM and agent traffic through a one-line proxy integration, logging requests, costs and latency with caching, rate limits and evaluations built in.",
      "provider": "Helicone",
      "categories": [
        "data-analytics",
        "infrastructure"
      ],
      "accessMethods": [
        "api"
      ],
      "verified": true,
      "official": false,
      "featured": false,
      "authType": "apiKey",
      "why": "Session Tracing: Groups multi-step agent calls into sessions so a whole run can be replayed and debugged as one unit."
    },
    {
      "name": "Arize Phoenix",
      "slug": "arize-phoenix",
      "url": "https://agentswitchboard.dev/agents/arize-phoenix",
      "iconUrl": "https://agentswitchboard.dev/icons/arize-phoenix.png?v=1d10df0f",
      "description": "Traces, evaluates and debugs LLM and agent applications from an open-source observability platform built on OpenTelemetry, runnable locally or self-hosted.",
      "provider": "Arize AI",
      "categories": [
        "data-analytics",
        "infrastructure"
      ],
      "accessMethods": [
        "api",
        "cli"
      ],
      "verified": true,
      "official": true,
      "featured": false,
      "authType": "none",
      "why": "Evaluation Runs: Runs LLM-as-judge evaluations over captured traces and datasets to quantify quality regressions across versions."
    },
    {
      "name": "Google Agents CLI",
      "slug": "google-agents-cli",
      "url": "https://agentswitchboard.dev/agents/google-agents-cli",
      "iconUrl": "https://agentswitchboard.dev/icons/google-agents-cli.png?v=1d11d93e",
      "description": "Turns any coding assistant into an expert at creating, evaluating, and deploying AI agents on Google Cloud. An open CLI plus skills that scaffold, run evals, and ship to Agent Engine or Cloud Run.",
      "provider": "Google",
      "categories": [
        "code-devtools"
      ],
      "accessMethods": [
        "cli"
      ],
      "verified": true,
      "official": true,
      "featured": false,
      "authType": "oauth2",
      "why": "Evaluation Runs: Runs evaluation suites against an agent and reports quality metrics before it is deployed to production."
    },
    {
      "name": "Laminar",
      "slug": "laminar",
      "url": "https://agentswitchboard.dev/agents/laminar",
      "iconUrl": "https://agentswitchboard.dev/icons/laminar.png?v=a56422fc",
      "description": "Trace, evaluate, and debug AI agents with open-source observability — captures LLM calls, tool calls, and sub-agent spans, then clusters recurring failure modes into eval datasets.",
      "provider": "Laminar",
      "categories": [
        "code-devtools",
        "infrastructure"
      ],
      "accessMethods": [
        "api",
        "cli",
        "mcp"
      ],
      "verified": true,
      "official": false,
      "featured": false,
      "authType": "apiKey",
      "why": "Agent Tracing: Records nested spans for LLM calls, tool invocations, and sub-agent runs so a full trajectory is reconstructable."
    },
    {
      "name": "Braintrust",
      "slug": "braintrust",
      "url": "https://agentswitchboard.dev/agents/braintrust",
      "iconUrl": "https://agentswitchboard.dev/icons/braintrust.png?v=cf3fabd0",
      "description": "Evaluates and observes LLM applications end to end: capture traces, score them with evals, iterate in a prompt playground, and gate deploys on eval results run from CI.",
      "provider": "Braintrust Data",
      "categories": [
        "infrastructure",
        "code-devtools"
      ],
      "accessMethods": [
        "api",
        "cli",
        "mcp"
      ],
      "verified": true,
      "official": false,
      "featured": false,
      "authType": "bearer",
      "why": "Experiment Based Evals: Runs eval suites over datasets with code and LLM-judge scorers, comparing scores across experiments and model versions."
    },
    {
      "name": "LangChain / LangSmith",
      "slug": "langchain-langsmith",
      "url": "https://agentswitchboard.dev/agents/langchain-langsmith",
      "iconUrl": "https://agentswitchboard.dev/icons/langchain-langsmith.png?v=4536470a",
      "description": "Framework for building LLM applications with chains, agents, and tool use — plus LangSmith for tracing, evaluation, and monitoring.",
      "provider": "LangChain",
      "categories": [
        "infrastructure",
        "code-devtools"
      ],
      "accessMethods": [
        "api",
        "mcp",
        "cli"
      ],
      "verified": true,
      "official": true,
      "featured": false,
      "authType": "apiKey",
      "why": null
    },
    {
      "name": "AgentScope",
      "slug": "agentscope",
      "url": "https://agentswitchboard.dev/agents/agentscope",
      "iconUrl": "https://agentswitchboard.dev/icons/agentscope.png?v=d9b61523",
      "description": "Builds multi-agent applications with a Python framework focused on transparency, observability, and tracing. Includes a runtime and studio for debugging. By Alibaba.",
      "provider": "Alibaba",
      "categories": [
        "orchestration"
      ],
      "accessMethods": [
        "cli",
        "api"
      ],
      "verified": true,
      "official": true,
      "featured": false,
      "authType": "apiKey",
      "why": null
    },
    {
      "name": "Armature",
      "slug": "armature",
      "url": "https://agentswitchboard.dev/agents/armature",
      "iconUrl": "https://agentswitchboard.dev/icons/armature.png?v=beced433",
      "description": "Product analytics and evaluation for agent sessions running against MCP servers, tracking tool calls, failures, and per-session outcomes.",
      "provider": "Armature",
      "categories": [
        "data-analytics",
        "code-devtools"
      ],
      "accessMethods": [
        "api",
        "mcp"
      ],
      "verified": true,
      "official": false,
      "featured": false,
      "authType": "apiKey",
      "why": null
    },
    {
      "name": "Genkit",
      "slug": "genkit",
      "url": "https://agentswitchboard.dev/agents/genkit",
      "iconUrl": "https://agentswitchboard.dev/icons/genkit.png?v=8bfef480",
      "description": "Builds AI features and agent flows in JavaScript, Go or Python with typed flows, tool calling and a local dev UI for tracing, from Google.",
      "provider": "Google",
      "categories": [
        "orchestration",
        "code-devtools"
      ],
      "accessMethods": [
        "api",
        "cli"
      ],
      "verified": true,
      "official": true,
      "featured": false,
      "authType": "apiKey",
      "why": null
    },
    {
      "name": "Cursor Agent SDK",
      "slug": "cursor-agent-sdk",
      "url": "https://agentswitchboard.dev/agents/cursor-agent-sdk",
      "iconUrl": "https://agentswitchboard.dev/icons/cursor-agent-sdk.png?v=8db0ad61",
      "description": "Lets developers define, test, evaluate, and deploy agents as Markdown and TypeScript code, on Cursor-managed hosting or their own infrastructure.",
      "provider": "Anysphere",
      "categories": [
        "orchestration",
        "code-devtools"
      ],
      "accessMethods": [
        "api",
        "cli"
      ],
      "verified": true,
      "official": true,
      "featured": false,
      "authType": "apiKey",
      "why": "Run Evals: Discovers and runs eval cases from the project's evals folder, locally or as remote batches, with agent-sdk eval."
    },
    {
      "name": "DeepEval",
      "slug": "deepeval",
      "url": "https://agentswitchboard.dev/agents/deepeval",
      "iconUrl": "https://agentswitchboard.dev/icons/deepeval.png?v=5b468c07",
      "description": "Unit-tests LLM and agent outputs in a pytest-style workflow, scoring answer relevancy, faithfulness, hallucination and tool correctness in CI.",
      "provider": "Confident AI",
      "categories": [
        "code-devtools",
        "data-analytics"
      ],
      "accessMethods": [
        "cli",
        "api"
      ],
      "verified": true,
      "official": false,
      "featured": false,
      "authType": "none",
      "why": "Evaluation Metric Suite: Scores outputs on faithfulness, relevancy, hallucination and bias using research-backed metric implementations."
    },
    {
      "name": "AgentOps",
      "slug": "agentops",
      "url": "https://agentswitchboard.dev/agents/agentops",
      "iconUrl": "https://agentswitchboard.dev/icons/agentops.png?v=e2b72d49",
      "description": "Observability platform purpose-built for AI agents. Track agent sessions, LLM calls, tool use, and costs. Integrates with LangChain, CrewAI, AutoGen, and OpenAI.",
      "provider": "AgentOps",
      "categories": [
        "infrastructure"
      ],
      "accessMethods": [
        "api"
      ],
      "verified": true,
      "official": false,
      "featured": false,
      "authType": "apiKey",
      "why": null
    },
    {
      "name": "promptfoo",
      "slug": "promptfoo",
      "url": "https://agentswitchboard.dev/agents/promptfoo",
      "iconUrl": "https://agentswitchboard.dev/icons/promptfoo.png?v=bc00c2f2",
      "description": "Tests and red-teams LLM apps, agents and RAG pipelines from declarative config, scanning for prompt injection, jailbreaks and other vulnerabilities in CI.",
      "provider": "promptfoo",
      "categories": [
        "security",
        "code-devtools"
      ],
      "accessMethods": [
        "cli",
        "api"
      ],
      "verified": true,
      "official": false,
      "featured": false,
      "authType": "none",
      "why": null
    },
    {
      "name": "NVIDIA NeMo Agent Toolkit",
      "slug": "nvidia-nemo-agent-toolkit",
      "url": "https://agentswitchboard.dev/agents/nvidia-nemo-agent-toolkit",
      "iconUrl": "https://agentswitchboard.dev/icons/nvidia-nemo-agent-toolkit.png?v=754270a8",
      "description": "Connects, profiles, evaluates, and optimizes agent workflows across frameworks like LangChain, LlamaIndex, and CrewAI, with MCP and A2A support and a `nat` CLI for Python developers.",
      "provider": "NVIDIA",
      "categories": [
        "orchestration"
      ],
      "accessMethods": [
        "api",
        "cli",
        "mcp"
      ],
      "verified": true,
      "official": true,
      "featured": false,
      "authType": "none",
      "why": "Workflow Evaluation: Scores the accuracy of agent workflows against datasets with `nat eval` and records performance characteristics per run."
    },
    {
      "name": "VoltAgent",
      "slug": "voltagent",
      "url": "https://agentswitchboard.dev/agents/voltagent",
      "iconUrl": "https://agentswitchboard.dev/icons/voltagent.png?v=36f64141",
      "description": "Builds TypeScript AI agents with tools, memory, and sub-agent supervision, paired with a local observability console that traces every step of a run.",
      "provider": "VoltAgent",
      "categories": [
        "orchestration",
        "code-devtools"
      ],
      "accessMethods": [
        "api",
        "cli"
      ],
      "verified": true,
      "official": false,
      "featured": false,
      "authType": "apiKey",
      "why": "Run Tracing: Streams every tool call, message, and state change into a local console for step-by-step debugging of agent runs."
    },
    {
      "name": "Hatchet",
      "slug": "hatchet",
      "url": "https://agentswitchboard.dev/agents/hatchet",
      "iconUrl": "https://agentswitchboard.dev/icons/hatchet.png?v=8ffb8229",
      "description": "Durable task orchestration engine for AI agents — parallel workloads, automatic retries, concurrency controls, and built-in OpenTelemetry tracing. MIT-licensed, self-hostable.",
      "provider": "Hatchet Technologies",
      "categories": [
        "orchestration"
      ],
      "accessMethods": [
        "api",
        "cli"
      ],
      "verified": true,
      "official": false,
      "featured": false,
      "authType": "apiKey",
      "why": null
    },
    {
      "name": "HyperProbe",
      "slug": "hyperprobe",
      "url": "https://agentswitchboard.dev/agents/hyperprobe",
      "iconUrl": "https://agentswitchboard.dev/icons/hyperprobe.png?v=36797b57",
      "description": "Agents that perform read-only production debugging, tracing failures across services without mutating state. Y Combinator S26.",
      "provider": "HyperProbe",
      "categories": [
        "code-devtools",
        "infrastructure"
      ],
      "accessMethods": [
        "api"
      ],
      "verified": true,
      "official": false,
      "featured": false,
      "authType": "apiKey",
      "why": null
    },
    {
      "name": "Cloudflare AI Gateway",
      "slug": "cloudflare-ai-gateway",
      "url": "https://agentswitchboard.dev/agents/cloudflare-ai-gateway",
      "iconUrl": "https://agentswitchboard.dev/icons/cloudflare-ai-gateway.png?v=7c4f193b",
      "description": "Proxies requests to OpenAI, Anthropic, Google, Workers AI, and other model providers, adding analytics, logging, caching, rate limiting, and retries with model fallback.",
      "provider": "Cloudflare",
      "categories": [
        "infrastructure"
      ],
      "accessMethods": [
        "api"
      ],
      "verified": true,
      "official": true,
      "featured": false,
      "authType": "bearer",
      "why": null
    },
    {
      "name": "Portkey",
      "slug": "portkey",
      "url": "https://agentswitchboard.dev/agents/portkey",
      "iconUrl": "https://agentswitchboard.dev/icons/portkey.png?v=d73e09d8",
      "description": "AI gateway with unified API for 200+ LLMs, plus observability, caching, fallbacks, and cost management. Drop-in replacement for the OpenAI SDK.",
      "provider": "Portkey",
      "categories": [
        "infrastructure"
      ],
      "accessMethods": [
        "api"
      ],
      "verified": true,
      "official": false,
      "featured": false,
      "authType": "apiKey",
      "why": null
    },
    {
      "name": "Vercel AI Gateway",
      "slug": "vercel-ai-gateway",
      "url": "https://agentswitchboard.dev/agents/vercel-ai-gateway",
      "iconUrl": "https://agentswitchboard.dev/icons/vercel-ai-gateway.png?v=40448473",
      "description": "Routes model calls from any app or coding agent across providers with one API key, adding fallbacks, request logs, spend budgets, and BYOK, via the AI SDK or OpenAI-compatible APIs.",
      "provider": "Vercel",
      "categories": [
        "infrastructure",
        "language"
      ],
      "accessMethods": [
        "api"
      ],
      "verified": true,
      "official": true,
      "featured": false,
      "authType": "apiKey",
      "why": null
    }
  ],
  "relatedTasks": [
    "monitor-apps-in-production",
    "run-code-in-a-sandbox",
    "deploy-an-app",
    "query-a-database"
  ],
  "stacks": [
    "agents-in-production"
  ]
}