benchmarkheaven.com

{
  "name": "search_benchmarks",
  "kind": "answer",
  "impl": "imperative",
  "description": "Find benchmarks tracked by Benchmark Heaven. Returns exact versioned benchmark ids with metric, unit, direction, category, maintainer, source link, last verification date and how many catalog models have a result. Read-only.",
  "inputSchema": {
    "type": "object",
    "additionalProperties": false,
    "properties": {
      "query": {
        "type": "string",
        "maxLength": 100,
        "description": "Words to match in the benchmark name, family, id or maintainer (case-insensitive). Empty lists everything."
      },
      "category": {
        "type": "string",
        "maxLength": 40,
        "description": "Exact category, e.g. \"Coding\", \"Math\", \"Agentic\"."
      },
      "limit": {
        "type": "integer",
        "minimum": 1,
        "maximum": 25,
        "default": 10
      },
      "cursor": {
        "type": "string",
        "pattern": "^[0-9]{1,4}$",
        "description": "next_cursor from the previous page."
      }
    }
  },
  "page": "/about"
}

{
  "name": "get_benchmark_results",
  "kind": "answer",
  "impl": "imperative",
  "description": "Published results of one exact benchmark version: per model the value in the benchmark's native unit, basis (measured, self-reported, derived, preliminary), run configuration and source (URL, retrieval date, locator). Read-only; never estimates.",
  "inputSchema": {
    "type": "object",
    "additionalProperties": false,
    "required": [
      "benchmark_id"
    ],
    "properties": {
      "benchmark_id": {
        "type": "string",
        "pattern": "^[A-Za-z0-9][A-Za-z0-9._:@%+-]{0,159}$",
        "description": "Exact versioned benchmark id from search_benchmarks, e.g. \"aa-gpqa-diamond::snapshot-2026-09-10\"."
      },
      "model_ids": {
        "type": "array",
        "maxItems": 10,
        "uniqueItems": true,
        "items": {
          "type": "string",
          "pattern": "^[A-Za-z0-9][A-Za-z0-9._:@%+-]{0,159}$"
        },
        "description": "Optional exact catalog model ids (e.g. \"claude-opus-5::max\"); omit to page through every published result."
      },
      "limit": {
        "type": "integer",
        "minimum": 1,
        "maximum": 50,
        "default": 25
      },
      "cursor": {
        "type": "string",
        "pattern": "^[0-9]{1,5}$",
        "description": "next_cursor from the previous page (only without model_ids)."
      }
    }
  },
  "page": "/about"
}

{
  "name": "get_model_benchmark_summary",
  "kind": "answer",
  "impl": "imperative",
  "description": "Published benchmark results for one exact catalog model (variant included), with unit, basis, benchmark version and source date. Requested benchmarks without a published result return value null. Read-only; never estimates.",
  "inputSchema": {
    "type": "object",
    "additionalProperties": false,
    "required": [
      "model_id"
    ],
    "properties": {
      "model_id": {
        "type": "string",
        "pattern": "^[A-Za-z0-9][A-Za-z0-9._:@%+-]{0,159}$",
        "description": "Exact catalog model id including the variant, e.g. \"kimi-k3::max\"."
      },
      "benchmark_ids": {
        "type": "array",
        "maxItems": 25,
        "uniqueItems": true,
        "items": {
          "type": "string",
          "pattern": "^[A-Za-z0-9][A-Za-z0-9._:@%+-]{0,159}$"
        },
        "description": "Optional exact benchmark ids; each one without a published result comes back with value null."
      },
      "limit": {
        "type": "integer",
        "minimum": 1,
        "maximum": 50,
        "default": 25
      }
    }
  },
  "page": "/about"
}

{
  "name": "list_models",
  "kind": "answer",
  "impl": "imperative",
  "description": "List systems on the current JevBench leaderboard (Jev-class decision models) with axis scores (intelligence, calibration, speed, cost; 0-100, higher is better), Capability score/rank/eligibility, Composite score/rank, listing flags, price per 1,000 decisions and median latency. From v1.7.0 (method v1.7) a decision over 10 s counts as failed: rows also give the measured p95, the number of decisions over 10 s, the method and the v1.6 composite. Read-only.",
  "inputSchema": {
    "type": "object",
    "additionalProperties": false,
    "properties": {
      "board": {
        "type": "string",
        "enum": [
          "all",
          "open",
          "api"
        ],
        "default": "all",
        "description": "\"open\" = open-weights board (/jev-models), \"api\" = hosted API board (/jev-models/api), \"all\" = both. The Jev reference row is on both boards."
      },
      "eligible_only": {
        "type": "boolean",
        "default": false,
        "description": "Only systems eligible for the official Capability ranking (ranked and within the cost and median-latency caps)."
      },
      "sort_by": {
        "type": "string",
        "enum": [
          "capability",
          "composite",
          "intelligence",
          "calibration",
          "speed",
          "cost"
        ],
        "default": "capability",
        "description": "Score to sort by, descending (higher is better on every axis). Missing scores sort last; the key breaks ties."
      },
      "limit": {
        "type": "integer",
        "minimum": 1,
        "maximum": 50,
        "default": 20
      },
      "cursor": {
        "type": "string",
        "pattern": "^[0-9]{1,4}$",
        "description": "next_cursor from the previous page."
      }
    }
  },
  "page": "/about"
}

{
  "name": "get_model",
  "kind": "answer",
  "impl": "imperative",
  "description": "The full public JevBench row of one system by exact key: axes, Capability eligibility and reasons, Composite and open-board ranks, listing flags, price with basis, raw and adjusted latency, source link. Read-only.",
  "inputSchema": {
    "type": "object",
    "additionalProperties": false,
    "required": [
      "key"
    ],
    "properties": {
      "key": {
        "type": "string",
        "minLength": 1,
        "maxLength": 128,
        "description": "Exact system key from list_models, e.g. \"sage-1.3.0\"."
      }
    }
  },
  "page": "/about"
}

{
  "name": "compare_models",
  "kind": "act",
  "impl": "imperative",
  "description": "Compare two JevBench systems by exact key: both public rows, per-axis and score differences (first minus second; null when either is missing), official ranks and a link to the side-by-side comparison page. Read-only.",
  "inputSchema": {
    "type": "object",
    "additionalProperties": false,
    "required": [
      "keys"
    ],
    "properties": {
      "keys": {
        "type": "array",
        "minItems": 2,
        "maxItems": 2,
        "uniqueItems": true,
        "items": {
          "type": "string",
          "minLength": 1,
          "maxLength": 128
        },
        "description": "Exactly two different system keys from list_models."
      }
    }
  },
  "page": "/about"
}