{"ok":true,"site":{"host":"benchmarkheaven.com","url":"https://benchmarkheaven.com/about","type":"live","desc":"Where every benchmark result and price comes from, how the adjusted cost per task is calculated, and how publishers can ask for removal.","tools":[{"name":"search_benchmarks","kind":"answer","impl":"imperative","description":"Find benchmarks tracked by Benchmark Heaven. Returns exact versioned benchmark ids with metric, unit, direction, category, maintainer, source link, last verification date and how many catalog models have a result. Read-only.","title":"search benchmarks","annotations":{"readOnlyHint":true,"untrustedContentHint":true},"inputSchema":{"type":"object","additionalProperties":false,"properties":{"query":{"type":"string","maxLength":100,"description":"Words to match in the benchmark name, family, id or maintainer (case-insensitive). Empty lists everything."},"category":{"type":"string","maxLength":40,"description":"Exact category, e.g. \"Coding\", \"Math\", \"Agentic\"."},"limit":{"type":"integer","minimum":1,"maximum":25,"default":10},"cursor":{"type":"string","pattern":"^[0-9]{1,4}$","description":"next_cursor from the previous page."}}},"hasHandler":true,"hasExecute":true,"__source":"spec","executable":true,"handlerField":"execute","page":"/about","pages":["/about","/decision-model-benchmarks","/benchmarks","/compare","/benchmaxxing","/jev-models/methodology"]},{"name":"get_benchmark_results","kind":"answer","impl":"imperative","description":"Published results of one exact benchmark version: per model the value in the benchmark's native unit, basis (measured, self-reported, derived, preliminary), run configuration and source (URL, retrieval date, locator). Read-only; never estimates.","title":"get benchmark results","annotations":{"readOnlyHint":true,"untrustedContentHint":true},"inputSchema":{"type":"object","additionalProperties":false,"required":["benchmark_id"],"properties":{"benchmark_id":{"type":"string","pattern":"^[A-Za-z0-9][A-Za-z0-9._:@%+-]{0,159}$","description":"Exact versioned benchmark id from search_benchmarks, e.g. \"aa-gpqa-diamond::snapshot-2026-09-10\"."},"model_ids":{"type":"array","maxItems":10,"uniqueItems":true,"items":{"type":"string","pattern":"^[A-Za-z0-9][A-Za-z0-9._:@%+-]{0,159}$"},"description":"Optional exact catalog model ids (e.g. \"claude-opus-5::max\"); omit to page through every published result."},"limit":{"type":"integer","minimum":1,"maximum":50,"default":25},"cursor":{"type":"string","pattern":"^[0-9]{1,5}$","description":"next_cursor from the previous page (only without model_ids)."}}},"hasHandler":true,"hasExecute":true,"__source":"spec","executable":true,"handlerField":"execute","page":"/about","pages":["/about","/decision-model-benchmarks","/benchmarks","/compare","/benchmaxxing","/jev-models/methodology"]},{"name":"get_model_benchmark_summary","kind":"answer","impl":"imperative","description":"Published benchmark results for one exact catalog model (variant included), with unit, basis, benchmark version and source date. Requested benchmarks without a published result return value null. Read-only; never estimates.","title":"get model benchmark summary","annotations":{"readOnlyHint":true,"untrustedContentHint":true},"inputSchema":{"type":"object","additionalProperties":false,"required":["model_id"],"properties":{"model_id":{"type":"string","pattern":"^[A-Za-z0-9][A-Za-z0-9._:@%+-]{0,159}$","description":"Exact catalog model id including the variant, e.g. \"kimi-k3::max\"."},"benchmark_ids":{"type":"array","maxItems":25,"uniqueItems":true,"items":{"type":"string","pattern":"^[A-Za-z0-9][A-Za-z0-9._:@%+-]{0,159}$"},"description":"Optional exact benchmark ids; each one without a published result comes back with value null."},"limit":{"type":"integer","minimum":1,"maximum":50,"default":25}}},"hasHandler":true,"hasExecute":true,"__source":"spec","executable":true,"handlerField":"execute","page":"/about","pages":["/about","/decision-model-benchmarks","/benchmarks","/compare","/benchmaxxing","/jev-models/methodology"]},{"name":"list_models","kind":"answer","impl":"imperative","description":"List systems on the current JevBench leaderboard (Jev-class decision models) with axis scores (intelligence, calibration, speed, cost; 0-100, higher is better), Capability score/rank/eligibility, Composite score/rank, listing flags, price per 1,000 decisions and median latency. From v1.7.0 (method v1.7) a decision over 10 s counts as failed: rows also give the measured p95, the number of decisions over 10 s, the method and the v1.6 composite. Read-only.","title":"List JevBench models","annotations":{"readOnlyHint":true,"untrustedContentHint":true},"inputSchema":{"type":"object","additionalProperties":false,"properties":{"board":{"type":"string","enum":["all","open","api"],"default":"all","description":"\"open\" = open-weights board (/jev-models), \"api\" = hosted API board (/jev-models/api), \"all\" = both. The Jev reference row is on both boards."},"eligible_only":{"type":"boolean","default":false,"description":"Only systems eligible for the official Capability ranking (ranked and within the cost and median-latency caps)."},"sort_by":{"type":"string","enum":["capability","composite","intelligence","calibration","speed","cost"],"default":"capability","description":"Score to sort by, descending (higher is better on every axis). Missing scores sort last; the key breaks ties."},"limit":{"type":"integer","minimum":1,"maximum":50,"default":20},"cursor":{"type":"string","pattern":"^[0-9]{1,4}$","description":"next_cursor from the previous page."}}},"hasHandler":true,"hasExecute":true,"__source":"spec","executable":true,"handlerField":"execute","page":"/about","pages":["/about","/decision-model-benchmarks","/benchmarks","/compare","/benchmaxxing","/jev-models/methodology"]},{"name":"get_model","kind":"answer","impl":"imperative","description":"The full public JevBench row of one system by exact key: axes, Capability eligibility and reasons, Composite and open-board ranks, listing flags, price with basis, raw and adjusted latency, source link. Read-only.","title":"Get a JevBench model","annotations":{"readOnlyHint":true,"untrustedContentHint":true},"inputSchema":{"type":"object","additionalProperties":false,"required":["key"],"properties":{"key":{"type":"string","minLength":1,"maxLength":128,"description":"Exact system key from list_models, e.g. \"sage-1.3.0\"."}}},"hasHandler":true,"hasExecute":true,"__source":"spec","executable":true,"handlerField":"execute","page":"/about","pages":["/about","/decision-model-benchmarks","/benchmarks","/compare","/benchmaxxing","/jev-models/methodology"]},{"name":"compare_models","kind":"act","impl":"imperative","description":"Compare two JevBench systems by exact key: both public rows, per-axis and score differences (first minus second; null when either is missing), official ranks and a link to the side-by-side comparison page. Read-only.","title":"Compare two JevBench models","annotations":{"readOnlyHint":true,"untrustedContentHint":true},"inputSchema":{"type":"object","additionalProperties":false,"required":["keys"],"properties":{"keys":{"type":"array","minItems":2,"maxItems":2,"uniqueItems":true,"items":{"type":"string","minLength":1,"maxLength":128},"description":"Exactly two different system keys from list_models."}}},"hasHandler":true,"hasExecute":true,"__source":"spec","executable":true,"handlerField":"execute","page":"/about","pages":["/about","/decision-model-benchmarks","/benchmarks","/compare","/benchmaxxing","/jev-models/methodology"]}],"category":"AI & Agents","favicon":"/favicons/benchmarkheaven.com.png","apiSurface":"spec","toolCount":6}}