{"leaderboard":{"id":"076032b9-6e20-458a-b56d-5aea2488b3f9","package_id":"e7236beb-3ccb-467a-81a7-4997c0bb91cf","package":"frontier-bench/frontier-bench","dataset_version_ids":["2f325140-b410-452e-87b3-978ffcb4e1ea"],"name":"frontier-bench","title":"FRONTIER-BENCH V0.1","description":"Leaderboard for the Frontier-Bench benchmark","metadata_schema":{"type":"object","required":["agent_display","model_display","agent_org","model_org","display_date","date","release_date","display_release_date"],"properties":{"pr":{"type":"string","minLength":1},"date":{"type":"string","pattern":"^\\d{4}-\\d{2}-\\d{2}$","description":"Sortable date key in YYYY-MM-DD format."},"judge":{"type":"string","minLength":1},"agent_org":{"type":"object","required":["url","label"],"properties":{"url":{"type":"string","format":"uri","minLength":1},"label":{"type":"string","minLength":1}},"description":"Link object used by type `link` cells (`{ url, label }`).","additionalProperties":false},"model_org":{"type":"object","required":["url","label"],"properties":{"url":{"type":"string","format":"uri","minLength":1},"label":{"type":"string","minLength":1}},"description":"Link object used by type `link` cells (`{ url, label }`).","additionalProperties":false},"display_date":{"type":"string","pattern":"^[A-Za-z]{3} [0-9]{1,2}, [0-9]{4}$","maxLength":12,"minLength":11,"description":"Human-friendly date display in \"Mon D, YYYY\" format."},"release_date":{"type":"string","pattern":"^\\d{4}-\\d{2}-\\d{2}$","description":"Model release/announce date in YYYY-MM-DD format."},"agent_display":{"type":"object","required":["url","label"],"properties":{"url":{"type":"string","format":"uri","minLength":1},"label":{"type":"string","minLength":1}},"description":"Link object used by type `link` cells (`{ url, label }`).","additionalProperties":false},"model_display":{"type":"object","required":["url","label"],"properties":{"url":{"type":"string","format":"uri","minLength":1},"label":{"type":"string","minLength":1}},"description":"Link object used by type `link` cells (`{ url, label }`).","additionalProperties":false},"reasoning_effort":{"type":"string"},"display_release_date":{"type":"string","pattern":"^[A-Za-z]{3} [0-9]{1,2}, [0-9]{4}$","maxLength":12,"minLength":11,"description":"Human-friendly model release date in \"Mon D, YYYY\" format."}},"additionalProperties":false},"metrics_schema":{"type":"object","required":["accuracy","accuracy_stderr","display_accuracy","display_cost","display_total_tokens"],"properties":{"accuracy":{"type":"number","maximum":100,"minimum":0},"display_cost":{"type":"string"},"total_tokens":{"type":"number","minimum":0},"total_cost_usd":{"type":"number","minimum":0},"accuracy_stderr":{"type":"number","maximum":100,"minimum":0},"display_accuracy":{"type":"string"},"display_total_tokens":{"type":"string"}},"additionalProperties":false},"columns":[{"id":"agent_display","type":"link","header":"Agent","accessor":"metadata.agent_display"},{"id":"model_display","type":"link","header":"Model","accessor":"metadata.model_display"},{"id":"reasoning_effort","type":"text","header":"Reasoning Effort","accessor":"metadata.reasoning_effort"},{"id":"accuracy","type":"number","align":"right","header":"Accuracy","accessor":"metrics.accuracy","display_type":"markdown","display_accessor":"metrics.display_accuracy"},{"id":"release_date","type":"date","header":"Release Date","accessor":"metadata.release_date","display_type":"text","display_accessor":"metadata.display_release_date"},{"id":"agent_org","type":"link","header":"Agent Org","accessor":"metadata.agent_org"},{"id":"model_org","type":"link","header":"Model Org","accessor":"metadata.model_org"},{"id":"total_tokens","type":"number","align":"right","header":"Tokens","accessor":"metrics.total_tokens","display_type":"text","display_accessor":"metrics.display_total_tokens"},{"id":"total_cost_usd","type":"number","align":"right","header":"Cost","accessor":"metrics.total_cost_usd","display_type":"text","display_accessor":"metrics.display_cost"}],"rank_by":[{"accessor":"metrics.accuracy","direction":"desc"}],"visibility":"public","created_by":"2c4c89e3-c97d-4e19-9e0b-aa695f0a0d66","created_at":"2026-07-21T21:03:02.008776+00:00","updated_at":"2026-07-28T19:49:17.632545+00:00"},"rows":[{"id":"e9d504e6-a814-4cc4-99cf-033054408797","leaderboard_id":"076032b9-6e20-458a-b56d-5aea2488b3f9","rank":1,"metadata":{"date":"2026-07-24","agent_org":{"url":"https://www.princeton.edu","label":"Princeton"},"model_org":{"url":"https://www.anthropic.com","label":"Anthropic"},"display_date":"Jul 24, 2026","release_date":"2026-07-24","agent_display":{"url":"https://github.com/SWE-agent/mini-swe-agent","label":"mini-SWE-agent"},"model_display":{"url":"https://www.anthropic.com/news/claude-opus-5","label":"Opus 5"},"reasoning_effort":"max","display_release_date":"Jul 24, 2026"},"metrics":{"accuracy":43.53,"display_cost":"$6.0k","total_tokens":7280891794,"total_cost_usd":5973.57,"accuracy_stderr":1.65,"display_accuracy":"**43.5%** ± 1.7%","display_total_tokens":"7.3B"},"status":"display","created_at":"2026-07-24T20:02:59.210369+00:00","updated_at":"2026-07-24T20:43:15.176386+00:00","n_trials":0},{"id":"4ce39786-946e-447b-bf36-9650ff7749ac","leaderboard_id":"076032b9-6e20-458a-b56d-5aea2488b3f9","rank":2,"metadata":{"date":"2026-07-13","agent_org":{"url":"https://openai.com","label":"OpenAI"},"model_org":{"url":"https://openai.com","label":"OpenAI"},"display_date":"Jul 13, 2026","release_date":"2026-07-09","agent_display":{"url":"https://openai.com/codex/","label":"Codex"},"model_display":{"url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","label":"GPT-5.6 Sol"},"reasoning_effort":"max","display_release_date":"Jul 9, 2026"},"metrics":{"accuracy":32.74,"display_cost":"$4.0k","total_tokens":5840709769,"total_cost_usd":4019.38,"accuracy_stderr":1.59,"display_accuracy":"**32.7%** ± 1.6%","display_total_tokens":"5.8B"},"status":"display","created_at":"2026-07-23T00:48:29.293634+00:00","updated_at":"2026-07-28T19:47:44.662588+00:00","n_trials":0},{"id":"fd93299a-0c43-4ec3-8aee-421fdbef4b8d","leaderboard_id":"076032b9-6e20-458a-b56d-5aea2488b3f9","rank":3,"metadata":{"date":"2026-07-13","agent_org":{"url":"https://www.anthropic.com","label":"Anthropic"},"model_org":{"url":"https://www.anthropic.com","label":"Anthropic"},"display_date":"Jul 13, 2026","release_date":"2026-06-09","agent_display":{"url":"https://claude.com/claude-code","label":"Claude Code"},"model_display":{"url":"https://www.anthropic.com/news/claude-fable-5-mythos-5","label":"Fable 5"},"reasoning_effort":"max","display_release_date":"Jun 9, 2026"},"metrics":{"accuracy":31.74,"display_cost":"$7.3k","total_tokens":3594392235,"total_cost_usd":7320.25,"accuracy_stderr":1.68,"display_accuracy":"**31.7%** ± 1.7%","display_total_tokens":"3.6B"},"status":"display","created_at":"2026-07-23T00:48:22.980255+00:00","updated_at":"2026-07-28T19:47:32.790399+00:00","n_trials":0},{"id":"46eaaf38-415f-443a-a68b-f81c46e49be6","leaderboard_id":"076032b9-6e20-458a-b56d-5aea2488b3f9","rank":4,"metadata":{"date":"2026-07-12","agent_org":{"url":"https://www.anthropic.com","label":"Anthropic"},"model_org":{"url":"https://www.anthropic.com","label":"Anthropic"},"display_date":"Jul 12, 2026","release_date":"2026-05-28","agent_display":{"url":"https://claude.com/claude-code","label":"Claude Code"},"model_display":{"url":"https://www.anthropic.com/news/claude-opus-4-8","label":"Opus 4.8"},"reasoning_effort":"max","display_release_date":"May 28, 2026"},"metrics":{"accuracy":21.08,"display_cost":"$5.7k","total_tokens":5208776503,"total_cost_usd":5701.74,"accuracy_stderr":1.55,"display_accuracy":"**21.1%** ± 1.6%","display_total_tokens":"5.2B"},"status":"display","created_at":"2026-07-23T00:48:35.616121+00:00","updated_at":"2026-07-28T19:48:42.571115+00:00","n_trials":0},{"id":"8519e479-95e6-44a4-a108-1834df711b28","leaderboard_id":"076032b9-6e20-458a-b56d-5aea2488b3f9","rank":5,"metadata":{"date":"2026-07-12","agent_org":{"url":"https://openai.com","label":"OpenAI"},"model_org":{"url":"https://openai.com","label":"OpenAI"},"display_date":"Jul 12, 2026","release_date":"2026-07-09","agent_display":{"url":"https://openai.com/codex/","label":"Codex"},"model_display":{"url":"https://developers.openai.com/api/docs/models/gpt-5.6-terra","label":"GPT-5.6 Terra"},"reasoning_effort":"max","display_release_date":"Jul 9, 2026"},"metrics":{"accuracy":20.81,"display_cost":"$2.5k","total_tokens":7024607919,"total_cost_usd":2484.52,"accuracy_stderr":1.4,"display_accuracy":"**20.8%** ± 1.4%","display_total_tokens":"7.0B"},"status":"display","created_at":"2026-07-23T00:48:42.022884+00:00","updated_at":"2026-07-23T05:55:34.36349+00:00","n_trials":0},{"id":"2ccf029d-62fe-4756-a624-b2d376895a32","leaderboard_id":"076032b9-6e20-458a-b56d-5aea2488b3f9","rank":6,"metadata":{"date":"2026-07-15","agent_org":{"url":"https://cursor.com","label":"Cursor"},"model_org":{"url":"https://x.ai","label":"xAI"},"display_date":"Jul 15, 2026","release_date":"2026-07-08","agent_display":{"url":"https://cursor.com/docs/cli/overview","label":"Cursor CLI"},"model_display":{"url":"https://docs.x.ai/developers/models/grok-4.5","label":"Grok 4.5"},"reasoning_effort":"xhigh","display_release_date":"Jul 8, 2026"},"metrics":{"accuracy":15.68,"display_cost":"$960","total_tokens":1238993118,"total_cost_usd":960.04,"accuracy_stderr":1.47,"display_accuracy":"**15.7%** ± 1.5%","display_total_tokens":"1.2B"},"status":"display","created_at":"2026-07-23T00:49:01.181248+00:00","updated_at":"2026-07-28T19:49:06.124157+00:00","n_trials":0},{"id":"95ef728a-41bc-4882-adfa-0d30ce50671c","leaderboard_id":"076032b9-6e20-458a-b56d-5aea2488b3f9","rank":7,"metadata":{"date":"2026-07-12","agent_org":{"url":"https://www.anthropic.com","label":"Anthropic"},"model_org":{"url":"https://www.anthropic.com","label":"Anthropic"},"display_date":"Jul 12, 2026","release_date":"2026-06-30","agent_display":{"url":"https://claude.com/claude-code","label":"Claude Code"},"model_display":{"url":"https://www.anthropic.com/news/claude-sonnet-5","label":"Sonnet 5"},"reasoning_effort":"max","display_release_date":"Jun 30, 2026"},"metrics":{"accuracy":14.59,"display_cost":"$7.9k","total_tokens":17948790860,"total_cost_usd":7944.04,"accuracy_stderr":1.5,"display_accuracy":"**14.6%** ± 1.5%","display_total_tokens":"17.9B"},"status":"display","created_at":"2026-07-23T00:48:48.31335+00:00","updated_at":"2026-07-28T19:47:56.288937+00:00","n_trials":0},{"id":"5651c446-0517-409c-9632-aafd5ca76e93","leaderboard_id":"076032b9-6e20-458a-b56d-5aea2488b3f9","rank":8,"metadata":{"date":"2026-07-12","agent_org":{"url":"https://openai.com","label":"OpenAI"},"model_org":{"url":"https://openai.com","label":"OpenAI"},"display_date":"Jul 12, 2026","release_date":"2026-07-09","agent_display":{"url":"https://openai.com/codex/","label":"Codex"},"model_display":{"url":"https://developers.openai.com/api/docs/models/gpt-5.6-luna","label":"GPT-5.6 Luna"},"reasoning_effort":"max","display_release_date":"Jul 9, 2026"},"metrics":{"accuracy":14.32,"display_cost":"$1.7k","total_tokens":12047530572,"total_cost_usd":1653.7,"accuracy_stderr":1.25,"display_accuracy":"**14.3%** ± 1.2%","display_total_tokens":"12.0B"},"status":"display","created_at":"2026-07-23T00:48:54.777793+00:00","updated_at":"2026-07-23T05:55:34.36349+00:00","n_trials":0},{"id":"95e83401-73ec-4317-8e4b-d0d579fcfb71","leaderboard_id":"076032b9-6e20-458a-b56d-5aea2488b3f9","rank":9,"metadata":{"date":"2026-07-19","agent_org":{"url":"https://www.anthropic.com","label":"Anthropic"},"model_org":{"url":"https://z.ai","label":"Z.ai"},"display_date":"Jul 19, 2026","release_date":"2026-06-13","agent_display":{"url":"https://claude.com/claude-code","label":"Claude Code"},"model_display":{"url":"https://z.ai/blog/glm-5.2","label":"GLM 5.2"},"reasoning_effort":"max","display_release_date":"Jun 13, 2026"},"metrics":{"accuracy":4.59,"display_cost":"$3.8k","total_tokens":3276045131,"total_cost_usd":3766.5,"accuracy_stderr":0.97,"display_accuracy":"**4.6%** ± 1.0%","display_total_tokens":"3.3B"},"status":"display","created_at":"2026-07-23T00:49:14.299058+00:00","updated_at":"2026-07-28T19:48:54.14958+00:00","n_trials":0}]}