{
  "document_type": "leaderboard",
  "schema_version": 3,
  "leaderboard": [
    {
      "rank": 1,
      "model": {
        "model_id": "M-0014",
        "display_name": "Claude Fable 5 (high)",
        "route": "anthropic/claude-fable-5",
        "provider": "anthropic",
        "reasoning_effort": "high",
        "seed_capability": "unsupported",
        "configuration_hash": "127681fed10b65b1941fd632054f269dc7f5fc9ba11f5f0b0785b5af97abb671"
      },
      "efficiency_rank": 10,
      "ideal_distance_rank": 9,
      "product_efficiency_rank": 10,
      "pareto_efficient": true,
      "status": "evaluated",
      "execution_id": "BX-20260805-official-M0014-011",
      "completed_at": "2026-08-05T18:13:13.469638Z",
      "question_score": "12.05714285714285714285714286",
      "question_score_confidence_interval": {
        "confidence_level": "0.95",
        "method": "stratified-welch-t-v1",
        "estimate": "12.05714285714285714285714286",
        "lower": "10.13061100248741697934536639261240857332",
        "upper": "13.98367471179829730636891932738759142668",
        "standard_error": "0.7188426071662356745782939976759210015063",
        "degrees_of_freedom": "4.398282131145291639796380400927539413567",
        "subject_count": 7,
        "trial_count": 35
      },
      "success_rate": "1",
      "total_cost_usd": "7.86635157",
      "guesser_cost_per_episode_usd": "0.1775935857142857142857142857",
      "full_cost_per_episode_usd": "0.224752902",
      "runtime_per_episode_ms": "274108.2571428571428571428571",
      "guesser_think_time_per_episode_ms": "118801.3428571428571428571429",
      "guesser_latency_per_call_ms": "9098.571115973741794310722101",
      "ideal_distance_score": "0.41974001094927431166301585027732611786568685380570",
      "normalized_question_score": "0",
      "normalized_guesser_cost": "0.41974001094927431166301585027732611786568685380570",
      "cost_adjusted_question_score": "2.141271233469387755102040817",
      "efficiency_status": "ranked",
      "successful": 35,
      "terminal_trials": 35,
      "contract": {
        "evaluated_outputs": 457,
        "valid_outputs": 457,
        "violations": 0,
        "counted_penalties": 0,
        "affected_trials": 0,
        "compliance_rate": "1",
        "status": "clean"
      }
    },
    {
      "rank": 2,
      "model": {
        "model_id": "M-0006",
        "display_name": "Claude Opus 5 (high)",
        "route": "anthropic/claude-opus-5",
        "provider": "anthropic",
        "reasoning_effort": "high",
        "seed_capability": "unsupported",
        "configuration_hash": "57d7ad4d350ae94172c661c4d094320a9a18e4bac29f37b6be6f38ed638beef2"
      },
      "efficiency_rank": 7,
      "ideal_distance_rank": 2,
      "product_efficiency_rank": 7,
      "pareto_efficient": true,
      "status": "evaluated",
      "execution_id": "BX-20260728-official-M0006-010",
      "completed_at": "2026-07-28T18:11:37.435939Z",
      "question_score": "12.34285714285714285714285714",
      "question_score_confidence_interval": {
        "confidence_level": "0.95",
        "method": "stratified-welch-t-v1",
        "estimate": "12.34285714285714285714285714",
        "lower": "11.41281499520431320224745713501628803018",
        "upper": "13.27289929050997251203825714498371196982",
        "standard_error": "0.4140393356054125306776761187762791920898",
        "degrees_of_freedom": "9.437697287464555133486704831202182868763",
        "subject_count": 7,
        "trial_count": 35
      },
      "success_rate": "1",
      "total_cost_usd": "5.4675744",
      "guesser_cost_per_episode_usd": "0.07052075714285714285714285714",
      "full_cost_per_episode_usd": "0.1562164114285714285714285714",
      "runtime_per_episode_ms": "213310.3142857142857142857143",
      "guesser_think_time_per_episode_ms": "72777.08571428571428571428571",
      "guesser_latency_per_call_ms": "5454.385438972162740899357602",
      "ideal_distance_score": "0.15451644480814609470456901859420805967070370273747",
      "normalized_question_score": "0.014164305949008498583569404816867160477975106131981",
      "normalized_guesser_cost": "0.15386586415814178464200849802250762656934321202553",
      "cost_adjusted_question_score": "0.8704276310204081632653061222",
      "efficiency_status": "ranked",
      "successful": 35,
      "terminal_trials": 35,
      "contract": {
        "evaluated_outputs": 467,
        "valid_outputs": 464,
        "violations": 3,
        "counted_penalties": 3,
        "affected_trials": 2,
        "compliance_rate": "0.9935760171306209850107066381",
        "status": "breached"
      }
    },
    {
      "rank": 3,
      "model": {
        "model_id": "M-0007",
        "display_name": "Kimi K3 (high)",
        "route": "moonshotai/kimi-k3",
        "provider": "moonshotai",
        "reasoning_effort": "high",
        "seed_capability": "unsupported",
        "configuration_hash": "4af95146032fe4fa46c22b423b26fa118dd7dba49793b661c17bfdbca2baf5f6"
      },
      "efficiency_rank": 12,
      "ideal_distance_rank": 12,
      "product_efficiency_rank": 12,
      "pareto_efficient": false,
      "status": "evaluated",
      "execution_id": "BX-20260728-official-M0007-010",
      "completed_at": "2026-07-28T22:10:44.445690Z",
      "question_score": "12.74285714285714285714285714",
      "question_score_confidence_interval": {
        "confidence_level": "0.95",
        "method": "stratified-welch-t-v1",
        "estimate": "12.74285714285714285714285714",
        "lower": "9.155108600456976432658316012024476218670",
        "upper": "16.33060568525730928162739826797552378133",
        "standard_error": "1.564399865371865277213939438899090055641",
        "degrees_of_freedom": "8.261285558588991805126397474922831970740",
        "subject_count": 7,
        "trial_count": 35
      },
      "success_rate": "0.9429",
      "total_cost_usd": "10.08658847",
      "guesser_cost_per_episode_usd": "0.2032166142857142857142857143",
      "full_cost_per_episode_usd": "0.288188242",
      "runtime_per_episode_ms": "543451.8857142857142857142857",
      "guesser_think_time_per_episode_ms": "414346.4857142857142857142857",
      "guesser_latency_per_call_ms": "30275.83924843423799582463466",
      "ideal_distance_score": "0.48455884846251196650981565156904684499987667360352",
      "normalized_question_score": "0.033994334277620396600566571957081751719378215056697",
      "normalized_guesser_cost": "0.48336493755788393812222999793425038205545065238846",
      "cost_adjusted_question_score": "2.589560284897959183673469387",
      "efficiency_status": "ranked",
      "successful": 33,
      "terminal_trials": 35,
      "contract": {
        "evaluated_outputs": 479,
        "valid_outputs": 468,
        "violations": 11,
        "counted_penalties": 11,
        "affected_trials": 10,
        "compliance_rate": "0.9770354906054279749478079332",
        "status": "breached"
      }
    },
    {
      "rank": 4,
      "model": {
        "model_id": "M-0002",
        "display_name": "gpt-oss-120B (high)",
        "route": "openai/gpt-oss-120b",
        "provider": "cerebras",
        "reasoning_effort": "high",
        "seed_capability": "supported",
        "configuration_hash": "875912ffd40d041d262b5692a395418650e137fedc2caf62622275d590169d3c"
      },
      "efficiency_rank": 5,
      "ideal_distance_rank": 1,
      "product_efficiency_rank": 5,
      "pareto_efficient": true,
      "status": "evaluated",
      "execution_id": "BX-20260728-official-M0002-011",
      "completed_at": "2026-07-28T22:21:35.165858Z",
      "question_score": "13.62857142857142857142857143",
      "question_score_confidence_interval": {
        "confidence_level": "0.95",
        "method": "stratified-welch-t-v1",
        "estimate": "13.62857142857142857142857143",
        "lower": "9.793058544800950239515896191828107287396",
        "upper": "17.46408431234190690334124666817189271260",
        "standard_error": "1.741920122820135023689009391098916960156",
        "degrees_of_freedom": "10.96315748963085280532347692363848157755",
        "subject_count": 7,
        "trial_count": 35
      },
      "success_rate": "0.9429",
      "total_cost_usd": "4.6420839",
      "guesser_cost_per_episode_usd": "0.03629296",
      "full_cost_per_episode_usd": "0.1326309685714285714285714286",
      "runtime_per_episode_ms": "228575.3428571428571428571429",
      "guesser_think_time_per_episode_ms": "92219.11428571428571428571429",
      "guesser_latency_per_call_ms": "6587.079591836734693877551020",
      "ideal_distance_score": "0.10398391237645142030439536976634507023556036741161",
      "normalized_question_score": "0.077903682719546742209631727980021507274755435000682",
      "normalized_guesser_cost": "0.068874307632423581176307438821030699937275270919459",
      "cost_adjusted_question_score": "0.4946211977142857142857142858",
      "efficiency_status": "ranked",
      "successful": 33,
      "terminal_trials": 35,
      "contract": {
        "evaluated_outputs": 490,
        "valid_outputs": 444,
        "violations": 46,
        "counted_penalties": 46,
        "affected_trials": 7,
        "compliance_rate": "0.9061224489795918367346938776",
        "status": "breached"
      }
    },
    {
      "rank": 5,
      "model": {
        "model_id": "M-0016",
        "display_name": "Gemini 3.7 Flash (high)",
        "route": "google/gemini-3.7-flash",
        "provider": "google-ai-studio",
        "reasoning_effort": "high",
        "seed_capability": "supported",
        "configuration_hash": "42421eaa1968d1e449c5c433c17513a75f928ecf76769abdd8c48bd68c09f4fd"
      },
      "efficiency_rank": 8,
      "ideal_distance_rank": 6,
      "product_efficiency_rank": 8,
      "pareto_efficient": false,
      "status": "evaluated",
      "execution_id": "BX-20260817-official-M0016-015",
      "completed_at": "2026-08-17T13:58:15.353152Z",
      "question_score": "14.02857142857142857142857143",
      "question_score_confidence_interval": {
        "confidence_level": "0.95",
        "method": "stratified-welch-t-v1",
        "estimate": "14.02857142857142857142857143",
        "lower": "10.78143322177796576595016291890033753087",
        "upper": "17.27570963536489137690697994109966246913",
        "standard_error": "1.243759934838795199897907176777057901294",
        "degrees_of_freedom": "4.756090635940583716330504940402764963378",
        "subject_count": 7,
        "trial_count": 35
      },
      "success_rate": "0.9714",
      "total_cost_usd": "9.57958324",
      "guesser_cost_per_episode_usd": "0.06525685714285714285714285714",
      "full_cost_per_episode_usd": "0.2737023782857142857142857143",
      "runtime_per_episode_ms": "274730.5142857142857142857143",
      "guesser_think_time_per_episode_ms": "76884.45714285714285714285714",
      "guesser_latency_per_call_ms": "5125.630476190476190476190476",
      "ideal_distance_score": "0.17139168276832627153974892229471218794315128226855",
      "normalized_question_score": "0.097733711048158640226628895120236098516158543925399",
      "normalized_guesser_cost": "0.14079499510605347630647884355797008093817090748914",
      "cost_adjusted_question_score": "0.9154604816326530612244897960",
      "efficiency_status": "ranked",
      "successful": 34,
      "terminal_trials": 35,
      "contract": {
        "evaluated_outputs": 525,
        "valid_outputs": 525,
        "violations": 0,
        "counted_penalties": 0,
        "affected_trials": 0,
        "compliance_rate": "1",
        "status": "clean"
      }
    },
    {
      "rank": 6,
      "model": {
        "model_id": "M-0015",
        "display_name": "Grok 4.6 (high)",
        "route": "x-ai/grok-4.6",
        "provider": "xai",
        "reasoning_effort": "high",
        "seed_capability": "supported",
        "configuration_hash": "b8abdbd29972661719449e8974792259da7eab3966f39ec93ac6a0338dc5aba8"
      },
      "efficiency_rank": 9,
      "ideal_distance_rank": 7,
      "product_efficiency_rank": 9,
      "pareto_efficient": false,
      "status": "evaluated",
      "execution_id": "BX-20260814-official-M0015-013",
      "completed_at": "2026-08-14T10:51:12.347510Z",
      "question_score": "14.28571428571428571428571429",
      "question_score_confidence_interval": {
        "confidence_level": "0.95",
        "method": "stratified-welch-t-v1",
        "estimate": "14.28571428571428571428571429",
        "lower": "12.35887402471360874992931866227721489627",
        "upper": "16.21255454671496267864210991772278510373",
        "standard_error": "0.7795864942652715854132301263740919409236",
        "degrees_of_freedom": "5.762029724681231219036790384146836676682",
        "subject_count": 7,
        "trial_count": 35
      },
      "success_rate": "1",
      "total_cost_usd": "8.2485804550000000060",
      "guesser_cost_per_episode_usd": "0.06531971428571428571428571429",
      "full_cost_per_episode_usd": "0.2356737272857142858857142857",
      "runtime_per_episode_ms": "480442.1714285714285714285714",
      "guesser_think_time_per_episode_ms": "187606.2",
      "guesser_latency_per_call_ms": "12273.30280373831775700934579",
      "ideal_distance_score": "0.17909044344801382968075139066680840236507930105134",
      "normalized_question_score": "0.11048158640226628895184135985201710951857409978412",
      "normalized_guesser_cost": "0.14095107662038203920437456586646866761025084905264",
      "cost_adjusted_question_score": "0.9331387755102040816326530616",
      "efficiency_status": "ranked",
      "successful": 35,
      "terminal_trials": 35,
      "contract": {
        "evaluated_outputs": 535,
        "valid_outputs": 535,
        "violations": 0,
        "counted_penalties": 0,
        "affected_trials": 0,
        "compliance_rate": "1",
        "status": "clean"
      }
    },
    {
      "rank": 7,
      "model": {
        "model_id": "M-0010",
        "display_name": "GPT-5.6 Sol (high)",
        "route": "openai/gpt-5.6-sol",
        "provider": "openai",
        "reasoning_effort": "high",
        "seed_capability": "supported",
        "configuration_hash": "61d05be229671aec00a68e3513e89d41dcf10f6dad000f829ded81d532a65e8c"
      },
      "efficiency_rank": 11,
      "ideal_distance_rank": 10,
      "product_efficiency_rank": 11,
      "pareto_efficient": false,
      "status": "evaluated",
      "execution_id": "BX-20260728-official-M0010-010",
      "completed_at": "2026-07-28T22:03:15.063709Z",
      "question_score": "14.4",
      "question_score_confidence_interval": {
        "confidence_level": "0.95",
        "method": "stratified-welch-t-v1",
        "estimate": "14.4",
        "lower": "12.84365336453153411059518615471635663365",
        "upper": "15.95634663546846588940481384528364336635",
        "standard_error": "0.7423074889580902686546198606453738715469",
        "degrees_of_freedom": "18.52868514960197639308262421081526214659",
        "subject_count": 7,
        "trial_count": 35
      },
      "success_rate": "1",
      "total_cost_usd": "9.97267112",
      "guesser_cost_per_episode_usd": "0.1710784071428571428571428571",
      "full_cost_per_episode_usd": "0.2849334605714285714285714286",
      "runtime_per_episode_ms": "303108.2",
      "guesser_think_time_per_episode_ms": "135788.8",
      "guesser_latency_per_call_ms": "8817.454545454545454545454545",
      "ideal_distance_score": "0.41994350137686804602826601600937605287476487275174",
      "normalized_question_score": "0.11614730878186968838526912167961383206670465215193",
      "normalized_guesser_cost": "0.40356207330644017152175043038122818800761547324678",
      "cost_adjusted_question_score": "2.463529062857142857142857142",
      "efficiency_status": "ranked",
      "successful": 35,
      "terminal_trials": 35,
      "contract": {
        "evaluated_outputs": 539,
        "valid_outputs": 539,
        "violations": 0,
        "counted_penalties": 0,
        "affected_trials": 0,
        "compliance_rate": "1",
        "status": "clean"
      }
    },
    {
      "rank": 8,
      "model": {
        "model_id": "M-0005",
        "display_name": "Claude Sonnet 5 (high)",
        "route": "anthropic/claude-sonnet-5",
        "provider": "anthropic",
        "reasoning_effort": "high",
        "seed_capability": "unsupported",
        "configuration_hash": "7b0bf3be712016bcc6b59bfb407b5af0bcc87646a16fa63e19e6a7f0b4e015b7"
      },
      "efficiency_rank": 6,
      "ideal_distance_rank": 4,
      "product_efficiency_rank": 6,
      "pareto_efficient": false,
      "status": "evaluated",
      "execution_id": "BX-20260728-official-M0005-012",
      "completed_at": "2026-07-29T00:01:47.634031Z",
      "question_score": "14.74285714285714285714285714",
      "question_score_confidence_interval": {
        "confidence_level": "0.95",
        "method": "stratified-welch-t-v1",
        "estimate": "14.74285714285714285714285714",
        "lower": "12.77962071191072794631663163499231187786",
        "upper": "16.70609357380355776796908264500768812214",
        "standard_error": "0.8270922757788644857571460398379391177520",
        "degrees_of_freedom": "6.870902250857341336816511831555053299481",
        "subject_count": 7,
        "trial_count": 35
      },
      "success_rate": "1",
      "total_cost_usd": "5.00817975",
      "guesser_cost_per_episode_usd": "0.04925074285714285714285714286",
      "full_cost_per_episode_usd": "0.14309085",
      "runtime_per_episode_ms": "232300.8857142857142857142857",
      "guesser_think_time_per_episode_ms": "84132.54285714285714285714286",
      "guesser_latency_per_call_ms": "5344.172413793103448275862069",
      "ideal_distance_score": "0.16714828277553630572575464425826936231111589580485",
      "normalized_question_score": "0.13314447592067988668555240765815470792639375968028",
      "normalized_guesser_cost": "0.10104997262057096026931785839171766744896600104312",
      "cost_adjusted_question_score": "0.7260966661224489795918367346",
      "efficiency_status": "ranked",
      "successful": 35,
      "terminal_trials": 35,
      "contract": {
        "evaluated_outputs": 551,
        "valid_outputs": 535,
        "violations": 16,
        "counted_penalties": 16,
        "affected_trials": 12,
        "compliance_rate": "0.9709618874773139745916515426",
        "status": "breached"
      }
    },
    {
      "rank": 9,
      "model": {
        "model_id": "M-0008",
        "display_name": "Grok 4.5 (high)",
        "route": "x-ai/grok-4.5",
        "provider": "xai",
        "reasoning_effort": "high",
        "seed_capability": "supported",
        "configuration_hash": "78cf2999036229a932924d1d0c7fc096c8c55f8cac0b8270ae94c3dc99e943f6"
      },
      "efficiency_rank": 4,
      "ideal_distance_rank": 3,
      "product_efficiency_rank": 4,
      "pareto_efficient": true,
      "status": "evaluated",
      "execution_id": "BX-20260728-official-M0008-010",
      "completed_at": "2026-07-28T20:29:07.162346Z",
      "question_score": "15.17142857142857142857142857",
      "question_score_confidence_interval": {
        "confidence_level": "0.95",
        "method": "stratified-welch-t-v1",
        "estimate": "15.17142857142857142857142857",
        "lower": "11.70380879887104659472604662091159815167",
        "upper": "18.63904834398609626241681051908840184833",
        "standard_error": "1.366060924329362345711242349678178953002",
        "degrees_of_freedom": "5.219477822096352365461608722641242775624",
        "subject_count": 7,
        "trial_count": 35
      },
      "success_rate": "0.9429",
      "total_cost_usd": "4.99134587",
      "guesser_cost_per_episode_usd": "0.02859845714285714285714285714",
      "full_cost_per_episode_usd": "0.142609882",
      "runtime_per_episode_ms": "230677.4857142857142857142857",
      "guesser_think_time_per_episode_ms": "69788.6",
      "guesser_latency_per_call_ms": "4330.852836879432624113475177",
      "ideal_distance_score": "0.16221409253047753985304776890735211445749331184614",
      "normalized_question_score": "0.15439093484419263456090651537920615685865386930318",
      "normalized_guesser_cost": "0.049767972165064087775697429545013306917812084446000",
      "cost_adjusted_question_score": "0.4338794497959183673469387754",
      "efficiency_status": "ranked",
      "successful": 33,
      "terminal_trials": 35,
      "contract": {
        "evaluated_outputs": 564,
        "valid_outputs": 554,
        "violations": 10,
        "counted_penalties": 10,
        "affected_trials": 9,
        "compliance_rate": "0.9822695035460992907801418440",
        "status": "breached"
      }
    },
    {
      "rank": 10,
      "model": {
        "model_id": "M-0003",
        "display_name": "GPT-5 Nano (medium)",
        "route": "openai/gpt-5-nano",
        "provider": "openai",
        "reasoning_effort": "medium",
        "seed_capability": "supported",
        "configuration_hash": "6035094f718b1498b7afb55dc6ed4347b1237fcc2deee07f355b7f52a0783104"
      },
      "efficiency_rank": 1,
      "ideal_distance_rank": 5,
      "product_efficiency_rank": 1,
      "pareto_efficient": true,
      "status": "evaluated",
      "execution_id": "BX-20260728-official-M0003-010",
      "completed_at": "2026-07-28T21:04:03.749958Z",
      "question_score": "15.48571428571428571428571429",
      "question_score_confidence_interval": {
        "confidence_level": "0.95",
        "method": "stratified-welch-t-v1",
        "estimate": "15.48571428571428571428571429",
        "lower": "13.84057397557049221774038879858546264372",
        "upper": "17.13085459585807921083103978141453735628",
        "standard_error": "0.7605046659546379865353935385239951220650",
        "degrees_of_freedom": "12.83346728898788646738965436245364148993",
        "subject_count": 7,
        "trial_count": 35
      },
      "success_rate": "0.9429",
      "total_cost_usd": "4.5882192",
      "guesser_cost_per_episode_usd": "0.01382635",
      "full_cost_per_episode_usd": "0.1310919771428571428571428571",
      "runtime_per_episode_ms": "506450.1142857142857142857143",
      "guesser_think_time_per_episode_ms": "332164.7142857142857142857143",
      "guesser_latency_per_call_ms": "20218.72173913043478260869565",
      "ideal_distance_score": "0.17047475496634093857051861554839155372016960617153",
      "normalized_question_score": "0.16997167138810198300283286127266088324278342655827",
      "normalized_guesser_cost": "0.013087131326958487057100407857387038763477894673651",
      "cost_adjusted_question_score": "0.2141109057142857142857142858",
      "efficiency_status": "ranked",
      "successful": 33,
      "terminal_trials": 35,
      "contract": {
        "evaluated_outputs": 575,
        "valid_outputs": 575,
        "violations": 0,
        "counted_penalties": 0,
        "affected_trials": 0,
        "compliance_rate": "1",
        "status": "clean"
      }
    },
    {
      "rank": 11,
      "model": {
        "model_id": "M-0004",
        "display_name": "Gemini 3.6 Flash (high)",
        "route": "google/gemini-3.6-flash",
        "provider": "google-vertex",
        "reasoning_effort": "high",
        "seed_capability": "supported",
        "configuration_hash": "aaef1936d2c9ecd713a2e0fa801e343d63fb6aa6478f273f6df834e9c94b8e5a"
      },
      "efficiency_rank": 13,
      "ideal_distance_rank": 11,
      "product_efficiency_rank": 13,
      "pareto_efficient": false,
      "status": "evaluated",
      "execution_id": "BX-20260728-official-M0004-010",
      "completed_at": "2026-07-28T18:59:41.161957Z",
      "question_score": "16.88571428571428571428571429",
      "question_score_confidence_interval": {
        "confidence_level": "0.95",
        "method": "stratified-welch-t-v1",
        "estimate": "16.88571428571428571428571429",
        "lower": "14.07891549212737330021754837173032759172",
        "upper": "19.69251307930119812835388020826967240828",
        "standard_error": "1.065028503978045784638362940337589849643",
        "degrees_of_freedom": "4.620400388102478318804917181222528789469",
        "subject_count": 7,
        "trial_count": 35
      },
      "success_rate": "0.9714",
      "total_cost_usd": "9.85554295",
      "guesser_cost_per_episode_usd": "0.1705019571428571428571428571",
      "full_cost_per_episode_usd": "0.2815869414285714285714285714",
      "runtime_per_episode_ms": "295106.7142857142857142857143",
      "guesser_think_time_per_episode_ms": "118662.7428571428571428571429",
      "guesser_latency_per_call_ms": "6645.1136",
      "ideal_distance_score": "0.46798538802948889349999906445928571724117311876104",
      "normalized_question_score": "0.23937677053824362606232294626341195258769430779478",
      "normalized_guesser_cost": "0.40213068166429721566392333083220889277421642027275",
      "cost_adjusted_question_score": "2.879047333469387755102040816",
      "efficiency_status": "ranked",
      "successful": 34,
      "terminal_trials": 35,
      "contract": {
        "evaluated_outputs": 625,
        "valid_outputs": 624,
        "violations": 1,
        "counted_penalties": 1,
        "affected_trials": 1,
        "compliance_rate": "0.9984",
        "status": "breached"
      }
    },
    {
      "rank": 12,
      "model": {
        "model_id": "M-0017",
        "display_name": "Ox Alpha (high)",
        "route": "stealth/ox-alpha",
        "provider": "stealth",
        "reasoning_effort": "high",
        "seed_capability": "unsupported",
        "configuration_hash": "31f8e00b27493c965f384fe48b31f03ae53bc81f5bb00864d9d7394e3abb074c"
      },
      "efficiency_rank": null,
      "ideal_distance_rank": null,
      "product_efficiency_rank": null,
      "pareto_efficient": false,
      "status": "evaluated",
      "execution_id": "BX-20260823-official-M0017-018",
      "completed_at": "2026-08-23T08:51:52.992917Z",
      "question_score": "17.6",
      "question_score_confidence_interval": {
        "confidence_level": "0.95",
        "method": "stratified-welch-t-v1",
        "estimate": "17.6",
        "lower": "14.60265808702477640601663840670676062548",
        "upper": "20.59734191297522359398336159329323937452",
        "standard_error": "1.385346048274360854504262799846750402387",
        "degrees_of_freedom": "12.81144713442084896219346621413716233404",
        "subject_count": 7,
        "trial_count": 35
      },
      "success_rate": "0.9143",
      "total_cost_usd": "9.7757258500000000465",
      "guesser_cost_per_episode_usd": "0",
      "full_cost_per_episode_usd": "0.2793064528571428584714285714",
      "runtime_per_episode_ms": "777163.0285714285714285714286",
      "guesser_think_time_per_episode_ms": "313864.4285714285714285714286",
      "guesser_latency_per_call_ms": "16952.55401234567901234567901",
      "ideal_distance_score": null,
      "normalized_question_score": null,
      "normalized_guesser_cost": null,
      "cost_adjusted_question_score": null,
      "efficiency_status": "recorded_guesser_cost_unavailable",
      "successful": 32,
      "terminal_trials": 35,
      "contract": {
        "evaluated_outputs": 648,
        "valid_outputs": 601,
        "violations": 47,
        "counted_penalties": 47,
        "affected_trials": 35,
        "compliance_rate": "0.9274691358024691358024691358",
        "status": "breached"
      }
    },
    {
      "rank": 13,
      "model": {
        "model_id": "M-0001",
        "display_name": "GPT-5.6 Luna (high)",
        "route": "openai/gpt-5.6-luna",
        "provider": "openai",
        "reasoning_effort": "high",
        "seed_capability": "unsupported",
        "configuration_hash": "273067b5c6676389f8e4cfa0e9aa5e507e60187bae55a51edcc53b98b55c177b"
      },
      "efficiency_rank": 3,
      "ideal_distance_rank": 8,
      "product_efficiency_rank": 3,
      "pareto_efficient": false,
      "status": "evaluated",
      "execution_id": "BX-20260728-official-M0001-010",
      "completed_at": "2026-07-28T18:53:33.209886Z",
      "question_score": "17.74285714285714285714285714",
      "question_score_confidence_interval": {
        "confidence_level": "0.95",
        "method": "stratified-welch-t-v1",
        "estimate": "17.74285714285714285714285714",
        "lower": "14.73749826504108472749724938685413090962",
        "upper": "20.74821602067320098678846489314586909038",
        "standard_error": "1.376478796622389097730749612342140426020",
        "degrees_of_freedom": "11.77842860734861653165414931181878805768",
        "subject_count": 7,
        "trial_count": 35
      },
      "success_rate": "0.9143",
      "total_cost_usd": "5.88215295",
      "guesser_cost_per_episode_usd": "0.01837530714285714285714285714",
      "full_cost_per_episode_usd": "0.1680615128571428571428571429",
      "runtime_per_episode_ms": "286590.8",
      "guesser_think_time_per_episode_ms": "77486.45714285714285714285714",
      "guesser_latency_per_call_ms": "4153.179173047473200612557427",
      "ideal_distance_score": "0.28292231800163026678473874334629572636667555479638",
      "normalized_question_score": "0.28186968838526912181303116120976414223691707661565",
      "normalized_guesser_cost": "0.024382715045845145668064308277675969654695546304851",
      "cost_adjusted_question_score": "0.3260304495918367346938775509",
      "efficiency_status": "ranked",
      "successful": 32,
      "terminal_trials": 35,
      "contract": {
        "evaluated_outputs": 653,
        "valid_outputs": 652,
        "violations": 1,
        "counted_penalties": 1,
        "affected_trials": 1,
        "compliance_rate": "0.9984686064318529862174578867",
        "status": "breached"
      }
    },
    {
      "rank": 14,
      "model": {
        "model_id": "M-0012",
        "display_name": "Mistral Medium 3.5 (high)",
        "route": "mistralai/mistral-medium-3-5",
        "provider": "mistral",
        "reasoning_effort": "high",
        "seed_capability": "supported",
        "configuration_hash": "446ba308ed23f88ca1eb236f30d10b028785217b8bf42f18d20714b940975977"
      },
      "efficiency_rank": 14,
      "ideal_distance_rank": 14,
      "product_efficiency_rank": 14,
      "pareto_efficient": false,
      "status": "evaluated",
      "execution_id": "BX-20260729-official-M0012-011",
      "completed_at": "2026-07-30T03:37:42.497575Z",
      "question_score": "20.05714285714285714285714286",
      "question_score_confidence_interval": {
        "confidence_level": "0.95",
        "method": "stratified-welch-t-v1",
        "estimate": "20.05714285714285714285714286",
        "lower": "16.10823560795740804633343913701758240912",
        "upper": "24.00605010632830623938084658298241759088",
        "standard_error": "1.823654102540908308168061929945158717913",
        "degrees_of_freedom": "12.71047219788390147158533359868564452611",
        "subject_count": 7,
        "trial_count": 35
      },
      "success_rate": "0.8857",
      "total_cost_usd": "18.5485533200000000048",
      "guesser_cost_per_episode_usd": "0.4112758714285714285714285714",
      "full_cost_per_episode_usd": "0.5299586662857142858514285714",
      "runtime_per_episode_ms": "1539875.771428571428571428571",
      "guesser_think_time_per_episode_ms": "872075.4857142857142857142857",
      "guesser_latency_per_call_ms": "41640.71214188267394270122783",
      "ideal_distance_score": "1.0757750737981523590319438733425597152232957777326",
      "normalized_question_score": "0.39660056657223796033994334280429182482806217849433",
      "normalized_guesser_cost": "1",
      "cost_adjusted_question_score": "8.249018906938775510204081633",
      "efficiency_status": "ranked",
      "successful": 31,
      "terminal_trials": 35,
      "contract": {
        "evaluated_outputs": 733,
        "valid_outputs": 731,
        "violations": 2,
        "counted_penalties": 2,
        "affected_trials": 1,
        "compliance_rate": "0.9972714870395634379263301501",
        "status": "breached"
      }
    },
    {
      "rank": 15,
      "model": {
        "model_id": "M-0009",
        "display_name": "Llama 4 Maverick (non-thinking)",
        "route": "meta-llama/llama-4-maverick",
        "provider": "parasail",
        "reasoning_effort": "none",
        "seed_capability": "supported",
        "configuration_hash": "2605a4929c3a1365791b158f49fc73eb121fca562b8266ec392dc79758bb5eaf"
      },
      "efficiency_rank": 2,
      "ideal_distance_rank": 13,
      "product_efficiency_rank": 2,
      "pareto_efficient": true,
      "status": "evaluated",
      "execution_id": "BX-20260728-official-M0009-010",
      "completed_at": "2026-07-28T20:47:52.048806Z",
      "question_score": "32.22857142857142857142857143",
      "question_score_confidence_interval": {
        "confidence_level": "0.95",
        "method": "stratified-welch-t-v1",
        "estimate": "32.22857142857142857142857143",
        "lower": "26.82281354334085392702551833194063310948",
        "upper": "37.63432931380200321583162452805936689052",
        "standard_error": "2.530548057118804274703150178278331482309",
        "degrees_of_freedom": "14.62645828552632483962953770680351834625",
        "subject_count": 7,
        "trial_count": 35
      },
      "success_rate": "0.4857",
      "total_cost_usd": "3.72512258",
      "guesser_cost_per_episode_usd": "0.008555900857142857142857142857",
      "full_cost_per_episode_usd": "0.1064320737142857142857142857",
      "runtime_per_episode_ms": "194476.2571428571428571428571",
      "guesser_think_time_per_episode_ms": "27893.4",
      "guesser_latency_per_call_ms": "862.4284452296819787985865724",
      "ideal_distance_score": "1",
      "normalized_question_score": "1",
      "normalized_guesser_cost": "0",
      "cost_adjusted_question_score": "0.2757444619102040816326530612",
      "efficiency_status": "ranked",
      "successful": 17,
      "terminal_trials": 35,
      "contract": {
        "evaluated_outputs": 1132,
        "valid_outputs": 1132,
        "violations": 0,
        "counted_penalties": 0,
        "affected_trials": 0,
        "compliance_rate": "1",
        "status": "clean"
      }
    }
  ]
}