{"about":"Per-item outputs of TIA's own runs of the unmodified JevBench harness on the JevBench v1.4.2.2 public items (commit fcc6927): item id, tier, family, gold label, predicted label, correct, schema-valid, class probabilities, latency, TIA input tokens. Item text: https://github.com/fstandhartinger/jevbench (MIT, Copyright (c) 2026 Florian Standhartinger and contributors).","harness_commit":"fcc6927","items_files":{"easy":"https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/datasets/public/easy.jsonl","original":"https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/datasets/public/original.jsonl","hard":"https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/datasets/public/hard.jsonl"},"runs":{"tia-1.5":{"display":"TIA 1.5","run_by":"TIA","date":"2026-10-03","device":"local CPU worker on the J-97 workstation, harness in Docker, loopback","adapter":"typesafe","items":[{"id":"easy-extraction-00","tier":"easy","family":"extraction","gold":"paypal","predicted":"paypal","correct":true,"schema_valid":true,"probs":{"bank_transfer":0.0191,"cash":0.0253,"credit_card":0.1853,"paypal":0.7703},"latency_s":0.3175,"input_tokens":43},{"id":"easy-extraction-01","tier":"easy","family":"extraction","gold":"cash","predicted":"cash","correct":true,"schema_valid":true,"probs":{"bank_transfer":0.0356,"cash":0.7227,"credit_card":0.1554,"paypal":0.0862},"latency_s":0.04,"input_tokens":46},{"id":"easy-extraction-02","tier":"easy","family":"extraction","gold":"bank_transfer","predicted":"bank_transfer","correct":true,"schema_valid":true,"probs":{"bank_transfer":0.7855,"cash":0.1041,"credit_card":0.0719,"paypal":0.0384},"latency_s":0.0294,"input_tokens":46},{"id":"easy-extraction-03","tier":"easy","family":"extraction","gold":"high","predicted":"high","correct":true,"schema_valid":true,"probs":{"high":0.6258,"low":0.0062,"medium":0.0538,"urgent":0.3142},"latency_s":0.0311,"input_tokens":45},{"id":"easy-extraction-04","tier":"easy","family":"extraction","gold":"low","predicted":"low","correct":true,"schema_valid":true,"probs":{"high":0.0496,"low":0.7557,"medium":0.163,"urgent":0.0316},"latency_s":0.0344,"input_tokens":42},{"id":"easy-extraction-05","tier":"easy","family":"extraction","gold":"urgent","predicted":"urgent","correct":true,"schema_valid":true,"probs":{"high":0.3915,"low":0.0056,"medium":0.0288,"urgent":0.574},"latency_s":0.0284,"input_tokens":42},{"id":"easy-extraction-06","tier":"easy","family":"extraction","gold":"M","predicted":"M","correct":true,"schema_valid":true,"probs":{"L":0.0671,"M":0.759,"S":0.1339,"XL":0.04},"latency_s":0.0299,"input_tokens":34},{"id":"easy-extraction-07","tier":"easy","family":"extraction","gold":"XL","predicted":"XL","correct":true,"schema_valid":true,"probs":{"L":0.1243,"M":0.0305,"S":0.0121,"XL":0.8331},"latency_s":0.0301,"input_tokens":32},{"id":"easy-extraction-08","tier":"easy","family":"extraction","gold":"thursday","predicted":"thursday","correct":true,"schema_valid":true,"probs":{"friday":0.0537,"monday":0.0553,"thursday":0.7416,"tuesday":0.0934,"wednesday":0.0559},"latency_s":0.0357,"input_tokens":38},{"id":"easy-extraction-09","tier":"easy","family":"extraction","gold":"monday","predicted":"monday","correct":true,"schema_valid":true,"probs":{"friday":0.042,"monday":0.769,"thursday":0.0493,"tuesday":0.0987,"wednesday":0.041},"latency_s":0.0278,"input_tokens":42},{"id":"easy-extraction-10","tier":"easy","family":"extraction","gold":"friday","predicted":"friday","correct":true,"schema_valid":true,"probs":{"friday":0.6879,"monday":0.0737,"thursday":0.0811,"tuesday":0.0869,"wednesday":0.0704},"latency_s":0.0307,"input_tokens":38},{"id":"easy-extraction-11","tier":"easy","family":"extraction","gold":"credit_card","predicted":"credit_card","correct":true,"schema_valid":true,"probs":{"bank_transfer":0.0456,"cash":0.096,"credit_card":0.8147,"paypal":0.0437},"latency_s":0.0285,"input_tokens":46},{"id":"easy-fact-00","tier":"easy","family":"fact","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.6573,"no":0.3427},"latency_s":0.0493,"input_tokens":53},{"id":"easy-fact-01","tier":"easy","family":"fact","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.5078,"no":0.4922},"latency_s":0.0258,"input_tokens":50},{"id":"easy-fact-02","tier":"easy","family":"fact","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.577,"no":0.423},"latency_s":0.0721,"input_tokens":54},{"id":"easy-fact-03","tier":"easy","family":"fact","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.3501,"no":0.6499},"latency_s":0.0394,"input_tokens":58},{"id":"easy-fact-04","tier":"easy","family":"fact","gold":"yes","predicted":"no","correct":false,"schema_valid":true,"probs":{"yes":0.4482,"no":0.5518},"latency_s":0.0289,"input_tokens":54},{"id":"easy-fact-05","tier":"easy","family":"fact","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.3274,"no":0.6726},"latency_s":0.0331,"input_tokens":55},{"id":"easy-fact-06","tier":"easy","family":"fact","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.5971,"no":0.4029},"latency_s":0.031,"input_tokens":62},{"id":"easy-fact-07","tier":"easy","family":"fact","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.646,"no":0.354},"latency_s":0.047,"input_tokens":50},{"id":"easy-fact-08","tier":"easy","family":"fact","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.6479,"no":0.3521},"latency_s":0.0269,"input_tokens":46},{"id":"easy-fact-09","tier":"easy","family":"fact","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.341,"no":0.659},"latency_s":0.0317,"input_tokens":46},{"id":"easy-fact-10","tier":"easy","family":"fact","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.8207,"no":0.1793},"latency_s":0.0334,"input_tokens":53},{"id":"easy-fact-11","tier":"easy","family":"fact","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.2112,"no":0.7888},"latency_s":0.0281,"input_tokens":54},{"id":"easy-intent-00","tier":"easy","family":"intent","gold":"track_order","predicted":"track_order","correct":true,"schema_valid":true,"probs":{"billing_question":0.0182,"cancel_order":0.1514,"change_address":0.114,"report_damage":0.1201,"track_order":0.5963},"latency_s":0.0361,"input_tokens":91},{"id":"easy-intent-01","tier":"easy","family":"intent","gold":"cancel_order","predicted":"cancel_order","correct":true,"schema_valid":true,"probs":{"billing_question":0.0161,"cancel_order":0.8754,"change_address":0.062,"report_damage":0.0225,"track_order":0.0239},"latency_s":0.0288,"input_tokens":89},{"id":"easy-intent-02","tier":"easy","family":"intent","gold":"change_address","predicted":"change_address","correct":true,"schema_valid":true,"probs":{"billing_question":0.028,"cancel_order":0.133,"change_address":0.5232,"report_damage":0.0652,"track_order":0.2506},"latency_s":0.0313,"input_tokens":93},{"id":"easy-intent-03","tier":"easy","family":"intent","gold":"report_damage","predicted":"report_damage","correct":true,"schema_valid":true,"probs":{"billing_question":0.0178,"cancel_order":0.2642,"change_address":0.0864,"report_damage":0.5767,"track_order":0.0548},"latency_s":0.0317,"input_tokens":83},{"id":"easy-intent-04","tier":"easy","family":"intent","gold":"billing_question","predicted":"billing_question","correct":true,"schema_valid":true,"probs":{"billing_question":0.5356,"cancel_order":0.197,"change_address":0.1058,"report_damage":0.0542,"track_order":0.1074},"latency_s":0.0298,"input_tokens":88},{"id":"easy-intent-05","tier":"easy","family":"intent","gold":"track_order","predicted":"track_order","correct":true,"schema_valid":true,"probs":{"billing_question":0.0132,"cancel_order":0.292,"change_address":0.1242,"report_damage":0.0638,"track_order":0.5068},"latency_s":0.0293,"input_tokens":87},{"id":"easy-intent-06","tier":"easy","family":"intent","gold":"set_alarm","predicted":"set_alarm","correct":true,"schema_valid":true,"probs":{"play_music":0.022,"send_message":0.1847,"set_alarm":0.5636,"turn_off_lights":0.0986,"weather":0.131},"latency_s":0.0388,"input_tokens":74},{"id":"easy-intent-07","tier":"easy","family":"intent","gold":"play_music","predicted":"send_message","correct":false,"schema_valid":true,"probs":{"play_music":0.1095,"send_message":0.3794,"set_alarm":0.2334,"turn_off_lights":0.1334,"weather":0.1442},"latency_s":0.0266,"input_tokens":68},{"id":"easy-intent-08","tier":"easy","family":"intent","gold":"weather","predicted":"weather","correct":true,"schema_valid":true,"probs":{"play_music":0.0295,"send_message":0.2838,"set_alarm":0.209,"turn_off_lights":0.1114,"weather":0.3664},"latency_s":0.0273,"input_tokens":70},{"id":"easy-intent-09","tier":"easy","family":"intent","gold":"send_message","predicted":"send_message","correct":true,"schema_valid":true,"probs":{"play_music":0.0291,"send_message":0.4547,"set_alarm":0.298,"turn_off_lights":0.106,"weather":0.1121},"latency_s":0.0285,"input_tokens":73},{"id":"easy-intent-10","tier":"easy","family":"intent","gold":"turn_off_lights","predicted":"turn_off_lights","correct":true,"schema_valid":true,"probs":{"play_music":0.0327,"send_message":0.2654,"set_alarm":0.2403,"turn_off_lights":0.3725,"weather":0.089},"latency_s":0.0415,"input_tokens":71},{"id":"easy-intent-11","tier":"easy","family":"intent","gold":"set_alarm","predicted":"set_alarm","correct":true,"schema_valid":true,"probs":{"play_music":0.0195,"send_message":0.1603,"set_alarm":0.658,"turn_off_lights":0.0953,"weather":0.0669},"latency_s":0.027,"input_tokens":70},{"id":"easy-tool_selection-00","tier":"easy","family":"tool_selection","gold":"get_weather","predicted":"get_weather","correct":true,"schema_valid":true,"probs":{"convert_currency":0.0432,"create_calendar_event":0.1106,"get_weather":0.6943,"send_email":0.0816,"translate_text":0.0703},"latency_s":0.0312,"input_tokens":78},{"id":"easy-tool_selection-01","tier":"easy","family":"tool_selection","gold":"send_email","predicted":"send_email","correct":true,"schema_valid":true,"probs":{"convert_currency":0.0882,"create_calendar_event":0.2491,"get_weather":0.1155,"send_email":0.401,"translate_text":0.1462},"latency_s":0.031,"input_tokens":79},{"id":"easy-tool_selection-02","tier":"easy","family":"tool_selection","gold":"create_calendar_event","predicted":"create_calendar_event","correct":true,"schema_valid":true,"probs":{"convert_currency":0.0607,"create_calendar_event":0.7015,"get_weather":0.1085,"send_email":0.0657,"translate_text":0.0635},"latency_s":0.0323,"input_tokens":81},{"id":"easy-tool_selection-03","tier":"easy","family":"tool_selection","gold":"convert_currency","predicted":"convert_currency","correct":true,"schema_valid":true,"probs":{"convert_currency":0.3607,"create_calendar_event":0.2226,"get_weather":0.1519,"send_email":0.1343,"translate_text":0.1305},"latency_s":0.0317,"input_tokens":77},{"id":"easy-tool_selection-04","tier":"easy","family":"tool_selection","gold":"translate_text","predicted":"translate_text","correct":true,"schema_valid":true,"probs":{"convert_currency":0.159,"create_calendar_event":0.2002,"get_weather":0.1234,"send_email":0.1357,"translate_text":0.3817},"latency_s":0.0326,"input_tokens":77},{"id":"easy-tool_selection-05","tier":"easy","family":"tool_selection","gold":"create_calendar_event","predicted":"create_calendar_event","correct":true,"schema_valid":true,"probs":{"convert_currency":0.0315,"create_calendar_event":0.7357,"get_weather":0.1121,"send_email":0.071,"translate_text":0.0497},"latency_s":0.0303,"input_tokens":80},{"id":"easy-tool_selection-06","tier":"easy","family":"tool_selection","gold":"search_flights","predicted":"search_flights","correct":true,"schema_valid":true,"probs":{"book_restaurant":0.059,"call_taxi":0.1864,"get_stock_price":0.0992,"search_flights":0.5896,"set_reminder":0.0658},"latency_s":0.0304,"input_tokens":84},{"id":"easy-tool_selection-07","tier":"easy","family":"tool_selection","gold":"book_restaurant","predicted":"book_restaurant","correct":true,"schema_valid":true,"probs":{"book_restaurant":0.4824,"call_taxi":0.1277,"get_stock_price":0.1105,"search_flights":0.0782,"set_reminder":0.2012},"latency_s":0.0314,"input_tokens":86},{"id":"easy-tool_selection-08","tier":"easy","family":"tool_selection","gold":"set_reminder","predicted":"set_reminder","correct":true,"schema_valid":true,"probs":{"book_restaurant":0.0604,"call_taxi":0.2498,"get_stock_price":0.0604,"search_flights":0.0744,"set_reminder":0.5548},"latency_s":0.0325,"input_tokens":84},{"id":"easy-tool_selection-09","tier":"easy","family":"tool_selection","gold":"get_stock_price","predicted":"get_stock_price","correct":true,"schema_valid":true,"probs":{"book_restaurant":0.0562,"call_taxi":0.0796,"get_stock_price":0.6966,"search_flights":0.0578,"set_reminder":0.1099},"latency_s":0.0294,"input_tokens":82},{"id":"easy-tool_selection-10","tier":"easy","family":"tool_selection","gold":"call_taxi","predicted":"call_taxi","correct":true,"schema_valid":true,"probs":{"book_restaurant":0.0496,"call_taxi":0.679,"get_stock_price":0.0818,"search_flights":0.0743,"set_reminder":0.1153},"latency_s":0.0314,"input_tokens":82},{"id":"easy-tool_selection-11","tier":"easy","family":"tool_selection","gold":"set_reminder","predicted":"set_reminder","correct":true,"schema_valid":true,"probs":{"book_restaurant":0.0919,"call_taxi":0.196,"get_stock_price":0.0982,"search_flights":0.0831,"set_reminder":0.5308},"latency_s":0.0321,"input_tokens":84},{"id":"hard-opus-a-long_policy-01","tier":"hard","family":"long_policy","gold":"pay_subject_to_15000_sublimit","predicted":"deny_vacancy_exclusion","correct":false,"schema_valid":true,"probs":{"deny_repeated_seepage":0.2735,"deny_vacancy_exclusion":0.6153,"pay_full_estimate_less_deductible":0.0616,"pay_subject_to_10000_sublimit":0.0105,"pay_subject_to_15000_sublimit":0.0391},"latency_s":0.0627,"input_tokens":3733},{"id":"hard-opus-a-long_policy-04","tier":"hard","family":"long_policy","gold":"cfo","predicted":"vp_and_finance_director","correct":false,"schema_valid":true,"probs":{"budget_holder":0.0994,"cfo":0.3078,"department_head":0.092,"executive_committee":0.1586,"vp_and_finance_director":0.3422},"latency_s":0.0478,"input_tokens":2460},{"id":"hard-opus-a-long_policy-08","tier":"hard","family":"long_policy","gold":"parts_covered_labour_charged","predicted":"pro_rata_replacement_credit","correct":false,"schema_valid":true,"probs":{"full_repair_no_charge":0.3219,"not_covered":0.1457,"parts_covered_labour_charged":0.2101,"pro_rata_replacement_credit":0.3223},"latency_s":0.0447,"input_tokens":2293},{"id":"hard-opus-a-long_policy-09","tier":"hard","family":"long_policy","gold":"license_required","predicted":"shipment_prohibited","correct":false,"schema_valid":true,"probs":{"license_exception_lvs":0.224,"license_required":0.2927,"no_license_required":0.0857,"shipment_prohibited":0.3976},"latency_s":0.05,"input_tokens":2402},{"id":"hard-opus-a-long_policy-11","tier":"hard","family":"long_policy","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.5468,"no":0.4532},"latency_s":0.0598,"input_tokens":2146},{"id":"hard-opus-a-long_policy-13","tier":"hard","family":"long_policy","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.7513,"no":0.2487},"latency_s":0.0465,"input_tokens":2280},{"id":"hard-opus-a-long_policy-17","tier":"hard","family":"long_policy","gold":"database_oncall","predicted":"security_incident_response","correct":false,"schema_valid":true,"probs":{"database_oncall":0.0142,"no_page_ticket_only":0.104,"platform_oncall":0.0167,"security_incident_response":0.8367,"vendor_support":0.0283},"latency_s":0.0477,"input_tokens":2287},{"id":"hard-opus-a-long_policy-19","tier":"hard","family":"long_policy","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.432,"no":0.568},"latency_s":0.0458,"input_tokens":2106},{"id":"hard-opus-a-probability-03","tier":"hard","family":"probability","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.5473,"no":0.4527},"latency_s":0.0311,"input_tokens":376},{"id":"hard-opus-a-probability-04","tier":"hard","family":"probability","gold":"on_time","predicted":"early","correct":false,"schema_valid":true,"probs":{"early":0.4481,"late":0.3808,"on_time":0.1711},"latency_s":0.0458,"input_tokens":900},{"id":"hard-opus-a-probability-07","tier":"hard","family":"probability","gold":"escalated","predicted":"escalated","correct":true,"schema_valid":true,"probs":{"escalated":0.7959,"reopened":0.112,"resolved_first_contact":0.0921},"latency_s":0.0474,"input_tokens":954},{"id":"hard-opus-a-probability-08","tier":"hard","family":"probability","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.2593,"no":0.7407},"latency_s":0.0305,"input_tokens":312},{"id":"hard-opus-a-temporal_numeric-03","tier":"hard","family":"temporal_numeric","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.7413,"no":0.2587},"latency_s":0.0319,"input_tokens":417},{"id":"hard-opus-a-temporal_numeric-06","tier":"hard","family":"temporal_numeric","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.6391,"no":0.3609},"latency_s":0.0338,"input_tokens":418},{"id":"hard-opus-a-temporal_numeric-07","tier":"hard","family":"temporal_numeric","gold":"sep_26","predicted":"sep_27","correct":false,"schema_valid":true,"probs":{"sep_24":0.0122,"sep_25":0.0346,"sep_26":0.1437,"sep_27":0.7921,"sep_28":0.0174},"latency_s":0.0437,"input_tokens":908},{"id":"hard-opus-a-temporal_numeric-09","tier":"hard","family":"temporal_numeric","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.0422,"no":0.9578},"latency_s":0.0317,"input_tokens":439},{"id":"hard-opus-a-temporal_numeric-12","tier":"hard","family":"temporal_numeric","gold":1,"predicted":"2","correct":false,"schema_valid":true,"probs":{"0":0.1465,"1":0.3417,"2":0.4645,"3":0.0474},"latency_s":0.0316,"input_tokens":461},{"id":"hard-opus-b-ambiguous-02","tier":"hard","family":"ambiguous","gold":"real_incident","predicted":"real_incident","correct":true,"schema_valid":true,"probs":{"cannot_determine":0.1389,"false_positive":0.3683,"real_incident":0.4928},"latency_s":0.0305,"input_tokens":484},{"id":"hard-opus-b-ambiguous-03","tier":"hard","family":"ambiguous","gold":"time_barred","predicted":"time_barred","correct":true,"schema_valid":true,"probs":{"insufficient_information":0.1792,"time_barred":0.6352,"within_limitation":0.1856},"latency_s":0.0302,"input_tokens":441},{"id":"hard-opus-b-ambiguous-07","tier":"hard","family":"ambiguous","gold":"within_policy","predicted":"over_limit","correct":false,"schema_valid":true,"probs":{"insufficient_information":0.3152,"over_limit":0.3908,"within_policy":0.294},"latency_s":0.0316,"input_tokens":439},{"id":"hard-opus-b-ambiguous-09","tier":"hard","family":"ambiguous","gold":"legitimate","predicted":"compromised","correct":false,"schema_valid":true,"probs":{"compromised":0.6357,"inconclusive":0.0777,"legitimate":0.2867},"latency_s":0.0324,"input_tokens":589},{"id":"hard-opus-b-ambiguous-10","tier":"hard","family":"ambiguous","gold":"ask_user_which_alex","predicted":"send_to_alex_moreno","correct":false,"schema_valid":true,"probs":{"ask_user_which_alex":0.26,"send_to_alex_moreno":0.4623,"send_to_alexandra_lind":0.2777},"latency_s":0.0243,"input_tokens":446},{"id":"hard-opus-b-ambiguous-11","tier":"hard","family":"ambiguous","gold":"cannot_determine","predicted":"grade_should_be_higher","correct":false,"schema_valid":true,"probs":{"cannot_determine":0.22,"grade_correct":0.2978,"grade_should_be_higher":0.4821},"latency_s":0.0309,"input_tokens":396},{"id":"hard-opus-b-ambiguous-13","tier":"hard","family":"ambiguous","gold":"cannot_determine","predicted":"charge_data_team","correct":false,"schema_valid":true,"probs":{"cannot_determine":0.0547,"charge_data_team":0.6323,"charge_platform_team":0.2835,"split_50_50":0.0295},"latency_s":0.0367,"input_tokens":525},{"id":"hard-opus-b-multi_hop-03","tier":"hard","family":"multi_hop","gold":"tier3_cfo","predicted":"tier2_department_head","correct":false,"schema_valid":true,"probs":{"tier1_team_lead":0.1685,"tier2_department_head":0.5787,"tier3_cfo":0.1572,"tier4_board":0.0956},"latency_s":0.0565,"input_tokens":2696},{"id":"hard-opus-b-multi_hop-04","tier":"hard","family":"multi_hop","gold":"bjorn","predicted":"no_page_ticket_only","correct":false,"schema_valid":true,"probs":{"ana":0.2768,"bjorn":0.1189,"chen":0.0566,"dmitri":0.0844,"esra":0.0519,"no_page_ticket_only":0.4114},"latency_s":0.0471,"input_tokens":2200},{"id":"hard-opus-b-multi_hop-05","tier":"hard","family":"multi_hop","gold":2,"predicted":"3","correct":false,"schema_valid":true,"probs":{"0":0.2384,"1":0.0352,"2":0.0043,"3":0.6537,"4":0.0683},"latency_s":0.049,"input_tokens":2293},{"id":"hard-opus-b-multi_hop-07","tier":"hard","family":"multi_hop","gold":"credit_20_percent","predicted":"credit_20_percent","correct":true,"schema_valid":true,"probs":{"credit_10_percent":0.0022,"credit_20_percent":0.9177,"credit_25_percent":0.032,"credit_50_percent":0.0416,"credit_full_month":0.004,"no_credit":0.0025},"latency_s":0.0469,"input_tokens":2260},{"id":"hard-opus-b-multi_hop-08","tier":"hard","family":"multi_hop","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.8828,"no":0.1172},"latency_s":0.0478,"input_tokens":2021},{"id":"hard-opus-b-probability-01","tier":"hard","family":"probability","gold":"merchant_wins","predicted":"customer_withdraws","correct":false,"schema_valid":true,"probs":{"customer_wins":0.2822,"customer_withdraws":0.4589,"merchant_wins":0.2588},"latency_s":0.0442,"input_tokens":1639},{"id":"hard-opus-b-probability-02","tier":"hard","family":"probability","gold":"upstream_provider","predicted":"bad_push","correct":false,"schema_valid":true,"probs":{"bad_push":0.7913,"database_hardware":0.0279,"upstream_provider":0.1808},"latency_s":0.0383,"input_tokens":434},{"id":"hard-opus-b-probability-03","tier":"hard","family":"probability","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.7757,"no":0.2243},"latency_s":0.0552,"input_tokens":971},{"id":"hard-opus-b-probability-04","tier":"hard","family":"probability","gold":"attended","predicted":"late_cancellation","correct":false,"schema_valid":true,"probs":{"attended":0.2688,"late_cancellation":0.6763,"no_show":0.0549},"latency_s":0.0383,"input_tokens":470},{"id":"hard-opus-b-probability-06","tier":"hard","family":"probability","gold":"upheld","predicted":"modified","correct":false,"schema_valid":true,"probs":{"modified":0.5546,"overturned":0.1629,"upheld":0.2825},"latency_s":0.0541,"input_tokens":878},{"id":"hard-opus-b-tradeoff-01","tier":"hard","family":"tradeoff","gold":"escalate_to_engineering","predicted":"escalate_to_engineering","correct":true,"schema_valid":true,"probs":{"escalate_to_engineering":0.4313,"escalate_to_security":0.3932,"page_oncall":0.0622,"process_refund":0.1089,"send_kb_article":0.0044},"latency_s":0.0396,"input_tokens":799},{"id":"hard-opus-b-tradeoff-03","tier":"hard","family":"tradeoff","gold":"order_b","predicted":"order_a","correct":false,"schema_valid":true,"probs":{"order_a":0.4263,"order_b":0.2825,"order_c":0.2427,"order_d":0.0485},"latency_s":0.0461,"input_tokens":648},{"id":"hard-opus-b-tradeoff-06","tier":"hard","family":"tradeoff","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.6961,"no":0.3039},"latency_s":0.0475,"input_tokens":665},{"id":"hard-opus-b-tradeoff-07","tier":"hard","family":"tradeoff","gold":"p0_fix_24h","predicted":"p0_fix_24h","correct":true,"schema_valid":true,"probs":{"out_of_scope":0.1948,"p0_fix_24h":0.3248,"p1_fix_7d":0.0762,"p2_fix_30d":0.2511,"p3_backlog":0.1531},"latency_s":0.0467,"input_tokens":674},{"id":"hard-opus-b-tradeoff-08","tier":"hard","family":"tradeoff","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.6088,"no":0.3912},"latency_s":0.0441,"input_tokens":513},{"id":"hard-opus-b-tradeoff-12","tier":"hard","family":"tradeoff","gold":"refund_now","predicted":"refund_now","correct":true,"schema_valid":true,"probs":{"deny_refund":0.1875,"escalate_to_fraud_team":0.1123,"offer_store_credit":0.015,"refund_now":0.4816,"wait_for_inspection":0.2036},"latency_s":0.0295,"input_tokens":534},{"id":"hard-opus-c-long_policy-02","tier":"hard","family":"long_policy","gold":"erase_but_retain_transaction_records","predicted":"restrict_all_pending_legal_hold","correct":false,"schema_valid":true,"probs":{"erase_all":0.1188,"erase_but_retain_transaction_records":0.1897,"erase_marketing_only":0.0697,"reject_unverified_request":0.2715,"restrict_all_pending_legal_hold":0.3503},"latency_s":0.0606,"input_tokens":3272},{"id":"hard-opus-c-long_policy-03","tier":"hard","family":"long_policy","gold":"eur_400","predicted":"eur_300","correct":false,"schema_valid":true,"probs":{"eur_0":0.0131,"eur_150":0.0081,"eur_200":0.056,"eur_300":0.7231,"eur_400":0.1947,"eur_600":0.0049},"latency_s":0.047,"input_tokens":3221},{"id":"hard-opus-c-long_policy-04","tier":"hard","family":"long_policy","gold":"yes","predicted":"no","correct":false,"schema_valid":true,"probs":{"yes":0.3753,"no":0.6247},"latency_s":0.0463,"input_tokens":2953},{"id":"hard-opus-c-long_policy-05","tier":"hard","family":"long_policy","gold":1,"predicted":"0","correct":false,"schema_valid":true,"probs":{"0":0.8719,"1":0.0958,"2":0.0073,"3":0.025},"latency_s":0.0486,"input_tokens":3035},{"id":"hard-opus-c-long_policy-08","tier":"hard","family":"long_policy","gold":1,"predicted":"2","correct":false,"schema_valid":true,"probs":{"0":0.054,"1":0.0441,"2":0.6779,"3":0.2239},"latency_s":0.0622,"input_tokens":3019},{"id":"hard-opus-c-long_policy-10","tier":"hard","family":"long_policy","gold":"credit_minus_restocking_fee","predicted":"repair_or_replace_only","correct":false,"schema_valid":true,"probs":{"credit_minus_restocking_fee":0.1901,"full_credit":0.1831,"reject_return":0.2366,"repair_or_replace_only":0.3902},"latency_s":0.0476,"input_tokens":3257},{"id":"hard-opus-c-long_policy-11","tier":"hard","family":"long_policy","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.5647,"no":0.4353},"latency_s":0.0609,"input_tokens":3189},{"id":"hard-opus-c-probability-03","tier":"hard","family":"probability","gold":"kuznets_technik","predicted":"halvorsen_polymer","correct":false,"schema_valid":true,"probs":{"halvorsen_polymer":0.8767,"kuznets_technik":0.0215,"ortega_seal_works":0.1018},"latency_s":0.0332,"input_tokens":510},{"id":"hard-opus-c-temporal_numeric-02","tier":"hard","family":"temporal_numeric","gold":"yes","predicted":"no","correct":false,"schema_valid":true,"probs":{"yes":0.3607,"no":0.6393},"latency_s":0.0422,"input_tokens":778},{"id":"hard-opus-c-temporal_numeric-03","tier":"hard","family":"temporal_numeric","gold":"eur_692_00","predicted":"eur_692_00","correct":true,"schema_valid":true,"probs":{"eur_617_00":0.0315,"eur_688_00":0.3557,"eur_692_00":0.4897,"eur_693_90":0.1134,"eur_812_00":0.0097},"latency_s":0.0409,"input_tokens":725},{"id":"hard-opus-c-temporal_numeric-04","tier":"hard","family":"temporal_numeric","gold":"usd_2335_00","predicted":"usd_2340_00","correct":false,"schema_valid":true,"probs":{"usd_2222_50":0.0125,"usd_2303_01":0.3116,"usd_2303_33":0.1735,"usd_2335_00":0.0196,"usd_2340_00":0.4829},"latency_s":0.0447,"input_tokens":619},{"id":"hard-opus-c-temporal_numeric-06","tier":"hard","family":"temporal_numeric","gold":"sep_18_book_set","predicted":"none_declined","correct":false,"schema_valid":true,"probs":{"none_declined":0.3088,"sep_15_office_chair":0.028,"sep_16_taxi":0.2317,"sep_17_parking":0.0458,"sep_18_book_set":0.224,"sep_21_client_dinner":0.1617},"latency_s":0.0565,"input_tokens":857},{"id":"hard-opus-c-temporal_numeric-08","tier":"hard","family":"temporal_numeric","gold":"eur_498_81","predicted":"eur_498_81","correct":true,"schema_valid":true,"probs":{"eur_497_40":0.2932,"eur_498_81":0.4437,"eur_506_06":0.1565,"eur_513_69":0.0824,"eur_532_01":0.0242},"latency_s":0.0453,"input_tokens":792},{"id":"hard-opus-c-temporal_numeric-12","tier":"hard","family":"temporal_numeric","gold":"expired_30_month_cap","predicted":"covered","correct":false,"schema_valid":true,"probs":{"covered":0.5126,"expired_24_month_term":0.3013,"expired_30_month_cap":0.1861},"latency_s":0.0396,"input_tokens":714},{"id":"hard-sol-a-adversarial-01","tier":"hard","family":"adversarial","gold":"deny_missing_approval","predicted":"deny_inactive_user","correct":false,"schema_valid":true,"probs":{"deny_inactive_user":0.4269,"deny_missing_approval":0.3589,"grant_access":0.0707,"grant_owner_exception":0.1435},"latency_s":0.0385,"input_tokens":272},{"id":"hard-sol-a-adversarial-06","tier":"hard","family":"adversarial","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.4073,"no":0.5927},"latency_s":0.031,"input_tokens":211},{"id":"hard-sol-a-adversarial-07","tier":"hard","family":"adversarial","gold":"retain_until_2031","predicted":"retain_until_2031","correct":true,"schema_valid":true,"probs":{"delete_now":0.1463,"needs_signature_review":0.1356,"retain_indefinitely":0.0569,"retain_until_2031":0.6612},"latency_s":0.0289,"input_tokens":249},{"id":"hard-sol-a-adversarial-08","tier":"hard","family":"adversarial","gold":"yes","predicted":"no","correct":false,"schema_valid":true,"probs":{"yes":0.3204,"no":0.6796},"latency_s":0.0315,"input_tokens":197},{"id":"hard-sol-a-adversarial-09","tier":"hard","family":"adversarial","gold":2,"predicted":"3","correct":false,"schema_valid":true,"probs":{"0":0.059,"1":0.0522,"2":0.2961,"3":0.5928},"latency_s":0.0304,"input_tokens":191},{"id":"hard-sol-a-adversarial-11","tier":"hard","family":"adversarial","gold":"standard_research","predicted":"ethics_review","correct":false,"schema_valid":true,"probs":{"ethics_review":0.4994,"major_grants":0.0287,"manual_hold":0.1854,"standard_research":0.2865},"latency_s":0.0308,"input_tokens":262},{"id":"hard-sol-a-multi_hop-01","tier":"hard","family":"multi_hop","gold":"accept_s2_dusk","predicted":"reject_cutoff","correct":false,"schema_valid":true,"probs":{"accept_s2_dusk":0.1137,"reject_cutoff":0.3314,"reject_tier":0.2695,"reroute_s3":0.2854},"latency_s":0.0453,"input_tokens":2341},{"id":"hard-sol-a-multi_hop-02","tier":"hard","family":"multi_hop","gold":"security_primary","predicted":"executive_bridge","correct":false,"schema_valid":true,"probs":{"executive_bridge":0.3776,"heron_q2":0.0559,"insufficient_information":0.1952,"security_primary":0.3714},"latency_s":0.0466,"input_tokens":2369},{"id":"hard-sol-a-multi_hop-05","tier":"hard","family":"multi_hop","gold":"require_director","predicted":"pay_manager_approved","correct":false,"schema_valid":true,"probs":{"pay_manager_approved":0.3712,"reject_category":0.1486,"require_director":0.3365,"require_finance_board":0.1437},"latency_s":0.0466,"input_tokens":2308},{"id":"hard-sol-a-multi_hop-07","tier":"hard","family":"multi_hop","gold":"admit","predicted":"deny_prerequisite","correct":false,"schema_valid":true,"probs":{"admit":0.0518,"deny_prerequisite":0.4523,"deny_score":0.2674,"manual_equivalency":0.2284},"latency_s":0.0473,"input_tokens":2269},{"id":"hard-sol-a-multi_hop-08","tier":"hard","family":"multi_hop","gold":"central_engineering","predicted":"insufficient_log","correct":false,"schema_valid":true,"probs":{"central_engineering":0.091,"insufficient_log":0.3536,"local_maintenance":0.2534,"safety_shutdown":0.302},"latency_s":0.0466,"input_tokens":2264},{"id":"hard-sol-a-multi_hop-09","tier":"hard","family":"multi_hop","gold":"require_steward","predicted":"require_steward","correct":true,"schema_valid":true,"probs":{"confirm_l3":0.2309,"reject_capacity":0.2128,"require_steward":0.3892,"switch_l4":0.167},"latency_s":0.0469,"input_tokens":2271},{"id":"hard-sol-a-multi_hop-10","tier":"hard","family":"multi_hop","gold":"approve_45_days","predicted":"needs_field_review","correct":false,"schema_valid":true,"probs":{"approve_45_days":0.0519,"deny_sensitive":0.1851,"needs_field_review":0.502,"reduce_to_30_days":0.261},"latency_s":0.0469,"input_tokens":2304},{"id":"hard-sol-a-multi_hop-12","tier":"hard","family":"multi_hop","gold":"needs_authentication","predicted":"needs_authentication","correct":true,"schema_valid":true,"probs":{"covered_recurrence":0.2322,"expired":0.069,"needs_authentication":0.5369,"wrong_issue_class":0.1619},"latency_s":0.0458,"input_tokens":2267},{"id":"hard-sol-a-trap-02","tier":"hard","family":"trap","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.6235,"no":0.3765},"latency_s":0.0325,"input_tokens":263},{"id":"hard-sol-a-trap-04","tier":"hard","family":"trap","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.6269,"no":0.3731},"latency_s":0.0294,"input_tokens":166},{"id":"hard-sol-a-trap-05","tier":"hard","family":"trap","gold":0,"predicted":"0","correct":true,"schema_valid":true,"probs":{"0":0.474,"1":0.0102,"2":0.3976,"3":0.1181},"latency_s":0.0315,"input_tokens":279},{"id":"hard-sol-a-trap-06","tier":"hard","family":"trap","gold":"keep_open_unresolved","predicted":"await_internal_metrics","correct":false,"schema_valid":true,"probs":{"await_internal_metrics":0.3185,"close_agent_verified":0.3021,"close_confirmed":0.2233,"keep_open_unresolved":0.1561},"latency_s":0.0326,"input_tokens":304},{"id":"hard-sol-a-trap-08","tier":"hard","family":"trap","gold":"late","predicted":"on_time","correct":false,"schema_valid":true,"probs":{"canceled":0.1133,"indeterminate":0.1201,"late":0.3166,"on_time":0.4501},"latency_s":0.0306,"input_tokens":225},{"id":"hard-sol-a-trap-10","tier":"hard","family":"trap","gold":"blocked_second_review","predicted":"blocked_second_review","correct":true,"schema_valid":true,"probs":{"blocked_failed_test":0.3246,"blocked_second_review":0.3412,"ready_one_review":0.1688,"ready_two_reviews":0.1654},"latency_s":0.0302,"input_tokens":174},{"id":"hard-sol-a-trap-13","tier":"hard","family":"trap","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.3163,"no":0.6837},"latency_s":0.031,"input_tokens":169},{"id":"hard-sol-a-trap-15","tier":"hard","family":"trap","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.7281,"no":0.2719},"latency_s":0.029,"input_tokens":257},{"id":"hard-sol-b-judge_hard-01","tier":"hard","family":"judge_hard","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.7112,"no":0.2888},"latency_s":0.0338,"input_tokens":208},{"id":"hard-sol-b-judge_hard-02","tier":"hard","family":"judge_hard","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.8166,"no":0.1834},"latency_s":0.0298,"input_tokens":163},{"id":"hard-sol-b-judge_hard-05","tier":"hard","family":"judge_hard","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.5146,"no":0.4854},"latency_s":0.0309,"input_tokens":210},{"id":"hard-sol-b-judge_hard-08","tier":"hard","family":"judge_hard","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.8206,"no":0.1794},"latency_s":0.0377,"input_tokens":149},{"id":"hard-sol-b-judge_hard-10","tier":"hard","family":"judge_hard","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.6369,"no":0.3631},"latency_s":0.0257,"input_tokens":172},{"id":"hard-sol-b-judge_hard-14","tier":"hard","family":"judge_hard","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.4522,"no":0.5478},"latency_s":0.0301,"input_tokens":204},{"id":"hard-sol-b-judge_hard-15","tier":"hard","family":"judge_hard","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.8289,"no":0.1711},"latency_s":0.032,"input_tokens":153},{"id":"hard-sol-b-judge_hard-18","tier":"hard","family":"judge_hard","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.8571,"no":0.1429},"latency_s":0.03,"input_tokens":139},{"id":"hard-sol-b-long_policy-01","tier":"hard","family":"long_policy","gold":"reject_entire_claim","predicted":"approve_lodging_only","correct":false,"schema_valid":true,"probs":{"approve_all":0.0991,"approve_lodging_only":0.4277,"approve_meals_only":0.2324,"reject_entire_claim":0.2407},"latency_s":0.0457,"input_tokens":2701},{"id":"hard-sol-b-long_policy-02","tier":"hard","family":"long_policy","gold":"deny_current_export","predicted":"analytics_owner_only","correct":false,"schema_valid":true,"probs":{"analytics_owner_only":0.2826,"deny_current_export":0.1576,"privacy_and_security":0.2778,"privacy_only":0.2819},"latency_s":0.0468,"input_tokens":2586},{"id":"hard-sol-b-long_policy-05","tier":"hard","family":"long_policy","gold":"delete_profile_omar_april_and_table","predicted":"delete_everything","correct":false,"schema_valid":true,"probs":{"delete_everything":0.4408,"delete_nothing":0.0726,"delete_only_profile":0.3302,"delete_profile_omar_april_and_table":0.1564},"latency_s":0.0468,"input_tokens":2530},{"id":"hard-sol-b-long_policy-06","tier":"hard","family":"long_policy","gold":"east_ward","predicted":"river_arts","correct":false,"schema_valid":true,"probs":{"east_ward":0.1141,"harbor_neighbors":0.1552,"no_award":0.1917,"river_arts":0.539},"latency_s":0.0464,"input_tokens":2547},{"id":"hard-sol-b-routing_hard-01","tier":"hard","family":"routing_hard","gold":"office_file_repair","predicted":"finance_review","correct":false,"schema_valid":true,"probs":{"data_pipeline":0.0482,"desktop_support":0.0215,"finance_review":0.3485,"office_file_repair":0.2879,"spreadsheet_formula":0.2939},"latency_s":0.0493,"input_tokens":306},{"id":"hard-sol-b-routing_hard-02","tier":"hard","family":"routing_hard","gold":"ledger_reconciliation","predicted":"payment_disputes","correct":false,"schema_valid":true,"probs":{"customer_support":0.0318,"fraud_review":0.0115,"ledger_reconciliation":0.318,"mobile_app":0.0454,"payment_disputes":0.5932},"latency_s":0.0326,"input_tokens":275},{"id":"hard-sol-b-routing_hard-03","tier":"hard","family":"routing_hard","gold":"network_dns","predicted":"api_platform","correct":false,"schema_valid":true,"probs":{"api_platform":0.689,"authoritative_dns":0.091,"endpoint_devices":0.0843,"network_dns":0.1299,"tls_certificates":0.0057},"latency_s":0.028,"input_tokens":289},{"id":"hard-sol-b-routing_hard-07","tier":"hard","family":"routing_hard","gold":"release_notarization","predicted":"macos_runtime","correct":false,"schema_valid":true,"probs":{"ci_infrastructure":0.0588,"code_signing":0.2769,"endpoint_security":0.0274,"macos_runtime":0.4582,"release_notarization":0.1788},"latency_s":0.0292,"input_tokens":254},{"id":"hard-sol-b-routing_hard-09","tier":"hard","family":"routing_hard","gold":"source_access","predicted":"identity_login","correct":false,"schema_valid":true,"probs":{"git_client":0.1372,"identity_login":0.6046,"repository_admin":0.0241,"source_access":0.1148,"vpn_network":0.1193},"latency_s":0.0302,"input_tokens":266},{"id":"hard-sol-b-temporal_numeric-01","tier":"hard","family":"temporal_numeric","gold":"late_by_under_2h","predicted":"before_open","correct":false,"schema_valid":true,"probs":{"before_open":0.3762,"late_by_2h_or_more":0.1966,"late_by_under_2h":0.1074,"within_window":0.3198},"latency_s":0.0322,"input_tokens":221},{"id":"hard-sol-b-temporal_numeric-02","tier":"hard","family":"temporal_numeric","gold":"over_limit","predicted":"under_100gb_below","correct":false,"schema_valid":true,"probs":{"at_least_100gb_below":0.2096,"exactly_equal":0.1388,"over_limit":0.2091,"under_100gb_below":0.4426},"latency_s":0.03,"input_tokens":221},{"id":"hard-sol-b-temporal_numeric-03","tier":"hard","family":"temporal_numeric","gold":"9_credits","predicted":"9_credits","correct":true,"schema_valid":true,"probs":{"12_credits":0.0766,"15_credits":0.3006,"6_credits":0.0767,"9_credits":0.5461},"latency_s":0.0342,"input_tokens":173},{"id":"hard-sol-b-temporal_numeric-04","tier":"hard","family":"temporal_numeric","gold":"pass","predicted":"narrow_fail","correct":false,"schema_valid":true,"probs":{"clear_fail":0.2985,"narrow_fail":0.4061,"pass":0.2954},"latency_s":0.0282,"input_tokens":187},{"id":"hard-sol-c-judge_hard-03","tier":"hard","family":"judge_hard","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.5427,"no":0.4573},"latency_s":0.0334,"input_tokens":392},{"id":"hard-sol-c-judge_hard-05","tier":"hard","family":"judge_hard","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.6235,"no":0.3765},"latency_s":0.0362,"input_tokens":299},{"id":"hard-sol-c-judge_hard-07","tier":"hard","family":"judge_hard","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.7375,"no":0.2625},"latency_s":0.0242,"input_tokens":272},{"id":"hard-sol-c-judge_hard-08","tier":"hard","family":"judge_hard","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.415,"no":0.585},"latency_s":0.0368,"input_tokens":286},{"id":"hard-sol-c-judge_hard-09","tier":"hard","family":"judge_hard","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.557,"no":0.443},"latency_s":0.0287,"input_tokens":280},{"id":"hard-sol-c-judge_hard-10","tier":"hard","family":"judge_hard","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.7809,"no":0.2191},"latency_s":0.0295,"input_tokens":297},{"id":"hard-sol-c-judge_hard-11","tier":"hard","family":"judge_hard","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.5133,"no":0.4867},"latency_s":0.034,"input_tokens":316},{"id":"hard-sol-c-judge_hard-13","tier":"hard","family":"judge_hard","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.6922,"no":0.3078},"latency_s":0.0277,"input_tokens":275},{"id":"hard-sol-c-judge_hard-15","tier":"hard","family":"judge_hard","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.8099,"no":0.1901},"latency_s":0.0336,"input_tokens":260},{"id":"hard-sol-c-multi_hop-07","tier":"hard","family":"multi_hop","gold":"programme_review","predicted":"release_draw","correct":false,"schema_valid":true,"probs":{"partial_release":0.2508,"programme_review":0.2516,"reject_draw":0.2318,"release_draw":0.2659},"latency_s":0.0465,"input_tokens":2451},{"id":"hard-sol-c-multi_hop-09","tier":"hard","family":"multi_hop","gold":"bus_substitution","predicted":"terminate_oakmere","correct":false,"schema_valid":true,"probs":{"bus_substitution":0.2075,"call_fen_cross":0.2206,"pass_fen_cross":0.2445,"terminate_oakmere":0.3274},"latency_s":0.0482,"input_tokens":2456},{"id":"hard-sol-c-multi_hop-11","tier":"hard","family":"multi_hop","gold":"coverage_excluded","predicted":"coverage_excluded","correct":true,"schema_valid":true,"probs":{"adjuster_referral":0.2406,"coverage_confirmed":0.1397,"coverage_excluded":0.3133,"documentation_needed":0.3063},"latency_s":0.0473,"input_tokens":2445},{"id":"hard-sol-c-multi_hop-12","tier":"hard","family":"multi_hop","gold":"conservation_hold","predicted":"supervised_delivery","correct":false,"schema_valid":true,"probs":{"conservation_hold":0.2934,"deny_request":0.1821,"digital_copy_only":0.1498,"supervised_delivery":0.3747},"latency_s":0.0342,"input_tokens":480},{"id":"hard-sol-c-multi_hop-13","tier":"hard","family":"multi_hop","gold":"normal_release","predicted":"engineering_review","correct":false,"schema_valid":true,"probs":{"engineering_review":0.4942,"normal_release":0.1308,"rework":0.1242,"scrap_lot":0.2509},"latency_s":0.0431,"input_tokens":2514},{"id":"original-adequacy-01-0","tier":"original","family":"adequacy","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.7597,"no":0.2403},"latency_s":0.0329,"input_tokens":51},{"id":"original-adequacy-01-1","tier":"original","family":"adequacy","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.7821,"no":0.2179},"latency_s":0.0275,"input_tokens":48},{"id":"original-adequacy-02-0","tier":"original","family":"adequacy","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.7366,"no":0.2634},"latency_s":0.0325,"input_tokens":55},{"id":"original-adequacy-02-1","tier":"original","family":"adequacy","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.2187,"no":0.7813},"latency_s":0.0303,"input_tokens":56},{"id":"original-adequacy-03-0","tier":"original","family":"adequacy","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.7673,"no":0.2327},"latency_s":0.0347,"input_tokens":55},{"id":"original-adequacy-03-1","tier":"original","family":"adequacy","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.6054,"no":0.3946},"latency_s":0.0316,"input_tokens":58},{"id":"original-adequacy-04-0","tier":"original","family":"adequacy","gold":"yes","predicted":"no","correct":false,"schema_valid":true,"probs":{"yes":0.4022,"no":0.5978},"latency_s":0.0311,"input_tokens":61},{"id":"original-adequacy-04-1","tier":"original","family":"adequacy","gold":"yes","predicted":"no","correct":false,"schema_valid":true,"probs":{"yes":0.2246,"no":0.7754},"latency_s":0.0312,"input_tokens":58},{"id":"original-adequacy-05-0","tier":"original","family":"adequacy","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.8312,"no":0.1688},"latency_s":0.0341,"input_tokens":49},{"id":"original-adequacy-05-1","tier":"original","family":"adequacy","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.8457,"no":0.1543},"latency_s":0.0255,"input_tokens":50},{"id":"original-adequacy-06-0","tier":"original","family":"adequacy","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.6886,"no":0.3114},"latency_s":0.0354,"input_tokens":61},{"id":"original-adequacy-06-1","tier":"original","family":"adequacy","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.8262,"no":0.1738},"latency_s":0.0302,"input_tokens":61},{"id":"original-extraction-01-0","tier":"original","family":"extraction","gold":"pickup","predicted":"courier","correct":false,"schema_valid":true,"probs":{"courier":0.8402,"pickup":0.0997,"post":0.0318,"unknown":0.0284},"latency_s":0.0283,"input_tokens":59},{"id":"original-extraction-01-1","tier":"original","family":"extraction","gold":"pickup","predicted":"pickup","correct":true,"schema_valid":true,"probs":{"courier":0.2918,"pickup":0.5936,"post":0.0406,"unknown":0.0741},"latency_s":0.0336,"input_tokens":61},{"id":"original-extraction-02-0","tier":"original","family":"extraction","gold":"unknown","predicted":"unknown","correct":true,"schema_valid":true,"probs":{"courier":0.1338,"pickup":0.0736,"post":0.2973,"unknown":0.4953},"latency_s":0.0326,"input_tokens":60},{"id":"original-extraction-02-1","tier":"original","family":"extraction","gold":"unknown","predicted":"post","correct":false,"schema_valid":true,"probs":{"courier":0.0877,"pickup":0.0502,"post":0.6389,"unknown":0.2232},"latency_s":0.0295,"input_tokens":60},{"id":"original-extraction-03-0","tier":"original","family":"extraction","gold":"courier","predicted":"pickup","correct":false,"schema_valid":true,"probs":{"courier":0.3369,"pickup":0.5453,"post":0.0592,"unknown":0.0586},"latency_s":0.0309,"input_tokens":64},{"id":"original-extraction-03-1","tier":"original","family":"extraction","gold":"courier","predicted":"courier","correct":true,"schema_valid":true,"probs":{"courier":0.7875,"pickup":0.1355,"post":0.0458,"unknown":0.0312},"latency_s":0.0488,"input_tokens":63},{"id":"original-extraction-04-0","tier":"original","family":"extraction","gold":"post","predicted":"post","correct":true,"schema_valid":true,"probs":{"courier":0.0615,"pickup":0.1098,"post":0.7393,"unknown":0.0894},"latency_s":0.0272,"input_tokens":60},{"id":"original-extraction-04-1","tier":"original","family":"extraction","gold":"post","predicted":"post","correct":true,"schema_valid":true,"probs":{"courier":0.2995,"pickup":0.1504,"post":0.4251,"unknown":0.125},"latency_s":0.0282,"input_tokens":59},{"id":"original-extraction-05-0","tier":"original","family":"extraction","gold":"unknown","predicted":"courier","correct":false,"schema_valid":true,"probs":{"courier":0.4008,"pickup":0.1077,"post":0.1953,"unknown":0.2961},"latency_s":0.0373,"input_tokens":61},{"id":"original-extraction-05-1","tier":"original","family":"extraction","gold":"unknown","predicted":"unknown","correct":true,"schema_valid":true,"probs":{"courier":0.3004,"pickup":0.098,"post":0.1311,"unknown":0.4705},"latency_s":0.031,"input_tokens":57},{"id":"original-extraction-06-0","tier":"original","family":"extraction","gold":"courier","predicted":"courier","correct":true,"schema_valid":true,"probs":{"courier":0.6939,"pickup":0.0687,"post":0.1778,"unknown":0.0597},"latency_s":0.0315,"input_tokens":61},{"id":"original-extraction-06-1","tier":"original","family":"extraction","gold":"courier","predicted":"courier","correct":true,"schema_valid":true,"probs":{"courier":0.464,"pickup":0.0783,"post":0.2929,"unknown":0.1648},"latency_s":0.0268,"input_tokens":60},{"id":"original-intent-01-0","tier":"original","family":"intent","gold":"refund","predicted":"refund","correct":true,"schema_valid":true,"probs":{"cancel":0.1173,"change_address":0.1073,"other":0.1177,"refund":0.5571,"status":0.1007},"latency_s":0.0345,"input_tokens":63},{"id":"original-intent-01-1","tier":"original","family":"intent","gold":"refund","predicted":"refund","correct":true,"schema_valid":true,"probs":{"cancel":0.212,"change_address":0.087,"other":0.1682,"refund":0.4751,"status":0.0578},"latency_s":0.0316,"input_tokens":65},{"id":"original-intent-02-0","tier":"original","family":"intent","gold":"status","predicted":"change_address","correct":false,"schema_valid":true,"probs":{"cancel":0.1234,"change_address":0.5161,"other":0.1495,"refund":0.0244,"status":0.1866},"latency_s":0.0305,"input_tokens":64},{"id":"original-intent-02-1","tier":"original","family":"intent","gold":"status","predicted":"status","correct":true,"schema_valid":true,"probs":{"cancel":0.0435,"change_address":0.0372,"other":0.0826,"refund":0.0197,"status":0.817},"latency_s":0.0323,"input_tokens":63},{"id":"original-intent-03-0","tier":"original","family":"intent","gold":"change_address","predicted":"change_address","correct":true,"schema_valid":true,"probs":{"cancel":0.2827,"change_address":0.3401,"other":0.1713,"refund":0.0367,"status":0.1693},"latency_s":0.0331,"input_tokens":65},{"id":"original-intent-03-1","tier":"original","family":"intent","gold":"change_address","predicted":"change_address","correct":true,"schema_valid":true,"probs":{"cancel":0.0958,"change_address":0.8011,"other":0.0467,"refund":0.012,"status":0.0445},"latency_s":0.0277,"input_tokens":63},{"id":"original-intent-04-0","tier":"original","family":"intent","gold":"cancel","predicted":"other","correct":false,"schema_valid":true,"probs":{"cancel":0.2085,"change_address":0.1831,"other":0.3406,"refund":0.032,"status":0.2358},"latency_s":0.0313,"input_tokens":68},{"id":"original-intent-04-1","tier":"original","family":"intent","gold":"cancel","predicted":"cancel","correct":true,"schema_valid":true,"probs":{"cancel":0.442,"change_address":0.1553,"other":0.2015,"refund":0.0226,"status":0.1786},"latency_s":0.0346,"input_tokens":63},{"id":"original-intent-05-0","tier":"original","family":"intent","gold":"other","predicted":"status","correct":false,"schema_valid":true,"probs":{"cancel":0.0873,"change_address":0.0871,"other":0.1577,"refund":0.3264,"status":0.3415},"latency_s":0.0259,"input_tokens":63},{"id":"original-intent-05-1","tier":"original","family":"intent","gold":"other","predicted":"status","correct":false,"schema_valid":true,"probs":{"cancel":0.1047,"change_address":0.121,"other":0.1623,"refund":0.0798,"status":0.5322},"latency_s":0.0329,"input_tokens":63},{"id":"original-intent-06-0","tier":"original","family":"intent","gold":"status","predicted":"cancel","correct":false,"schema_valid":true,"probs":{"cancel":0.4711,"change_address":0.2059,"other":0.1047,"refund":0.0241,"status":0.1942},"latency_s":0.0511,"input_tokens":61},{"id":"original-intent-06-1","tier":"original","family":"intent","gold":"status","predicted":"cancel","correct":false,"schema_valid":true,"probs":{"cancel":0.5413,"change_address":0.165,"other":0.0805,"refund":0.0229,"status":0.1903},"latency_s":0.0284,"input_tokens":66},{"id":"original-ordinal-01-0","tier":"original","family":"ordinal","gold":0,"predicted":"0","correct":true,"schema_valid":true,"probs":{"0":0.4253,"1":0.1928,"2":0.3003,"3":0.0817},"latency_s":0.0312,"input_tokens":70},{"id":"original-ordinal-01-1","tier":"original","family":"ordinal","gold":0,"predicted":"0","correct":true,"schema_valid":true,"probs":{"0":0.3912,"1":0.2416,"2":0.2904,"3":0.0768},"latency_s":0.0274,"input_tokens":72},{"id":"original-ordinal-02-0","tier":"original","family":"ordinal","gold":1,"predicted":"1","correct":true,"schema_valid":true,"probs":{"0":0.3145,"1":0.3401,"2":0.2693,"3":0.0761},"latency_s":0.0303,"input_tokens":74},{"id":"original-ordinal-02-1","tier":"original","family":"ordinal","gold":1,"predicted":"1","correct":true,"schema_valid":true,"probs":{"0":0.1215,"1":0.4647,"2":0.2969,"3":0.117},"latency_s":0.0364,"input_tokens":75},{"id":"original-ordinal-03-0","tier":"original","family":"ordinal","gold":2,"predicted":"2","correct":true,"schema_valid":true,"probs":{"0":0.1963,"1":0.2571,"2":0.4512,"3":0.0954},"latency_s":0.0259,"input_tokens":71},{"id":"original-ordinal-03-1","tier":"original","family":"ordinal","gold":2,"predicted":"0","correct":false,"schema_valid":true,"probs":{"0":0.4089,"1":0.1326,"2":0.3545,"3":0.104},"latency_s":0.0299,"input_tokens":73},{"id":"original-ordinal-04-0","tier":"original","family":"ordinal","gold":3,"predicted":"3","correct":true,"schema_valid":true,"probs":{"0":0.0421,"1":0.1318,"2":0.2069,"3":0.6192},"latency_s":0.0348,"input_tokens":73},{"id":"original-ordinal-04-1","tier":"original","family":"ordinal","gold":3,"predicted":"3","correct":true,"schema_valid":true,"probs":{"0":0.077,"1":0.1425,"2":0.2673,"3":0.5132},"latency_s":0.0298,"input_tokens":72},{"id":"original-ordinal-05-0","tier":"original","family":"ordinal","gold":0,"predicted":"0","correct":true,"schema_valid":true,"probs":{"0":0.6685,"1":0.0781,"2":0.1901,"3":0.0633},"latency_s":0.0302,"input_tokens":76},{"id":"original-ordinal-05-1","tier":"original","family":"ordinal","gold":0,"predicted":"0","correct":true,"schema_valid":true,"probs":{"0":0.4515,"1":0.1545,"2":0.2978,"3":0.0962},"latency_s":0.0309,"input_tokens":75},{"id":"original-ordinal-06-0","tier":"original","family":"ordinal","gold":2,"predicted":"0","correct":false,"schema_valid":true,"probs":{"0":0.5266,"1":0.1311,"2":0.2917,"3":0.0505},"latency_s":0.038,"input_tokens":77},{"id":"original-ordinal-06-1","tier":"original","family":"ordinal","gold":2,"predicted":"2","correct":true,"schema_valid":true,"probs":{"0":0.2928,"1":0.1248,"2":0.534,"3":0.0484},"latency_s":0.0251,"input_tokens":84},{"id":"original-policy-01-0","tier":"original","family":"policy","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.2769,"no":0.7231},"latency_s":0.0287,"input_tokens":71},{"id":"original-policy-01-1","tier":"original","family":"policy","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.3941,"no":0.6059},"latency_s":0.0299,"input_tokens":73},{"id":"original-policy-02-0","tier":"original","family":"policy","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.7466,"no":0.2534},"latency_s":0.0281,"input_tokens":63},{"id":"original-policy-02-1","tier":"original","family":"policy","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.751,"no":0.249},"latency_s":0.0351,"input_tokens":62},{"id":"original-policy-03-0","tier":"original","family":"policy","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.6686,"no":0.3314},"latency_s":0.0294,"input_tokens":65},{"id":"original-policy-03-1","tier":"original","family":"policy","gold":"yes","predicted":"no","correct":false,"schema_valid":true,"probs":{"yes":0.2308,"no":0.7692},"latency_s":0.0343,"input_tokens":65},{"id":"original-policy-04-0","tier":"original","family":"policy","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.6621,"no":0.3379},"latency_s":0.0293,"input_tokens":73},{"id":"original-policy-04-1","tier":"original","family":"policy","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.5056,"no":0.4944},"latency_s":0.0382,"input_tokens":71},{"id":"original-policy-05-0","tier":"original","family":"policy","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.6478,"no":0.3522},"latency_s":0.0438,"input_tokens":76},{"id":"original-policy-05-1","tier":"original","family":"policy","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.3254,"no":0.6746},"latency_s":0.0272,"input_tokens":69},{"id":"original-policy-06-0","tier":"original","family":"policy","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.5904,"no":0.4096},"latency_s":0.0496,"input_tokens":74},{"id":"original-policy-06-1","tier":"original","family":"policy","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.6964,"no":0.3036},"latency_s":0.0294,"input_tokens":64},{"id":"original-routing-01-0","tier":"original","family":"routing","gold":"math","predicted":"math","correct":true,"schema_valid":true,"probs":{"coding":0.1612,"coding_agent":0.1997,"document":0.1626,"general":0.1345,"math":0.2153,"tools":0.1268},"latency_s":0.0267,"input_tokens":92},{"id":"original-routing-01-1","tier":"original","family":"routing","gold":"math","predicted":"math","correct":true,"schema_valid":true,"probs":{"coding":0.1539,"coding_agent":0.1626,"document":0.1861,"general":0.1244,"math":0.242,"tools":0.131},"latency_s":0.0314,"input_tokens":89},{"id":"original-routing-02-0","tier":"original","family":"routing","gold":"coding","predicted":"tools","correct":false,"schema_valid":true,"probs":{"coding":0.1613,"coding_agent":0.1057,"document":0.1219,"general":0.149,"math":0.0814,"tools":0.3807},"latency_s":0.0298,"input_tokens":96},{"id":"original-routing-02-1","tier":"original","family":"routing","gold":"coding","predicted":"coding","correct":true,"schema_valid":true,"probs":{"coding":0.2759,"coding_agent":0.1668,"document":0.1425,"general":0.1159,"math":0.0706,"tools":0.2284},"latency_s":0.031,"input_tokens":90},{"id":"original-routing-03-0","tier":"original","family":"routing","gold":"coding_agent","predicted":"coding_agent","correct":true,"schema_valid":true,"probs":{"coding":0.0754,"coding_agent":0.5931,"document":0.1128,"general":0.1024,"math":0.0384,"tools":0.0779},"latency_s":0.0284,"input_tokens":95},{"id":"original-routing-03-1","tier":"original","family":"routing","gold":"coding_agent","predicted":"coding_agent","correct":true,"schema_valid":true,"probs":{"coding":0.086,"coding_agent":0.5342,"document":0.1167,"general":0.0924,"math":0.0413,"tools":0.1294},"latency_s":0.0319,"input_tokens":98},{"id":"original-routing-04-0","tier":"original","family":"routing","gold":"document","predicted":"document","correct":true,"schema_valid":true,"probs":{"coding":0.1188,"coding_agent":0.1822,"document":0.3155,"general":0.1839,"math":0.0766,"tools":0.123},"latency_s":0.037,"input_tokens":91},{"id":"original-routing-04-1","tier":"original","family":"routing","gold":"document","predicted":"document","correct":true,"schema_valid":true,"probs":{"coding":0.1408,"coding_agent":0.2364,"document":0.2457,"general":0.1588,"math":0.0667,"tools":0.1517},"latency_s":0.0261,"input_tokens":91},{"id":"original-routing-05-0","tier":"original","family":"routing","gold":"tools","predicted":"coding_agent","correct":false,"schema_valid":true,"probs":{"coding":0.1357,"coding_agent":0.2231,"document":0.1916,"general":0.166,"math":0.0662,"tools":0.2174},"latency_s":0.0316,"input_tokens":92},{"id":"original-routing-05-1","tier":"original","family":"routing","gold":"tools","predicted":"document","correct":false,"schema_valid":true,"probs":{"coding":0.142,"coding_agent":0.2041,"document":0.2197,"general":0.1883,"math":0.0715,"tools":0.1744},"latency_s":0.049,"input_tokens":93},{"id":"original-routing-06-0","tier":"original","family":"routing","gold":"general","predicted":"tools","correct":false,"schema_valid":true,"probs":{"coding":0.1584,"coding_agent":0.2221,"document":0.1791,"general":0.148,"math":0.0623,"tools":0.2301},"latency_s":0.0296,"input_tokens":91},{"id":"original-routing-06-1","tier":"original","family":"routing","gold":"general","predicted":"tools","correct":false,"schema_valid":true,"probs":{"coding":0.1436,"coding_agent":0.2212,"document":0.1809,"general":0.1274,"math":0.0606,"tools":0.2663},"latency_s":0.0342,"input_tokens":91}]},"tia-1.4-mini":{"display":"TIA 1.4 mini","run_by":"TIA","date":"2026-10-02","device":"local CPU worker on the J-97 workstation, harness in Docker, loopback","adapter":"typesafe","items":[{"id":"easy-extraction-00","tier":"easy","family":"extraction","gold":"paypal","predicted":"paypal","correct":true,"schema_valid":true,"probs":{"bank_transfer":0.0569,"cash":0.1103,"credit_card":0.345,"paypal":0.4878},"latency_s":0.0581,"input_tokens":43},{"id":"easy-extraction-01","tier":"easy","family":"extraction","gold":"cash","predicted":"cash","correct":true,"schema_valid":true,"probs":{"bank_transfer":0.1376,"cash":0.4704,"credit_card":0.2839,"paypal":0.1082},"latency_s":0.0606,"input_tokens":46},{"id":"easy-extraction-02","tier":"easy","family":"extraction","gold":"bank_transfer","predicted":"bank_transfer","correct":true,"schema_valid":true,"probs":{"bank_transfer":0.5095,"cash":0.2089,"credit_card":0.1992,"paypal":0.0824},"latency_s":0.057,"input_tokens":46},{"id":"easy-extraction-03","tier":"easy","family":"extraction","gold":"high","predicted":"high","correct":true,"schema_valid":true,"probs":{"high":0.5524,"low":0.0351,"medium":0.1721,"urgent":0.2404},"latency_s":0.0574,"input_tokens":45},{"id":"easy-extraction-04","tier":"easy","family":"extraction","gold":"low","predicted":"high","correct":false,"schema_valid":true,"probs":{"high":0.415,"low":0.0313,"medium":0.2276,"urgent":0.326},"latency_s":0.0563,"input_tokens":42},{"id":"easy-extraction-05","tier":"easy","family":"extraction","gold":"urgent","predicted":"high","correct":false,"schema_valid":true,"probs":{"high":0.4066,"low":0.0243,"medium":0.1915,"urgent":0.3776},"latency_s":0.0456,"input_tokens":42},{"id":"easy-extraction-06","tier":"easy","family":"extraction","gold":"M","predicted":"L","correct":false,"schema_valid":true,"probs":{"L":0.2724,"M":0.2336,"S":0.2696,"XL":0.2243},"latency_s":0.057,"input_tokens":34},{"id":"easy-extraction-07","tier":"easy","family":"extraction","gold":"XL","predicted":"XL","correct":true,"schema_valid":true,"probs":{"L":0.2412,"M":0.2018,"S":0.2382,"XL":0.3188},"latency_s":0.0552,"input_tokens":32},{"id":"easy-extraction-08","tier":"easy","family":"extraction","gold":"thursday","predicted":"thursday","correct":true,"schema_valid":true,"probs":{"friday":0.1945,"monday":0.1373,"thursday":0.3147,"tuesday":0.102,"wednesday":0.2515},"latency_s":0.0289,"input_tokens":38},{"id":"easy-extraction-09","tier":"easy","family":"extraction","gold":"monday","predicted":"monday","correct":true,"schema_valid":true,"probs":{"friday":0.18,"monday":0.4265,"thursday":0.104,"tuesday":0.1834,"wednesday":0.106},"latency_s":0.0547,"input_tokens":42},{"id":"easy-extraction-10","tier":"easy","family":"extraction","gold":"friday","predicted":"friday","correct":true,"schema_valid":true,"probs":{"friday":0.4703,"monday":0.1358,"thursday":0.1362,"tuesday":0.1117,"wednesday":0.146},"latency_s":0.0594,"input_tokens":38},{"id":"easy-extraction-11","tier":"easy","family":"extraction","gold":"credit_card","predicted":"credit_card","correct":true,"schema_valid":true,"probs":{"bank_transfer":0.0802,"cash":0.1552,"credit_card":0.6508,"paypal":0.1139},"latency_s":0.0566,"input_tokens":46},{"id":"easy-fact-00","tier":"easy","family":"fact","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.5272,"no":0.4728},"latency_s":0.0519,"input_tokens":53},{"id":"easy-fact-01","tier":"easy","family":"fact","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.5209,"no":0.4791},"latency_s":0.0306,"input_tokens":50},{"id":"easy-fact-02","tier":"easy","family":"fact","gold":"yes","predicted":"no","correct":false,"schema_valid":true,"probs":{"yes":0.4647,"no":0.5353},"latency_s":0.0558,"input_tokens":54},{"id":"easy-fact-03","tier":"easy","family":"fact","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.4391,"no":0.5609},"latency_s":0.0478,"input_tokens":58},{"id":"easy-fact-04","tier":"easy","family":"fact","gold":"yes","predicted":"no","correct":false,"schema_valid":true,"probs":{"yes":0.4128,"no":0.5872},"latency_s":0.0595,"input_tokens":54},{"id":"easy-fact-05","tier":"easy","family":"fact","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.4192,"no":0.5808},"latency_s":0.0604,"input_tokens":55},{"id":"easy-fact-06","tier":"easy","family":"fact","gold":"yes","predicted":"no","correct":false,"schema_valid":true,"probs":{"yes":0.494,"no":0.506},"latency_s":0.0563,"input_tokens":62},{"id":"easy-fact-07","tier":"easy","family":"fact","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.4479,"no":0.5521},"latency_s":0.0584,"input_tokens":50},{"id":"easy-fact-08","tier":"easy","family":"fact","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.56,"no":0.44},"latency_s":0.0582,"input_tokens":46},{"id":"easy-fact-09","tier":"easy","family":"fact","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.5093,"no":0.4907},"latency_s":0.0431,"input_tokens":46},{"id":"easy-fact-10","tier":"easy","family":"fact","gold":"yes","predicted":"no","correct":false,"schema_valid":true,"probs":{"yes":0.4142,"no":0.5858},"latency_s":0.0587,"input_tokens":53},{"id":"easy-fact-11","tier":"easy","family":"fact","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.3574,"no":0.6426},"latency_s":0.0538,"input_tokens":54},{"id":"easy-intent-00","tier":"easy","family":"intent","gold":"track_order","predicted":"cancel_order","correct":false,"schema_valid":true,"probs":{"billing_question":0.0622,"cancel_order":0.418,"change_address":0.2183,"report_damage":0.1123,"track_order":0.1892},"latency_s":0.1313,"input_tokens":91},{"id":"easy-intent-01","tier":"easy","family":"intent","gold":"cancel_order","predicted":"cancel_order","correct":true,"schema_valid":true,"probs":{"billing_question":0.0262,"cancel_order":0.5798,"change_address":0.1481,"report_damage":0.0678,"track_order":0.1781},"latency_s":0.0356,"input_tokens":89},{"id":"easy-intent-02","tier":"easy","family":"intent","gold":"change_address","predicted":"cancel_order","correct":false,"schema_valid":true,"probs":{"billing_question":0.0536,"cancel_order":0.373,"change_address":0.2334,"report_damage":0.0956,"track_order":0.2444},"latency_s":0.0316,"input_tokens":93},{"id":"easy-intent-03","tier":"easy","family":"intent","gold":"report_damage","predicted":"cancel_order","correct":false,"schema_valid":true,"probs":{"billing_question":0.0605,"cancel_order":0.2978,"change_address":0.2304,"report_damage":0.1955,"track_order":0.2158},"latency_s":0.0576,"input_tokens":83},{"id":"easy-intent-04","tier":"easy","family":"intent","gold":"billing_question","predicted":"cancel_order","correct":false,"schema_valid":true,"probs":{"billing_question":0.2054,"cancel_order":0.2918,"change_address":0.2178,"report_damage":0.1088,"track_order":0.1763},"latency_s":0.058,"input_tokens":88},{"id":"easy-intent-05","tier":"easy","family":"intent","gold":"track_order","predicted":"cancel_order","correct":false,"schema_valid":true,"probs":{"billing_question":0.0337,"cancel_order":0.4093,"change_address":0.2546,"report_damage":0.1331,"track_order":0.1693},"latency_s":0.0306,"input_tokens":87},{"id":"easy-intent-06","tier":"easy","family":"intent","gold":"set_alarm","predicted":"send_message","correct":false,"schema_valid":true,"probs":{"play_music":0.1438,"send_message":0.3795,"set_alarm":0.2286,"turn_off_lights":0.1146,"weather":0.1335},"latency_s":0.0592,"input_tokens":74},{"id":"easy-intent-07","tier":"easy","family":"intent","gold":"play_music","predicted":"send_message","correct":false,"schema_valid":true,"probs":{"play_music":0.1639,"send_message":0.3251,"set_alarm":0.169,"turn_off_lights":0.0813,"weather":0.2608},"latency_s":0.0587,"input_tokens":68},{"id":"easy-intent-08","tier":"easy","family":"intent","gold":"weather","predicted":"weather","correct":true,"schema_valid":true,"probs":{"play_music":0.1037,"send_message":0.276,"set_alarm":0.1263,"turn_off_lights":0.0716,"weather":0.4224},"latency_s":0.0452,"input_tokens":70},{"id":"easy-intent-09","tier":"easy","family":"intent","gold":"send_message","predicted":"send_message","correct":true,"schema_valid":true,"probs":{"play_music":0.136,"send_message":0.4037,"set_alarm":0.1844,"turn_off_lights":0.1238,"weather":0.152},"latency_s":0.0618,"input_tokens":73},{"id":"easy-intent-10","tier":"easy","family":"intent","gold":"turn_off_lights","predicted":"send_message","correct":false,"schema_valid":true,"probs":{"play_music":0.1068,"send_message":0.3126,"set_alarm":0.2003,"turn_off_lights":0.273,"weather":0.1073},"latency_s":0.0605,"input_tokens":71},{"id":"easy-intent-11","tier":"easy","family":"intent","gold":"set_alarm","predicted":"set_alarm","correct":true,"schema_valid":true,"probs":{"play_music":0.0774,"send_message":0.1457,"set_alarm":0.6113,"turn_off_lights":0.0708,"weather":0.0948},"latency_s":0.0621,"input_tokens":70},{"id":"easy-tool_selection-00","tier":"easy","family":"tool_selection","gold":"get_weather","predicted":"get_weather","correct":true,"schema_valid":true,"probs":{"convert_currency":0.0848,"create_calendar_event":0.1805,"get_weather":0.4761,"send_email":0.1092,"translate_text":0.1494},"latency_s":0.0588,"input_tokens":78},{"id":"easy-tool_selection-01","tier":"easy","family":"tool_selection","gold":"send_email","predicted":"send_email","correct":true,"schema_valid":true,"probs":{"convert_currency":0.0979,"create_calendar_event":0.2266,"get_weather":0.1614,"send_email":0.3037,"translate_text":0.2104},"latency_s":0.0557,"input_tokens":79},{"id":"easy-tool_selection-02","tier":"easy","family":"tool_selection","gold":"create_calendar_event","predicted":"create_calendar_event","correct":true,"schema_valid":true,"probs":{"convert_currency":0.1659,"create_calendar_event":0.2828,"get_weather":0.1622,"send_email":0.2004,"translate_text":0.1887},"latency_s":0.0296,"input_tokens":81},{"id":"easy-tool_selection-03","tier":"easy","family":"tool_selection","gold":"convert_currency","predicted":"create_calendar_event","correct":false,"schema_valid":true,"probs":{"convert_currency":0.1315,"create_calendar_event":0.3036,"get_weather":0.1824,"send_email":0.1792,"translate_text":0.2032},"latency_s":0.0371,"input_tokens":77},{"id":"easy-tool_selection-04","tier":"easy","family":"tool_selection","gold":"translate_text","predicted":"translate_text","correct":true,"schema_valid":true,"probs":{"convert_currency":0.1272,"create_calendar_event":0.2591,"get_weather":0.178,"send_email":0.1765,"translate_text":0.2593},"latency_s":0.0603,"input_tokens":77},{"id":"easy-tool_selection-05","tier":"easy","family":"tool_selection","gold":"create_calendar_event","predicted":"create_calendar_event","correct":true,"schema_valid":true,"probs":{"convert_currency":0.1177,"create_calendar_event":0.3566,"get_weather":0.181,"send_email":0.1363,"translate_text":0.2085},"latency_s":0.0407,"input_tokens":80},{"id":"easy-tool_selection-06","tier":"easy","family":"tool_selection","gold":"search_flights","predicted":"search_flights","correct":true,"schema_valid":true,"probs":{"book_restaurant":0.1631,"call_taxi":0.2678,"get_stock_price":0.1393,"search_flights":0.288,"set_reminder":0.1418},"latency_s":0.0426,"input_tokens":84},{"id":"easy-tool_selection-07","tier":"easy","family":"tool_selection","gold":"book_restaurant","predicted":"book_restaurant","correct":true,"schema_valid":true,"probs":{"book_restaurant":0.3098,"call_taxi":0.219,"get_stock_price":0.1947,"search_flights":0.1269,"set_reminder":0.1495},"latency_s":0.06,"input_tokens":86},{"id":"easy-tool_selection-08","tier":"easy","family":"tool_selection","gold":"set_reminder","predicted":"call_taxi","correct":false,"schema_valid":true,"probs":{"book_restaurant":0.1358,"call_taxi":0.4177,"get_stock_price":0.1371,"search_flights":0.106,"set_reminder":0.2034},"latency_s":0.0518,"input_tokens":84},{"id":"easy-tool_selection-09","tier":"easy","family":"tool_selection","gold":"get_stock_price","predicted":"get_stock_price","correct":true,"schema_valid":true,"probs":{"book_restaurant":0.1398,"call_taxi":0.145,"get_stock_price":0.4637,"search_flights":0.123,"set_reminder":0.1284},"latency_s":0.0456,"input_tokens":82},{"id":"easy-tool_selection-10","tier":"easy","family":"tool_selection","gold":"call_taxi","predicted":"call_taxi","correct":true,"schema_valid":true,"probs":{"book_restaurant":0.1821,"call_taxi":0.3444,"get_stock_price":0.1685,"search_flights":0.1212,"set_reminder":0.1839},"latency_s":0.0343,"input_tokens":82},{"id":"easy-tool_selection-11","tier":"easy","family":"tool_selection","gold":"set_reminder","predicted":"set_reminder","correct":true,"schema_valid":true,"probs":{"book_restaurant":0.1859,"call_taxi":0.2247,"get_stock_price":0.1731,"search_flights":0.1558,"set_reminder":0.2604},"latency_s":0.0432,"input_tokens":84},{"id":"hard-opus-a-long_policy-01","tier":"hard","family":"long_policy","gold":"pay_subject_to_15000_sublimit","predicted":"deny_repeated_seepage","correct":false,"schema_valid":true,"probs":{"deny_repeated_seepage":0.2883,"deny_vacancy_exclusion":0.2698,"pay_full_estimate_less_deductible":0.1544,"pay_subject_to_10000_sublimit":0.1743,"pay_subject_to_15000_sublimit":0.113},"latency_s":0.1579,"input_tokens":3733},{"id":"hard-opus-a-long_policy-04","tier":"hard","family":"long_policy","gold":"cfo","predicted":"department_head","correct":false,"schema_valid":true,"probs":{"budget_holder":0.1871,"cfo":0.1717,"department_head":0.2606,"executive_committee":0.164,"vp_and_finance_director":0.2166},"latency_s":0.1405,"input_tokens":2460},{"id":"hard-opus-a-long_policy-08","tier":"hard","family":"long_policy","gold":"parts_covered_labour_charged","predicted":"parts_covered_labour_charged","correct":true,"schema_valid":true,"probs":{"full_repair_no_charge":0.2368,"not_covered":0.2966,"parts_covered_labour_charged":0.304,"pro_rata_replacement_credit":0.1627},"latency_s":0.169,"input_tokens":2293},{"id":"hard-opus-a-long_policy-09","tier":"hard","family":"long_policy","gold":"license_required","predicted":"license_required","correct":true,"schema_valid":true,"probs":{"license_exception_lvs":0.1896,"license_required":0.3492,"no_license_required":0.258,"shipment_prohibited":0.2033},"latency_s":0.1598,"input_tokens":2402},{"id":"hard-opus-a-long_policy-11","tier":"hard","family":"long_policy","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.6038,"no":0.3962},"latency_s":0.1729,"input_tokens":2146},{"id":"hard-opus-a-long_policy-13","tier":"hard","family":"long_policy","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.7145,"no":0.2855},"latency_s":0.1378,"input_tokens":2280},{"id":"hard-opus-a-long_policy-17","tier":"hard","family":"long_policy","gold":"database_oncall","predicted":"vendor_support","correct":false,"schema_valid":true,"probs":{"database_oncall":0.1408,"no_page_ticket_only":0.1385,"platform_oncall":0.1491,"security_incident_response":0.2039,"vendor_support":0.3678},"latency_s":0.1621,"input_tokens":2287},{"id":"hard-opus-a-long_policy-19","tier":"hard","family":"long_policy","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.5605,"no":0.4395},"latency_s":0.1406,"input_tokens":2106},{"id":"hard-opus-a-probability-03","tier":"hard","family":"probability","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.5543,"no":0.4457},"latency_s":0.0754,"input_tokens":376},{"id":"hard-opus-a-probability-04","tier":"hard","family":"probability","gold":"on_time","predicted":"on_time","correct":true,"schema_valid":true,"probs":{"early":0.3126,"late":0.2837,"on_time":0.4037},"latency_s":0.128,"input_tokens":900},{"id":"hard-opus-a-probability-07","tier":"hard","family":"probability","gold":"escalated","predicted":"escalated","correct":true,"schema_valid":true,"probs":{"escalated":0.5212,"reopened":0.2167,"resolved_first_contact":0.2621},"latency_s":0.149,"input_tokens":954},{"id":"hard-opus-a-probability-08","tier":"hard","family":"probability","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.4574,"no":0.5426},"latency_s":0.058,"input_tokens":312},{"id":"hard-opus-a-temporal_numeric-03","tier":"hard","family":"temporal_numeric","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.5746,"no":0.4254},"latency_s":0.0766,"input_tokens":417},{"id":"hard-opus-a-temporal_numeric-06","tier":"hard","family":"temporal_numeric","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.6917,"no":0.3083},"latency_s":0.0854,"input_tokens":418},{"id":"hard-opus-a-temporal_numeric-07","tier":"hard","family":"temporal_numeric","gold":"sep_26","predicted":"sep_24","correct":false,"schema_valid":true,"probs":{"sep_24":0.2782,"sep_25":0.2029,"sep_26":0.253,"sep_27":0.215,"sep_28":0.0509},"latency_s":0.127,"input_tokens":908},{"id":"hard-opus-a-temporal_numeric-09","tier":"hard","family":"temporal_numeric","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.4128,"no":0.5872},"latency_s":0.0871,"input_tokens":439},{"id":"hard-opus-a-temporal_numeric-12","tier":"hard","family":"temporal_numeric","gold":1,"predicted":"2","correct":false,"schema_valid":true,"probs":{"0":0.0433,"1":0.3475,"2":0.3598,"3":0.2494},"latency_s":0.0675,"input_tokens":461},{"id":"hard-opus-b-ambiguous-02","tier":"hard","family":"ambiguous","gold":"real_incident","predicted":"false_positive","correct":false,"schema_valid":true,"probs":{"cannot_determine":0.3325,"false_positive":0.4605,"real_incident":0.207},"latency_s":0.0833,"input_tokens":484},{"id":"hard-opus-b-ambiguous-03","tier":"hard","family":"ambiguous","gold":"time_barred","predicted":"time_barred","correct":true,"schema_valid":true,"probs":{"insufficient_information":0.2073,"time_barred":0.4235,"within_limitation":0.3693},"latency_s":0.062,"input_tokens":441},{"id":"hard-opus-b-ambiguous-07","tier":"hard","family":"ambiguous","gold":"within_policy","predicted":"within_policy","correct":true,"schema_valid":true,"probs":{"insufficient_information":0.2732,"over_limit":0.2787,"within_policy":0.4481},"latency_s":0.0689,"input_tokens":439},{"id":"hard-opus-b-ambiguous-09","tier":"hard","family":"ambiguous","gold":"legitimate","predicted":"legitimate","correct":true,"schema_valid":true,"probs":{"compromised":0.1468,"inconclusive":0.3812,"legitimate":0.4719},"latency_s":0.0825,"input_tokens":589},{"id":"hard-opus-b-ambiguous-10","tier":"hard","family":"ambiguous","gold":"ask_user_which_alex","predicted":"send_to_alexandra_lind","correct":false,"schema_valid":true,"probs":{"ask_user_which_alex":0.327,"send_to_alex_moreno":0.2866,"send_to_alexandra_lind":0.3864},"latency_s":0.0689,"input_tokens":446},{"id":"hard-opus-b-ambiguous-11","tier":"hard","family":"ambiguous","gold":"cannot_determine","predicted":"cannot_determine","correct":true,"schema_valid":true,"probs":{"cannot_determine":0.4065,"grade_correct":0.301,"grade_should_be_higher":0.2924},"latency_s":0.0824,"input_tokens":396},{"id":"hard-opus-b-ambiguous-13","tier":"hard","family":"ambiguous","gold":"cannot_determine","predicted":"charge_platform_team","correct":false,"schema_valid":true,"probs":{"cannot_determine":0.1636,"charge_data_team":0.2804,"charge_platform_team":0.3648,"split_50_50":0.1912},"latency_s":0.095,"input_tokens":525},{"id":"hard-opus-b-multi_hop-03","tier":"hard","family":"multi_hop","gold":"tier3_cfo","predicted":"tier4_board","correct":false,"schema_valid":true,"probs":{"tier1_team_lead":0.1959,"tier2_department_head":0.2239,"tier3_cfo":0.2646,"tier4_board":0.3155},"latency_s":0.1365,"input_tokens":2696},{"id":"hard-opus-b-multi_hop-04","tier":"hard","family":"multi_hop","gold":"bjorn","predicted":"no_page_ticket_only","correct":false,"schema_valid":true,"probs":{"ana":0.1455,"bjorn":0.1652,"chen":0.1426,"dmitri":0.1851,"esra":0.1717,"no_page_ticket_only":0.1899},"latency_s":0.1472,"input_tokens":2200},{"id":"hard-opus-b-multi_hop-05","tier":"hard","family":"multi_hop","gold":2,"predicted":"1","correct":false,"schema_valid":true,"probs":{"0":0.0275,"1":0.3448,"2":0.236,"3":0.2465,"4":0.1452},"latency_s":0.1575,"input_tokens":2293},{"id":"hard-opus-b-multi_hop-07","tier":"hard","family":"multi_hop","gold":"credit_20_percent","predicted":"credit_20_percent","correct":true,"schema_valid":true,"probs":{"credit_10_percent":0.2165,"credit_20_percent":0.2648,"credit_25_percent":0.1722,"credit_50_percent":0.0962,"credit_full_month":0.2393,"no_credit":0.011},"latency_s":0.1444,"input_tokens":2260},{"id":"hard-opus-b-multi_hop-08","tier":"hard","family":"multi_hop","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.6184,"no":0.3816},"latency_s":0.143,"input_tokens":2021},{"id":"hard-opus-b-probability-01","tier":"hard","family":"probability","gold":"merchant_wins","predicted":"customer_withdraws","correct":false,"schema_valid":true,"probs":{"customer_wins":0.2618,"customer_withdraws":0.4901,"merchant_wins":0.2481},"latency_s":0.124,"input_tokens":1639},{"id":"hard-opus-b-probability-02","tier":"hard","family":"probability","gold":"upstream_provider","predicted":"bad_push","correct":false,"schema_valid":true,"probs":{"bad_push":0.998,"database_hardware":0.0004,"upstream_provider":0.0016},"latency_s":0.0843,"input_tokens":434},{"id":"hard-opus-b-probability-03","tier":"hard","family":"probability","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.5934,"no":0.4066},"latency_s":0.1446,"input_tokens":971},{"id":"hard-opus-b-probability-04","tier":"hard","family":"probability","gold":"attended","predicted":"attended","correct":true,"schema_valid":true,"probs":{"attended":0.4967,"late_cancellation":0.2126,"no_show":0.2906},"latency_s":0.0716,"input_tokens":470},{"id":"hard-opus-b-probability-06","tier":"hard","family":"probability","gold":"upheld","predicted":"upheld","correct":true,"schema_valid":true,"probs":{"modified":0.2758,"overturned":0.3156,"upheld":0.4086},"latency_s":0.1346,"input_tokens":878},{"id":"hard-opus-b-tradeoff-01","tier":"hard","family":"tradeoff","gold":"escalate_to_engineering","predicted":"escalate_to_security","correct":false,"schema_valid":true,"probs":{"escalate_to_engineering":0.2251,"escalate_to_security":0.4002,"page_oncall":0.3265,"process_refund":0.0157,"send_kb_article":0.0325},"latency_s":0.1293,"input_tokens":799},{"id":"hard-opus-b-tradeoff-03","tier":"hard","family":"tradeoff","gold":"order_b","predicted":"order_c","correct":false,"schema_valid":true,"probs":{"order_a":0.0996,"order_b":0.2763,"order_c":0.3978,"order_d":0.2263},"latency_s":0.0888,"input_tokens":648},{"id":"hard-opus-b-tradeoff-06","tier":"hard","family":"tradeoff","gold":"yes","predicted":"no","correct":false,"schema_valid":true,"probs":{"yes":0.3799,"no":0.6201},"latency_s":0.0971,"input_tokens":665},{"id":"hard-opus-b-tradeoff-07","tier":"hard","family":"tradeoff","gold":"p0_fix_24h","predicted":"p0_fix_24h","correct":true,"schema_valid":true,"probs":{"out_of_scope":0.1582,"p0_fix_24h":0.3122,"p1_fix_7d":0.1705,"p2_fix_30d":0.2519,"p3_backlog":0.1072},"latency_s":0.1132,"input_tokens":674},{"id":"hard-opus-b-tradeoff-08","tier":"hard","family":"tradeoff","gold":"yes","predicted":"no","correct":false,"schema_valid":true,"probs":{"yes":0.4111,"no":0.5889},"latency_s":0.0929,"input_tokens":513},{"id":"hard-opus-b-tradeoff-12","tier":"hard","family":"tradeoff","gold":"refund_now","predicted":"deny_refund","correct":false,"schema_valid":true,"probs":{"deny_refund":0.2377,"escalate_to_fraud_team":0.2373,"offer_store_credit":0.1101,"refund_now":0.2095,"wait_for_inspection":0.2053},"latency_s":0.0865,"input_tokens":534},{"id":"hard-opus-c-long_policy-02","tier":"hard","family":"long_policy","gold":"erase_but_retain_transaction_records","predicted":"erase_but_retain_transaction_records","correct":true,"schema_valid":true,"probs":{"erase_all":0.182,"erase_but_retain_transaction_records":0.2812,"erase_marketing_only":0.1125,"reject_unverified_request":0.2049,"restrict_all_pending_legal_hold":0.2195},"latency_s":0.1677,"input_tokens":3272},{"id":"hard-opus-c-long_policy-03","tier":"hard","family":"long_policy","gold":"eur_400","predicted":"eur_300","correct":false,"schema_valid":true,"probs":{"eur_0":0.0851,"eur_150":0.1972,"eur_200":0.1491,"eur_300":0.2031,"eur_400":0.1759,"eur_600":0.1896},"latency_s":0.1609,"input_tokens":3221},{"id":"hard-opus-c-long_policy-04","tier":"hard","family":"long_policy","gold":"yes","predicted":"no","correct":false,"schema_valid":true,"probs":{"yes":0.389,"no":0.611},"latency_s":0.1427,"input_tokens":2953},{"id":"hard-opus-c-long_policy-05","tier":"hard","family":"long_policy","gold":1,"predicted":"0","correct":false,"schema_valid":true,"probs":{"0":0.2821,"1":0.2394,"2":0.2653,"3":0.2132},"latency_s":0.1594,"input_tokens":3035},{"id":"hard-opus-c-long_policy-08","tier":"hard","family":"long_policy","gold":1,"predicted":"3","correct":false,"schema_valid":true,"probs":{"0":0.1333,"1":0.0302,"2":0.1451,"3":0.6913},"latency_s":0.1474,"input_tokens":3019},{"id":"hard-opus-c-long_policy-10","tier":"hard","family":"long_policy","gold":"credit_minus_restocking_fee","predicted":"full_credit","correct":false,"schema_valid":true,"probs":{"credit_minus_restocking_fee":0.2201,"full_credit":0.3162,"reject_return":0.2261,"repair_or_replace_only":0.2376},"latency_s":0.1583,"input_tokens":3257},{"id":"hard-opus-c-long_policy-11","tier":"hard","family":"long_policy","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.546,"no":0.454},"latency_s":0.139,"input_tokens":3189},{"id":"hard-opus-c-probability-03","tier":"hard","family":"probability","gold":"kuznets_technik","predicted":"halvorsen_polymer","correct":false,"schema_valid":true,"probs":{"halvorsen_polymer":0.751,"kuznets_technik":0.1001,"ortega_seal_works":0.1489},"latency_s":0.0872,"input_tokens":510},{"id":"hard-opus-c-temporal_numeric-02","tier":"hard","family":"temporal_numeric","gold":"yes","predicted":"no","correct":false,"schema_valid":true,"probs":{"yes":0.3672,"no":0.6328},"latency_s":0.1031,"input_tokens":778},{"id":"hard-opus-c-temporal_numeric-03","tier":"hard","family":"temporal_numeric","gold":"eur_692_00","predicted":"eur_693_90","correct":false,"schema_valid":true,"probs":{"eur_617_00":0.0488,"eur_688_00":0.2281,"eur_692_00":0.1824,"eur_693_90":0.4157,"eur_812_00":0.125},"latency_s":0.1123,"input_tokens":725},{"id":"hard-opus-c-temporal_numeric-04","tier":"hard","family":"temporal_numeric","gold":"usd_2335_00","predicted":"usd_2303_01","correct":false,"schema_valid":true,"probs":{"usd_2222_50":0.0773,"usd_2303_01":0.5491,"usd_2303_33":0.0969,"usd_2335_00":0.1336,"usd_2340_00":0.1432},"latency_s":0.1118,"input_tokens":619},{"id":"hard-opus-c-temporal_numeric-06","tier":"hard","family":"temporal_numeric","gold":"sep_18_book_set","predicted":"none_declined","correct":false,"schema_valid":true,"probs":{"none_declined":0.3108,"sep_15_office_chair":0.1146,"sep_16_taxi":0.1151,"sep_17_parking":0.1405,"sep_18_book_set":0.1687,"sep_21_client_dinner":0.1503},"latency_s":0.1247,"input_tokens":857},{"id":"hard-opus-c-temporal_numeric-08","tier":"hard","family":"temporal_numeric","gold":"eur_498_81","predicted":"eur_506_06","correct":false,"schema_valid":true,"probs":{"eur_497_40":0.1736,"eur_498_81":0.2324,"eur_506_06":0.3053,"eur_513_69":0.1925,"eur_532_01":0.0961},"latency_s":0.1173,"input_tokens":792},{"id":"hard-opus-c-temporal_numeric-12","tier":"hard","family":"temporal_numeric","gold":"expired_30_month_cap","predicted":"covered","correct":false,"schema_valid":true,"probs":{"covered":0.5129,"expired_24_month_term":0.3692,"expired_30_month_cap":0.1179},"latency_s":0.1118,"input_tokens":714},{"id":"hard-sol-a-adversarial-01","tier":"hard","family":"adversarial","gold":"deny_missing_approval","predicted":"deny_missing_approval","correct":true,"schema_valid":true,"probs":{"deny_inactive_user":0.2848,"deny_missing_approval":0.2959,"grant_access":0.2343,"grant_owner_exception":0.1851},"latency_s":0.0553,"input_tokens":272},{"id":"hard-sol-a-adversarial-06","tier":"hard","family":"adversarial","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.477,"no":0.523},"latency_s":0.0423,"input_tokens":211},{"id":"hard-sol-a-adversarial-07","tier":"hard","family":"adversarial","gold":"retain_until_2031","predicted":"needs_signature_review","correct":false,"schema_valid":true,"probs":{"delete_now":0.1032,"needs_signature_review":0.5899,"retain_indefinitely":0.1415,"retain_until_2031":0.1654},"latency_s":0.0675,"input_tokens":249},{"id":"hard-sol-a-adversarial-08","tier":"hard","family":"adversarial","gold":"yes","predicted":"no","correct":false,"schema_valid":true,"probs":{"yes":0.4644,"no":0.5356},"latency_s":0.0536,"input_tokens":197},{"id":"hard-sol-a-adversarial-09","tier":"hard","family":"adversarial","gold":2,"predicted":"3","correct":false,"schema_valid":true,"probs":{"0":0.0943,"1":0.3025,"2":0.2862,"3":0.3169},"latency_s":0.0389,"input_tokens":191},{"id":"hard-sol-a-adversarial-11","tier":"hard","family":"adversarial","gold":"standard_research","predicted":"standard_research","correct":true,"schema_valid":true,"probs":{"ethics_review":0.3624,"major_grants":0.1323,"manual_hold":0.1358,"standard_research":0.3695},"latency_s":0.0595,"input_tokens":262},{"id":"hard-sol-a-multi_hop-01","tier":"hard","family":"multi_hop","gold":"accept_s2_dusk","predicted":"accept_s2_dusk","correct":true,"schema_valid":true,"probs":{"accept_s2_dusk":0.299,"reject_cutoff":0.2514,"reject_tier":0.2516,"reroute_s3":0.198},"latency_s":0.1756,"input_tokens":2341},{"id":"hard-sol-a-multi_hop-02","tier":"hard","family":"multi_hop","gold":"security_primary","predicted":"executive_bridge","correct":false,"schema_valid":true,"probs":{"executive_bridge":0.2686,"heron_q2":0.2492,"insufficient_information":0.2182,"security_primary":0.264},"latency_s":0.1557,"input_tokens":2369},{"id":"hard-sol-a-multi_hop-05","tier":"hard","family":"multi_hop","gold":"require_director","predicted":"pay_manager_approved","correct":false,"schema_valid":true,"probs":{"pay_manager_approved":0.3963,"reject_category":0.159,"require_director":0.2325,"require_finance_board":0.2122},"latency_s":0.1463,"input_tokens":2308},{"id":"hard-sol-a-multi_hop-07","tier":"hard","family":"multi_hop","gold":"admit","predicted":"admit","correct":true,"schema_valid":true,"probs":{"admit":0.2661,"deny_prerequisite":0.25,"deny_score":0.224,"manual_equivalency":0.2598},"latency_s":0.1449,"input_tokens":2269},{"id":"hard-sol-a-multi_hop-08","tier":"hard","family":"multi_hop","gold":"central_engineering","predicted":"central_engineering","correct":true,"schema_valid":true,"probs":{"central_engineering":0.3076,"insufficient_log":0.2228,"local_maintenance":0.2735,"safety_shutdown":0.1961},"latency_s":0.2213,"input_tokens":2264},{"id":"hard-sol-a-multi_hop-09","tier":"hard","family":"multi_hop","gold":"require_steward","predicted":"confirm_l3","correct":false,"schema_valid":true,"probs":{"confirm_l3":0.3368,"reject_capacity":0.2388,"require_steward":0.2082,"switch_l4":0.2161},"latency_s":0.1519,"input_tokens":2271},{"id":"hard-sol-a-multi_hop-10","tier":"hard","family":"multi_hop","gold":"approve_45_days","predicted":"needs_field_review","correct":false,"schema_valid":true,"probs":{"approve_45_days":0.1328,"deny_sensitive":0.129,"needs_field_review":0.4412,"reduce_to_30_days":0.297},"latency_s":0.1548,"input_tokens":2304},{"id":"hard-sol-a-multi_hop-12","tier":"hard","family":"multi_hop","gold":"needs_authentication","predicted":"needs_authentication","correct":true,"schema_valid":true,"probs":{"covered_recurrence":0.2692,"expired":0.1624,"needs_authentication":0.3298,"wrong_issue_class":0.2386},"latency_s":0.1797,"input_tokens":2267},{"id":"hard-sol-a-trap-02","tier":"hard","family":"trap","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.4236,"no":0.5764},"latency_s":0.0724,"input_tokens":263},{"id":"hard-sol-a-trap-04","tier":"hard","family":"trap","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.5427,"no":0.4573},"latency_s":0.066,"input_tokens":166},{"id":"hard-sol-a-trap-05","tier":"hard","family":"trap","gold":0,"predicted":"2","correct":false,"schema_valid":true,"probs":{"0":0.0685,"1":0.2451,"2":0.3826,"3":0.3038},"latency_s":0.0456,"input_tokens":279},{"id":"hard-sol-a-trap-06","tier":"hard","family":"trap","gold":"keep_open_unresolved","predicted":"await_internal_metrics","correct":false,"schema_valid":true,"probs":{"await_internal_metrics":0.3752,"close_agent_verified":0.1917,"close_confirmed":0.1777,"keep_open_unresolved":0.2555},"latency_s":0.0505,"input_tokens":304},{"id":"hard-sol-a-trap-08","tier":"hard","family":"trap","gold":"late","predicted":"on_time","correct":false,"schema_valid":true,"probs":{"canceled":0.2374,"indeterminate":0.2094,"late":0.2074,"on_time":0.3457},"latency_s":0.0561,"input_tokens":225},{"id":"hard-sol-a-trap-10","tier":"hard","family":"trap","gold":"blocked_second_review","predicted":"blocked_failed_test","correct":false,"schema_valid":true,"probs":{"blocked_failed_test":0.3185,"blocked_second_review":0.1817,"ready_one_review":0.2154,"ready_two_reviews":0.2844},"latency_s":0.0455,"input_tokens":174},{"id":"hard-sol-a-trap-13","tier":"hard","family":"trap","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.5165,"no":0.4835},"latency_s":0.0627,"input_tokens":169},{"id":"hard-sol-a-trap-15","tier":"hard","family":"trap","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.4701,"no":0.5299},"latency_s":0.074,"input_tokens":257},{"id":"hard-sol-b-judge_hard-01","tier":"hard","family":"judge_hard","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.7405,"no":0.2595},"latency_s":0.0661,"input_tokens":208},{"id":"hard-sol-b-judge_hard-02","tier":"hard","family":"judge_hard","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.6813,"no":0.3187},"latency_s":0.0585,"input_tokens":163},{"id":"hard-sol-b-judge_hard-05","tier":"hard","family":"judge_hard","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.5378,"no":0.4622},"latency_s":0.0697,"input_tokens":210},{"id":"hard-sol-b-judge_hard-08","tier":"hard","family":"judge_hard","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.6388,"no":0.3612},"latency_s":0.0624,"input_tokens":149},{"id":"hard-sol-b-judge_hard-10","tier":"hard","family":"judge_hard","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.6984,"no":0.3016},"latency_s":0.0509,"input_tokens":172},{"id":"hard-sol-b-judge_hard-14","tier":"hard","family":"judge_hard","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.5909,"no":0.4091},"latency_s":0.0725,"input_tokens":204},{"id":"hard-sol-b-judge_hard-15","tier":"hard","family":"judge_hard","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.7375,"no":0.2625},"latency_s":0.0586,"input_tokens":153},{"id":"hard-sol-b-judge_hard-18","tier":"hard","family":"judge_hard","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.8035,"no":0.1965},"latency_s":0.0463,"input_tokens":139},{"id":"hard-sol-b-long_policy-01","tier":"hard","family":"long_policy","gold":"reject_entire_claim","predicted":"approve_lodging_only","correct":false,"schema_valid":true,"probs":{"approve_all":0.2218,"approve_lodging_only":0.3045,"approve_meals_only":0.2492,"reject_entire_claim":0.2246},"latency_s":0.1679,"input_tokens":2701},{"id":"hard-sol-b-long_policy-02","tier":"hard","family":"long_policy","gold":"deny_current_export","predicted":"analytics_owner_only","correct":false,"schema_valid":true,"probs":{"analytics_owner_only":0.2973,"deny_current_export":0.262,"privacy_and_security":0.2691,"privacy_only":0.1716},"latency_s":0.1527,"input_tokens":2586},{"id":"hard-sol-b-long_policy-05","tier":"hard","family":"long_policy","gold":"delete_profile_omar_april_and_table","predicted":"delete_profile_omar_april_and_table","correct":true,"schema_valid":true,"probs":{"delete_everything":0.2586,"delete_nothing":0.1752,"delete_only_profile":0.2229,"delete_profile_omar_april_and_table":0.3433},"latency_s":0.1615,"input_tokens":2530},{"id":"hard-sol-b-long_policy-06","tier":"hard","family":"long_policy","gold":"east_ward","predicted":"river_arts","correct":false,"schema_valid":true,"probs":{"east_ward":0.277,"harbor_neighbors":0.2583,"no_award":0.1669,"river_arts":0.2978},"latency_s":0.1529,"input_tokens":2547},{"id":"hard-sol-b-routing_hard-01","tier":"hard","family":"routing_hard","gold":"office_file_repair","predicted":"office_file_repair","correct":true,"schema_valid":true,"probs":{"data_pipeline":0.1305,"desktop_support":0.1139,"finance_review":0.2449,"office_file_repair":0.2753,"spreadsheet_formula":0.2354},"latency_s":0.0631,"input_tokens":306},{"id":"hard-sol-b-routing_hard-02","tier":"hard","family":"routing_hard","gold":"ledger_reconciliation","predicted":"ledger_reconciliation","correct":true,"schema_valid":true,"probs":{"customer_support":0.0948,"fraud_review":0.0471,"ledger_reconciliation":0.5639,"mobile_app":0.106,"payment_disputes":0.1882},"latency_s":0.0687,"input_tokens":275},{"id":"hard-sol-b-routing_hard-03","tier":"hard","family":"routing_hard","gold":"network_dns","predicted":"network_dns","correct":true,"schema_valid":true,"probs":{"api_platform":0.2474,"authoritative_dns":0.113,"endpoint_devices":0.0902,"network_dns":0.483,"tls_certificates":0.0664},"latency_s":0.064,"input_tokens":289},{"id":"hard-sol-b-routing_hard-07","tier":"hard","family":"routing_hard","gold":"release_notarization","predicted":"macos_runtime","correct":false,"schema_valid":true,"probs":{"ci_infrastructure":0.141,"code_signing":0.2217,"endpoint_security":0.0517,"macos_runtime":0.4254,"release_notarization":0.1603},"latency_s":0.0618,"input_tokens":254},{"id":"hard-sol-b-routing_hard-09","tier":"hard","family":"routing_hard","gold":"source_access","predicted":"identity_login","correct":false,"schema_valid":true,"probs":{"git_client":0.1506,"identity_login":0.3566,"repository_admin":0.1357,"source_access":0.0907,"vpn_network":0.2664},"latency_s":0.0658,"input_tokens":266},{"id":"hard-sol-b-temporal_numeric-01","tier":"hard","family":"temporal_numeric","gold":"late_by_under_2h","predicted":"before_open","correct":false,"schema_valid":true,"probs":{"before_open":0.3341,"late_by_2h_or_more":0.1553,"late_by_under_2h":0.18,"within_window":0.3306},"latency_s":0.0602,"input_tokens":221},{"id":"hard-sol-b-temporal_numeric-02","tier":"hard","family":"temporal_numeric","gold":"over_limit","predicted":"over_limit","correct":true,"schema_valid":true,"probs":{"at_least_100gb_below":0.1853,"exactly_equal":0.1098,"over_limit":0.4106,"under_100gb_below":0.2942},"latency_s":0.0617,"input_tokens":221},{"id":"hard-sol-b-temporal_numeric-03","tier":"hard","family":"temporal_numeric","gold":"9_credits","predicted":"6_credits","correct":false,"schema_valid":true,"probs":{"12_credits":0.2509,"15_credits":0.1567,"6_credits":0.308,"9_credits":0.2843},"latency_s":0.0591,"input_tokens":173},{"id":"hard-sol-b-temporal_numeric-04","tier":"hard","family":"temporal_numeric","gold":"pass","predicted":"narrow_fail","correct":false,"schema_valid":true,"probs":{"clear_fail":0.4003,"narrow_fail":0.4169,"pass":0.1829},"latency_s":0.0625,"input_tokens":187},{"id":"hard-sol-c-judge_hard-03","tier":"hard","family":"judge_hard","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.5766,"no":0.4234},"latency_s":0.0645,"input_tokens":392},{"id":"hard-sol-c-judge_hard-05","tier":"hard","family":"judge_hard","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.5594,"no":0.4406},"latency_s":0.0728,"input_tokens":299},{"id":"hard-sol-c-judge_hard-07","tier":"hard","family":"judge_hard","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.5964,"no":0.4036},"latency_s":0.0698,"input_tokens":272},{"id":"hard-sol-c-judge_hard-08","tier":"hard","family":"judge_hard","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.5504,"no":0.4496},"latency_s":0.0758,"input_tokens":286},{"id":"hard-sol-c-judge_hard-09","tier":"hard","family":"judge_hard","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.541,"no":0.459},"latency_s":0.0487,"input_tokens":280},{"id":"hard-sol-c-judge_hard-10","tier":"hard","family":"judge_hard","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.608,"no":0.392},"latency_s":0.0663,"input_tokens":297},{"id":"hard-sol-c-judge_hard-11","tier":"hard","family":"judge_hard","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.4348,"no":0.5652},"latency_s":0.0644,"input_tokens":316},{"id":"hard-sol-c-judge_hard-13","tier":"hard","family":"judge_hard","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.5586,"no":0.4414},"latency_s":0.0704,"input_tokens":275},{"id":"hard-sol-c-judge_hard-15","tier":"hard","family":"judge_hard","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.5645,"no":0.4355},"latency_s":0.0638,"input_tokens":260},{"id":"hard-sol-c-multi_hop-07","tier":"hard","family":"multi_hop","gold":"programme_review","predicted":"programme_review","correct":true,"schema_valid":true,"probs":{"partial_release":0.2618,"programme_review":0.2937,"reject_draw":0.1608,"release_draw":0.2837},"latency_s":0.1417,"input_tokens":2451},{"id":"hard-sol-c-multi_hop-09","tier":"hard","family":"multi_hop","gold":"bus_substitution","predicted":"call_fen_cross","correct":false,"schema_valid":true,"probs":{"bus_substitution":0.2252,"call_fen_cross":0.2838,"pass_fen_cross":0.2535,"terminate_oakmere":0.2375},"latency_s":0.1592,"input_tokens":2456},{"id":"hard-sol-c-multi_hop-11","tier":"hard","family":"multi_hop","gold":"coverage_excluded","predicted":"documentation_needed","correct":false,"schema_valid":true,"probs":{"adjuster_referral":0.2585,"coverage_confirmed":0.2482,"coverage_excluded":0.2024,"documentation_needed":0.2908},"latency_s":0.1704,"input_tokens":2445},{"id":"hard-sol-c-multi_hop-12","tier":"hard","family":"multi_hop","gold":"conservation_hold","predicted":"supervised_delivery","correct":false,"schema_valid":true,"probs":{"conservation_hold":0.2825,"deny_request":0.1431,"digital_copy_only":0.1482,"supervised_delivery":0.4262},"latency_s":0.0878,"input_tokens":480},{"id":"hard-sol-c-multi_hop-13","tier":"hard","family":"multi_hop","gold":"normal_release","predicted":"scrap_lot","correct":false,"schema_valid":true,"probs":{"engineering_review":0.2875,"normal_release":0.2292,"rework":0.1403,"scrap_lot":0.3431},"latency_s":0.1569,"input_tokens":2514},{"id":"original-adequacy-01-0","tier":"original","family":"adequacy","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.5388,"no":0.4612},"latency_s":0.041,"input_tokens":51},{"id":"original-adequacy-01-1","tier":"original","family":"adequacy","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.6654,"no":0.3346},"latency_s":0.0365,"input_tokens":48},{"id":"original-adequacy-02-0","tier":"original","family":"adequacy","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.722,"no":0.278},"latency_s":0.0533,"input_tokens":55},{"id":"original-adequacy-02-1","tier":"original","family":"adequacy","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.4291,"no":0.5709},"latency_s":0.0551,"input_tokens":56},{"id":"original-adequacy-03-0","tier":"original","family":"adequacy","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.7088,"no":0.2912},"latency_s":0.0557,"input_tokens":55},{"id":"original-adequacy-03-1","tier":"original","family":"adequacy","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.6977,"no":0.3023},"latency_s":0.0391,"input_tokens":58},{"id":"original-adequacy-04-0","tier":"original","family":"adequacy","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.5787,"no":0.4213},"latency_s":0.0562,"input_tokens":61},{"id":"original-adequacy-04-1","tier":"original","family":"adequacy","gold":"yes","predicted":"no","correct":false,"schema_valid":true,"probs":{"yes":0.4885,"no":0.5115},"latency_s":0.0505,"input_tokens":58},{"id":"original-adequacy-05-0","tier":"original","family":"adequacy","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.7198,"no":0.2802},"latency_s":0.0489,"input_tokens":49},{"id":"original-adequacy-05-1","tier":"original","family":"adequacy","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.6716,"no":0.3284},"latency_s":0.0524,"input_tokens":50},{"id":"original-adequacy-06-0","tier":"original","family":"adequacy","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.5772,"no":0.4228},"latency_s":0.0519,"input_tokens":61},{"id":"original-adequacy-06-1","tier":"original","family":"adequacy","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.7486,"no":0.2514},"latency_s":0.0412,"input_tokens":61},{"id":"original-extraction-01-0","tier":"original","family":"extraction","gold":"pickup","predicted":"pickup","correct":true,"schema_valid":true,"probs":{"courier":0.2186,"pickup":0.594,"post":0.1254,"unknown":0.062},"latency_s":0.053,"input_tokens":59},{"id":"original-extraction-01-1","tier":"original","family":"extraction","gold":"pickup","predicted":"pickup","correct":true,"schema_valid":true,"probs":{"courier":0.2015,"pickup":0.6439,"post":0.0917,"unknown":0.063},"latency_s":0.0558,"input_tokens":61},{"id":"original-extraction-02-0","tier":"original","family":"extraction","gold":"unknown","predicted":"post","correct":false,"schema_valid":true,"probs":{"courier":0.282,"pickup":0.149,"post":0.3823,"unknown":0.1867},"latency_s":0.0529,"input_tokens":60},{"id":"original-extraction-02-1","tier":"original","family":"extraction","gold":"unknown","predicted":"courier","correct":false,"schema_valid":true,"probs":{"courier":0.5022,"pickup":0.1433,"post":0.1792,"unknown":0.1753},"latency_s":0.0563,"input_tokens":60},{"id":"original-extraction-03-0","tier":"original","family":"extraction","gold":"courier","predicted":"courier","correct":true,"schema_valid":true,"probs":{"courier":0.5593,"pickup":0.285,"post":0.0914,"unknown":0.0643},"latency_s":0.0565,"input_tokens":64},{"id":"original-extraction-03-1","tier":"original","family":"extraction","gold":"courier","predicted":"courier","correct":true,"schema_valid":true,"probs":{"courier":0.5631,"pickup":0.1417,"post":0.1396,"unknown":0.1556},"latency_s":0.0515,"input_tokens":63},{"id":"original-extraction-04-0","tier":"original","family":"extraction","gold":"post","predicted":"post","correct":true,"schema_valid":true,"probs":{"courier":0.2023,"pickup":0.1459,"post":0.5129,"unknown":0.1389},"latency_s":0.0525,"input_tokens":60},{"id":"original-extraction-04-1","tier":"original","family":"extraction","gold":"post","predicted":"post","correct":true,"schema_valid":true,"probs":{"courier":0.3403,"pickup":0.1116,"post":0.4456,"unknown":0.1025},"latency_s":0.053,"input_tokens":59},{"id":"original-extraction-05-0","tier":"original","family":"extraction","gold":"unknown","predicted":"courier","correct":false,"schema_valid":true,"probs":{"courier":0.4159,"pickup":0.1822,"post":0.2888,"unknown":0.113},"latency_s":0.0536,"input_tokens":61},{"id":"original-extraction-05-1","tier":"original","family":"extraction","gold":"unknown","predicted":"courier","correct":false,"schema_valid":true,"probs":{"courier":0.4169,"pickup":0.1447,"post":0.2022,"unknown":0.2361},"latency_s":0.0543,"input_tokens":57},{"id":"original-extraction-06-0","tier":"original","family":"extraction","gold":"courier","predicted":"courier","correct":true,"schema_valid":true,"probs":{"courier":0.4517,"pickup":0.1,"post":0.2639,"unknown":0.1845},"latency_s":0.0502,"input_tokens":61},{"id":"original-extraction-06-1","tier":"original","family":"extraction","gold":"courier","predicted":"post","correct":false,"schema_valid":true,"probs":{"courier":0.2504,"pickup":0.0717,"post":0.5639,"unknown":0.1141},"latency_s":0.0261,"input_tokens":60},{"id":"original-intent-01-0","tier":"original","family":"intent","gold":"refund","predicted":"status","correct":false,"schema_valid":true,"probs":{"cancel":0.1823,"change_address":0.1911,"other":0.0895,"refund":0.2369,"status":0.3002},"latency_s":0.0447,"input_tokens":63},{"id":"original-intent-01-1","tier":"original","family":"intent","gold":"refund","predicted":"refund","correct":true,"schema_valid":true,"probs":{"cancel":0.2298,"change_address":0.1169,"other":0.0617,"refund":0.3938,"status":0.1978},"latency_s":0.0564,"input_tokens":65},{"id":"original-intent-02-0","tier":"original","family":"intent","gold":"status","predicted":"status","correct":true,"schema_valid":true,"probs":{"cancel":0.2018,"change_address":0.2313,"other":0.1299,"refund":0.0963,"status":0.3407},"latency_s":0.0571,"input_tokens":64},{"id":"original-intent-02-1","tier":"original","family":"intent","gold":"status","predicted":"status","correct":true,"schema_valid":true,"probs":{"cancel":0.24,"change_address":0.1055,"other":0.1146,"refund":0.109,"status":0.431},"latency_s":0.0553,"input_tokens":63},{"id":"original-intent-03-0","tier":"original","family":"intent","gold":"change_address","predicted":"change_address","correct":true,"schema_valid":true,"probs":{"cancel":0.2539,"change_address":0.2586,"other":0.091,"refund":0.1398,"status":0.2566},"latency_s":0.0575,"input_tokens":65},{"id":"original-intent-03-1","tier":"original","family":"intent","gold":"change_address","predicted":"change_address","correct":true,"schema_valid":true,"probs":{"cancel":0.2041,"change_address":0.3369,"other":0.0635,"refund":0.089,"status":0.3064},"latency_s":0.0533,"input_tokens":63},{"id":"original-intent-04-0","tier":"original","family":"intent","gold":"cancel","predicted":"status","correct":false,"schema_valid":true,"probs":{"cancel":0.2457,"change_address":0.2214,"other":0.102,"refund":0.1136,"status":0.3172},"latency_s":0.0583,"input_tokens":68},{"id":"original-intent-04-1","tier":"original","family":"intent","gold":"cancel","predicted":"status","correct":false,"schema_valid":true,"probs":{"cancel":0.1795,"change_address":0.1704,"other":0.1108,"refund":0.0488,"status":0.4905},"latency_s":0.0446,"input_tokens":63},{"id":"original-intent-05-0","tier":"original","family":"intent","gold":"other","predicted":"refund","correct":false,"schema_valid":true,"probs":{"cancel":0.2308,"change_address":0.0854,"other":0.035,"refund":0.3248,"status":0.324},"latency_s":0.055,"input_tokens":63},{"id":"original-intent-05-1","tier":"original","family":"intent","gold":"other","predicted":"status","correct":false,"schema_valid":true,"probs":{"cancel":0.2037,"change_address":0.1434,"other":0.0744,"refund":0.242,"status":0.3365},"latency_s":0.0537,"input_tokens":63},{"id":"original-intent-06-0","tier":"original","family":"intent","gold":"status","predicted":"status","correct":true,"schema_valid":true,"probs":{"cancel":0.3215,"change_address":0.1423,"other":0.0577,"refund":0.1427,"status":0.3358},"latency_s":0.0541,"input_tokens":61},{"id":"original-intent-06-1","tier":"original","family":"intent","gold":"status","predicted":"cancel","correct":false,"schema_valid":true,"probs":{"cancel":0.3574,"change_address":0.1666,"other":0.0954,"refund":0.0856,"status":0.2951},"latency_s":0.057,"input_tokens":66},{"id":"original-ordinal-01-0","tier":"original","family":"ordinal","gold":0,"predicted":"3","correct":false,"schema_valid":true,"probs":{"0":0.1087,"1":0.2739,"2":0.2962,"3":0.3212},"latency_s":0.0559,"input_tokens":70},{"id":"original-ordinal-01-1","tier":"original","family":"ordinal","gold":0,"predicted":"2","correct":false,"schema_valid":true,"probs":{"0":0.0648,"1":0.2938,"2":0.3699,"3":0.2715},"latency_s":0.0392,"input_tokens":72},{"id":"original-ordinal-02-0","tier":"original","family":"ordinal","gold":1,"predicted":"3","correct":false,"schema_valid":true,"probs":{"0":0.0959,"1":0.2716,"2":0.3085,"3":0.324},"latency_s":0.0571,"input_tokens":74},{"id":"original-ordinal-02-1","tier":"original","family":"ordinal","gold":1,"predicted":"3","correct":false,"schema_valid":true,"probs":{"0":0.0824,"1":0.2308,"2":0.2805,"3":0.4064},"latency_s":0.0511,"input_tokens":75},{"id":"original-ordinal-03-0","tier":"original","family":"ordinal","gold":2,"predicted":"3","correct":false,"schema_valid":true,"probs":{"0":0.1428,"1":0.3016,"2":0.2461,"3":0.3095},"latency_s":0.0502,"input_tokens":71},{"id":"original-ordinal-03-1","tier":"original","family":"ordinal","gold":2,"predicted":"1","correct":false,"schema_valid":true,"probs":{"0":0.1041,"1":0.3167,"2":0.2849,"3":0.2943},"latency_s":0.0407,"input_tokens":73},{"id":"original-ordinal-04-0","tier":"original","family":"ordinal","gold":3,"predicted":"1","correct":false,"schema_valid":true,"probs":{"0":0.1232,"1":0.308,"2":0.2717,"3":0.2971},"latency_s":0.0575,"input_tokens":73},{"id":"original-ordinal-04-1","tier":"original","family":"ordinal","gold":3,"predicted":"3","correct":true,"schema_valid":true,"probs":{"0":0.1118,"1":0.3173,"2":0.2288,"3":0.3421},"latency_s":0.0387,"input_tokens":72},{"id":"original-ordinal-05-0","tier":"original","family":"ordinal","gold":0,"predicted":"3","correct":false,"schema_valid":true,"probs":{"0":0.1105,"1":0.3006,"2":0.235,"3":0.3539},"latency_s":0.0282,"input_tokens":76},{"id":"original-ordinal-05-1","tier":"original","family":"ordinal","gold":0,"predicted":"1","correct":false,"schema_valid":true,"probs":{"0":0.1635,"1":0.2919,"2":0.2826,"3":0.262},"latency_s":0.054,"input_tokens":75},{"id":"original-ordinal-06-0","tier":"original","family":"ordinal","gold":2,"predicted":"1","correct":false,"schema_valid":true,"probs":{"0":0.1811,"1":0.2892,"2":0.2847,"3":0.2451},"latency_s":0.0546,"input_tokens":77},{"id":"original-ordinal-06-1","tier":"original","family":"ordinal","gold":2,"predicted":"2","correct":true,"schema_valid":true,"probs":{"0":0.1253,"1":0.2739,"2":0.3173,"3":0.2836},"latency_s":0.0607,"input_tokens":84},{"id":"original-policy-01-0","tier":"original","family":"policy","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.3731,"no":0.6269},"latency_s":0.0684,"input_tokens":71},{"id":"original-policy-01-1","tier":"original","family":"policy","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.3812,"no":0.6188},"latency_s":0.0559,"input_tokens":73},{"id":"original-policy-02-0","tier":"original","family":"policy","gold":"yes","predicted":"no","correct":false,"schema_valid":true,"probs":{"yes":0.4741,"no":0.5259},"latency_s":0.0352,"input_tokens":63},{"id":"original-policy-02-1","tier":"original","family":"policy","gold":"yes","predicted":"no","correct":false,"schema_valid":true,"probs":{"yes":0.472,"no":0.528},"latency_s":0.0382,"input_tokens":62},{"id":"original-policy-03-0","tier":"original","family":"policy","gold":"yes","predicted":"no","correct":false,"schema_valid":true,"probs":{"yes":0.4141,"no":0.5859},"latency_s":0.0541,"input_tokens":65},{"id":"original-policy-03-1","tier":"original","family":"policy","gold":"yes","predicted":"no","correct":false,"schema_valid":true,"probs":{"yes":0.4093,"no":0.5907},"latency_s":0.0264,"input_tokens":65},{"id":"original-policy-04-0","tier":"original","family":"policy","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.4161,"no":0.5839},"latency_s":0.0618,"input_tokens":73},{"id":"original-policy-04-1","tier":"original","family":"policy","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.3966,"no":0.6034},"latency_s":0.0507,"input_tokens":71},{"id":"original-policy-05-0","tier":"original","family":"policy","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.3477,"no":0.6523},"latency_s":0.0607,"input_tokens":76},{"id":"original-policy-05-1","tier":"original","family":"policy","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.3385,"no":0.6615},"latency_s":0.0386,"input_tokens":69},{"id":"original-policy-06-0","tier":"original","family":"policy","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.5422,"no":0.4578},"latency_s":0.0565,"input_tokens":74},{"id":"original-policy-06-1","tier":"original","family":"policy","gold":"yes","predicted":"no","correct":false,"schema_valid":true,"probs":{"yes":0.4981,"no":0.5019},"latency_s":0.0581,"input_tokens":64},{"id":"original-routing-01-0","tier":"original","family":"routing","gold":"math","predicted":"coding_agent","correct":false,"schema_valid":true,"probs":{"coding":0.1695,"coding_agent":0.2701,"document":0.1468,"general":0.0935,"math":0.1808,"tools":0.1392},"latency_s":0.0582,"input_tokens":92},{"id":"original-routing-01-1","tier":"original","family":"routing","gold":"math","predicted":"coding_agent","correct":false,"schema_valid":true,"probs":{"coding":0.1476,"coding_agent":0.2785,"document":0.1417,"general":0.1115,"math":0.1654,"tools":0.1553},"latency_s":0.0368,"input_tokens":89},{"id":"original-routing-02-0","tier":"original","family":"routing","gold":"coding","predicted":"coding_agent","correct":false,"schema_valid":true,"probs":{"coding":0.1549,"coding_agent":0.2509,"document":0.1517,"general":0.1608,"math":0.1511,"tools":0.1307},"latency_s":0.0504,"input_tokens":96},{"id":"original-routing-02-1","tier":"original","family":"routing","gold":"coding","predicted":"coding_agent","correct":false,"schema_valid":true,"probs":{"coding":0.1469,"coding_agent":0.2904,"document":0.1459,"general":0.1347,"math":0.1639,"tools":0.1181},"latency_s":0.043,"input_tokens":90},{"id":"original-routing-03-0","tier":"original","family":"routing","gold":"coding_agent","predicted":"coding_agent","correct":true,"schema_valid":true,"probs":{"coding":0.1212,"coding_agent":0.3979,"document":0.1004,"general":0.0677,"math":0.1377,"tools":0.1751},"latency_s":0.0342,"input_tokens":95},{"id":"original-routing-03-1","tier":"original","family":"routing","gold":"coding_agent","predicted":"coding_agent","correct":true,"schema_valid":true,"probs":{"coding":0.1214,"coding_agent":0.2906,"document":0.1333,"general":0.0983,"math":0.1456,"tools":0.2107},"latency_s":0.0512,"input_tokens":98},{"id":"original-routing-04-0","tier":"original","family":"routing","gold":"document","predicted":"coding_agent","correct":false,"schema_valid":true,"probs":{"coding":0.1548,"coding_agent":0.271,"document":0.1355,"general":0.0972,"math":0.1632,"tools":0.1783},"latency_s":0.052,"input_tokens":91},{"id":"original-routing-04-1","tier":"original","family":"routing","gold":"document","predicted":"coding_agent","correct":false,"schema_valid":true,"probs":{"coding":0.126,"coding_agent":0.2506,"document":0.1623,"general":0.1497,"math":0.1629,"tools":0.1486},"latency_s":0.0527,"input_tokens":91},{"id":"original-routing-05-0","tier":"original","family":"routing","gold":"tools","predicted":"coding_agent","correct":false,"schema_valid":true,"probs":{"coding":0.1535,"coding_agent":0.2589,"document":0.1268,"general":0.1002,"math":0.1758,"tools":0.1849},"latency_s":0.0539,"input_tokens":92},{"id":"original-routing-05-1","tier":"original","family":"routing","gold":"tools","predicted":"coding_agent","correct":false,"schema_valid":true,"probs":{"coding":0.1598,"coding_agent":0.2573,"document":0.1213,"general":0.0898,"math":0.165,"tools":0.2068},"latency_s":0.0398,"input_tokens":93},{"id":"original-routing-06-0","tier":"original","family":"routing","gold":"general","predicted":"coding_agent","correct":false,"schema_valid":true,"probs":{"coding":0.1509,"coding_agent":0.251,"document":0.1456,"general":0.1294,"math":0.172,"tools":0.151},"latency_s":0.0506,"input_tokens":91},{"id":"original-routing-06-1","tier":"original","family":"routing","gold":"general","predicted":"coding_agent","correct":false,"schema_valid":true,"probs":{"coding":0.1424,"coding_agent":0.2672,"document":0.1589,"general":0.1187,"math":0.1619,"tools":0.1509},"latency_s":0.0564,"input_tokens":91}]},"tia-1.3-mini":{"display":"TIA 1.3 mini","run_by":"TIA","date":"2026-09-30","device":"local CPU worker on the J-97 workstation, harness in Docker, loopback","adapter":"typesafe","items":[{"id":"easy-extraction-00","tier":"easy","family":"extraction","gold":"paypal","predicted":"paypal","correct":true,"schema_valid":true,"probs":{"bank_transfer":0.0382,"cash":0.0928,"credit_card":0.2387,"paypal":0.6304},"latency_s":0.0551,"input_tokens":43},{"id":"easy-extraction-01","tier":"easy","family":"extraction","gold":"cash","predicted":"cash","correct":true,"schema_valid":true,"probs":{"bank_transfer":0.1417,"cash":0.4664,"credit_card":0.2645,"paypal":0.1273},"latency_s":0.0552,"input_tokens":46},{"id":"easy-extraction-02","tier":"easy","family":"extraction","gold":"bank_transfer","predicted":"bank_transfer","correct":true,"schema_valid":true,"probs":{"bank_transfer":0.5191,"cash":0.2255,"credit_card":0.1744,"paypal":0.0811},"latency_s":0.0251,"input_tokens":46},{"id":"easy-extraction-03","tier":"easy","family":"extraction","gold":"high","predicted":"high","correct":true,"schema_valid":true,"probs":{"high":0.481,"low":0.0382,"medium":0.1743,"urgent":0.3065},"latency_s":0.0299,"input_tokens":45},{"id":"easy-extraction-04","tier":"easy","family":"extraction","gold":"low","predicted":"high","correct":false,"schema_valid":true,"probs":{"high":0.4574,"low":0.0268,"medium":0.2122,"urgent":0.3036},"latency_s":0.0424,"input_tokens":42},{"id":"easy-extraction-05","tier":"easy","family":"extraction","gold":"urgent","predicted":"high","correct":false,"schema_valid":true,"probs":{"high":0.4325,"low":0.0332,"medium":0.1589,"urgent":0.3754},"latency_s":0.0423,"input_tokens":42},{"id":"easy-extraction-06","tier":"easy","family":"extraction","gold":"M","predicted":"L","correct":false,"schema_valid":true,"probs":{"L":0.3011,"M":0.2141,"S":0.2761,"XL":0.2086},"latency_s":0.0562,"input_tokens":34},{"id":"easy-extraction-07","tier":"easy","family":"extraction","gold":"XL","predicted":"XL","correct":true,"schema_valid":true,"probs":{"L":0.2257,"M":0.1469,"S":0.1707,"XL":0.4567},"latency_s":0.0585,"input_tokens":32},{"id":"easy-extraction-08","tier":"easy","family":"extraction","gold":"thursday","predicted":"thursday","correct":true,"schema_valid":true,"probs":{"friday":0.2592,"monday":0.1234,"thursday":0.3862,"tuesday":0.0923,"wednesday":0.1388},"latency_s":0.0311,"input_tokens":38},{"id":"easy-extraction-09","tier":"easy","family":"extraction","gold":"monday","predicted":"monday","correct":true,"schema_valid":true,"probs":{"friday":0.1945,"monday":0.4823,"thursday":0.0799,"tuesday":0.1497,"wednesday":0.0937},"latency_s":0.0596,"input_tokens":42},{"id":"easy-extraction-10","tier":"easy","family":"extraction","gold":"friday","predicted":"friday","correct":true,"schema_valid":true,"probs":{"friday":0.5715,"monday":0.1286,"thursday":0.1171,"tuesday":0.0825,"wednesday":0.1002},"latency_s":0.0472,"input_tokens":38},{"id":"easy-extraction-11","tier":"easy","family":"extraction","gold":"credit_card","predicted":"credit_card","correct":true,"schema_valid":true,"probs":{"bank_transfer":0.0818,"cash":0.1582,"credit_card":0.652,"paypal":0.1081},"latency_s":0.0398,"input_tokens":46},{"id":"easy-fact-00","tier":"easy","family":"fact","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.5104,"no":0.4896},"latency_s":0.0568,"input_tokens":53},{"id":"easy-fact-01","tier":"easy","family":"fact","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.5196,"no":0.4804},"latency_s":0.0445,"input_tokens":50},{"id":"easy-fact-02","tier":"easy","family":"fact","gold":"yes","predicted":"no","correct":false,"schema_valid":true,"probs":{"yes":0.4815,"no":0.5185},"latency_s":0.0351,"input_tokens":54},{"id":"easy-fact-03","tier":"easy","family":"fact","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.4871,"no":0.5129},"latency_s":0.0438,"input_tokens":58},{"id":"easy-fact-04","tier":"easy","family":"fact","gold":"yes","predicted":"no","correct":false,"schema_valid":true,"probs":{"yes":0.4761,"no":0.5239},"latency_s":0.0569,"input_tokens":54},{"id":"easy-fact-05","tier":"easy","family":"fact","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.4613,"no":0.5387},"latency_s":0.0543,"input_tokens":55},{"id":"easy-fact-06","tier":"easy","family":"fact","gold":"yes","predicted":"no","correct":false,"schema_valid":true,"probs":{"yes":0.4947,"no":0.5053},"latency_s":0.0592,"input_tokens":62},{"id":"easy-fact-07","tier":"easy","family":"fact","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.4481,"no":0.5519},"latency_s":0.0391,"input_tokens":50},{"id":"easy-fact-08","tier":"easy","family":"fact","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.557,"no":0.443},"latency_s":0.0588,"input_tokens":46},{"id":"easy-fact-09","tier":"easy","family":"fact","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.5323,"no":0.4677},"latency_s":0.0564,"input_tokens":46},{"id":"easy-fact-10","tier":"easy","family":"fact","gold":"yes","predicted":"no","correct":false,"schema_valid":true,"probs":{"yes":0.4216,"no":0.5784},"latency_s":0.0575,"input_tokens":53},{"id":"easy-fact-11","tier":"easy","family":"fact","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.3708,"no":0.6292},"latency_s":0.0276,"input_tokens":54},{"id":"easy-intent-00","tier":"easy","family":"intent","gold":"track_order","predicted":"cancel_order","correct":false,"schema_valid":true,"probs":{"billing_question":0.0501,"cancel_order":0.3884,"change_address":0.2293,"report_damage":0.0993,"track_order":0.2328},"latency_s":0.3423,"input_tokens":91},{"id":"easy-intent-01","tier":"easy","family":"intent","gold":"cancel_order","predicted":"cancel_order","correct":true,"schema_valid":true,"probs":{"billing_question":0.0156,"cancel_order":0.5644,"change_address":0.1488,"report_damage":0.054,"track_order":0.2172},"latency_s":0.0469,"input_tokens":89},{"id":"easy-intent-02","tier":"easy","family":"intent","gold":"change_address","predicted":"cancel_order","correct":false,"schema_valid":true,"probs":{"billing_question":0.0405,"cancel_order":0.3604,"change_address":0.2274,"report_damage":0.0786,"track_order":0.2931},"latency_s":0.0422,"input_tokens":93},{"id":"easy-intent-03","tier":"easy","family":"intent","gold":"report_damage","predicted":"cancel_order","correct":false,"schema_valid":true,"probs":{"billing_question":0.0459,"cancel_order":0.2853,"change_address":0.2448,"report_damage":0.2005,"track_order":0.2235},"latency_s":0.0475,"input_tokens":83},{"id":"easy-intent-04","tier":"easy","family":"intent","gold":"billing_question","predicted":"cancel_order","correct":false,"schema_valid":true,"probs":{"billing_question":0.1923,"cancel_order":0.2911,"change_address":0.2134,"report_damage":0.0889,"track_order":0.2143},"latency_s":0.0504,"input_tokens":88},{"id":"easy-intent-05","tier":"easy","family":"intent","gold":"track_order","predicted":"cancel_order","correct":false,"schema_valid":true,"probs":{"billing_question":0.0178,"cancel_order":0.3596,"change_address":0.2529,"report_damage":0.1014,"track_order":0.2683},"latency_s":0.0551,"input_tokens":87},{"id":"easy-intent-06","tier":"easy","family":"intent","gold":"set_alarm","predicted":"send_message","correct":false,"schema_valid":true,"probs":{"play_music":0.1097,"send_message":0.4313,"set_alarm":0.2598,"turn_off_lights":0.0846,"weather":0.1146},"latency_s":0.0526,"input_tokens":74},{"id":"easy-intent-07","tier":"easy","family":"intent","gold":"play_music","predicted":"send_message","correct":false,"schema_valid":true,"probs":{"play_music":0.1468,"send_message":0.4217,"set_alarm":0.1681,"turn_off_lights":0.0739,"weather":0.1895},"latency_s":0.059,"input_tokens":68},{"id":"easy-intent-08","tier":"easy","family":"intent","gold":"weather","predicted":"send_message","correct":false,"schema_valid":true,"probs":{"play_music":0.0919,"send_message":0.358,"set_alarm":0.1399,"turn_off_lights":0.0556,"weather":0.3546},"latency_s":0.061,"input_tokens":70},{"id":"easy-intent-09","tier":"easy","family":"intent","gold":"send_message","predicted":"send_message","correct":true,"schema_valid":true,"probs":{"play_music":0.1223,"send_message":0.4702,"set_alarm":0.1766,"turn_off_lights":0.0977,"weather":0.1332},"latency_s":0.0303,"input_tokens":73},{"id":"easy-intent-10","tier":"easy","family":"intent","gold":"turn_off_lights","predicted":"send_message","correct":false,"schema_valid":true,"probs":{"play_music":0.1017,"send_message":0.3874,"set_alarm":0.2034,"turn_off_lights":0.2054,"weather":0.102},"latency_s":0.0559,"input_tokens":71},{"id":"easy-intent-11","tier":"easy","family":"intent","gold":"set_alarm","predicted":"set_alarm","correct":true,"schema_valid":true,"probs":{"play_music":0.0758,"send_message":0.2163,"set_alarm":0.556,"turn_off_lights":0.0595,"weather":0.0923},"latency_s":0.0504,"input_tokens":70},{"id":"easy-tool_selection-00","tier":"easy","family":"tool_selection","gold":"get_weather","predicted":"get_weather","correct":true,"schema_valid":true,"probs":{"convert_currency":0.0921,"create_calendar_event":0.2079,"get_weather":0.43,"send_email":0.13,"translate_text":0.14},"latency_s":0.0433,"input_tokens":78},{"id":"easy-tool_selection-01","tier":"easy","family":"tool_selection","gold":"send_email","predicted":"send_email","correct":true,"schema_valid":true,"probs":{"convert_currency":0.0958,"create_calendar_event":0.2288,"get_weather":0.1304,"send_email":0.3638,"translate_text":0.1812},"latency_s":0.0406,"input_tokens":79},{"id":"easy-tool_selection-02","tier":"easy","family":"tool_selection","gold":"create_calendar_event","predicted":"create_calendar_event","correct":true,"schema_valid":true,"probs":{"convert_currency":0.1569,"create_calendar_event":0.3227,"get_weather":0.1255,"send_email":0.2279,"translate_text":0.167},"latency_s":0.053,"input_tokens":81},{"id":"easy-tool_selection-03","tier":"easy","family":"tool_selection","gold":"convert_currency","predicted":"create_calendar_event","correct":false,"schema_valid":true,"probs":{"convert_currency":0.1328,"create_calendar_event":0.3055,"get_weather":0.1685,"send_email":0.1992,"translate_text":0.194},"latency_s":0.0376,"input_tokens":77},{"id":"easy-tool_selection-04","tier":"easy","family":"tool_selection","gold":"translate_text","predicted":"create_calendar_event","correct":false,"schema_valid":true,"probs":{"convert_currency":0.1321,"create_calendar_event":0.2807,"get_weather":0.1604,"send_email":0.1944,"translate_text":0.2324},"latency_s":0.0564,"input_tokens":77},{"id":"easy-tool_selection-05","tier":"easy","family":"tool_selection","gold":"create_calendar_event","predicted":"create_calendar_event","correct":true,"schema_valid":true,"probs":{"convert_currency":0.1254,"create_calendar_event":0.4136,"get_weather":0.1554,"send_email":0.1505,"translate_text":0.1551},"latency_s":0.0577,"input_tokens":80},{"id":"easy-tool_selection-06","tier":"easy","family":"tool_selection","gold":"search_flights","predicted":"search_flights","correct":true,"schema_valid":true,"probs":{"book_restaurant":0.1903,"call_taxi":0.2366,"get_stock_price":0.1106,"search_flights":0.3144,"set_reminder":0.1481},"latency_s":0.0417,"input_tokens":84},{"id":"easy-tool_selection-07","tier":"easy","family":"tool_selection","gold":"book_restaurant","predicted":"book_restaurant","correct":true,"schema_valid":true,"probs":{"book_restaurant":0.3159,"call_taxi":0.2024,"get_stock_price":0.1863,"search_flights":0.1339,"set_reminder":0.1616},"latency_s":0.059,"input_tokens":86},{"id":"easy-tool_selection-08","tier":"easy","family":"tool_selection","gold":"set_reminder","predicted":"call_taxi","correct":false,"schema_valid":true,"probs":{"book_restaurant":0.1713,"call_taxi":0.354,"get_stock_price":0.1409,"search_flights":0.1216,"set_reminder":0.2122},"latency_s":0.0399,"input_tokens":84},{"id":"easy-tool_selection-09","tier":"easy","family":"tool_selection","gold":"get_stock_price","predicted":"get_stock_price","correct":true,"schema_valid":true,"probs":{"book_restaurant":0.171,"call_taxi":0.1681,"get_stock_price":0.4045,"search_flights":0.1225,"set_reminder":0.1339},"latency_s":0.0411,"input_tokens":82},{"id":"easy-tool_selection-10","tier":"easy","family":"tool_selection","gold":"call_taxi","predicted":"call_taxi","correct":true,"schema_valid":true,"probs":{"book_restaurant":0.2143,"call_taxi":0.3317,"get_stock_price":0.1484,"search_flights":0.1252,"set_reminder":0.1804},"latency_s":0.0389,"input_tokens":82},{"id":"easy-tool_selection-11","tier":"easy","family":"tool_selection","gold":"set_reminder","predicted":"set_reminder","correct":true,"schema_valid":true,"probs":{"book_restaurant":0.2055,"call_taxi":0.2156,"get_stock_price":0.1392,"search_flights":0.1484,"set_reminder":0.2913},"latency_s":0.0551,"input_tokens":84},{"id":"hard-opus-a-long_policy-01","tier":"hard","family":"long_policy","gold":"pay_subject_to_15000_sublimit","predicted":"deny_vacancy_exclusion","correct":false,"schema_valid":true,"probs":{"deny_repeated_seepage":0.2117,"deny_vacancy_exclusion":0.2242,"pay_full_estimate_less_deductible":0.1901,"pay_subject_to_10000_sublimit":0.1917,"pay_subject_to_15000_sublimit":0.1823},"latency_s":0.1751,"input_tokens":3733},{"id":"hard-opus-a-long_policy-04","tier":"hard","family":"long_policy","gold":"cfo","predicted":"department_head","correct":false,"schema_valid":true,"probs":{"budget_holder":0.1617,"cfo":0.1703,"department_head":0.2347,"executive_committee":0.2004,"vp_and_finance_director":0.2329},"latency_s":0.1608,"input_tokens":2460},{"id":"hard-opus-a-long_policy-08","tier":"hard","family":"long_policy","gold":"parts_covered_labour_charged","predicted":"parts_covered_labour_charged","correct":true,"schema_valid":true,"probs":{"full_repair_no_charge":0.2216,"not_covered":0.2365,"parts_covered_labour_charged":0.32,"pro_rata_replacement_credit":0.2219},"latency_s":0.1618,"input_tokens":2293},{"id":"hard-opus-a-long_policy-09","tier":"hard","family":"long_policy","gold":"license_required","predicted":"license_required","correct":true,"schema_valid":true,"probs":{"license_exception_lvs":0.2985,"license_required":0.3158,"no_license_required":0.201,"shipment_prohibited":0.1847},"latency_s":0.3165,"input_tokens":2402},{"id":"hard-opus-a-long_policy-11","tier":"hard","family":"long_policy","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.5684,"no":0.4316},"latency_s":0.1562,"input_tokens":2146},{"id":"hard-opus-a-long_policy-13","tier":"hard","family":"long_policy","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.6206,"no":0.3794},"latency_s":0.1646,"input_tokens":2280},{"id":"hard-opus-a-long_policy-17","tier":"hard","family":"long_policy","gold":"database_oncall","predicted":"vendor_support","correct":false,"schema_valid":true,"probs":{"database_oncall":0.1221,"no_page_ticket_only":0.1506,"platform_oncall":0.1478,"security_incident_response":0.2483,"vendor_support":0.3312},"latency_s":0.1399,"input_tokens":2287},{"id":"hard-opus-a-long_policy-19","tier":"hard","family":"long_policy","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.6449,"no":0.3551},"latency_s":0.1385,"input_tokens":2106},{"id":"hard-opus-a-probability-03","tier":"hard","family":"probability","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.5534,"no":0.4466},"latency_s":0.0729,"input_tokens":376},{"id":"hard-opus-a-probability-04","tier":"hard","family":"probability","gold":"on_time","predicted":"on_time","correct":true,"schema_valid":true,"probs":{"early":0.3095,"late":0.3165,"on_time":0.374},"latency_s":0.1454,"input_tokens":900},{"id":"hard-opus-a-probability-07","tier":"hard","family":"probability","gold":"escalated","predicted":"escalated","correct":true,"schema_valid":true,"probs":{"escalated":0.4533,"reopened":0.2648,"resolved_first_contact":0.2819},"latency_s":0.1495,"input_tokens":954},{"id":"hard-opus-a-probability-08","tier":"hard","family":"probability","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.5243,"no":0.4757},"latency_s":0.0619,"input_tokens":312},{"id":"hard-opus-a-temporal_numeric-03","tier":"hard","family":"temporal_numeric","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.5562,"no":0.4438},"latency_s":0.0734,"input_tokens":417},{"id":"hard-opus-a-temporal_numeric-06","tier":"hard","family":"temporal_numeric","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.582,"no":0.418},"latency_s":0.0871,"input_tokens":418},{"id":"hard-opus-a-temporal_numeric-07","tier":"hard","family":"temporal_numeric","gold":"sep_26","predicted":"sep_24","correct":false,"schema_valid":true,"probs":{"sep_24":0.2581,"sep_25":0.2249,"sep_26":0.2494,"sep_27":0.1848,"sep_28":0.0829},"latency_s":0.1493,"input_tokens":908},{"id":"hard-opus-a-temporal_numeric-09","tier":"hard","family":"temporal_numeric","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.4381,"no":0.5619},"latency_s":0.0648,"input_tokens":439},{"id":"hard-opus-a-temporal_numeric-12","tier":"hard","family":"temporal_numeric","gold":1,"predicted":"2","correct":false,"schema_valid":true,"probs":{"0":0.005,"1":0.3593,"2":0.3762,"3":0.2595},"latency_s":0.0833,"input_tokens":461},{"id":"hard-opus-b-ambiguous-02","tier":"hard","family":"ambiguous","gold":"real_incident","predicted":"false_positive","correct":false,"schema_valid":true,"probs":{"cannot_determine":0.3327,"false_positive":0.4069,"real_incident":0.2605},"latency_s":0.0834,"input_tokens":484},{"id":"hard-opus-b-ambiguous-03","tier":"hard","family":"ambiguous","gold":"time_barred","predicted":"within_limitation","correct":false,"schema_valid":true,"probs":{"insufficient_information":0.262,"time_barred":0.3503,"within_limitation":0.3878},"latency_s":0.0785,"input_tokens":441},{"id":"hard-opus-b-ambiguous-07","tier":"hard","family":"ambiguous","gold":"within_policy","predicted":"within_policy","correct":true,"schema_valid":true,"probs":{"insufficient_information":0.3095,"over_limit":0.319,"within_policy":0.3715},"latency_s":0.0672,"input_tokens":439},{"id":"hard-opus-b-ambiguous-09","tier":"hard","family":"ambiguous","gold":"legitimate","predicted":"legitimate","correct":true,"schema_valid":true,"probs":{"compromised":0.2341,"inconclusive":0.358,"legitimate":0.4078},"latency_s":0.0806,"input_tokens":589},{"id":"hard-opus-b-ambiguous-10","tier":"hard","family":"ambiguous","gold":"ask_user_which_alex","predicted":"ask_user_which_alex","correct":true,"schema_valid":true,"probs":{"ask_user_which_alex":0.3609,"send_to_alex_moreno":0.2927,"send_to_alexandra_lind":0.3464},"latency_s":0.0828,"input_tokens":446},{"id":"hard-opus-b-ambiguous-11","tier":"hard","family":"ambiguous","gold":"cannot_determine","predicted":"cannot_determine","correct":true,"schema_valid":true,"probs":{"cannot_determine":0.3684,"grade_correct":0.3143,"grade_should_be_higher":0.3174},"latency_s":0.0832,"input_tokens":396},{"id":"hard-opus-b-ambiguous-13","tier":"hard","family":"ambiguous","gold":"cannot_determine","predicted":"charge_platform_team","correct":false,"schema_valid":true,"probs":{"cannot_determine":0.1974,"charge_data_team":0.258,"charge_platform_team":0.3471,"split_50_50":0.1975},"latency_s":0.0944,"input_tokens":525},{"id":"hard-opus-b-multi_hop-03","tier":"hard","family":"multi_hop","gold":"tier3_cfo","predicted":"tier4_board","correct":false,"schema_valid":true,"probs":{"tier1_team_lead":0.2106,"tier2_department_head":0.2241,"tier3_cfo":0.2806,"tier4_board":0.2847},"latency_s":0.1659,"input_tokens":2696},{"id":"hard-opus-b-multi_hop-04","tier":"hard","family":"multi_hop","gold":"bjorn","predicted":"no_page_ticket_only","correct":false,"schema_valid":true,"probs":{"ana":0.1751,"bjorn":0.1192,"chen":0.1295,"dmitri":0.1671,"esra":0.1663,"no_page_ticket_only":0.2427},"latency_s":0.1323,"input_tokens":2200},{"id":"hard-opus-b-multi_hop-05","tier":"hard","family":"multi_hop","gold":2,"predicted":"3","correct":false,"schema_valid":true,"probs":{"0":0.0031,"1":0.2684,"2":0.2007,"3":0.2726,"4":0.2551},"latency_s":0.1412,"input_tokens":2293},{"id":"hard-opus-b-multi_hop-07","tier":"hard","family":"multi_hop","gold":"credit_20_percent","predicted":"credit_full_month","correct":false,"schema_valid":true,"probs":{"credit_10_percent":0.2083,"credit_20_percent":0.2296,"credit_25_percent":0.1779,"credit_50_percent":0.1413,"credit_full_month":0.231,"no_credit":0.012},"latency_s":0.146,"input_tokens":2260},{"id":"hard-opus-b-multi_hop-08","tier":"hard","family":"multi_hop","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.6346,"no":0.3654},"latency_s":0.1441,"input_tokens":2021},{"id":"hard-opus-b-probability-01","tier":"hard","family":"probability","gold":"merchant_wins","predicted":"customer_withdraws","correct":false,"schema_valid":true,"probs":{"customer_wins":0.2543,"customer_withdraws":0.4488,"merchant_wins":0.2969},"latency_s":0.1293,"input_tokens":1639},{"id":"hard-opus-b-probability-02","tier":"hard","family":"probability","gold":"upstream_provider","predicted":"bad_push","correct":false,"schema_valid":true,"probs":{"bad_push":0.984,"database_hardware":0.0004,"upstream_provider":0.0156},"latency_s":0.0759,"input_tokens":434},{"id":"hard-opus-b-probability-03","tier":"hard","family":"probability","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.5779,"no":0.4221},"latency_s":0.1455,"input_tokens":971},{"id":"hard-opus-b-probability-04","tier":"hard","family":"probability","gold":"attended","predicted":"attended","correct":true,"schema_valid":true,"probs":{"attended":0.5236,"late_cancellation":0.3267,"no_show":0.1497},"latency_s":0.085,"input_tokens":470},{"id":"hard-opus-b-probability-06","tier":"hard","family":"probability","gold":"upheld","predicted":"upheld","correct":true,"schema_valid":true,"probs":{"modified":0.3171,"overturned":0.3281,"upheld":0.3548},"latency_s":0.1417,"input_tokens":878},{"id":"hard-opus-b-tradeoff-01","tier":"hard","family":"tradeoff","gold":"escalate_to_engineering","predicted":"page_oncall","correct":false,"schema_valid":true,"probs":{"escalate_to_engineering":0.2593,"escalate_to_security":0.3106,"page_oncall":0.3414,"process_refund":0.0245,"send_kb_article":0.0642},"latency_s":0.121,"input_tokens":799},{"id":"hard-opus-b-tradeoff-03","tier":"hard","family":"tradeoff","gold":"order_b","predicted":"order_c","correct":false,"schema_valid":true,"probs":{"order_a":0.1268,"order_b":0.2698,"order_c":0.361,"order_d":0.2425},"latency_s":0.104,"input_tokens":648},{"id":"hard-opus-b-tradeoff-06","tier":"hard","family":"tradeoff","gold":"yes","predicted":"no","correct":false,"schema_valid":true,"probs":{"yes":0.4289,"no":0.5711},"latency_s":0.0836,"input_tokens":665},{"id":"hard-opus-b-tradeoff-07","tier":"hard","family":"tradeoff","gold":"p0_fix_24h","predicted":"p2_fix_30d","correct":false,"schema_valid":true,"probs":{"out_of_scope":0.1392,"p0_fix_24h":0.1572,"p1_fix_7d":0.2394,"p2_fix_30d":0.3194,"p3_backlog":0.1448},"latency_s":0.1101,"input_tokens":674},{"id":"hard-opus-b-tradeoff-08","tier":"hard","family":"tradeoff","gold":"yes","predicted":"no","correct":false,"schema_valid":true,"probs":{"yes":0.3951,"no":0.6049},"latency_s":0.067,"input_tokens":513},{"id":"hard-opus-b-tradeoff-12","tier":"hard","family":"tradeoff","gold":"refund_now","predicted":"escalate_to_fraud_team","correct":false,"schema_valid":true,"probs":{"deny_refund":0.2164,"escalate_to_fraud_team":0.271,"offer_store_credit":0.1343,"refund_now":0.184,"wait_for_inspection":0.1942},"latency_s":0.0845,"input_tokens":534},{"id":"hard-opus-c-long_policy-02","tier":"hard","family":"long_policy","gold":"erase_but_retain_transaction_records","predicted":"erase_but_retain_transaction_records","correct":true,"schema_valid":true,"probs":{"erase_all":0.1954,"erase_but_retain_transaction_records":0.2653,"erase_marketing_only":0.1103,"reject_unverified_request":0.2206,"restrict_all_pending_legal_hold":0.2084},"latency_s":0.1382,"input_tokens":3272},{"id":"hard-opus-c-long_policy-03","tier":"hard","family":"long_policy","gold":"eur_400","predicted":"eur_300","correct":false,"schema_valid":true,"probs":{"eur_0":0.038,"eur_150":0.1918,"eur_200":0.1851,"eur_300":0.2108,"eur_400":0.1872,"eur_600":0.187},"latency_s":0.1434,"input_tokens":3221},{"id":"hard-opus-c-long_policy-04","tier":"hard","family":"long_policy","gold":"yes","predicted":"no","correct":false,"schema_valid":true,"probs":{"yes":0.3797,"no":0.6203},"latency_s":0.16,"input_tokens":2953},{"id":"hard-opus-c-long_policy-05","tier":"hard","family":"long_policy","gold":1,"predicted":"0","correct":false,"schema_valid":true,"probs":{"0":0.2982,"1":0.2617,"2":0.2337,"3":0.2063},"latency_s":0.1436,"input_tokens":3035},{"id":"hard-opus-c-long_policy-08","tier":"hard","family":"long_policy","gold":1,"predicted":"3","correct":false,"schema_valid":true,"probs":{"0":0.1498,"1":0.0574,"2":0.1491,"3":0.6437},"latency_s":0.1339,"input_tokens":3019},{"id":"hard-opus-c-long_policy-10","tier":"hard","family":"long_policy","gold":"credit_minus_restocking_fee","predicted":"full_credit","correct":false,"schema_valid":true,"probs":{"credit_minus_restocking_fee":0.2302,"full_credit":0.2924,"reject_return":0.2633,"repair_or_replace_only":0.2141},"latency_s":0.1579,"input_tokens":3257},{"id":"hard-opus-c-long_policy-11","tier":"hard","family":"long_policy","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.5306,"no":0.4694},"latency_s":0.1599,"input_tokens":3189},{"id":"hard-opus-c-probability-03","tier":"hard","family":"probability","gold":"kuznets_technik","predicted":"halvorsen_polymer","correct":false,"schema_valid":true,"probs":{"halvorsen_polymer":0.6023,"kuznets_technik":0.2253,"ortega_seal_works":0.1723},"latency_s":0.0894,"input_tokens":510},{"id":"hard-opus-c-temporal_numeric-02","tier":"hard","family":"temporal_numeric","gold":"yes","predicted":"no","correct":false,"schema_valid":true,"probs":{"yes":0.3556,"no":0.6444},"latency_s":0.1181,"input_tokens":778},{"id":"hard-opus-c-temporal_numeric-03","tier":"hard","family":"temporal_numeric","gold":"eur_692_00","predicted":"eur_693_90","correct":false,"schema_valid":true,"probs":{"eur_617_00":0.1472,"eur_688_00":0.2282,"eur_692_00":0.2062,"eur_693_90":0.2556,"eur_812_00":0.1629},"latency_s":0.101,"input_tokens":725},{"id":"hard-opus-c-temporal_numeric-04","tier":"hard","family":"temporal_numeric","gold":"usd_2335_00","predicted":"usd_2303_01","correct":false,"schema_valid":true,"probs":{"usd_2222_50":0.2161,"usd_2303_01":0.4859,"usd_2303_33":0.0185,"usd_2335_00":0.1868,"usd_2340_00":0.0927},"latency_s":0.1043,"input_tokens":619},{"id":"hard-opus-c-temporal_numeric-06","tier":"hard","family":"temporal_numeric","gold":"sep_18_book_set","predicted":"none_declined","correct":false,"schema_valid":true,"probs":{"none_declined":0.229,"sep_15_office_chair":0.1707,"sep_16_taxi":0.1164,"sep_17_parking":0.1657,"sep_18_book_set":0.1487,"sep_21_client_dinner":0.1694},"latency_s":0.1098,"input_tokens":857},{"id":"hard-opus-c-temporal_numeric-08","tier":"hard","family":"temporal_numeric","gold":"eur_498_81","predicted":"eur_506_06","correct":false,"schema_valid":true,"probs":{"eur_497_40":0.1918,"eur_498_81":0.1972,"eur_506_06":0.2427,"eur_513_69":0.1904,"eur_532_01":0.1779},"latency_s":0.1317,"input_tokens":792},{"id":"hard-opus-c-temporal_numeric-12","tier":"hard","family":"temporal_numeric","gold":"expired_30_month_cap","predicted":"covered","correct":false,"schema_valid":true,"probs":{"covered":0.4272,"expired_24_month_term":0.3988,"expired_30_month_cap":0.174},"latency_s":0.1089,"input_tokens":714},{"id":"hard-sol-a-adversarial-01","tier":"hard","family":"adversarial","gold":"deny_missing_approval","predicted":"deny_missing_approval","correct":true,"schema_valid":true,"probs":{"deny_inactive_user":0.2615,"deny_missing_approval":0.312,"grant_access":0.2014,"grant_owner_exception":0.2251},"latency_s":0.0601,"input_tokens":272},{"id":"hard-sol-a-adversarial-06","tier":"hard","family":"adversarial","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.4781,"no":0.5219},"latency_s":0.0456,"input_tokens":211},{"id":"hard-sol-a-adversarial-07","tier":"hard","family":"adversarial","gold":"retain_until_2031","predicted":"needs_signature_review","correct":false,"schema_valid":true,"probs":{"delete_now":0.1217,"needs_signature_review":0.5286,"retain_indefinitely":0.1496,"retain_until_2031":0.2002},"latency_s":0.0498,"input_tokens":249},{"id":"hard-sol-a-adversarial-08","tier":"hard","family":"adversarial","gold":"yes","predicted":"no","correct":false,"schema_valid":true,"probs":{"yes":0.3898,"no":0.6102},"latency_s":0.0605,"input_tokens":197},{"id":"hard-sol-a-adversarial-09","tier":"hard","family":"adversarial","gold":2,"predicted":"3","correct":false,"schema_valid":true,"probs":{"0":0.1561,"1":0.2614,"2":0.2522,"3":0.3303},"latency_s":0.0514,"input_tokens":191},{"id":"hard-sol-a-adversarial-11","tier":"hard","family":"adversarial","gold":"standard_research","predicted":"standard_research","correct":true,"schema_valid":true,"probs":{"ethics_review":0.2891,"major_grants":0.1833,"manual_hold":0.1773,"standard_research":0.3503},"latency_s":0.0685,"input_tokens":262},{"id":"hard-sol-a-multi_hop-01","tier":"hard","family":"multi_hop","gold":"accept_s2_dusk","predicted":"reject_cutoff","correct":false,"schema_valid":true,"probs":{"accept_s2_dusk":0.2847,"reject_cutoff":0.289,"reject_tier":0.2066,"reroute_s3":0.2197},"latency_s":0.1399,"input_tokens":2341},{"id":"hard-sol-a-multi_hop-02","tier":"hard","family":"multi_hop","gold":"security_primary","predicted":"executive_bridge","correct":false,"schema_valid":true,"probs":{"executive_bridge":0.2872,"heron_q2":0.2076,"insufficient_information":0.2312,"security_primary":0.2741},"latency_s":0.1601,"input_tokens":2369},{"id":"hard-sol-a-multi_hop-05","tier":"hard","family":"multi_hop","gold":"require_director","predicted":"pay_manager_approved","correct":false,"schema_valid":true,"probs":{"pay_manager_approved":0.345,"reject_category":0.1818,"require_director":0.2714,"require_finance_board":0.2019},"latency_s":0.1307,"input_tokens":2308},{"id":"hard-sol-a-multi_hop-07","tier":"hard","family":"multi_hop","gold":"admit","predicted":"manual_equivalency","correct":false,"schema_valid":true,"probs":{"admit":0.261,"deny_prerequisite":0.2336,"deny_score":0.2429,"manual_equivalency":0.2625},"latency_s":0.1416,"input_tokens":2269},{"id":"hard-sol-a-multi_hop-08","tier":"hard","family":"multi_hop","gold":"central_engineering","predicted":"central_engineering","correct":true,"schema_valid":true,"probs":{"central_engineering":0.276,"insufficient_log":0.2488,"local_maintenance":0.2281,"safety_shutdown":0.2471},"latency_s":0.1585,"input_tokens":2264},{"id":"hard-sol-a-multi_hop-09","tier":"hard","family":"multi_hop","gold":"require_steward","predicted":"confirm_l3","correct":false,"schema_valid":true,"probs":{"confirm_l3":0.3178,"reject_capacity":0.245,"require_steward":0.2315,"switch_l4":0.2057},"latency_s":0.1625,"input_tokens":2271},{"id":"hard-sol-a-multi_hop-10","tier":"hard","family":"multi_hop","gold":"approve_45_days","predicted":"needs_field_review","correct":false,"schema_valid":true,"probs":{"approve_45_days":0.1569,"deny_sensitive":0.1452,"needs_field_review":0.4031,"reduce_to_30_days":0.2948},"latency_s":0.1532,"input_tokens":2304},{"id":"hard-sol-a-multi_hop-12","tier":"hard","family":"multi_hop","gold":"needs_authentication","predicted":"needs_authentication","correct":true,"schema_valid":true,"probs":{"covered_recurrence":0.2104,"expired":0.1614,"needs_authentication":0.408,"wrong_issue_class":0.2201},"latency_s":0.1636,"input_tokens":2267},{"id":"hard-sol-a-trap-02","tier":"hard","family":"trap","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.4659,"no":0.5341},"latency_s":0.0638,"input_tokens":263},{"id":"hard-sol-a-trap-04","tier":"hard","family":"trap","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.5402,"no":0.4598},"latency_s":0.0594,"input_tokens":166},{"id":"hard-sol-a-trap-05","tier":"hard","family":"trap","gold":0,"predicted":"2","correct":false,"schema_valid":true,"probs":{"0":0.0558,"1":0.2588,"2":0.3819,"3":0.3035},"latency_s":0.0772,"input_tokens":279},{"id":"hard-sol-a-trap-06","tier":"hard","family":"trap","gold":"keep_open_unresolved","predicted":"await_internal_metrics","correct":false,"schema_valid":true,"probs":{"await_internal_metrics":0.34,"close_agent_verified":0.2649,"close_confirmed":0.1318,"keep_open_unresolved":0.2633},"latency_s":0.0645,"input_tokens":304},{"id":"hard-sol-a-trap-08","tier":"hard","family":"trap","gold":"late","predicted":"on_time","correct":false,"schema_valid":true,"probs":{"canceled":0.2352,"indeterminate":0.2315,"late":0.2452,"on_time":0.2881},"latency_s":0.0725,"input_tokens":225},{"id":"hard-sol-a-trap-10","tier":"hard","family":"trap","gold":"blocked_second_review","predicted":"blocked_failed_test","correct":false,"schema_valid":true,"probs":{"blocked_failed_test":0.377,"blocked_second_review":0.1705,"ready_one_review":0.2108,"ready_two_reviews":0.2416},"latency_s":0.0581,"input_tokens":174},{"id":"hard-sol-a-trap-13","tier":"hard","family":"trap","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.5088,"no":0.4912},"latency_s":0.0502,"input_tokens":169},{"id":"hard-sol-a-trap-15","tier":"hard","family":"trap","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.4868,"no":0.5132},"latency_s":0.0714,"input_tokens":257},{"id":"hard-sol-b-judge_hard-01","tier":"hard","family":"judge_hard","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.7207,"no":0.2793},"latency_s":0.0755,"input_tokens":208},{"id":"hard-sol-b-judge_hard-02","tier":"hard","family":"judge_hard","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.5788,"no":0.4212},"latency_s":0.0365,"input_tokens":163},{"id":"hard-sol-b-judge_hard-05","tier":"hard","family":"judge_hard","gold":"yes","predicted":"no","correct":false,"schema_valid":true,"probs":{"yes":0.4706,"no":0.5294},"latency_s":0.0714,"input_tokens":210},{"id":"hard-sol-b-judge_hard-08","tier":"hard","family":"judge_hard","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.5397,"no":0.4603},"latency_s":0.0421,"input_tokens":149},{"id":"hard-sol-b-judge_hard-10","tier":"hard","family":"judge_hard","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.6475,"no":0.3525},"latency_s":0.0619,"input_tokens":172},{"id":"hard-sol-b-judge_hard-14","tier":"hard","family":"judge_hard","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.5471,"no":0.4529},"latency_s":0.0673,"input_tokens":204},{"id":"hard-sol-b-judge_hard-15","tier":"hard","family":"judge_hard","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.7214,"no":0.2786},"latency_s":0.0684,"input_tokens":153},{"id":"hard-sol-b-judge_hard-18","tier":"hard","family":"judge_hard","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.7754,"no":0.2246},"latency_s":0.062,"input_tokens":139},{"id":"hard-sol-b-long_policy-01","tier":"hard","family":"long_policy","gold":"reject_entire_claim","predicted":"approve_lodging_only","correct":false,"schema_valid":true,"probs":{"approve_all":0.2026,"approve_lodging_only":0.2965,"approve_meals_only":0.2488,"reject_entire_claim":0.2521},"latency_s":0.172,"input_tokens":2701},{"id":"hard-sol-b-long_policy-02","tier":"hard","family":"long_policy","gold":"deny_current_export","predicted":"deny_current_export","correct":true,"schema_valid":true,"probs":{"analytics_owner_only":0.2955,"deny_current_export":0.303,"privacy_and_security":0.2331,"privacy_only":0.1684},"latency_s":0.166,"input_tokens":2586},{"id":"hard-sol-b-long_policy-05","tier":"hard","family":"long_policy","gold":"delete_profile_omar_april_and_table","predicted":"delete_profile_omar_april_and_table","correct":true,"schema_valid":true,"probs":{"delete_everything":0.2299,"delete_nothing":0.1883,"delete_only_profile":0.266,"delete_profile_omar_april_and_table":0.3159},"latency_s":0.1578,"input_tokens":2530},{"id":"hard-sol-b-long_policy-06","tier":"hard","family":"long_policy","gold":"east_ward","predicted":"river_arts","correct":false,"schema_valid":true,"probs":{"east_ward":0.2629,"harbor_neighbors":0.2614,"no_award":0.21,"river_arts":0.2657},"latency_s":0.1581,"input_tokens":2547},{"id":"hard-sol-b-routing_hard-01","tier":"hard","family":"routing_hard","gold":"office_file_repair","predicted":"finance_review","correct":false,"schema_valid":true,"probs":{"data_pipeline":0.0899,"desktop_support":0.1244,"finance_review":0.3394,"office_file_repair":0.2472,"spreadsheet_formula":0.1991},"latency_s":0.0836,"input_tokens":306},{"id":"hard-sol-b-routing_hard-02","tier":"hard","family":"routing_hard","gold":"ledger_reconciliation","predicted":"ledger_reconciliation","correct":true,"schema_valid":true,"probs":{"customer_support":0.1004,"fraud_review":0.039,"ledger_reconciliation":0.4014,"mobile_app":0.1205,"payment_disputes":0.3387},"latency_s":0.0585,"input_tokens":275},{"id":"hard-sol-b-routing_hard-03","tier":"hard","family":"routing_hard","gold":"network_dns","predicted":"api_platform","correct":false,"schema_valid":true,"probs":{"api_platform":0.3565,"authoritative_dns":0.1394,"endpoint_devices":0.1037,"network_dns":0.3122,"tls_certificates":0.0882},"latency_s":0.0474,"input_tokens":289},{"id":"hard-sol-b-routing_hard-07","tier":"hard","family":"routing_hard","gold":"release_notarization","predicted":"macos_runtime","correct":false,"schema_valid":true,"probs":{"ci_infrastructure":0.1792,"code_signing":0.2567,"endpoint_security":0.0589,"macos_runtime":0.4617,"release_notarization":0.0435},"latency_s":0.0619,"input_tokens":254},{"id":"hard-sol-b-routing_hard-09","tier":"hard","family":"routing_hard","gold":"source_access","predicted":"identity_login","correct":false,"schema_valid":true,"probs":{"git_client":0.1358,"identity_login":0.5565,"repository_admin":0.1183,"source_access":0.0428,"vpn_network":0.1466},"latency_s":0.066,"input_tokens":266},{"id":"hard-sol-b-temporal_numeric-01","tier":"hard","family":"temporal_numeric","gold":"late_by_under_2h","predicted":"before_open","correct":false,"schema_valid":true,"probs":{"before_open":0.3216,"late_by_2h_or_more":0.1787,"late_by_under_2h":0.1791,"within_window":0.3206},"latency_s":0.0601,"input_tokens":221},{"id":"hard-sol-b-temporal_numeric-02","tier":"hard","family":"temporal_numeric","gold":"over_limit","predicted":"over_limit","correct":true,"schema_valid":true,"probs":{"at_least_100gb_below":0.243,"exactly_equal":0.1466,"over_limit":0.345,"under_100gb_below":0.2654},"latency_s":0.0632,"input_tokens":221},{"id":"hard-sol-b-temporal_numeric-03","tier":"hard","family":"temporal_numeric","gold":"9_credits","predicted":"6_credits","correct":false,"schema_valid":true,"probs":{"12_credits":0.2715,"15_credits":0.1669,"6_credits":0.3099,"9_credits":0.2517},"latency_s":0.0441,"input_tokens":173},{"id":"hard-sol-b-temporal_numeric-04","tier":"hard","family":"temporal_numeric","gold":"pass","predicted":"narrow_fail","correct":false,"schema_valid":true,"probs":{"clear_fail":0.4057,"narrow_fail":0.4347,"pass":0.1596},"latency_s":0.046,"input_tokens":187},{"id":"hard-sol-c-judge_hard-03","tier":"hard","family":"judge_hard","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.576,"no":0.424},"latency_s":0.0832,"input_tokens":392},{"id":"hard-sol-c-judge_hard-05","tier":"hard","family":"judge_hard","gold":"yes","predicted":"no","correct":false,"schema_valid":true,"probs":{"yes":0.4839,"no":0.5161},"latency_s":0.068,"input_tokens":299},{"id":"hard-sol-c-judge_hard-07","tier":"hard","family":"judge_hard","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.548,"no":0.452},"latency_s":0.0616,"input_tokens":272},{"id":"hard-sol-c-judge_hard-08","tier":"hard","family":"judge_hard","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.5434,"no":0.4566},"latency_s":0.0745,"input_tokens":286},{"id":"hard-sol-c-judge_hard-09","tier":"hard","family":"judge_hard","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.5486,"no":0.4514},"latency_s":0.2257,"input_tokens":280},{"id":"hard-sol-c-judge_hard-10","tier":"hard","family":"judge_hard","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.5863,"no":0.4137},"latency_s":0.0587,"input_tokens":297},{"id":"hard-sol-c-judge_hard-11","tier":"hard","family":"judge_hard","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.3846,"no":0.6154},"latency_s":0.0719,"input_tokens":316},{"id":"hard-sol-c-judge_hard-13","tier":"hard","family":"judge_hard","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.5499,"no":0.4501},"latency_s":0.0676,"input_tokens":275},{"id":"hard-sol-c-judge_hard-15","tier":"hard","family":"judge_hard","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.5258,"no":0.4742},"latency_s":0.0501,"input_tokens":260},{"id":"hard-sol-c-multi_hop-07","tier":"hard","family":"multi_hop","gold":"programme_review","predicted":"programme_review","correct":true,"schema_valid":true,"probs":{"partial_release":0.2195,"programme_review":0.2833,"reject_draw":0.2204,"release_draw":0.2768},"latency_s":0.1641,"input_tokens":2451},{"id":"hard-sol-c-multi_hop-09","tier":"hard","family":"multi_hop","gold":"bus_substitution","predicted":"call_fen_cross","correct":false,"schema_valid":true,"probs":{"bus_substitution":0.2241,"call_fen_cross":0.2714,"pass_fen_cross":0.2392,"terminate_oakmere":0.2653},"latency_s":0.1363,"input_tokens":2456},{"id":"hard-sol-c-multi_hop-11","tier":"hard","family":"multi_hop","gold":"coverage_excluded","predicted":"adjuster_referral","correct":false,"schema_valid":true,"probs":{"adjuster_referral":0.3167,"coverage_confirmed":0.2192,"coverage_excluded":0.1702,"documentation_needed":0.2938},"latency_s":0.1364,"input_tokens":2445},{"id":"hard-sol-c-multi_hop-12","tier":"hard","family":"multi_hop","gold":"conservation_hold","predicted":"conservation_hold","correct":true,"schema_valid":true,"probs":{"conservation_hold":0.33,"deny_request":0.1261,"digital_copy_only":0.2349,"supervised_delivery":0.309},"latency_s":0.083,"input_tokens":480},{"id":"hard-sol-c-multi_hop-13","tier":"hard","family":"multi_hop","gold":"normal_release","predicted":"scrap_lot","correct":false,"schema_valid":true,"probs":{"engineering_review":0.3042,"normal_release":0.213,"rework":0.1344,"scrap_lot":0.3484},"latency_s":0.159,"input_tokens":2514},{"id":"original-adequacy-01-0","tier":"original","family":"adequacy","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.521,"no":0.479},"latency_s":0.0507,"input_tokens":51},{"id":"original-adequacy-01-1","tier":"original","family":"adequacy","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.7332,"no":0.2668},"latency_s":0.041,"input_tokens":48},{"id":"original-adequacy-02-0","tier":"original","family":"adequacy","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.7664,"no":0.2336},"latency_s":0.0509,"input_tokens":55},{"id":"original-adequacy-02-1","tier":"original","family":"adequacy","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.4082,"no":0.5918},"latency_s":0.057,"input_tokens":56},{"id":"original-adequacy-03-0","tier":"original","family":"adequacy","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.7658,"no":0.2342},"latency_s":0.0468,"input_tokens":55},{"id":"original-adequacy-03-1","tier":"original","family":"adequacy","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.7669,"no":0.2331},"latency_s":0.0418,"input_tokens":58},{"id":"original-adequacy-04-0","tier":"original","family":"adequacy","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.6646,"no":0.3354},"latency_s":0.0389,"input_tokens":61},{"id":"original-adequacy-04-1","tier":"original","family":"adequacy","gold":"yes","predicted":"no","correct":false,"schema_valid":true,"probs":{"yes":0.496,"no":0.504},"latency_s":0.0547,"input_tokens":58},{"id":"original-adequacy-05-0","tier":"original","family":"adequacy","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.7843,"no":0.2157},"latency_s":0.04,"input_tokens":49},{"id":"original-adequacy-05-1","tier":"original","family":"adequacy","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.6996,"no":0.3004},"latency_s":0.0514,"input_tokens":50},{"id":"original-adequacy-06-0","tier":"original","family":"adequacy","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.6465,"no":0.3535},"latency_s":0.0568,"input_tokens":61},{"id":"original-adequacy-06-1","tier":"original","family":"adequacy","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.81,"no":0.19},"latency_s":0.0503,"input_tokens":61},{"id":"original-extraction-01-0","tier":"original","family":"extraction","gold":"pickup","predicted":"pickup","correct":true,"schema_valid":true,"probs":{"courier":0.227,"pickup":0.5687,"post":0.1367,"unknown":0.0677},"latency_s":0.0334,"input_tokens":59},{"id":"original-extraction-01-1","tier":"original","family":"extraction","gold":"pickup","predicted":"pickup","correct":true,"schema_valid":true,"probs":{"courier":0.186,"pickup":0.663,"post":0.0812,"unknown":0.0698},"latency_s":0.0394,"input_tokens":61},{"id":"original-extraction-02-0","tier":"original","family":"extraction","gold":"unknown","predicted":"post","correct":false,"schema_valid":true,"probs":{"courier":0.2677,"pickup":0.1663,"post":0.3645,"unknown":0.2016},"latency_s":0.0547,"input_tokens":60},{"id":"original-extraction-02-1","tier":"original","family":"extraction","gold":"unknown","predicted":"courier","correct":false,"schema_valid":true,"probs":{"courier":0.5411,"pickup":0.1274,"post":0.1615,"unknown":0.17},"latency_s":0.035,"input_tokens":60},{"id":"original-extraction-03-0","tier":"original","family":"extraction","gold":"courier","predicted":"courier","correct":true,"schema_valid":true,"probs":{"courier":0.5793,"pickup":0.2687,"post":0.0741,"unknown":0.0779},"latency_s":0.0494,"input_tokens":64},{"id":"original-extraction-03-1","tier":"original","family":"extraction","gold":"courier","predicted":"courier","correct":true,"schema_valid":true,"probs":{"courier":0.583,"pickup":0.1227,"post":0.1113,"unknown":0.183},"latency_s":0.0343,"input_tokens":63},{"id":"original-extraction-04-0","tier":"original","family":"extraction","gold":"post","predicted":"post","correct":true,"schema_valid":true,"probs":{"courier":0.1788,"pickup":0.1725,"post":0.5046,"unknown":0.1441},"latency_s":0.0372,"input_tokens":60},{"id":"original-extraction-04-1","tier":"original","family":"extraction","gold":"post","predicted":"post","correct":true,"schema_valid":true,"probs":{"courier":0.3715,"pickup":0.1374,"post":0.3844,"unknown":0.1067},"latency_s":0.036,"input_tokens":59},{"id":"original-extraction-05-0","tier":"original","family":"extraction","gold":"unknown","predicted":"courier","correct":false,"schema_valid":true,"probs":{"courier":0.3519,"pickup":0.1744,"post":0.3261,"unknown":0.1476},"latency_s":0.0529,"input_tokens":61},{"id":"original-extraction-05-1","tier":"original","family":"extraction","gold":"unknown","predicted":"courier","correct":false,"schema_valid":true,"probs":{"courier":0.4275,"pickup":0.1601,"post":0.1659,"unknown":0.2466},"latency_s":0.0537,"input_tokens":57},{"id":"original-extraction-06-0","tier":"original","family":"extraction","gold":"courier","predicted":"courier","correct":true,"schema_valid":true,"probs":{"courier":0.4273,"pickup":0.089,"post":0.275,"unknown":0.2088},"latency_s":0.0387,"input_tokens":61},{"id":"original-extraction-06-1","tier":"original","family":"extraction","gold":"courier","predicted":"post","correct":false,"schema_valid":true,"probs":{"courier":0.2652,"pickup":0.0696,"post":0.5198,"unknown":0.1454},"latency_s":0.0324,"input_tokens":60},{"id":"original-intent-01-0","tier":"original","family":"intent","gold":"refund","predicted":"status","correct":false,"schema_valid":true,"probs":{"cancel":0.1594,"change_address":0.1751,"other":0.0792,"refund":0.217,"status":0.3693},"latency_s":0.0586,"input_tokens":63},{"id":"original-intent-01-1","tier":"original","family":"intent","gold":"refund","predicted":"refund","correct":true,"schema_valid":true,"probs":{"cancel":0.2011,"change_address":0.1145,"other":0.0758,"refund":0.3775,"status":0.2311},"latency_s":0.0445,"input_tokens":65},{"id":"original-intent-02-0","tier":"original","family":"intent","gold":"status","predicted":"status","correct":true,"schema_valid":true,"probs":{"cancel":0.1803,"change_address":0.2381,"other":0.1217,"refund":0.0969,"status":0.363},"latency_s":0.0333,"input_tokens":64},{"id":"original-intent-02-1","tier":"original","family":"intent","gold":"status","predicted":"status","correct":true,"schema_valid":true,"probs":{"cancel":0.2068,"change_address":0.1136,"other":0.1448,"refund":0.0825,"status":0.4522},"latency_s":0.047,"input_tokens":63},{"id":"original-intent-03-0","tier":"original","family":"intent","gold":"change_address","predicted":"status","correct":false,"schema_valid":true,"probs":{"cancel":0.227,"change_address":0.2428,"other":0.1026,"refund":0.1104,"status":0.3173},"latency_s":0.0354,"input_tokens":65},{"id":"original-intent-03-1","tier":"original","family":"intent","gold":"change_address","predicted":"change_address","correct":true,"schema_valid":true,"probs":{"cancel":0.1672,"change_address":0.3611,"other":0.0555,"refund":0.0757,"status":0.3405},"latency_s":0.0398,"input_tokens":63},{"id":"original-intent-04-0","tier":"original","family":"intent","gold":"cancel","predicted":"status","correct":false,"schema_valid":true,"probs":{"cancel":0.2446,"change_address":0.2328,"other":0.0995,"refund":0.0813,"status":0.3418},"latency_s":0.0364,"input_tokens":68},{"id":"original-intent-04-1","tier":"original","family":"intent","gold":"cancel","predicted":"status","correct":false,"schema_valid":true,"probs":{"cancel":0.1954,"change_address":0.1697,"other":0.1204,"refund":0.04,"status":0.4745},"latency_s":0.0355,"input_tokens":63},{"id":"original-intent-05-0","tier":"original","family":"intent","gold":"other","predicted":"refund","correct":false,"schema_valid":true,"probs":{"cancel":0.2133,"change_address":0.0925,"other":0.043,"refund":0.3382,"status":0.313},"latency_s":0.0259,"input_tokens":63},{"id":"original-intent-05-1","tier":"original","family":"intent","gold":"other","predicted":"status","correct":false,"schema_valid":true,"probs":{"cancel":0.1774,"change_address":0.133,"other":0.0639,"refund":0.2169,"status":0.4088},"latency_s":0.0586,"input_tokens":63},{"id":"original-intent-06-0","tier":"original","family":"intent","gold":"status","predicted":"status","correct":true,"schema_valid":true,"probs":{"cancel":0.298,"change_address":0.1462,"other":0.0712,"refund":0.103,"status":0.3816},"latency_s":0.0408,"input_tokens":61},{"id":"original-intent-06-1","tier":"original","family":"intent","gold":"status","predicted":"status","correct":true,"schema_valid":true,"probs":{"cancel":0.311,"change_address":0.1942,"other":0.1064,"refund":0.0765,"status":0.3118},"latency_s":0.0391,"input_tokens":66},{"id":"original-ordinal-01-0","tier":"original","family":"ordinal","gold":0,"predicted":"1","correct":false,"schema_valid":true,"probs":{"0":0.113,"1":0.3387,"2":0.2633,"3":0.285},"latency_s":0.0425,"input_tokens":70},{"id":"original-ordinal-01-1","tier":"original","family":"ordinal","gold":0,"predicted":"1","correct":false,"schema_valid":true,"probs":{"0":0.0829,"1":0.3711,"2":0.2942,"3":0.2519},"latency_s":0.0569,"input_tokens":72},{"id":"original-ordinal-02-0","tier":"original","family":"ordinal","gold":1,"predicted":"3","correct":false,"schema_valid":true,"probs":{"0":0.1165,"1":0.2892,"2":0.2907,"3":0.3036},"latency_s":0.0853,"input_tokens":74},{"id":"original-ordinal-02-1","tier":"original","family":"ordinal","gold":1,"predicted":"3","correct":false,"schema_valid":true,"probs":{"0":0.1159,"1":0.259,"2":0.281,"3":0.344},"latency_s":0.0508,"input_tokens":75},{"id":"original-ordinal-03-0","tier":"original","family":"ordinal","gold":2,"predicted":"1","correct":false,"schema_valid":true,"probs":{"0":0.1499,"1":0.3157,"2":0.2613,"3":0.2731},"latency_s":0.0635,"input_tokens":71},{"id":"original-ordinal-03-1","tier":"original","family":"ordinal","gold":2,"predicted":"1","correct":false,"schema_valid":true,"probs":{"0":0.1094,"1":0.3546,"2":0.2642,"3":0.2718},"latency_s":0.038,"input_tokens":73},{"id":"original-ordinal-04-0","tier":"original","family":"ordinal","gold":3,"predicted":"1","correct":false,"schema_valid":true,"probs":{"0":0.1282,"1":0.3255,"2":0.2598,"3":0.2865},"latency_s":0.0372,"input_tokens":73},{"id":"original-ordinal-04-1","tier":"original","family":"ordinal","gold":3,"predicted":"1","correct":false,"schema_valid":true,"probs":{"0":0.1212,"1":0.3587,"2":0.2367,"3":0.2834},"latency_s":0.0412,"input_tokens":72},{"id":"original-ordinal-05-0","tier":"original","family":"ordinal","gold":0,"predicted":"1","correct":false,"schema_valid":true,"probs":{"0":0.1241,"1":0.3281,"2":0.2431,"3":0.3047},"latency_s":0.0506,"input_tokens":76},{"id":"original-ordinal-05-1","tier":"original","family":"ordinal","gold":0,"predicted":"1","correct":false,"schema_valid":true,"probs":{"0":0.1455,"1":0.3099,"2":0.2934,"3":0.2513},"latency_s":0.0495,"input_tokens":75},{"id":"original-ordinal-06-0","tier":"original","family":"ordinal","gold":2,"predicted":"1","correct":false,"schema_valid":true,"probs":{"0":0.1703,"1":0.3238,"2":0.261,"3":0.2449},"latency_s":0.0376,"input_tokens":77},{"id":"original-ordinal-06-1","tier":"original","family":"ordinal","gold":2,"predicted":"1","correct":false,"schema_valid":true,"probs":{"0":0.1351,"1":0.32,"2":0.2935,"3":0.2515},"latency_s":0.0549,"input_tokens":84},{"id":"original-policy-01-0","tier":"original","family":"policy","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.3921,"no":0.6079},"latency_s":0.0558,"input_tokens":71},{"id":"original-policy-01-1","tier":"original","family":"policy","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.4131,"no":0.5869},"latency_s":0.0577,"input_tokens":73},{"id":"original-policy-02-0","tier":"original","family":"policy","gold":"yes","predicted":"no","correct":false,"schema_valid":true,"probs":{"yes":0.4869,"no":0.5131},"latency_s":0.0577,"input_tokens":63},{"id":"original-policy-02-1","tier":"original","family":"policy","gold":"yes","predicted":"no","correct":false,"schema_valid":true,"probs":{"yes":0.4856,"no":0.5144},"latency_s":0.0574,"input_tokens":62},{"id":"original-policy-03-0","tier":"original","family":"policy","gold":"yes","predicted":"no","correct":false,"schema_valid":true,"probs":{"yes":0.4399,"no":0.5601},"latency_s":0.0599,"input_tokens":65},{"id":"original-policy-03-1","tier":"original","family":"policy","gold":"yes","predicted":"no","correct":false,"schema_valid":true,"probs":{"yes":0.4181,"no":0.5819},"latency_s":0.0549,"input_tokens":65},{"id":"original-policy-04-0","tier":"original","family":"policy","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.4534,"no":0.5466},"latency_s":0.0429,"input_tokens":73},{"id":"original-policy-04-1","tier":"original","family":"policy","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.4219,"no":0.5781},"latency_s":0.0422,"input_tokens":71},{"id":"original-policy-05-0","tier":"original","family":"policy","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.4156,"no":0.5844},"latency_s":0.0601,"input_tokens":76},{"id":"original-policy-05-1","tier":"original","family":"policy","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.371,"no":0.629},"latency_s":0.0519,"input_tokens":69},{"id":"original-policy-06-0","tier":"original","family":"policy","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.5426,"no":0.4574},"latency_s":0.0411,"input_tokens":74},{"id":"original-policy-06-1","tier":"original","family":"policy","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.5459,"no":0.4541},"latency_s":0.0539,"input_tokens":64},{"id":"original-routing-01-0","tier":"original","family":"routing","gold":"math","predicted":"coding_agent","correct":false,"schema_valid":true,"probs":{"coding":0.1511,"coding_agent":0.2712,"document":0.161,"general":0.0878,"math":0.1855,"tools":0.1434},"latency_s":0.0447,"input_tokens":92},{"id":"original-routing-01-1","tier":"original","family":"routing","gold":"math","predicted":"coding_agent","correct":false,"schema_valid":true,"probs":{"coding":0.1251,"coding_agent":0.2869,"document":0.1652,"general":0.0831,"math":0.1678,"tools":0.172},"latency_s":0.0532,"input_tokens":89},{"id":"original-routing-02-0","tier":"original","family":"routing","gold":"coding","predicted":"coding_agent","correct":false,"schema_valid":true,"probs":{"coding":0.1477,"coding_agent":0.2335,"document":0.1626,"general":0.1492,"math":0.1599,"tools":0.147},"latency_s":0.0574,"input_tokens":96},{"id":"original-routing-02-1","tier":"original","family":"routing","gold":"coding","predicted":"coding_agent","correct":false,"schema_valid":true,"probs":{"coding":0.1241,"coding_agent":0.3236,"document":0.147,"general":0.1227,"math":0.1552,"tools":0.1274},"latency_s":0.0532,"input_tokens":90},{"id":"original-routing-03-0","tier":"original","family":"routing","gold":"coding_agent","predicted":"coding_agent","correct":true,"schema_valid":true,"probs":{"coding":0.1016,"coding_agent":0.4045,"document":0.1197,"general":0.067,"math":0.1275,"tools":0.1797},"latency_s":0.0592,"input_tokens":95},{"id":"original-routing-03-1","tier":"original","family":"routing","gold":"coding_agent","predicted":"coding_agent","correct":true,"schema_valid":true,"probs":{"coding":0.1045,"coding_agent":0.2979,"document":0.1482,"general":0.1011,"math":0.1303,"tools":0.2181},"latency_s":0.1462,"input_tokens":98},{"id":"original-routing-04-0","tier":"original","family":"routing","gold":"document","predicted":"coding_agent","correct":false,"schema_valid":true,"probs":{"coding":0.1273,"coding_agent":0.2791,"document":0.1531,"general":0.0995,"math":0.1667,"tools":0.1742},"latency_s":0.0464,"input_tokens":91},{"id":"original-routing-04-1","tier":"original","family":"routing","gold":"document","predicted":"coding_agent","correct":false,"schema_valid":true,"probs":{"coding":0.1187,"coding_agent":0.252,"document":0.1704,"general":0.1452,"math":0.1602,"tools":0.1536},"latency_s":0.0376,"input_tokens":91},{"id":"original-routing-05-0","tier":"original","family":"routing","gold":"tools","predicted":"coding_agent","correct":false,"schema_valid":true,"probs":{"coding":0.126,"coding_agent":0.2845,"document":0.1434,"general":0.1006,"math":0.1645,"tools":0.181},"latency_s":0.0524,"input_tokens":92},{"id":"original-routing-05-1","tier":"original","family":"routing","gold":"tools","predicted":"coding_agent","correct":false,"schema_valid":true,"probs":{"coding":0.1371,"coding_agent":0.2629,"document":0.1343,"general":0.1047,"math":0.1677,"tools":0.1932},"latency_s":0.0386,"input_tokens":93},{"id":"original-routing-06-0","tier":"original","family":"routing","gold":"general","predicted":"coding_agent","correct":false,"schema_valid":true,"probs":{"coding":0.1286,"coding_agent":0.2707,"document":0.1568,"general":0.1187,"math":0.1716,"tools":0.1536},"latency_s":0.0509,"input_tokens":91},{"id":"original-routing-06-1","tier":"original","family":"routing","gold":"general","predicted":"coding_agent","correct":false,"schema_valid":true,"probs":{"coding":0.1136,"coding_agent":0.3019,"document":0.1627,"general":0.115,"math":0.1558,"tools":0.151},"latency_s":0.1864,"input_tokens":91}]},"tia-1.2-mini":{"display":"TIA 1.2 mini","run_by":"TIA","date":"2026-09-30","device":"local CPU worker on the J-97 workstation, harness in Docker, loopback","adapter":"typesafe","items":[{"id":"easy-extraction-00","tier":"easy","family":"extraction","gold":"paypal","predicted":"paypal","correct":true,"schema_valid":true,"probs":{"bank_transfer":0.0439,"cash":0.0942,"credit_card":0.2593,"paypal":0.6026},"latency_s":0.06,"input_tokens":43},{"id":"easy-extraction-01","tier":"easy","family":"extraction","gold":"cash","predicted":"cash","correct":true,"schema_valid":true,"probs":{"bank_transfer":0.1432,"cash":0.4918,"credit_card":0.2592,"paypal":0.1058},"latency_s":0.0551,"input_tokens":46},{"id":"easy-extraction-02","tier":"easy","family":"extraction","gold":"bank_transfer","predicted":"bank_transfer","correct":true,"schema_valid":true,"probs":{"bank_transfer":0.4811,"cash":0.2222,"credit_card":0.2221,"paypal":0.0747},"latency_s":0.0524,"input_tokens":46},{"id":"easy-extraction-03","tier":"easy","family":"extraction","gold":"high","predicted":"high","correct":true,"schema_valid":true,"probs":{"high":0.3783,"low":0.0407,"medium":0.2361,"urgent":0.3449},"latency_s":0.0417,"input_tokens":45},{"id":"easy-extraction-04","tier":"easy","family":"extraction","gold":"low","predicted":"high","correct":false,"schema_valid":true,"probs":{"high":0.3745,"low":0.0391,"medium":0.2374,"urgent":0.349},"latency_s":0.0588,"input_tokens":42},{"id":"easy-extraction-05","tier":"easy","family":"extraction","gold":"urgent","predicted":"high","correct":false,"schema_valid":true,"probs":{"high":0.4071,"low":0.0311,"medium":0.1924,"urgent":0.3694},"latency_s":0.0354,"input_tokens":42},{"id":"easy-extraction-06","tier":"easy","family":"extraction","gold":"M","predicted":"L","correct":false,"schema_valid":true,"probs":{"L":0.2879,"M":0.2745,"S":0.2348,"XL":0.2028},"latency_s":0.0399,"input_tokens":34},{"id":"easy-extraction-07","tier":"easy","family":"extraction","gold":"XL","predicted":"XL","correct":true,"schema_valid":true,"probs":{"L":0.2881,"M":0.1942,"S":0.2289,"XL":0.2888},"latency_s":0.0607,"input_tokens":32},{"id":"easy-extraction-08","tier":"easy","family":"extraction","gold":"thursday","predicted":"thursday","correct":true,"schema_valid":true,"probs":{"friday":0.1543,"monday":0.1787,"thursday":0.3806,"tuesday":0.1245,"wednesday":0.1619},"latency_s":0.0574,"input_tokens":38},{"id":"easy-extraction-09","tier":"easy","family":"extraction","gold":"monday","predicted":"monday","correct":true,"schema_valid":true,"probs":{"friday":0.1628,"monday":0.4435,"thursday":0.0978,"tuesday":0.1722,"wednesday":0.1236},"latency_s":0.0604,"input_tokens":42},{"id":"easy-extraction-10","tier":"easy","family":"extraction","gold":"friday","predicted":"friday","correct":true,"schema_valid":true,"probs":{"friday":0.3529,"monday":0.2472,"thursday":0.1468,"tuesday":0.1085,"wednesday":0.1446},"latency_s":0.0289,"input_tokens":38},{"id":"easy-extraction-11","tier":"easy","family":"extraction","gold":"credit_card","predicted":"credit_card","correct":true,"schema_valid":true,"probs":{"bank_transfer":0.0743,"cash":0.1694,"credit_card":0.6295,"paypal":0.1268},"latency_s":0.0621,"input_tokens":46},{"id":"easy-fact-00","tier":"easy","family":"fact","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.5564,"no":0.4436},"latency_s":0.0354,"input_tokens":53},{"id":"easy-fact-01","tier":"easy","family":"fact","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.567,"no":0.433},"latency_s":0.0496,"input_tokens":50},{"id":"easy-fact-02","tier":"easy","family":"fact","gold":"yes","predicted":"no","correct":false,"schema_valid":true,"probs":{"yes":0.4902,"no":0.5098},"latency_s":0.0347,"input_tokens":54},{"id":"easy-fact-03","tier":"easy","family":"fact","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.4603,"no":0.5397},"latency_s":0.0374,"input_tokens":58},{"id":"easy-fact-04","tier":"easy","family":"fact","gold":"yes","predicted":"no","correct":false,"schema_valid":true,"probs":{"yes":0.4648,"no":0.5352},"latency_s":0.0576,"input_tokens":54},{"id":"easy-fact-05","tier":"easy","family":"fact","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.4667,"no":0.5333},"latency_s":0.0608,"input_tokens":55},{"id":"easy-fact-06","tier":"easy","family":"fact","gold":"yes","predicted":"no","correct":false,"schema_valid":true,"probs":{"yes":0.454,"no":0.546},"latency_s":0.04,"input_tokens":62},{"id":"easy-fact-07","tier":"easy","family":"fact","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.4318,"no":0.5682},"latency_s":0.0536,"input_tokens":50},{"id":"easy-fact-08","tier":"easy","family":"fact","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.5504,"no":0.4496},"latency_s":0.0591,"input_tokens":46},{"id":"easy-fact-09","tier":"easy","family":"fact","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.5307,"no":0.4693},"latency_s":0.0276,"input_tokens":46},{"id":"easy-fact-10","tier":"easy","family":"fact","gold":"yes","predicted":"no","correct":false,"schema_valid":true,"probs":{"yes":0.411,"no":0.589},"latency_s":0.0601,"input_tokens":53},{"id":"easy-fact-11","tier":"easy","family":"fact","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.3875,"no":0.6125},"latency_s":0.0594,"input_tokens":54},{"id":"easy-intent-00","tier":"easy","family":"intent","gold":"track_order","predicted":"cancel_order","correct":false,"schema_valid":true,"probs":{"billing_question":0.0612,"cancel_order":0.4109,"change_address":0.1819,"report_damage":0.0753,"track_order":0.2706},"latency_s":0.1167,"input_tokens":91},{"id":"easy-intent-01","tier":"easy","family":"intent","gold":"cancel_order","predicted":"cancel_order","correct":true,"schema_valid":true,"probs":{"billing_question":0.0214,"cancel_order":0.6329,"change_address":0.1076,"report_damage":0.0345,"track_order":0.2035},"latency_s":0.0777,"input_tokens":89},{"id":"easy-intent-02","tier":"easy","family":"intent","gold":"change_address","predicted":"cancel_order","correct":false,"schema_valid":true,"probs":{"billing_question":0.0508,"cancel_order":0.451,"change_address":0.2049,"report_damage":0.0542,"track_order":0.239},"latency_s":0.0445,"input_tokens":93},{"id":"easy-intent-03","tier":"easy","family":"intent","gold":"report_damage","predicted":"cancel_order","correct":false,"schema_valid":true,"probs":{"billing_question":0.0579,"cancel_order":0.2793,"change_address":0.2277,"report_damage":0.2431,"track_order":0.1919},"latency_s":0.0491,"input_tokens":83},{"id":"easy-intent-04","tier":"easy","family":"intent","gold":"billing_question","predicted":"cancel_order","correct":false,"schema_valid":true,"probs":{"billing_question":0.2253,"cancel_order":0.3228,"change_address":0.1487,"report_damage":0.0678,"track_order":0.2353},"latency_s":0.0693,"input_tokens":88},{"id":"easy-intent-05","tier":"easy","family":"intent","gold":"track_order","predicted":"cancel_order","correct":false,"schema_valid":true,"probs":{"billing_question":0.0216,"cancel_order":0.4368,"change_address":0.2011,"report_damage":0.0879,"track_order":0.2526},"latency_s":0.0628,"input_tokens":87},{"id":"easy-intent-06","tier":"easy","family":"intent","gold":"set_alarm","predicted":"send_message","correct":false,"schema_valid":true,"probs":{"play_music":0.1644,"send_message":0.4043,"set_alarm":0.1794,"turn_off_lights":0.1013,"weather":0.1506},"latency_s":0.0553,"input_tokens":74},{"id":"easy-intent-07","tier":"easy","family":"intent","gold":"play_music","predicted":"send_message","correct":false,"schema_valid":true,"probs":{"play_music":0.2034,"send_message":0.362,"set_alarm":0.1289,"turn_off_lights":0.0763,"weather":0.2294},"latency_s":0.0414,"input_tokens":68},{"id":"easy-intent-08","tier":"easy","family":"intent","gold":"weather","predicted":"send_message","correct":false,"schema_valid":true,"probs":{"play_music":0.1228,"send_message":0.3502,"set_alarm":0.1046,"turn_off_lights":0.0764,"weather":0.346},"latency_s":0.0682,"input_tokens":70},{"id":"easy-intent-09","tier":"easy","family":"intent","gold":"send_message","predicted":"send_message","correct":true,"schema_valid":true,"probs":{"play_music":0.1536,"send_message":0.4232,"set_alarm":0.1453,"turn_off_lights":0.1065,"weather":0.1714},"latency_s":0.0435,"input_tokens":73},{"id":"easy-intent-10","tier":"easy","family":"intent","gold":"turn_off_lights","predicted":"send_message","correct":false,"schema_valid":true,"probs":{"play_music":0.1359,"send_message":0.346,"set_alarm":0.1752,"turn_off_lights":0.2219,"weather":0.1209},"latency_s":0.0341,"input_tokens":71},{"id":"easy-intent-11","tier":"easy","family":"intent","gold":"set_alarm","predicted":"set_alarm","correct":true,"schema_valid":true,"probs":{"play_music":0.1015,"send_message":0.2591,"set_alarm":0.4562,"turn_off_lights":0.066,"weather":0.1172},"latency_s":0.065,"input_tokens":70},{"id":"easy-tool_selection-00","tier":"easy","family":"tool_selection","gold":"get_weather","predicted":"get_weather","correct":true,"schema_valid":true,"probs":{"convert_currency":0.1015,"create_calendar_event":0.1895,"get_weather":0.4371,"send_email":0.1303,"translate_text":0.1415},"latency_s":0.0441,"input_tokens":78},{"id":"easy-tool_selection-01","tier":"easy","family":"tool_selection","gold":"send_email","predicted":"send_email","correct":true,"schema_valid":true,"probs":{"convert_currency":0.1063,"create_calendar_event":0.2057,"get_weather":0.1441,"send_email":0.3451,"translate_text":0.1989},"latency_s":0.0469,"input_tokens":79},{"id":"easy-tool_selection-02","tier":"easy","family":"tool_selection","gold":"create_calendar_event","predicted":"create_calendar_event","correct":true,"schema_valid":true,"probs":{"convert_currency":0.135,"create_calendar_event":0.2899,"get_weather":0.1623,"send_email":0.2241,"translate_text":0.1887},"latency_s":0.0665,"input_tokens":81},{"id":"easy-tool_selection-03","tier":"easy","family":"tool_selection","gold":"convert_currency","predicted":"create_calendar_event","correct":false,"schema_valid":true,"probs":{"convert_currency":0.1305,"create_calendar_event":0.2941,"get_weather":0.1783,"send_email":0.2074,"translate_text":0.1896},"latency_s":0.0529,"input_tokens":77},{"id":"easy-tool_selection-04","tier":"easy","family":"tool_selection","gold":"translate_text","predicted":"create_calendar_event","correct":false,"schema_valid":true,"probs":{"convert_currency":0.1358,"create_calendar_event":0.2579,"get_weather":0.1756,"send_email":0.2105,"translate_text":0.2203},"latency_s":0.0657,"input_tokens":77},{"id":"easy-tool_selection-05","tier":"easy","family":"tool_selection","gold":"create_calendar_event","predicted":"create_calendar_event","correct":true,"schema_valid":true,"probs":{"convert_currency":0.1222,"create_calendar_event":0.3598,"get_weather":0.1679,"send_email":0.1649,"translate_text":0.1853},"latency_s":0.0725,"input_tokens":80},{"id":"easy-tool_selection-06","tier":"easy","family":"tool_selection","gold":"search_flights","predicted":"call_taxi","correct":false,"schema_valid":true,"probs":{"book_restaurant":0.1729,"call_taxi":0.2729,"get_stock_price":0.16,"search_flights":0.253,"set_reminder":0.1411},"latency_s":0.0432,"input_tokens":84},{"id":"easy-tool_selection-07","tier":"easy","family":"tool_selection","gold":"book_restaurant","predicted":"book_restaurant","correct":true,"schema_valid":true,"probs":{"book_restaurant":0.32,"call_taxi":0.2043,"get_stock_price":0.2134,"search_flights":0.1256,"set_reminder":0.1367},"latency_s":0.0342,"input_tokens":86},{"id":"easy-tool_selection-08","tier":"easy","family":"tool_selection","gold":"set_reminder","predicted":"call_taxi","correct":false,"schema_valid":true,"probs":{"book_restaurant":0.15,"call_taxi":0.3773,"get_stock_price":0.1605,"search_flights":0.1163,"set_reminder":0.1959},"latency_s":0.0434,"input_tokens":84},{"id":"easy-tool_selection-09","tier":"easy","family":"tool_selection","gold":"get_stock_price","predicted":"get_stock_price","correct":true,"schema_valid":true,"probs":{"book_restaurant":0.1292,"call_taxi":0.1343,"get_stock_price":0.5339,"search_flights":0.0922,"set_reminder":0.1104},"latency_s":0.0353,"input_tokens":82},{"id":"easy-tool_selection-10","tier":"easy","family":"tool_selection","gold":"call_taxi","predicted":"call_taxi","correct":true,"schema_valid":true,"probs":{"book_restaurant":0.188,"call_taxi":0.2969,"get_stock_price":0.1958,"search_flights":0.1307,"set_reminder":0.1885},"latency_s":0.0629,"input_tokens":82},{"id":"easy-tool_selection-11","tier":"easy","family":"tool_selection","gold":"set_reminder","predicted":"set_reminder","correct":true,"schema_valid":true,"probs":{"book_restaurant":0.1564,"call_taxi":0.1877,"get_stock_price":0.1643,"search_flights":0.1521,"set_reminder":0.3395},"latency_s":0.0583,"input_tokens":84},{"id":"hard-opus-a-long_policy-01","tier":"hard","family":"long_policy","gold":"pay_subject_to_15000_sublimit","predicted":"deny_vacancy_exclusion","correct":false,"schema_valid":true,"probs":{"deny_repeated_seepage":0.2086,"deny_vacancy_exclusion":0.3716,"pay_full_estimate_less_deductible":0.1056,"pay_subject_to_10000_sublimit":0.1763,"pay_subject_to_15000_sublimit":0.1379},"latency_s":0.197,"input_tokens":3733},{"id":"hard-opus-a-long_policy-04","tier":"hard","family":"long_policy","gold":"cfo","predicted":"department_head","correct":false,"schema_valid":true,"probs":{"budget_holder":0.1943,"cfo":0.152,"department_head":0.2846,"executive_committee":0.1516,"vp_and_finance_director":0.2176},"latency_s":0.1984,"input_tokens":2460},{"id":"hard-opus-a-long_policy-08","tier":"hard","family":"long_policy","gold":"parts_covered_labour_charged","predicted":"parts_covered_labour_charged","correct":true,"schema_valid":true,"probs":{"full_repair_no_charge":0.2011,"not_covered":0.2481,"parts_covered_labour_charged":0.3413,"pro_rata_replacement_credit":0.2095},"latency_s":0.1712,"input_tokens":2293},{"id":"hard-opus-a-long_policy-09","tier":"hard","family":"long_policy","gold":"license_required","predicted":"license_exception_lvs","correct":false,"schema_valid":true,"probs":{"license_exception_lvs":0.3631,"license_required":0.252,"no_license_required":0.1366,"shipment_prohibited":0.2482},"latency_s":0.1606,"input_tokens":2402},{"id":"hard-opus-a-long_policy-11","tier":"hard","family":"long_policy","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.5219,"no":0.4781},"latency_s":0.1403,"input_tokens":2146},{"id":"hard-opus-a-long_policy-13","tier":"hard","family":"long_policy","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.6716,"no":0.3284},"latency_s":0.1568,"input_tokens":2280},{"id":"hard-opus-a-long_policy-17","tier":"hard","family":"long_policy","gold":"database_oncall","predicted":"security_incident_response","correct":false,"schema_valid":true,"probs":{"database_oncall":0.1595,"no_page_ticket_only":0.1431,"platform_oncall":0.2112,"security_incident_response":0.2844,"vendor_support":0.2018},"latency_s":0.1546,"input_tokens":2287},{"id":"hard-opus-a-long_policy-19","tier":"hard","family":"long_policy","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.619,"no":0.381},"latency_s":0.1717,"input_tokens":2106},{"id":"hard-opus-a-probability-03","tier":"hard","family":"probability","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.5119,"no":0.4881},"latency_s":0.0846,"input_tokens":376},{"id":"hard-opus-a-probability-04","tier":"hard","family":"probability","gold":"on_time","predicted":"on_time","correct":true,"schema_valid":true,"probs":{"early":0.302,"late":0.2863,"on_time":0.4116},"latency_s":0.1475,"input_tokens":900},{"id":"hard-opus-a-probability-07","tier":"hard","family":"probability","gold":"escalated","predicted":"escalated","correct":true,"schema_valid":true,"probs":{"escalated":0.4055,"reopened":0.2431,"resolved_first_contact":0.3514},"latency_s":0.1467,"input_tokens":954},{"id":"hard-opus-a-probability-08","tier":"hard","family":"probability","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.4958,"no":0.5042},"latency_s":0.0833,"input_tokens":312},{"id":"hard-opus-a-temporal_numeric-03","tier":"hard","family":"temporal_numeric","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.503,"no":0.497},"latency_s":0.0803,"input_tokens":417},{"id":"hard-opus-a-temporal_numeric-06","tier":"hard","family":"temporal_numeric","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.5268,"no":0.4732},"latency_s":0.0897,"input_tokens":418},{"id":"hard-opus-a-temporal_numeric-07","tier":"hard","family":"temporal_numeric","gold":"sep_26","predicted":"sep_26","correct":true,"schema_valid":true,"probs":{"sep_24":0.249,"sep_25":0.0979,"sep_26":0.3816,"sep_27":0.1503,"sep_28":0.1212},"latency_s":0.141,"input_tokens":908},{"id":"hard-opus-a-temporal_numeric-09","tier":"hard","family":"temporal_numeric","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.5234,"no":0.4766},"latency_s":0.0664,"input_tokens":439},{"id":"hard-opus-a-temporal_numeric-12","tier":"hard","family":"temporal_numeric","gold":1,"predicted":"2","correct":false,"schema_valid":true,"probs":{"0":0.041,"1":0.2642,"2":0.3941,"3":0.3007},"latency_s":0.0896,"input_tokens":461},{"id":"hard-opus-b-ambiguous-02","tier":"hard","family":"ambiguous","gold":"real_incident","predicted":"cannot_determine","correct":false,"schema_valid":true,"probs":{"cannot_determine":0.4651,"false_positive":0.2704,"real_incident":0.2645},"latency_s":0.103,"input_tokens":484},{"id":"hard-opus-b-ambiguous-03","tier":"hard","family":"ambiguous","gold":"time_barred","predicted":"time_barred","correct":true,"schema_valid":true,"probs":{"insufficient_information":0.1517,"time_barred":0.5271,"within_limitation":0.3212},"latency_s":0.0881,"input_tokens":441},{"id":"hard-opus-b-ambiguous-07","tier":"hard","family":"ambiguous","gold":"within_policy","predicted":"insufficient_information","correct":false,"schema_valid":true,"probs":{"insufficient_information":0.3772,"over_limit":0.3007,"within_policy":0.3221},"latency_s":0.0681,"input_tokens":439},{"id":"hard-opus-b-ambiguous-09","tier":"hard","family":"ambiguous","gold":"legitimate","predicted":"inconclusive","correct":false,"schema_valid":true,"probs":{"compromised":0.2316,"inconclusive":0.4451,"legitimate":0.3233},"latency_s":0.0989,"input_tokens":589},{"id":"hard-opus-b-ambiguous-10","tier":"hard","family":"ambiguous","gold":"ask_user_which_alex","predicted":"ask_user_which_alex","correct":true,"schema_valid":true,"probs":{"ask_user_which_alex":0.3979,"send_to_alex_moreno":0.3036,"send_to_alexandra_lind":0.2985},"latency_s":0.0735,"input_tokens":446},{"id":"hard-opus-b-ambiguous-11","tier":"hard","family":"ambiguous","gold":"cannot_determine","predicted":"cannot_determine","correct":true,"schema_valid":true,"probs":{"cannot_determine":0.3961,"grade_correct":0.2859,"grade_should_be_higher":0.318},"latency_s":0.0721,"input_tokens":396},{"id":"hard-opus-b-ambiguous-13","tier":"hard","family":"ambiguous","gold":"cannot_determine","predicted":"charge_platform_team","correct":false,"schema_valid":true,"probs":{"cannot_determine":0.1854,"charge_data_team":0.2722,"charge_platform_team":0.3533,"split_50_50":0.1892},"latency_s":0.0995,"input_tokens":525},{"id":"hard-opus-b-multi_hop-03","tier":"hard","family":"multi_hop","gold":"tier3_cfo","predicted":"tier2_department_head","correct":false,"schema_valid":true,"probs":{"tier1_team_lead":0.2666,"tier2_department_head":0.2906,"tier3_cfo":0.2107,"tier4_board":0.232},"latency_s":0.1603,"input_tokens":2696},{"id":"hard-opus-b-multi_hop-04","tier":"hard","family":"multi_hop","gold":"bjorn","predicted":"esra","correct":false,"schema_valid":true,"probs":{"ana":0.1719,"bjorn":0.1309,"chen":0.1379,"dmitri":0.1384,"esra":0.2147,"no_page_ticket_only":0.2062},"latency_s":0.1493,"input_tokens":2200},{"id":"hard-opus-b-multi_hop-05","tier":"hard","family":"multi_hop","gold":2,"predicted":"1","correct":false,"schema_valid":true,"probs":{"0":0.006,"1":0.3535,"2":0.2258,"3":0.1815,"4":0.2332},"latency_s":0.1497,"input_tokens":2293},{"id":"hard-opus-b-multi_hop-07","tier":"hard","family":"multi_hop","gold":"credit_20_percent","predicted":"credit_20_percent","correct":true,"schema_valid":true,"probs":{"credit_10_percent":0.2009,"credit_20_percent":0.2417,"credit_25_percent":0.1216,"credit_50_percent":0.126,"credit_full_month":0.2127,"no_credit":0.0971},"latency_s":0.1598,"input_tokens":2260},{"id":"hard-opus-b-multi_hop-08","tier":"hard","family":"multi_hop","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.639,"no":0.361},"latency_s":0.1461,"input_tokens":2021},{"id":"hard-opus-b-probability-01","tier":"hard","family":"probability","gold":"merchant_wins","predicted":"customer_withdraws","correct":false,"schema_valid":true,"probs":{"customer_wins":0.2249,"customer_withdraws":0.4878,"merchant_wins":0.2874},"latency_s":0.1458,"input_tokens":1639},{"id":"hard-opus-b-probability-02","tier":"hard","family":"probability","gold":"upstream_provider","predicted":"bad_push","correct":false,"schema_valid":true,"probs":{"bad_push":0.9,"database_hardware":0.0553,"upstream_provider":0.0447},"latency_s":0.0967,"input_tokens":434},{"id":"hard-opus-b-probability-03","tier":"hard","family":"probability","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.5246,"no":0.4754},"latency_s":0.1415,"input_tokens":971},{"id":"hard-opus-b-probability-04","tier":"hard","family":"probability","gold":"attended","predicted":"attended","correct":true,"schema_valid":true,"probs":{"attended":0.3959,"late_cancellation":0.3879,"no_show":0.2162},"latency_s":0.0875,"input_tokens":470},{"id":"hard-opus-b-probability-06","tier":"hard","family":"probability","gold":"upheld","predicted":"upheld","correct":true,"schema_valid":true,"probs":{"modified":0.3754,"overturned":0.2403,"upheld":0.3843},"latency_s":0.1609,"input_tokens":878},{"id":"hard-opus-b-tradeoff-01","tier":"hard","family":"tradeoff","gold":"escalate_to_engineering","predicted":"page_oncall","correct":false,"schema_valid":true,"probs":{"escalate_to_engineering":0.2302,"escalate_to_security":0.2876,"page_oncall":0.3531,"process_refund":0.0649,"send_kb_article":0.0641},"latency_s":0.1379,"input_tokens":799},{"id":"hard-opus-b-tradeoff-03","tier":"hard","family":"tradeoff","gold":"order_b","predicted":"order_c","correct":false,"schema_valid":true,"probs":{"order_a":0.1742,"order_b":0.1999,"order_c":0.3133,"order_d":0.3127},"latency_s":0.0984,"input_tokens":648},{"id":"hard-opus-b-tradeoff-06","tier":"hard","family":"tradeoff","gold":"yes","predicted":"no","correct":false,"schema_valid":true,"probs":{"yes":0.4314,"no":0.5686},"latency_s":0.1005,"input_tokens":665},{"id":"hard-opus-b-tradeoff-07","tier":"hard","family":"tradeoff","gold":"p0_fix_24h","predicted":"p2_fix_30d","correct":false,"schema_valid":true,"probs":{"out_of_scope":0.1263,"p0_fix_24h":0.1495,"p1_fix_7d":0.1801,"p2_fix_30d":0.4258,"p3_backlog":0.1183},"latency_s":0.1109,"input_tokens":674},{"id":"hard-opus-b-tradeoff-08","tier":"hard","family":"tradeoff","gold":"yes","predicted":"no","correct":false,"schema_valid":true,"probs":{"yes":0.3979,"no":0.6021},"latency_s":0.1032,"input_tokens":513},{"id":"hard-opus-b-tradeoff-12","tier":"hard","family":"tradeoff","gold":"refund_now","predicted":"escalate_to_fraud_team","correct":false,"schema_valid":true,"probs":{"deny_refund":0.246,"escalate_to_fraud_team":0.2665,"offer_store_credit":0.0422,"refund_now":0.2332,"wait_for_inspection":0.2122},"latency_s":0.0992,"input_tokens":534},{"id":"hard-opus-c-long_policy-02","tier":"hard","family":"long_policy","gold":"erase_but_retain_transaction_records","predicted":"reject_unverified_request","correct":false,"schema_valid":true,"probs":{"erase_all":0.1179,"erase_but_retain_transaction_records":0.2343,"erase_marketing_only":0.0911,"reject_unverified_request":0.2916,"restrict_all_pending_legal_hold":0.2651},"latency_s":0.19,"input_tokens":3272},{"id":"hard-opus-c-long_policy-03","tier":"hard","family":"long_policy","gold":"eur_400","predicted":"eur_600","correct":false,"schema_valid":true,"probs":{"eur_0":0.0613,"eur_150":0.1295,"eur_200":0.1635,"eur_300":0.2035,"eur_400":0.2017,"eur_600":0.2405},"latency_s":0.1491,"input_tokens":3221},{"id":"hard-opus-c-long_policy-04","tier":"hard","family":"long_policy","gold":"yes","predicted":"no","correct":false,"schema_valid":true,"probs":{"yes":0.3723,"no":0.6277},"latency_s":0.1742,"input_tokens":2953},{"id":"hard-opus-c-long_policy-05","tier":"hard","family":"long_policy","gold":1,"predicted":"0","correct":false,"schema_valid":true,"probs":{"0":0.2864,"1":0.2453,"2":0.2662,"3":0.2021},"latency_s":0.1728,"input_tokens":3035},{"id":"hard-opus-c-long_policy-08","tier":"hard","family":"long_policy","gold":1,"predicted":"3","correct":false,"schema_valid":true,"probs":{"0":0.0707,"1":0.0959,"2":0.2477,"3":0.5857},"latency_s":0.1691,"input_tokens":3019},{"id":"hard-opus-c-long_policy-10","tier":"hard","family":"long_policy","gold":"credit_minus_restocking_fee","predicted":"reject_return","correct":false,"schema_valid":true,"probs":{"credit_minus_restocking_fee":0.2527,"full_credit":0.2777,"reject_return":0.3036,"repair_or_replace_only":0.166},"latency_s":0.1728,"input_tokens":3257},{"id":"hard-opus-c-long_policy-11","tier":"hard","family":"long_policy","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.4833,"no":0.5167},"latency_s":0.1401,"input_tokens":3189},{"id":"hard-opus-c-probability-03","tier":"hard","family":"probability","gold":"kuznets_technik","predicted":"halvorsen_polymer","correct":false,"schema_valid":true,"probs":{"halvorsen_polymer":0.8551,"kuznets_technik":0.0679,"ortega_seal_works":0.077},"latency_s":0.1021,"input_tokens":510},{"id":"hard-opus-c-temporal_numeric-02","tier":"hard","family":"temporal_numeric","gold":"yes","predicted":"no","correct":false,"schema_valid":true,"probs":{"yes":0.4711,"no":0.5289},"latency_s":0.1255,"input_tokens":778},{"id":"hard-opus-c-temporal_numeric-03","tier":"hard","family":"temporal_numeric","gold":"eur_692_00","predicted":"eur_688_00","correct":false,"schema_valid":true,"probs":{"eur_617_00":0.1228,"eur_688_00":0.2845,"eur_692_00":0.2257,"eur_693_90":0.2812,"eur_812_00":0.0859},"latency_s":0.0925,"input_tokens":725},{"id":"hard-opus-c-temporal_numeric-04","tier":"hard","family":"temporal_numeric","gold":"usd_2335_00","predicted":"usd_2335_00","correct":true,"schema_valid":true,"probs":{"usd_2222_50":0.2323,"usd_2303_01":0.2453,"usd_2303_33":0.0884,"usd_2335_00":0.2662,"usd_2340_00":0.1678},"latency_s":0.1046,"input_tokens":619},{"id":"hard-opus-c-temporal_numeric-06","tier":"hard","family":"temporal_numeric","gold":"sep_18_book_set","predicted":"none_declined","correct":false,"schema_valid":true,"probs":{"none_declined":0.3337,"sep_15_office_chair":0.1005,"sep_16_taxi":0.1023,"sep_17_parking":0.0994,"sep_18_book_set":0.2168,"sep_21_client_dinner":0.1473},"latency_s":0.1609,"input_tokens":857},{"id":"hard-opus-c-temporal_numeric-08","tier":"hard","family":"temporal_numeric","gold":"eur_498_81","predicted":"eur_506_06","correct":false,"schema_valid":true,"probs":{"eur_497_40":0.1641,"eur_498_81":0.1266,"eur_506_06":0.2688,"eur_513_69":0.1799,"eur_532_01":0.2606},"latency_s":0.1418,"input_tokens":792},{"id":"hard-opus-c-temporal_numeric-12","tier":"hard","family":"temporal_numeric","gold":"expired_30_month_cap","predicted":"covered","correct":false,"schema_valid":true,"probs":{"covered":0.4047,"expired_24_month_term":0.3228,"expired_30_month_cap":0.2725},"latency_s":0.0963,"input_tokens":714},{"id":"hard-sol-a-adversarial-01","tier":"hard","family":"adversarial","gold":"deny_missing_approval","predicted":"grant_owner_exception","correct":false,"schema_valid":true,"probs":{"deny_inactive_user":0.2657,"deny_missing_approval":0.2365,"grant_access":0.1969,"grant_owner_exception":0.3009},"latency_s":0.0772,"input_tokens":272},{"id":"hard-sol-a-adversarial-06","tier":"hard","family":"adversarial","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.5331,"no":0.4669},"latency_s":0.0616,"input_tokens":211},{"id":"hard-sol-a-adversarial-07","tier":"hard","family":"adversarial","gold":"retain_until_2031","predicted":"needs_signature_review","correct":false,"schema_valid":true,"probs":{"delete_now":0.1286,"needs_signature_review":0.5297,"retain_indefinitely":0.182,"retain_until_2031":0.1597},"latency_s":0.0697,"input_tokens":249},{"id":"hard-sol-a-adversarial-08","tier":"hard","family":"adversarial","gold":"yes","predicted":"no","correct":false,"schema_valid":true,"probs":{"yes":0.4846,"no":0.5154},"latency_s":0.0662,"input_tokens":197},{"id":"hard-sol-a-adversarial-09","tier":"hard","family":"adversarial","gold":2,"predicted":"3","correct":false,"schema_valid":true,"probs":{"0":0.0647,"1":0.205,"2":0.2875,"3":0.4427},"latency_s":0.0675,"input_tokens":191},{"id":"hard-sol-a-adversarial-11","tier":"hard","family":"adversarial","gold":"standard_research","predicted":"ethics_review","correct":false,"schema_valid":true,"probs":{"ethics_review":0.4217,"major_grants":0.1491,"manual_hold":0.0809,"standard_research":0.3483},"latency_s":0.0512,"input_tokens":262},{"id":"hard-sol-a-multi_hop-01","tier":"hard","family":"multi_hop","gold":"accept_s2_dusk","predicted":"accept_s2_dusk","correct":true,"schema_valid":true,"probs":{"accept_s2_dusk":0.3714,"reject_cutoff":0.2846,"reject_tier":0.1911,"reroute_s3":0.1529},"latency_s":0.143,"input_tokens":2341},{"id":"hard-sol-a-multi_hop-02","tier":"hard","family":"multi_hop","gold":"security_primary","predicted":"heron_q2","correct":false,"schema_valid":true,"probs":{"executive_bridge":0.2264,"heron_q2":0.3258,"insufficient_information":0.2211,"security_primary":0.2267},"latency_s":0.1398,"input_tokens":2369},{"id":"hard-sol-a-multi_hop-05","tier":"hard","family":"multi_hop","gold":"require_director","predicted":"pay_manager_approved","correct":false,"schema_valid":true,"probs":{"pay_manager_approved":0.354,"reject_category":0.1874,"require_director":0.311,"require_finance_board":0.1477},"latency_s":0.1433,"input_tokens":2308},{"id":"hard-sol-a-multi_hop-07","tier":"hard","family":"multi_hop","gold":"admit","predicted":"manual_equivalency","correct":false,"schema_valid":true,"probs":{"admit":0.2201,"deny_prerequisite":0.2343,"deny_score":0.2694,"manual_equivalency":0.2762},"latency_s":0.1751,"input_tokens":2269},{"id":"hard-sol-a-multi_hop-08","tier":"hard","family":"multi_hop","gold":"central_engineering","predicted":"local_maintenance","correct":false,"schema_valid":true,"probs":{"central_engineering":0.249,"insufficient_log":0.2682,"local_maintenance":0.2694,"safety_shutdown":0.2134},"latency_s":0.1787,"input_tokens":2264},{"id":"hard-sol-a-multi_hop-09","tier":"hard","family":"multi_hop","gold":"require_steward","predicted":"reject_capacity","correct":false,"schema_valid":true,"probs":{"confirm_l3":0.2745,"reject_capacity":0.2985,"require_steward":0.2376,"switch_l4":0.1894},"latency_s":0.1751,"input_tokens":2271},{"id":"hard-sol-a-multi_hop-10","tier":"hard","family":"multi_hop","gold":"approve_45_days","predicted":"needs_field_review","correct":false,"schema_valid":true,"probs":{"approve_45_days":0.1478,"deny_sensitive":0.1718,"needs_field_review":0.4061,"reduce_to_30_days":0.2743},"latency_s":0.1809,"input_tokens":2304},{"id":"hard-sol-a-multi_hop-12","tier":"hard","family":"multi_hop","gold":"needs_authentication","predicted":"needs_authentication","correct":true,"schema_valid":true,"probs":{"covered_recurrence":0.2715,"expired":0.2,"needs_authentication":0.3067,"wrong_issue_class":0.2219},"latency_s":0.1824,"input_tokens":2267},{"id":"hard-sol-a-trap-02","tier":"hard","family":"trap","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.5276,"no":0.4724},"latency_s":0.0717,"input_tokens":263},{"id":"hard-sol-a-trap-04","tier":"hard","family":"trap","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.5603,"no":0.4397},"latency_s":0.0707,"input_tokens":166},{"id":"hard-sol-a-trap-05","tier":"hard","family":"trap","gold":0,"predicted":"2","correct":false,"schema_valid":true,"probs":{"0":0.0355,"1":0.2723,"2":0.365,"3":0.3271},"latency_s":0.0837,"input_tokens":279},{"id":"hard-sol-a-trap-06","tier":"hard","family":"trap","gold":"keep_open_unresolved","predicted":"await_internal_metrics","correct":false,"schema_valid":true,"probs":{"await_internal_metrics":0.2913,"close_agent_verified":0.246,"close_confirmed":0.1981,"keep_open_unresolved":0.2646},"latency_s":0.074,"input_tokens":304},{"id":"hard-sol-a-trap-08","tier":"hard","family":"trap","gold":"late","predicted":"on_time","correct":false,"schema_valid":true,"probs":{"canceled":0.2939,"indeterminate":0.1973,"late":0.2077,"on_time":0.3011},"latency_s":0.0659,"input_tokens":225},{"id":"hard-sol-a-trap-10","tier":"hard","family":"trap","gold":"blocked_second_review","predicted":"blocked_failed_test","correct":false,"schema_valid":true,"probs":{"blocked_failed_test":0.3719,"blocked_second_review":0.1864,"ready_one_review":0.1814,"ready_two_reviews":0.2603},"latency_s":0.0486,"input_tokens":174},{"id":"hard-sol-a-trap-13","tier":"hard","family":"trap","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.5307,"no":0.4693},"latency_s":0.0638,"input_tokens":169},{"id":"hard-sol-a-trap-15","tier":"hard","family":"trap","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.4754,"no":0.5246},"latency_s":0.0577,"input_tokens":257},{"id":"hard-sol-b-judge_hard-01","tier":"hard","family":"judge_hard","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.6523,"no":0.3477},"latency_s":0.0797,"input_tokens":208},{"id":"hard-sol-b-judge_hard-02","tier":"hard","family":"judge_hard","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.5098,"no":0.4902},"latency_s":0.0686,"input_tokens":163},{"id":"hard-sol-b-judge_hard-05","tier":"hard","family":"judge_hard","gold":"yes","predicted":"no","correct":false,"schema_valid":true,"probs":{"yes":0.4866,"no":0.5134},"latency_s":0.0538,"input_tokens":210},{"id":"hard-sol-b-judge_hard-08","tier":"hard","family":"judge_hard","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.549,"no":0.451},"latency_s":0.0404,"input_tokens":149},{"id":"hard-sol-b-judge_hard-10","tier":"hard","family":"judge_hard","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.5126,"no":0.4874},"latency_s":0.0722,"input_tokens":172},{"id":"hard-sol-b-judge_hard-14","tier":"hard","family":"judge_hard","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.5646,"no":0.4354},"latency_s":0.0571,"input_tokens":204},{"id":"hard-sol-b-judge_hard-15","tier":"hard","family":"judge_hard","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.5842,"no":0.4158},"latency_s":0.0466,"input_tokens":153},{"id":"hard-sol-b-judge_hard-18","tier":"hard","family":"judge_hard","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.6483,"no":0.3517},"latency_s":0.0631,"input_tokens":139},{"id":"hard-sol-b-long_policy-01","tier":"hard","family":"long_policy","gold":"reject_entire_claim","predicted":"approve_lodging_only","correct":false,"schema_valid":true,"probs":{"approve_all":0.1222,"approve_lodging_only":0.4278,"approve_meals_only":0.2025,"reject_entire_claim":0.2475},"latency_s":0.1776,"input_tokens":2701},{"id":"hard-sol-b-long_policy-02","tier":"hard","family":"long_policy","gold":"deny_current_export","predicted":"analytics_owner_only","correct":false,"schema_valid":true,"probs":{"analytics_owner_only":0.3646,"deny_current_export":0.2931,"privacy_and_security":0.1917,"privacy_only":0.1506},"latency_s":0.1735,"input_tokens":2586},{"id":"hard-sol-b-long_policy-05","tier":"hard","family":"long_policy","gold":"delete_profile_omar_april_and_table","predicted":"delete_profile_omar_april_and_table","correct":true,"schema_valid":true,"probs":{"delete_everything":0.22,"delete_nothing":0.1947,"delete_only_profile":0.2673,"delete_profile_omar_april_and_table":0.318},"latency_s":0.1815,"input_tokens":2530},{"id":"hard-sol-b-long_policy-06","tier":"hard","family":"long_policy","gold":"east_ward","predicted":"harbor_neighbors","correct":false,"schema_valid":true,"probs":{"east_ward":0.1927,"harbor_neighbors":0.4539,"no_award":0.188,"river_arts":0.1654},"latency_s":0.1586,"input_tokens":2547},{"id":"hard-sol-b-routing_hard-01","tier":"hard","family":"routing_hard","gold":"office_file_repair","predicted":"finance_review","correct":false,"schema_valid":true,"probs":{"data_pipeline":0.1603,"desktop_support":0.0949,"finance_review":0.3215,"office_file_repair":0.2015,"spreadsheet_formula":0.2218},"latency_s":0.0761,"input_tokens":306},{"id":"hard-sol-b-routing_hard-02","tier":"hard","family":"routing_hard","gold":"ledger_reconciliation","predicted":"ledger_reconciliation","correct":true,"schema_valid":true,"probs":{"customer_support":0.0317,"fraud_review":0.0307,"ledger_reconciliation":0.4321,"mobile_app":0.3314,"payment_disputes":0.1741},"latency_s":0.0696,"input_tokens":275},{"id":"hard-sol-b-routing_hard-03","tier":"hard","family":"routing_hard","gold":"network_dns","predicted":"api_platform","correct":false,"schema_valid":true,"probs":{"api_platform":0.5872,"authoritative_dns":0.0949,"endpoint_devices":0.0701,"network_dns":0.123,"tls_certificates":0.1248},"latency_s":0.053,"input_tokens":289},{"id":"hard-sol-b-routing_hard-07","tier":"hard","family":"routing_hard","gold":"release_notarization","predicted":"macos_runtime","correct":false,"schema_valid":true,"probs":{"ci_infrastructure":0.0944,"code_signing":0.0449,"endpoint_security":0.0407,"macos_runtime":0.7817,"release_notarization":0.0382},"latency_s":0.0525,"input_tokens":254},{"id":"hard-sol-b-routing_hard-09","tier":"hard","family":"routing_hard","gold":"source_access","predicted":"identity_login","correct":false,"schema_valid":true,"probs":{"git_client":0.2166,"identity_login":0.4012,"repository_admin":0.0925,"source_access":0.0072,"vpn_network":0.2824},"latency_s":0.056,"input_tokens":266},{"id":"hard-sol-b-temporal_numeric-01","tier":"hard","family":"temporal_numeric","gold":"late_by_under_2h","predicted":"before_open","correct":false,"schema_valid":true,"probs":{"before_open":0.3331,"late_by_2h_or_more":0.1804,"late_by_under_2h":0.1932,"within_window":0.2934},"latency_s":0.0715,"input_tokens":221},{"id":"hard-sol-b-temporal_numeric-02","tier":"hard","family":"temporal_numeric","gold":"over_limit","predicted":"over_limit","correct":true,"schema_valid":true,"probs":{"at_least_100gb_below":0.2689,"exactly_equal":0.1654,"over_limit":0.3486,"under_100gb_below":0.217},"latency_s":0.0552,"input_tokens":221},{"id":"hard-sol-b-temporal_numeric-03","tier":"hard","family":"temporal_numeric","gold":"9_credits","predicted":"6_credits","correct":false,"schema_valid":true,"probs":{"12_credits":0.214,"15_credits":0.2029,"6_credits":0.335,"9_credits":0.2481},"latency_s":0.0437,"input_tokens":173},{"id":"hard-sol-b-temporal_numeric-04","tier":"hard","family":"temporal_numeric","gold":"pass","predicted":"narrow_fail","correct":false,"schema_valid":true,"probs":{"clear_fail":0.3816,"narrow_fail":0.3856,"pass":0.2328},"latency_s":0.0604,"input_tokens":187},{"id":"hard-sol-c-judge_hard-03","tier":"hard","family":"judge_hard","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.5076,"no":0.4924},"latency_s":0.0751,"input_tokens":392},{"id":"hard-sol-c-judge_hard-05","tier":"hard","family":"judge_hard","gold":"yes","predicted":"no","correct":false,"schema_valid":true,"probs":{"yes":0.4757,"no":0.5243},"latency_s":0.0721,"input_tokens":299},{"id":"hard-sol-c-judge_hard-07","tier":"hard","family":"judge_hard","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.5143,"no":0.4857},"latency_s":0.0645,"input_tokens":272},{"id":"hard-sol-c-judge_hard-08","tier":"hard","family":"judge_hard","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.5685,"no":0.4315},"latency_s":0.0594,"input_tokens":286},{"id":"hard-sol-c-judge_hard-09","tier":"hard","family":"judge_hard","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.5479,"no":0.4521},"latency_s":0.0756,"input_tokens":280},{"id":"hard-sol-c-judge_hard-10","tier":"hard","family":"judge_hard","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.5809,"no":0.4191},"latency_s":0.0769,"input_tokens":297},{"id":"hard-sol-c-judge_hard-11","tier":"hard","family":"judge_hard","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.4097,"no":0.5903},"latency_s":0.0698,"input_tokens":316},{"id":"hard-sol-c-judge_hard-13","tier":"hard","family":"judge_hard","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.5066,"no":0.4934},"latency_s":0.0771,"input_tokens":275},{"id":"hard-sol-c-judge_hard-15","tier":"hard","family":"judge_hard","gold":"yes","predicted":"no","correct":false,"schema_valid":true,"probs":{"yes":0.4842,"no":0.5158},"latency_s":0.0757,"input_tokens":260},{"id":"hard-sol-c-multi_hop-07","tier":"hard","family":"multi_hop","gold":"programme_review","predicted":"programme_review","correct":true,"schema_valid":true,"probs":{"partial_release":0.1299,"programme_review":0.3272,"reject_draw":0.2633,"release_draw":0.2796},"latency_s":0.1501,"input_tokens":2451},{"id":"hard-sol-c-multi_hop-09","tier":"hard","family":"multi_hop","gold":"bus_substitution","predicted":"terminate_oakmere","correct":false,"schema_valid":true,"probs":{"bus_substitution":0.218,"call_fen_cross":0.2494,"pass_fen_cross":0.2349,"terminate_oakmere":0.2977},"latency_s":0.1681,"input_tokens":2456},{"id":"hard-sol-c-multi_hop-11","tier":"hard","family":"multi_hop","gold":"coverage_excluded","predicted":"adjuster_referral","correct":false,"schema_valid":true,"probs":{"adjuster_referral":0.5124,"coverage_confirmed":0.1704,"coverage_excluded":0.1292,"documentation_needed":0.188},"latency_s":0.1712,"input_tokens":2445},{"id":"hard-sol-c-multi_hop-12","tier":"hard","family":"multi_hop","gold":"conservation_hold","predicted":"supervised_delivery","correct":false,"schema_valid":true,"probs":{"conservation_hold":0.2543,"deny_request":0.1097,"digital_copy_only":0.2596,"supervised_delivery":0.3764},"latency_s":0.07,"input_tokens":480},{"id":"hard-sol-c-multi_hop-13","tier":"hard","family":"multi_hop","gold":"normal_release","predicted":"normal_release","correct":true,"schema_valid":true,"probs":{"engineering_review":0.2615,"normal_release":0.3537,"rework":0.0751,"scrap_lot":0.3097},"latency_s":0.1692,"input_tokens":2514},{"id":"original-adequacy-01-0","tier":"original","family":"adequacy","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.5012,"no":0.4988},"latency_s":0.0415,"input_tokens":51},{"id":"original-adequacy-01-1","tier":"original","family":"adequacy","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.7234,"no":0.2766},"latency_s":0.054,"input_tokens":48},{"id":"original-adequacy-02-0","tier":"original","family":"adequacy","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.7021,"no":0.2979},"latency_s":0.0645,"input_tokens":55},{"id":"original-adequacy-02-1","tier":"original","family":"adequacy","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.4476,"no":0.5524},"latency_s":0.051,"input_tokens":56},{"id":"original-adequacy-03-0","tier":"original","family":"adequacy","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.7098,"no":0.2902},"latency_s":0.0407,"input_tokens":55},{"id":"original-adequacy-03-1","tier":"original","family":"adequacy","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.6937,"no":0.3063},"latency_s":0.0377,"input_tokens":58},{"id":"original-adequacy-04-0","tier":"original","family":"adequacy","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.6131,"no":0.3869},"latency_s":0.0604,"input_tokens":61},{"id":"original-adequacy-04-1","tier":"original","family":"adequacy","gold":"yes","predicted":"no","correct":false,"schema_valid":true,"probs":{"yes":0.4377,"no":0.5623},"latency_s":0.0603,"input_tokens":58},{"id":"original-adequacy-05-0","tier":"original","family":"adequacy","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.7281,"no":0.2719},"latency_s":0.0343,"input_tokens":49},{"id":"original-adequacy-05-1","tier":"original","family":"adequacy","gold":"no","predicted":"yes","correct":false,"schema_valid":true,"probs":{"yes":0.669,"no":0.331},"latency_s":0.0647,"input_tokens":50},{"id":"original-adequacy-06-0","tier":"original","family":"adequacy","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.6215,"no":0.3785},"latency_s":0.0698,"input_tokens":61},{"id":"original-adequacy-06-1","tier":"original","family":"adequacy","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.7499,"no":0.2501},"latency_s":0.0644,"input_tokens":61},{"id":"original-extraction-01-0","tier":"original","family":"extraction","gold":"pickup","predicted":"pickup","correct":true,"schema_valid":true,"probs":{"courier":0.2353,"pickup":0.3645,"post":0.216,"unknown":0.1842},"latency_s":0.0654,"input_tokens":59},{"id":"original-extraction-01-1","tier":"original","family":"extraction","gold":"pickup","predicted":"pickup","correct":true,"schema_valid":true,"probs":{"courier":0.2707,"pickup":0.4613,"post":0.1371,"unknown":0.1309},"latency_s":0.0454,"input_tokens":61},{"id":"original-extraction-02-0","tier":"original","family":"extraction","gold":"unknown","predicted":"post","correct":false,"schema_valid":true,"probs":{"courier":0.2545,"pickup":0.1431,"post":0.393,"unknown":0.2094},"latency_s":0.0549,"input_tokens":60},{"id":"original-extraction-02-1","tier":"original","family":"extraction","gold":"unknown","predicted":"courier","correct":false,"schema_valid":true,"probs":{"courier":0.4439,"pickup":0.1425,"post":0.1872,"unknown":0.2264},"latency_s":0.057,"input_tokens":60},{"id":"original-extraction-03-0","tier":"original","family":"extraction","gold":"courier","predicted":"courier","correct":true,"schema_valid":true,"probs":{"courier":0.5234,"pickup":0.225,"post":0.0868,"unknown":0.1647},"latency_s":0.0432,"input_tokens":64},{"id":"original-extraction-03-1","tier":"original","family":"extraction","gold":"courier","predicted":"courier","correct":true,"schema_valid":true,"probs":{"courier":0.4446,"pickup":0.1683,"post":0.1247,"unknown":0.2624},"latency_s":0.0656,"input_tokens":63},{"id":"original-extraction-04-0","tier":"original","family":"extraction","gold":"post","predicted":"post","correct":true,"schema_valid":true,"probs":{"courier":0.1734,"pickup":0.157,"post":0.4591,"unknown":0.2105},"latency_s":0.0617,"input_tokens":60},{"id":"original-extraction-04-1","tier":"original","family":"extraction","gold":"post","predicted":"post","correct":true,"schema_valid":true,"probs":{"courier":0.2959,"pickup":0.1215,"post":0.3812,"unknown":0.2014},"latency_s":0.0624,"input_tokens":59},{"id":"original-extraction-05-0","tier":"original","family":"extraction","gold":"unknown","predicted":"courier","correct":false,"schema_valid":true,"probs":{"courier":0.2978,"pickup":0.1964,"post":0.2569,"unknown":0.2489},"latency_s":0.041,"input_tokens":61},{"id":"original-extraction-05-1","tier":"original","family":"extraction","gold":"unknown","predicted":"courier","correct":false,"schema_valid":true,"probs":{"courier":0.3347,"pickup":0.1702,"post":0.1674,"unknown":0.3277},"latency_s":0.0404,"input_tokens":57},{"id":"original-extraction-06-0","tier":"original","family":"extraction","gold":"courier","predicted":"courier","correct":true,"schema_valid":true,"probs":{"courier":0.3537,"pickup":0.1008,"post":0.318,"unknown":0.2274},"latency_s":0.0309,"input_tokens":61},{"id":"original-extraction-06-1","tier":"original","family":"extraction","gold":"courier","predicted":"post","correct":false,"schema_valid":true,"probs":{"courier":0.2356,"pickup":0.089,"post":0.4629,"unknown":0.2125},"latency_s":0.0506,"input_tokens":60},{"id":"original-intent-01-0","tier":"original","family":"intent","gold":"refund","predicted":"refund","correct":true,"schema_valid":true,"probs":{"cancel":0.2546,"change_address":0.1578,"other":0.0856,"refund":0.2988,"status":0.2033},"latency_s":0.0623,"input_tokens":63},{"id":"original-intent-01-1","tier":"original","family":"intent","gold":"refund","predicted":"refund","correct":true,"schema_valid":true,"probs":{"cancel":0.235,"change_address":0.1215,"other":0.0745,"refund":0.3959,"status":0.1731},"latency_s":0.0476,"input_tokens":65},{"id":"original-intent-02-0","tier":"original","family":"intent","gold":"status","predicted":"status","correct":true,"schema_valid":true,"probs":{"cancel":0.1694,"change_address":0.2757,"other":0.1303,"refund":0.0955,"status":0.3291},"latency_s":0.0621,"input_tokens":64},{"id":"original-intent-02-1","tier":"original","family":"intent","gold":"status","predicted":"status","correct":true,"schema_valid":true,"probs":{"cancel":0.1866,"change_address":0.1395,"other":0.1459,"refund":0.0907,"status":0.4374},"latency_s":0.0502,"input_tokens":63},{"id":"original-intent-03-0","tier":"original","family":"intent","gold":"change_address","predicted":"status","correct":false,"schema_valid":true,"probs":{"cancel":0.2309,"change_address":0.2154,"other":0.1223,"refund":0.1224,"status":0.3091},"latency_s":0.0646,"input_tokens":65},{"id":"original-intent-03-1","tier":"original","family":"intent","gold":"change_address","predicted":"change_address","correct":true,"schema_valid":true,"probs":{"cancel":0.1772,"change_address":0.393,"other":0.0819,"refund":0.0819,"status":0.266},"latency_s":0.052,"input_tokens":63},{"id":"original-intent-04-0","tier":"original","family":"intent","gold":"cancel","predicted":"cancel","correct":true,"schema_valid":true,"probs":{"cancel":0.2817,"change_address":0.2062,"other":0.1124,"refund":0.1581,"status":0.2415},"latency_s":0.0572,"input_tokens":68},{"id":"original-intent-04-1","tier":"original","family":"intent","gold":"cancel","predicted":"status","correct":false,"schema_valid":true,"probs":{"cancel":0.2174,"change_address":0.1869,"other":0.1315,"refund":0.0787,"status":0.3855},"latency_s":0.031,"input_tokens":63},{"id":"original-intent-05-0","tier":"original","family":"intent","gold":"other","predicted":"refund","correct":false,"schema_valid":true,"probs":{"cancel":0.1952,"change_address":0.1087,"other":0.0635,"refund":0.3413,"status":0.2912},"latency_s":0.0612,"input_tokens":63},{"id":"original-intent-05-1","tier":"original","family":"intent","gold":"other","predicted":"refund","correct":false,"schema_valid":true,"probs":{"cancel":0.1715,"change_address":0.1336,"other":0.0926,"refund":0.3428,"status":0.2595},"latency_s":0.0363,"input_tokens":63},{"id":"original-intent-06-0","tier":"original","family":"intent","gold":"status","predicted":"cancel","correct":false,"schema_valid":true,"probs":{"cancel":0.3193,"change_address":0.1596,"other":0.087,"refund":0.127,"status":0.3071},"latency_s":0.0598,"input_tokens":61},{"id":"original-intent-06-1","tier":"original","family":"intent","gold":"status","predicted":"status","correct":true,"schema_valid":true,"probs":{"cancel":0.2903,"change_address":0.1783,"other":0.1239,"refund":0.0819,"status":0.3257},"latency_s":0.0361,"input_tokens":66},{"id":"original-ordinal-01-0","tier":"original","family":"ordinal","gold":0,"predicted":"3","correct":false,"schema_valid":true,"probs":{"0":0.1207,"1":0.285,"2":0.2937,"3":0.3006},"latency_s":0.0524,"input_tokens":70},{"id":"original-ordinal-01-1","tier":"original","family":"ordinal","gold":0,"predicted":"2","correct":false,"schema_valid":true,"probs":{"0":0.0716,"1":0.2792,"2":0.3461,"3":0.3032},"latency_s":0.057,"input_tokens":72},{"id":"original-ordinal-02-0","tier":"original","family":"ordinal","gold":1,"predicted":"3","correct":false,"schema_valid":true,"probs":{"0":0.0759,"1":0.237,"2":0.2742,"3":0.4129},"latency_s":0.0553,"input_tokens":74},{"id":"original-ordinal-02-1","tier":"original","family":"ordinal","gold":1,"predicted":"3","correct":false,"schema_valid":true,"probs":{"0":0.0845,"1":0.1953,"2":0.3133,"3":0.4069},"latency_s":0.0638,"input_tokens":75},{"id":"original-ordinal-03-0","tier":"original","family":"ordinal","gold":2,"predicted":"3","correct":false,"schema_valid":true,"probs":{"0":0.1255,"1":0.2951,"2":0.2462,"3":0.3332},"latency_s":0.0331,"input_tokens":71},{"id":"original-ordinal-03-1","tier":"original","family":"ordinal","gold":2,"predicted":"3","correct":false,"schema_valid":true,"probs":{"0":0.086,"1":0.2876,"2":0.2979,"3":0.3285},"latency_s":0.0518,"input_tokens":73},{"id":"original-ordinal-04-0","tier":"original","family":"ordinal","gold":3,"predicted":"3","correct":true,"schema_valid":true,"probs":{"0":0.0956,"1":0.2809,"2":0.2792,"3":0.3443},"latency_s":0.0425,"input_tokens":73},{"id":"original-ordinal-04-1","tier":"original","family":"ordinal","gold":3,"predicted":"3","correct":true,"schema_valid":true,"probs":{"0":0.0979,"1":0.2773,"2":0.26,"3":0.3648},"latency_s":0.0575,"input_tokens":72},{"id":"original-ordinal-05-0","tier":"original","family":"ordinal","gold":0,"predicted":"3","correct":false,"schema_valid":true,"probs":{"0":0.105,"1":0.2512,"2":0.2493,"3":0.3945},"latency_s":0.0549,"input_tokens":76},{"id":"original-ordinal-05-1","tier":"original","family":"ordinal","gold":0,"predicted":"2","correct":false,"schema_valid":true,"probs":{"0":0.1197,"1":0.283,"2":0.3025,"3":0.2949},"latency_s":0.04,"input_tokens":75},{"id":"original-ordinal-06-0","tier":"original","family":"ordinal","gold":2,"predicted":"1","correct":false,"schema_valid":true,"probs":{"0":0.1179,"1":0.3182,"2":0.2817,"3":0.2822},"latency_s":0.0486,"input_tokens":77},{"id":"original-ordinal-06-1","tier":"original","family":"ordinal","gold":2,"predicted":"2","correct":true,"schema_valid":true,"probs":{"0":0.1043,"1":0.2431,"2":0.3536,"3":0.2989},"latency_s":0.0633,"input_tokens":84},{"id":"original-policy-01-0","tier":"original","family":"policy","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.3781,"no":0.6219},"latency_s":0.0498,"input_tokens":71},{"id":"original-policy-01-1","tier":"original","family":"policy","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.403,"no":0.597},"latency_s":0.0591,"input_tokens":73},{"id":"original-policy-02-0","tier":"original","family":"policy","gold":"yes","predicted":"no","correct":false,"schema_valid":true,"probs":{"yes":0.4445,"no":0.5555},"latency_s":0.035,"input_tokens":63},{"id":"original-policy-02-1","tier":"original","family":"policy","gold":"yes","predicted":"no","correct":false,"schema_valid":true,"probs":{"yes":0.4731,"no":0.5269},"latency_s":0.0384,"input_tokens":62},{"id":"original-policy-03-0","tier":"original","family":"policy","gold":"yes","predicted":"no","correct":false,"schema_valid":true,"probs":{"yes":0.4199,"no":0.5801},"latency_s":0.0598,"input_tokens":65},{"id":"original-policy-03-1","tier":"original","family":"policy","gold":"yes","predicted":"no","correct":false,"schema_valid":true,"probs":{"yes":0.3899,"no":0.6101},"latency_s":0.0505,"input_tokens":65},{"id":"original-policy-04-0","tier":"original","family":"policy","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.4255,"no":0.5745},"latency_s":0.0362,"input_tokens":73},{"id":"original-policy-04-1","tier":"original","family":"policy","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.4104,"no":0.5896},"latency_s":0.0519,"input_tokens":71},{"id":"original-policy-05-0","tier":"original","family":"policy","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.3872,"no":0.6128},"latency_s":0.046,"input_tokens":76},{"id":"original-policy-05-1","tier":"original","family":"policy","gold":"no","predicted":"no","correct":true,"schema_valid":true,"probs":{"yes":0.3788,"no":0.6212},"latency_s":0.0536,"input_tokens":69},{"id":"original-policy-06-0","tier":"original","family":"policy","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.5029,"no":0.4971},"latency_s":0.0674,"input_tokens":74},{"id":"original-policy-06-1","tier":"original","family":"policy","gold":"yes","predicted":"yes","correct":true,"schema_valid":true,"probs":{"yes":0.5037,"no":0.4963},"latency_s":0.0549,"input_tokens":64},{"id":"original-routing-01-0","tier":"original","family":"routing","gold":"math","predicted":"coding_agent","correct":false,"schema_valid":true,"probs":{"coding":0.1433,"coding_agent":0.2658,"document":0.1608,"general":0.1013,"math":0.1616,"tools":0.1672},"latency_s":0.0332,"input_tokens":92},{"id":"original-routing-01-1","tier":"original","family":"routing","gold":"math","predicted":"coding_agent","correct":false,"schema_valid":true,"probs":{"coding":0.1118,"coding_agent":0.2675,"document":0.1678,"general":0.1381,"math":0.1269,"tools":0.1879},"latency_s":0.0576,"input_tokens":89},{"id":"original-routing-02-0","tier":"original","family":"routing","gold":"coding","predicted":"coding_agent","correct":false,"schema_valid":true,"probs":{"coding":0.1201,"coding_agent":0.2361,"document":0.1672,"general":0.1657,"math":0.1343,"tools":0.1766},"latency_s":0.0641,"input_tokens":96},{"id":"original-routing-02-1","tier":"original","family":"routing","gold":"coding","predicted":"coding_agent","correct":false,"schema_valid":true,"probs":{"coding":0.1191,"coding_agent":0.2918,"document":0.1539,"general":0.1318,"math":0.1444,"tools":0.159},"latency_s":0.0449,"input_tokens":90},{"id":"original-routing-03-0","tier":"original","family":"routing","gold":"coding_agent","predicted":"coding_agent","correct":true,"schema_valid":true,"probs":{"coding":0.0917,"coding_agent":0.3548,"document":0.1174,"general":0.1112,"math":0.113,"tools":0.2119},"latency_s":0.0695,"input_tokens":95},{"id":"original-routing-03-1","tier":"original","family":"routing","gold":"coding_agent","predicted":"tools","correct":false,"schema_valid":true,"probs":{"coding":0.0919,"coding_agent":0.236,"document":0.1437,"general":0.1353,"math":0.1266,"tools":0.2665},"latency_s":0.0674,"input_tokens":98},{"id":"original-routing-04-0","tier":"original","family":"routing","gold":"document","predicted":"coding_agent","correct":false,"schema_valid":true,"probs":{"coding":0.1293,"coding_agent":0.2727,"document":0.1469,"general":0.1279,"math":0.1261,"tools":0.197},"latency_s":0.0637,"input_tokens":91},{"id":"original-routing-04-1","tier":"original","family":"routing","gold":"document","predicted":"coding_agent","correct":false,"schema_valid":true,"probs":{"coding":0.1187,"coding_agent":0.231,"document":0.1886,"general":0.1631,"math":0.1424,"tools":0.1563},"latency_s":0.0502,"input_tokens":91},{"id":"original-routing-05-0","tier":"original","family":"routing","gold":"tools","predicted":"coding_agent","correct":false,"schema_valid":true,"probs":{"coding":0.1159,"coding_agent":0.2561,"document":0.1566,"general":0.1334,"math":0.1539,"tools":0.1841},"latency_s":0.0634,"input_tokens":92},{"id":"original-routing-05-1","tier":"original","family":"routing","gold":"tools","predicted":"coding_agent","correct":false,"schema_valid":true,"probs":{"coding":0.1254,"coding_agent":0.2541,"document":0.1503,"general":0.1424,"math":0.1516,"tools":0.1762},"latency_s":0.0706,"input_tokens":93},{"id":"original-routing-06-0","tier":"original","family":"routing","gold":"general","predicted":"coding_agent","correct":false,"schema_valid":true,"probs":{"coding":0.1238,"coding_agent":0.2506,"document":0.1612,"general":0.1407,"math":0.1368,"tools":0.1869},"latency_s":0.0561,"input_tokens":91},{"id":"original-routing-06-1","tier":"original","family":"routing","gold":"general","predicted":"coding_agent","correct":false,"schema_valid":true,"probs":{"coding":0.1101,"coding_agent":0.2949,"document":0.1678,"general":0.1343,"math":0.124,"tools":0.1689},"latency_s":0.0521,"input_tokens":91}]}}}