{"title":"Same data, same scoring: JevBench v1.4.2.2 public items","reference":"tia-1.5","benchmark":{"name":"JevBench","revision":"v1.4.2.2","repo":"https://github.com/fstandhartinger/jevbench","commit":"fcc6927","commit_full":"fcc69279a933f882ccc4489eb14bf467647dbdb7","results_file":"https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json","per_item_file":"https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.2/jevbench-v1.2-per-task.json","items_files":{"easy":"https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/datasets/public/easy.jsonl","original":"https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/datasets/public/original.jsonl","hard":"https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/datasets/public/hard.jsonl"},"published_utc":"2026-09-27T18:49:51+00:00","licence":"MIT (harness and every public item: provenance.license MIT on all 231)","items":{"easy":48,"original":72,"hard":111,"total":231},"published_tier_sizes":{"easy":72,"judge":146,"standard":96,"hard":220,"sealed":308}},"method":{"paired":"exact two-sided McNemar of TIA 1.5 vs the system on the items the system answered (all 231 unless its run was partial); 95% paired bootstrap CI of the accuracy difference (10,000 resamples, seed 13); Holm over the paired family; verdict at Holm p < 0.05","unpaired":"no per-item outcomes published: Wilson 95% intervals of both public accuracies over the same 231 items; a verdict only when the intervals are disjoint (conservative)","published_tiers":"JevBench's published tiers (easy, standard, judge, hard) include held-out items TIA was never run on, so they are shown with Wilson intervals but not compared","price":"US$ per 1,000 decisions, JevBench's unit (each row's official figure, estimate or measured as published); correct_per_usd = public accuracy x 1000 / price","latency":"p50 seconds per decision as published (self-hosted rows carry JevBench's x2 + 0.15 s rule); correct_per_second = public accuracy / p50"},"tia":{"id":"tia-1.5","display":"TIA 1.5","author":"TIA","kind":"local evaluation","ours":true,"listing":"not on the official board","measured_by":"TIA","measured_on":"2026-10-03","public":{"correct":125,"n":231,"acc":0.5411255411255411,"wilson95":[0.47671740174484795,0.6041882445175688]},"answered":231,"published_tiers":null,"latency":{"p50_s_raw":0.0322,"p50_s_adjusted":0.2144,"basis":"TIA measurement through the unmodified harness, local CPU worker over loopback; JevBench's self-host rule applied (x2 + 0.15 s). Not measured by JevBench."},"cost":{"usd_per_1000":0.010945637695637696,"kind":"estimated list price","basis":"Hypothetical API price, not a deployed-model tariff: 500 VND per 1M input tokens x 569 measured input tokens per decision, at 26,000 VND/USD"},"per_item":"TIA run","sources":[],"comparable":true,"public_tiers":{"easy":{"correct":44,"n":48,"acc":0.9166666666666666,"wilson95":[0.8044669809657894,0.967116243410477]},"original":{"correct":44,"n":72,"acc":0.6111111111111112,"wilson95":[0.49562354903304096,0.7153428568880604]},"hard":{"correct":37,"n":111,"acc":0.3333333333333333,"wilson95":[0.2525113744596184,0.425305327274862]}},"vs_ref":null,"correct_per_usd":49437.55276508035,"correct_per_second":2.52390644181689},"ranks":{"overall":{"rank":79,"of":98,"basis":"public accuracy over the same 231 items, every comparable row","jevbench_ranked_only":{"rank":77,"of":94,"excluded":["classifier-dev-fast","qwen3.8-27b","needle-3-tools","needle-3"]}},"easy":{"rank":42,"of":56,"basis":"public easy items, rows with per-item outcomes"},"original":{"rank":41,"of":56,"basis":"public original items, rows with per-item outcomes"},"hard":{"rank":47,"of":56,"basis":"public hard items, rows with per-item outcomes"},"correct_per_usd":{"rank":24,"of":96},"correct_per_second":{"rank":14,"of":94}},"verdicts":{"better":10,"not distinguishable":24,"worse":63},"verdicts_by_test":{"paired":{"better":8,"not distinguishable":14,"worse":33},"unpaired":{"better":2,"not distinguishable":10,"worse":30}},"verdicts_jevbench_ranked_only":{"better":8,"not distinguishable":24,"worse":61},"verdict_keys":{"better":["tia-1.3-mini","tia-1.2-mini","clm-8b","mirror","mxbai-rerank-base-v2","bge-reranker-v2-m3","gte-reranker-modernbert-base","certo","needle-3-tools","needle-3"],"not distinguishable":["tia-1.4-mini","raw-phi-4-mini","decision-fast","jeff","laya","lev-350m","von-395m","opendecision","openjev-verdict-1.4","kev-0.5b","openjev-verdict","raw-qwen3-1.7b","jevact","gliner2-large","mghafiri-qwen3.5-0.8b-decision-model","open-jev-deberta-v3-large","smalljev","gliner2","open-jev-zefan-2b","gliner2.5-multi","simplejev-qwen3.5-0.8b","gliner2.5-small","raw-qwen3-0.6b","verdict-small"],"worse":["imajev_4b","plumb-4b","decider-4b-v2","jev-1.13.0","jevk5-v02","cygnet","hopper","winnow-12b","reflex-4b","djev","jev-omni","metask-jev-4b","semif-qwen3.5-4b","jobe-qwen3.5-4b","localjev-qwen3.5-4b","system-one-open","spark-s1-4b-v6","malkuth-4b","jqv","qwen3-reranker-4b","decider-35b-a3b","raw-qwen3-4b-instruct-2507","opensourcejev-qwen35-4b-q4km","zerank-2","decision-machine-1","malkuth-2b","jev-qwen3.5-9b-base-nvfp4","openjev-razorback16","kev-4b","decision-2b","qwen35-9b-jev-data-mix-v2","gpt-6-luna-low","simplejev-qwen3.8-27b","ninfer-qwen3.8-flash-next","swanone","gpt-6-luna","open-alternative-jev","jev-local","decider-2b","standardone-8b","openjev-sglang","ninfer-qwen3.8-27b-t1.5","ninfer-qwen3.8-27b","kev-8b","jevone","typecastlm","simplejev-qwen3.6-35b-a3b","kev-0.6b","raw-qwen3-8b","system-one-sg","litjev","nimble-9b","gpt-5.6-luna","reflex-27b","djev-thinking","openjev-thinking","gemini-3.1-flash-lite","instinct","open-jev-zefan-9b","deepseek-flash","open-jev-json-canvas-joshuasp","classifier-dev-fast","qwen3.8-27b"]},"compared":97,"not_compared":[{"id":"kushal-gemma4-31b-it-autoloops","display":"Autoloops – Gemma 4 31B IT","published_rate":0.9283819628647216,"reason":"published public accuracy 0.9283819628647215 is not k/231: measured on a different item set"}],"holm_family":55,"holm_family_split":{"official":52,"tia_run":3},"rows":[{"id":"gpt-6-luna","display":"GPT-6 Luna (default medium reasoning effort)","author":"OpenAI","kind":"hosted API","class":"llm-baseline","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"GPT-6 API measurement 2026-09-23","comparable":true,"public":{"correct":230,"n":231,"acc":0.9956709956709956,"wilson95":[0.975890531785928,0.9992354152716204]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":1,"n":96,"correct":96,"wilson95":[0.9615244115586972,1]},"judge":{"acc":0.9726027397260274,"n":146,"correct":142,"wilson95":[0.9316781374425416,0.9892952769268636]},"hard":{"acc":0.9863636363636364,"n":220,"correct":217,"wilson95":[0.9606820346448648,0.995351762979216]}},"latency":{"p50_s_raw":1.477266326546669,"p50_s_adjusted":1.477266326546669,"basis":"none (production API)"},"cost":{"usd_per_1000":0.13544588014981276,"kind":"measured"},"per_item":null,"sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L7206"],"public_tiers":null,"vs_ref":{"test":"Wilson 95% intervals (unpaired)","n":231,"on":"official public accuracy","diff":-0.4545454545454546,"ref_wilson95":[0.47671740174484795,0.6041882445175688],"system_wilson95":[0.975890531785928,0.9992354152716204],"verdict":"worse"},"correct_per_usd":7351.061505670847,"correct_per_second":0.6739955942802308},{"id":"gpt-6-luna-low","display":"GPT-6 Luna (low reasoning effort)","author":"OpenAI","kind":"hosted API","class":"llm-baseline","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"GPT-6 API measurement 2026-09-23","comparable":true,"public":{"correct":229,"n":231,"acc":0.9913419913419912,"wilson95":[0.9689871214788128,0.9976224417616386]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.9895833333333334,"n":96,"correct":95,"wilson95":[0.9433338231810272,0.9981588298035302]},"judge":{"acc":0.9726027397260274,"n":146,"correct":142,"wilson95":[0.9316781374425416,0.9892952769268636]},"hard":{"acc":0.9727272727272728,"n":220,"correct":214,"wilson95":[0.9417870993871404,0.9874420123222468]}},"latency":{"p50_s_raw":1.4418320655822754,"p50_s_adjusted":1.4418320655822754,"basis":"none (production API)"},"cost":{"usd_per_1000":0.12699831460674169,"kind":"measured"},"per_item":null,"sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L6514"],"public_tiers":null,"vs_ref":{"test":"Wilson 95% intervals (unpaired)","n":231,"on":"official public accuracy","diff":-0.45021645021645024,"ref_wilson95":[0.47671740174484795,0.6041882445175688],"system_wilson95":[0.9689871214788128,0.9976224417616386],"verdict":"worse"},"correct_per_usd":7805.946042762413,"correct_per_second":0.687557181592881},{"id":"deepseek-flash","display":"DeepSeek V4.1 Flash (thinking default)","author":"DeepSeek","kind":"open weights","class":"llm-baseline","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.3.0 public","comparable":true,"public":{"correct":226,"n":231,"acc":0.9783549783549784,"wilson95":[0.9503404687870832,0.9907199430020788]},"published_tiers":{"easy":{"acc":0.9861111111111112,"n":72,"correct":71,"wilson95":[0.9254339777605758,0.997544048144242]},"standard":{"acc":0.9895833333333334,"n":96,"correct":95,"wilson95":[0.9433338231810272,0.9981588298035302]},"judge":{"acc":0.9315068493150684,"n":146,"correct":136,"wilson95":[0.8785152238371042,0.9623735458045264]},"hard":{"acc":0.95,"n":220,"correct":209,"wilson95":[0.9127000990708214,0.971854536114076]}},"latency":{"p50_s_raw":1.4163860343396664,"p50_s_adjusted":1.4163860343396664,"basis":"none (production API)"},"cost":{"usd_per_1000":0.593682584269663,"kind":"measured"},"per_item":"official","sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L16406","https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.2/jevbench-v1.2-per-task.json#L12884"],"answered":231,"public_tiers":{"easy":{"correct":48,"n":48,"acc":1,"wilson95":[0.9258998692515198,1.0000000000000002]},"original":{"correct":71,"n":72,"acc":0.9861111111111112,"wilson95":[0.9254339777605758,0.997544048144242]},"hard":{"correct":107,"n":111,"acc":0.963963963963964,"wilson95":[0.9109899435288228,0.9858986975887052]}},"vs_ref":{"test":"exact McNemar (paired)","n":231,"on":"all 231 public items","diff":-0.4372294372294372,"ci95":[-0.5021645021645021,-0.3722943722943723],"only_ref":1,"only_system":102,"p":2.051038353574631e-29,"p_holm":1.1280710944660467e-27,"verdict":"worse"},"correct_per_usd":1647.942864213428,"correct_per_second":0.6907403452414704},{"id":"gpt-5.6-luna","display":"GPT-5.6 Luna (low reasoning effort)","author":"OpenAI","kind":"hosted API","class":"llm-baseline","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.3.0 public","comparable":true,"public":{"correct":225,"n":231,"acc":0.974025974025974,"wilson95":[0.9445013291748412,0.9880426987972244]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.9791666666666666,"n":96,"correct":94,"wilson95":[0.9271928642112758,0.9942680301991424]},"judge":{"acc":0.9657534246575342,"n":146,"correct":141,"wilson95":[0.9223410969879317,0.9852848765935112]},"hard":{"acc":0.9454545454545454,"n":220,"correct":208,"wilson95":[0.9070939990822572,0.968525740797742]}},"latency":{"p50_s_raw":0.9680032916367054,"p50_s_adjusted":0.9680032916367054,"basis":"none (production API)"},"cost":{"usd_per_1000":0.24191123595505612,"kind":"measured"},"per_item":"official","sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L12117","https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.2/jevbench-v1.2-per-task.json#L9512"],"answered":231,"public_tiers":{"easy":{"correct":48,"n":48,"acc":1,"wilson95":[0.9258998692515198,1.0000000000000002]},"original":{"correct":70,"n":72,"acc":0.9722222222222222,"wilson95":[0.9042582470003044,0.9923489781643758]},"hard":{"correct":107,"n":111,"acc":0.963963963963964,"wilson95":[0.9109899435288228,0.9858986975887052]}},"vs_ref":{"test":"exact McNemar (paired)","n":231,"on":"all 231 public items","diff":-0.4329004329004329,"ci95":[-0.49783549783549785,-0.3636363636363637],"only_ref":3,"only_system":103,"p":4.8956707778015985e-27,"p_holm":2.6436622200128636e-25,"verdict":"worse"},"correct_per_usd":4026.3775685348287,"correct_per_second":1.0062217581709725},{"id":"jevone","display":"JevOne","author":"Juspay","kind":"open weights","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"round 4 unpublished","comparable":true,"public":{"correct":207,"n":231,"acc":0.8961038961038961,"wilson95":[0.8500656331827298,0.929183486081599]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.96875,"n":96,"correct":93,"wilson95":[0.9121132061243772,0.9893159297119012]},"judge":{"acc":0.9246575342465754,"n":146,"correct":135,"wilson95":[0.8701296486178092,0.9574116802358594]},"hard":{"acc":0.75,"n":220,"correct":165,"wilson95":[0.6888220854359439,0.8025971563334435]}},"latency":{"p50_s_raw":0.08668531733565032,"p50_s_adjusted":0.32337063467130067,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.1373250936329588,"kind":"estimate"},"per_item":null,"sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L9994"],"public_tiers":null,"vs_ref":{"test":"Wilson 95% intervals (unpaired)","n":231,"on":"official public accuracy","diff":-0.354978354978355,"ref_wilson95":[0.47671740174484795,0.6041882445175688],"system_wilson95":[0.8500656331827298,0.929183486081599],"verdict":"worse"},"correct_per_usd":6525.41988064464,"correct_per_second":2.7711356568130143},{"id":"ninfer-qwen3.8-flash-next","display":"NInfer Qwen3.8-Flash-Next mixed","author":"Igor L. / NInfer contributors","kind":"open weights","class":"native-logit","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"round 4 unpublished","comparable":true,"public":{"correct":207,"n":231,"acc":0.8961038961038961,"wilson95":[0.8500656331827298,0.929183486081599]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.9895833333333334,"n":96,"correct":95,"wilson95":[0.9433338231810272,0.9981588298035302]},"judge":{"acc":0.952054794520548,"n":146,"correct":139,"wilson95":[0.9043471165859538,0.9765839754195642]},"hard":{"acc":0.7727272727272727,"n":220,"correct":170,"wilson95":[0.712948683536588,0.8231450347572892]}},"latency":{"p50_s_raw":0.07892056694254279,"p50_s_adjusted":0.3078411338850856,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.10869606741573032,"kind":"estimate"},"per_item":null,"sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L6883"],"public_tiers":null,"vs_ref":{"test":"Wilson 95% intervals (unpaired)","n":231,"on":"official public accuracy","diff":-0.354978354978355,"ref_wilson95":[0.47671740174484795,0.6041882445175688],"system_wilson95":[0.8500656331827298,0.929183486081599],"verdict":"worse"},"correct_per_usd":8244.124349748216,"correct_per_second":2.910929688942751},{"id":"plumb-4b","display":"Plumb-4B (crh225, JevK5 v0.2 + LoRA)","author":"crh225","kind":"open weights","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.4.2.1 fast-lane addition (official Plumb-4B v1.4 protocol measurement)","comparable":true,"public":{"correct":207,"n":231,"acc":0.8961038961038961,"wilson95":[0.8500656331827298,0.929183486081599]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.9791666666666666,"n":96,"correct":94,"wilson95":[0.9271928642112758,0.9942680301991424]},"judge":{"acc":0.952054794520548,"n":146,"correct":139,"wilson95":[0.9043471165859538,0.9765839754195642]},"hard":{"acc":0.7772727272727272,"n":220,"correct":171,"wilson95":[0.7178001271148297,0.8272284864839455]}},"latency":{"p50_s_raw":0.016472740564495325,"p50_s_adjusted":0.18294548112899064,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.0298085393258427,"kind":"estimate"},"per_item":null,"sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L758"],"public_tiers":null,"vs_ref":{"test":"Wilson 95% intervals (unpaired)","n":231,"on":"official public accuracy","diff":-0.354978354978355,"ref_wilson95":[0.47671740174484795,0.6041882445175688],"system_wilson95":[0.8500656331827298,0.929183486081599],"verdict":"worse"},"correct_per_usd":30061.98614123347,"correct_per_second":4.898201860870637},{"id":"jev-omni","display":"Jev-Omni (akhilaaa3, Gemma-4-12B merged)","author":"akhilaaa3","kind":"open weights","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"round 8 new (add-requests run 8, 23 Sep 2026)","comparable":true,"public":{"correct":205,"n":231,"acc":0.8874458874458875,"wilson95":[0.8401928539554603,0.9220234976746756]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.96875,"n":96,"correct":93,"wilson95":[0.9121132061243772,0.9893159297119012]},"judge":{"acc":0.9246575342465754,"n":146,"correct":135,"wilson95":[0.8701296486178092,0.9574116802358594]},"hard":{"acc":0.75,"n":220,"correct":165,"wilson95":[0.6888220854359439,0.8025971563334435]}},"latency":{"p50_s_raw":0.2170813046395779,"p50_s_adjusted":0.5841626092791558,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.03682059925093633,"kind":"estimate"},"per_item":null,"sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L2305"],"public_tiers":null,"vs_ref":{"test":"Wilson 95% intervals (unpaired)","n":231,"on":"official public accuracy","diff":-0.3463203463203463,"ref_wilson95":[0.47671740174484795,0.6041882445175688],"system_wilson95":[0.8401928539554603,0.9220234976746756],"verdict":"worse"},"correct_per_usd":24101.886050192956,"correct_per_second":1.5191761221091793},{"id":"openjev-thinking","display":"OpenJev (thinking, BF16)","author":"razorback16","kind":"open weights","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.3.0 public","comparable":true,"public":{"correct":205,"n":231,"acc":0.8874458874458875,"wilson95":[0.8401928539554603,0.9220234976746756]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":1,"n":96,"correct":96,"wilson95":[0.9615244115586972,1]},"judge":{"acc":0.9452054794520548,"n":146,"correct":138,"wilson95":[0.895607327679061,0.9719763235384946]},"hard":{"acc":0.7818181818181819,"n":220,"correct":172,"wilson95":[0.7226606998035965,0.8313028091000767]}},"latency":{"p50_s_raw":0.46296589844860137,"p50_s_adjusted":1.0759317968972026,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.25463898876404495,"kind":"estimate"},"per_item":"official","sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L13510","https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.2/jevbench-v1.2-per-task.json#L47728"],"answered":231,"public_tiers":{"easy":{"correct":48,"n":48,"acc":1,"wilson95":[0.9258998692515198,1.0000000000000002]},"original":{"correct":72,"n":72,"acc":1,"wilson95":[0.9493488266449552,1]},"hard":{"correct":85,"n":111,"acc":0.7657657657657657,"wilson95":[0.6789083968937019,0.8348433489891537]}},"vs_ref":{"test":"exact McNemar (paired)","n":231,"on":"all 231 public items","diff":-0.34632034632034636,"ci95":[-0.4155844155844156,-0.27705627705627706],"only_ref":5,"only_system":85,"p":7.532842063484042e-20,"p_holm":3.992406293646542e-18,"verdict":"worse"},"correct_per_usd":3485.113932290305,"correct_per_second":0.8248161175319149},{"id":"swanone","display":"swanOne (blockbrain, Qwen3.8-Flash-Next NVFP4)","author":"blockbrain","kind":"open weights","class":"system-one-open","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.4.2 addition (official offline measurement on an evaluator-owned Lium pod, 24 Sep 2026, jevbench-add-requests run 12)","comparable":true,"public":{"correct":205,"n":231,"acc":0.8874458874458875,"wilson95":[0.8401928539554603,0.9220234976746756]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.9583333333333334,"n":96,"correct":92,"wilson95":[0.8977180846850661,0.983679292577073]},"judge":{"acc":0.9315068493150684,"n":146,"correct":136,"wilson95":[0.8785152238371042,0.9623735458045264]},"hard":{"acc":0.7545454545454545,"n":220,"correct":166,"wilson95":[0.6936305468978045,0.806723590176481]}},"latency":{"p50_s_raw":0.2812396320514381,"p50_s_adjusted":0.7124792641028762,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.11099550561797752,"kind":"estimate"},"per_item":null,"sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L7082"],"public_tiers":null,"vs_ref":{"test":"Wilson 95% intervals (unpaired)","n":231,"on":"official public accuracy","diff":-0.3463203463203463,"ref_wilson95":[0.47671740174484795,0.6041882445175688],"system_wilson95":[0.8401928539554603,0.9220234976746756],"verdict":"worse"},"correct_per_usd":7995.331725414936,"correct_per_second":1.2455743376101225},{"id":"cygnet","display":"Cygnet (blockbrain, frozen Gemma-4-12B-it)","author":"blockbrain","kind":"open weights","class":"system-one-open","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.4.2 addition (official offline measurement on an evaluator-owned Lium pod, 24 Sep 2026, jevbench-add-requests run 12)","comparable":true,"public":{"correct":203,"n":231,"acc":0.8787878787878788,"wilson95":[0.8303921924629958,0.9147913915329472]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.96875,"n":96,"correct":93,"wilson95":[0.9121132061243772,0.9893159297119012]},"judge":{"acc":0.952054794520548,"n":146,"correct":139,"wilson95":[0.9043471165859538,0.9765839754195642]},"hard":{"acc":0.7545454545454545,"n":220,"correct":166,"wilson95":[0.6936305468978045,0.806723590176481]}},"latency":{"p50_s_raw":0.035219401121139526,"p50_s_adjusted":0.22043880224227905,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.03736151685393259,"kind":"estimate"},"per_item":null,"sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L1385"],"public_tiers":null,"vs_ref":{"test":"Wilson 95% intervals (unpaired)","n":231,"on":"official public accuracy","diff":-0.33766233766233766,"ref_wilson95":[0.47671740174484795,0.6041882445175688],"system_wilson95":[0.8303921924629958,0.9147913915329472],"verdict":"worse"},"correct_per_usd":23521.204511678698,"correct_per_second":3.9865389842847354},{"id":"djev-thinking","display":"djev (thinking)","author":"David Villalon / Maisa","kind":"open weights","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.3.0 public","comparable":true,"public":{"correct":202,"n":231,"acc":0.8744588744588745,"wilson95":[0.8255168636268739,0.9111503365519724]},"published_tiers":{"easy":{"acc":0.9583333333333334,"n":72,"correct":69,"wilson95":[0.8845072317804198,0.9857291926441228]},"standard":{"acc":0.9895833333333334,"n":96,"correct":95,"wilson95":[0.9433338231810272,0.9981588298035302]},"judge":{"acc":0.8013698630136986,"n":146,"correct":117,"wilson95":[0.7292972462782585,0.8579901483920869]},"hard":{"acc":0.7772727272727272,"n":220,"correct":171,"wilson95":[0.7178001271148297,0.8272284864839455]}},"latency":{"p50_s_raw":0.4260098780505359,"p50_s_adjusted":1.0020197561010715,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.27429398876404487,"kind":"estimate"},"per_item":"official","sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L13112","https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.2/jevbench-v1.2-per-task.json#L25248"],"answered":231,"public_tiers":{"easy":{"correct":46,"n":48,"acc":0.9583333333333334,"wilson95":[0.8602434401996508,0.9884981066142424]},"original":{"correct":71,"n":72,"acc":0.9861111111111112,"wilson95":[0.9254339777605758,0.997544048144242]},"hard":{"correct":85,"n":111,"acc":0.7657657657657657,"wilson95":[0.6789083968937019,0.8348433489891537]}},"vs_ref":{"test":"exact McNemar (paired)","n":231,"on":"all 231 public items","diff":-0.33333333333333337,"ci95":[-0.4069264069264069,-0.26406926406926406],"only_ref":9,"only_system":86,"p":6.606356572585723e-17,"p_holm":3.435305417744576e-15,"verdict":"worse"},"correct_per_usd":3188.0351384991804,"correct_per_second":0.8726962409019304},{"id":"gemini-3.1-flash-lite","display":"Gemini 3.1 Flash-Lite","author":"Google","kind":"hosted API","class":"llm-baseline","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.3.0 public","comparable":true,"public":{"correct":201,"n":231,"acc":0.8701298701298701,"wilson95":[0.8206572486627326,0.9074935676990172]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.9895833333333334,"n":96,"correct":95,"wilson95":[0.9433338231810272,0.9981588298035302]},"judge":{"acc":0.9315068493150684,"n":146,"correct":136,"wilson95":[0.8785152238371042,0.9623735458045264]},"hard":{"acc":0.75,"n":220,"correct":165,"wilson95":[0.6888220854359439,0.8025971563334435]}},"latency":{"p50_s_raw":0.756230715662241,"p50_s_adjusted":0.756230715662241,"basis":"none (production API)"},"cost":{"usd_per_1000":0.26378698501872655,"kind":"measured"},"per_item":"official","sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L13882","https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.2/jevbench-v1.2-per-task.json#L10636"],"answered":231,"public_tiers":{"easy":{"correct":48,"n":48,"acc":1,"wilson95":[0.9258998692515198,1.0000000000000002]},"original":{"correct":71,"n":72,"acc":0.9861111111111112,"wilson95":[0.9254339777605758,0.997544048144242]},"hard":{"correct":82,"n":111,"acc":0.7387387387387387,"wilson95":[0.6500078009210981,0.8114980047024842]}},"vs_ref":{"test":"exact McNemar (paired)","n":231,"on":"all 231 public items","diff":-0.32900432900432897,"ci95":[-0.3982683982683982,-0.2597402597402597],"only_ref":10,"only_system":86,"p":3.2114796492801454e-16,"p_holm":1.5736250281472713e-14,"verdict":"worse"},"correct_per_usd":3298.6080418944803,"correct_per_second":1.150614292845651},{"id":"reflex-27b","display":"reflex-27b (Qwen3.8-27B)","author":"kshetrajna12","kind":"open weights","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.3.0 public","comparable":true,"public":{"correct":201,"n":231,"acc":0.8701298701298701,"wilson95":[0.8206572486627326,0.9074935676990172]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.9583333333333334,"n":96,"correct":92,"wilson95":[0.8977180846850661,0.983679292577073]},"judge":{"acc":0.958904109589041,"n":146,"correct":140,"wilson95":[0.9132470509865872,0.9810314818068931]},"hard":{"acc":0.759090909090909,"n":220,"correct":167,"wilson95":[0.6984472783022945,0.8108417540768887]}},"latency":{"p50_s_raw":1.8879061974585056,"p50_s_adjusted":3.925812394917011,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.18111733707865169,"kind":"estimate"},"per_item":"official","sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L12714","https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.2/jevbench-v1.2-per-task.json#L52224"],"answered":231,"public_tiers":{"easy":{"correct":48,"n":48,"acc":1,"wilson95":[0.9258998692515198,1.0000000000000002]},"original":{"correct":69,"n":72,"acc":0.9583333333333334,"wilson95":[0.8845072317804198,0.9857291926441228]},"hard":{"correct":84,"n":111,"acc":0.7567567567567568,"wilson95":[0.6692307503570539,0.827105682106044]}},"vs_ref":{"test":"exact McNemar (paired)","n":231,"on":"all 231 public items","diff":-0.32900432900432897,"ci95":[-0.3982683982683982,-0.2554112554112554],"only_ref":10,"only_system":86,"p":3.2114796492801454e-16,"p_holm":1.5736250281472713e-14,"verdict":"worse"},"correct_per_usd":4804.232903181483,"correct_per_second":0.2216432632533537},{"id":"instinct","display":"Instinct (ZooWork, Qwen3.8-27B)","author":"rayrain-srp (ZooWork)","kind":"hosted demo","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.4.2 addition (official full-protocol measurement, jevbench-add-requests run 11, 24 Sep 2026)","comparable":true,"public":{"correct":200,"n":231,"acc":0.8658008658008658,"wilson95":[0.8158127032670599,0.9038217292775934]},"published_tiers":{"easy":{"acc":0.9861111111111112,"n":72,"correct":71,"wilson95":[0.9254339777605758,0.997544048144242]},"standard":{"acc":0.9583333333333334,"n":96,"correct":92,"wilson95":[0.8977180846850661,0.983679292577073]},"judge":{"acc":0.9452054794520548,"n":146,"correct":138,"wilson95":[0.895607327679061,0.9719763235384946]},"hard":{"acc":0.759090909090909,"n":220,"correct":167,"wilson95":[0.6984472783022945,0.8108417540768887]}},"latency":{"p50_s_raw":0.25864607095718384,"p50_s_adjusted":0.5172921419143677,"basis":"x2 (free evaluation/demo endpoint: no SLA, no status page, no terms, no bookable price)"},"cost":{"usd_per_1000":0.3253253932584269,"kind":"estimate"},"per_item":null,"sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L14687"],"public_tiers":null,"vs_ref":{"test":"Wilson 95% intervals (unpaired)","n":231,"on":"official public accuracy","diff":-0.3246753246753247,"ref_wilson95":[0.47671740174484795,0.6041882445175688],"system_wilson95":[0.8158127032670599,0.9038217292775934],"verdict":"worse"},"correct_per_usd":2661.3381056089415,"correct_per_second":1.6737174135233437},{"id":"jev-1.13.0","display":"Jev 1.13.0 (TypeSafe AI)","author":"TypeSafe AI","kind":"hosted API","class":"jev","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.3.0 public","comparable":true,"public":{"correct":200,"n":231,"acc":0.8658008658008658,"wilson95":[0.8158127032670599,0.9038217292775934]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.9895833333333334,"n":96,"correct":95,"wilson95":[0.9433338231810272,0.9981588298035302]},"judge":{"acc":0.9452054794520548,"n":146,"correct":138,"wilson95":[0.895607327679061,0.9719763235384946]},"hard":{"acc":0.740909090909091,"n":220,"correct":163,"wilson95":[0.6792293878102094,0.7943200633493823]}},"latency":{"p50_s_raw":0.6524335257709026,"p50_s_adjusted":0.6524335257709026,"basis":"none (production API)"},"cost":{"usd_per_1000":0.03991423595505617,"kind":"measured"},"per_item":"official","sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L1016","https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.2/jevbench-v1.2-per-task.json#L5016"],"answered":231,"public_tiers":{"easy":{"correct":48,"n":48,"acc":1,"wilson95":[0.9258998692515198,1.0000000000000002]},"original":{"correct":71,"n":72,"acc":0.9861111111111112,"wilson95":[0.9254339777605758,0.997544048144242]},"hard":{"correct":81,"n":111,"acc":0.7297297297297297,"wilson95":[0.6404595657250507,0.8036309264787738]}},"vs_ref":{"test":"exact McNemar (paired)","n":231,"on":"all 231 public items","diff":-0.3246753246753247,"ci95":[-0.3939393939393939,-0.2554112554112554],"only_ref":9,"only_system":84,"p":2.1688614442545302e-16,"p_holm":1.1061193365698104e-14,"verdict":"worse"},"correct_per_usd":21691.5304798961,"correct_per_second":1.3270330717261234},{"id":"simplejev-qwen3.8-27b","display":"SimpleJev Qwen3.8-27B","author":"Featherless AI","kind":"open weights","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.3.0 public","comparable":true,"public":{"correct":200,"n":231,"acc":0.8658008658008658,"wilson95":[0.8158127032670599,0.9038217292775934]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.96875,"n":96,"correct":93,"wilson95":[0.9121132061243772,0.9893159297119012]},"judge":{"acc":0.9315068493150684,"n":146,"correct":136,"wilson95":[0.8785152238371042,0.9623735458045264]},"hard":{"acc":0.75,"n":220,"correct":165,"wilson95":[0.6888220854359439,0.8025971563334435]}},"latency":{"p50_s_raw":1.0138170085847378,"p50_s_adjusted":2.0276340171694756,"basis":"x2 (assumption, not measured)"},"cost":{"usd_per_1000":0.10399651685393256,"kind":"estimate"},"per_item":"official","sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L6684","https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.2/jevbench-v1.2-per-task.json#L55596"],"answered":231,"public_tiers":{"easy":{"correct":48,"n":48,"acc":1,"wilson95":[0.9258998692515198,1.0000000000000002]},"original":{"correct":70,"n":72,"acc":0.9722222222222222,"wilson95":[0.9042582470003044,0.9923489781643758]},"hard":{"correct":82,"n":111,"acc":0.7387387387387387,"wilson95":[0.6500078009210981,0.8114980047024842]}},"vs_ref":{"test":"exact McNemar (paired)","n":231,"on":"all 231 public items","diff":-0.3246753246753247,"ci95":[-0.3939393939393939,-0.2510822510822511],"only_ref":11,"only_system":86,"p":1.4342435250828828e-15,"p_holm":6.597520215381261e-14,"verdict":"worse"},"correct_per_usd":8325.28715376996,"correct_per_second":0.4270005624631911},{"id":"imajev_4b","display":"Imajev-4B","author":"mohit67890","kind":"open weights","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.4.2.2 fast-lane addition (official Imajev-4B v1.4 protocol measurement; one rotation)","comparable":true,"public":{"correct":199,"n":231,"acc":0.8614718614718615,"wilson95":[0.8109826327650083,0.9001354159625488]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.9895833333333334,"n":96,"correct":95,"wilson95":[0.9433338231810272,0.9981588298035302]},"judge":{"acc":0.8904109589041096,"n":146,"correct":130,"wilson95":[0.8293942123918577,0.9314099125219988]},"hard_public":{"acc":0.7207207207207207,"n":111,"correct":80,"wilson95":[0.6309517480050978,0.7957234307789688]}},"latency":{"p50_s_raw":0.03957595250176382,"p50_s_adjusted":0.22915190500352764,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.02200522471910112,"kind":"estimate"},"per_item":null,"sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L569"],"public_tiers":null,"vs_ref":{"test":"Wilson 95% intervals (unpaired)","n":231,"on":"official public accuracy","diff":-0.3203463203463203,"ref_wilson95":[0.47671740174484795,0.6041882445175688],"system_wilson95":[0.8109826327650083,0.9001354159625488],"verdict":"worse"},"correct_per_usd":39148.51461271745,"correct_per_second":3.7593921004435886},{"id":"litjev","display":"LitJev (Qwen3.8-27B)","author":"Zhengxu Yu","kind":"open weights","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.3.0 public","comparable":true,"public":{"correct":199,"n":231,"acc":0.8614718614718615,"wilson95":[0.8109826327650083,0.9001354159625488]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.9791666666666666,"n":96,"correct":94,"wilson95":[0.9271928642112758,0.9942680301991424]},"judge":{"acc":0.8835616438356164,"n":146,"correct":129,"wilson95":[0.8214385528954438,0.9260181312304504]},"hard":{"acc":0.7318181818181818,"n":220,"correct":161,"wilson95":[0.6696680830524655,0.7860115774973302]}},"latency":{"p50_s_raw":2.025244139134884,"p50_s_adjusted":4.200488278269768,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.16303594007490638,"kind":"estimate"},"per_item":"official","sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L11321","https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.2/jevbench-v1.2-per-task.json#L42108"],"answered":231,"public_tiers":{"easy":{"correct":48,"n":48,"acc":1,"wilson95":[0.9258998692515198,1.0000000000000002]},"original":{"correct":71,"n":72,"acc":0.9861111111111112,"wilson95":[0.9254339777605758,0.997544048144242]},"hard":{"correct":80,"n":111,"acc":0.7207207207207207,"wilson95":[0.6309517480050978,0.7957234307789688]}},"vs_ref":{"test":"exact McNemar (paired)","n":231,"on":"all 231 public items","diff":-0.3203463203463204,"ci95":[-0.3896103896103896,-0.2510822510822511],"only_ref":9,"only_system":83,"p":3.923298087974169e-16,"p_holm":1.843950101347859e-14,"verdict":"worse"},"correct_per_usd":5283.9383823963035,"correct_per_second":0.2050885050503491},{"id":"winnow-12b","display":"Winnow-12B Q8","author":"Eldan Ring","kind":"open weights","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.3.0 public","comparable":true,"public":{"correct":198,"n":231,"acc":0.8571428571428571,"wilson95":[0.8061664869370828,0.8964351779733775]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.96875,"n":96,"correct":93,"wilson95":[0.9121132061243772,0.9893159297119012]},"judge":{"acc":0.910958904109589,"n":146,"correct":133,"wilson95":[0.8536207185523637,0.94722572872538]},"hard":{"acc":0.7090909090909091,"n":220,"correct":156,"wilson95":[0.645895791605641,0.7651093924196649]}},"latency":{"p50_s_raw":0.22503555566072464,"p50_s_adjusted":0.6000711113214493,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.037087359550561805,"kind":"estimate"},"per_item":"official","sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L1708","https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.2/jevbench-v1.2-per-task.json#L57844"],"answered":231,"public_tiers":{"easy":{"correct":48,"n":48,"acc":1,"wilson95":[0.9258998692515198,1.0000000000000002]},"original":{"correct":69,"n":72,"acc":0.9583333333333334,"wilson95":[0.8845072317804198,0.9857291926441228]},"hard":{"correct":81,"n":111,"acc":0.7297297297297297,"wilson95":[0.6404595657250507,0.8036309264787738]}},"vs_ref":{"test":"exact McNemar (paired)","n":231,"on":"all 231 public items","diff":-0.316017316017316,"ci95":[-0.3852813852813853,-0.24242424242424243],"only_ref":12,"only_system":85,"p":1.0431765969668068e-14,"p_holm":4.58997702665395e-13,"verdict":"worse"},"correct_per_usd":23111.455426593533,"correct_per_second":1.428402135965679},{"id":"classifier-dev-fast","display":"classifier.dev (fast tier)","author":"mrmps (@michael_chomsky)","kind":"hosted API","class":"jev-service","listing":"honorable_mention","ranked":false,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.3.0 public","comparable":true,"public":{"correct":197,"n":231,"acc":0.8528138528138528,"wilson95":[0.8013637555232439,0.8927215255701201]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.9895833333333334,"n":96,"correct":95,"wilson95":[0.9433338231810272,0.9981588298035302]},"judge":{"acc":0.9726027397260274,"n":146,"correct":142,"wilson95":[0.9316781374425416,0.9892952769268636]},"hard":{"acc":0.7045454545454546,"n":220,"correct":155,"wilson95":[0.6411629187420985,0.7609073699783094]}},"latency":{"p50_s_raw":0.3863608464598656,"p50_s_adjusted":0.3863608464598656,"basis":"none (production API)"},"cost":{"usd_per_1000":0.003333333333333333,"kind":"estimate"},"per_item":"official","sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L17799","https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.2/jevbench-v1.2-per-task.json#L20752"],"answered":231,"public_tiers":{"easy":{"correct":48,"n":48,"acc":1,"wilson95":[0.9258998692515198,1.0000000000000002]},"original":{"correct":71,"n":72,"acc":0.9861111111111112,"wilson95":[0.9254339777605758,0.997544048144242]},"hard":{"correct":78,"n":111,"acc":0.7027027027027027,"wilson95":[0.612052796042019,0.779791755902532]}},"vs_ref":{"test":"exact McNemar (paired)","n":231,"on":"all 231 public items","diff":-0.3116883116883117,"ci95":[-0.380952380952381,-0.24242424242424243],"only_ref":10,"only_system":82,"p":3.3046946373720168e-15,"p_holm":1.4871125868174078e-13,"verdict":"worse"},"correct_per_usd":255844.15584415584,"correct_per_second":2.207298851910041},{"id":"jevk5-v02","display":"JevK5 v0.2.0","author":"allebee","kind":"not stated","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"round 5 new","comparable":true,"public":{"correct":197,"n":231,"acc":0.8528138528138528,"wilson95":[0.8013637555232439,0.8927215255701201]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.9583333333333334,"n":96,"correct":92,"wilson95":[0.8977180846850661,0.983679292577073]},"judge":{"acc":0.9452054794520548,"n":146,"correct":138,"wilson95":[0.895607327679061,0.9719763235384946]},"hard":{"acc":0.7,"n":220,"correct":154,"wilson95":[0.6364370126708406,0.7566983807446693]}},"latency":null,"cost":{"usd_per_1000":0.022356404494382014,"kind":"estimate"},"per_item":null,"sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L1215"],"public_tiers":null,"vs_ref":{"test":"Wilson 95% intervals (unpaired)","n":231,"on":"official public accuracy","diff":-0.3116883116883117,"ref_wilson95":[0.47671740174484795,0.6041882445175688],"system_wilson95":[0.8013637555232439,0.8927215255701201],"verdict":"worse"},"correct_per_usd":38146.28837245086,"correct_per_second":null},{"id":"openjev-sglang","display":"openjev-sglang (Qwen3.6-35B-A3B on SGLang)","author":"ekzhang","kind":"open weights","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.3.0 public","comparable":true,"public":{"correct":197,"n":231,"acc":0.8528138528138528,"wilson95":[0.8013637555232439,0.8927215255701201]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.9583333333333334,"n":96,"correct":92,"wilson95":[0.8977180846850661,0.983679292577073]},"judge":{"acc":0.952054794520548,"n":146,"correct":139,"wilson95":[0.9043471165859538,0.9765839754195642]},"hard":{"acc":0.7136363636363636,"n":220,"correct":157,"wilson95":[0.6506357247568185,0.7693043545733852]}},"latency":{"p50_s_raw":0.6776718497276306,"p50_s_adjusted":1.3553436994552612,"basis":"x2 (assumption, not measured)"},"cost":{"usd_per_1000":0.13127546816479402,"kind":"estimate"},"per_item":"official","sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L9019","https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.2/jevbench-v1.2-per-task.json#L8388"],"answered":231,"public_tiers":{"easy":{"correct":48,"n":48,"acc":1,"wilson95":[0.9258998692515198,1.0000000000000002]},"original":{"correct":68,"n":72,"acc":0.9444444444444444,"wilson95":[0.8656798395141685,0.9781857841702364]},"hard":{"correct":81,"n":111,"acc":0.7297297297297297,"wilson95":[0.6404595657250507,0.8036309264787738]}},"vs_ref":{"test":"exact McNemar (paired)","n":231,"on":"all 231 public items","diff":-0.3116883116883117,"ci95":[-0.3852813852813853,-0.23809523809523808],"only_ref":12,"only_system":84,"p":1.8316192854098385e-14,"p_holm":7.875962927262306e-13,"verdict":"worse"},"correct_per_usd":6496.368778843661,"correct_per_second":0.6292233129918375},{"id":"open-jev-json-canvas-joshuasp","display":"Open Jev JSON Canvas (JoshuaSP)","author":"JoshuaSP","kind":"open weights","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"round 4 unpublished","comparable":true,"public":{"correct":195,"n":231,"acc":0.8441558441558441,"wilson95":[0.7917966716362469,0.8852558418229238]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.9895833333333334,"n":96,"correct":95,"wilson95":[0.9433338231810272,0.9981588298035302]},"judge":{"acc":0.9246575342465754,"n":146,"correct":135,"wilson95":[0.8701296486178092,0.9574116802358594]},"hard":{"acc":0.7045454545454546,"n":220,"correct":155,"wilson95":[0.6411629187420985,0.7609073699783094]}},"latency":{"p50_s_raw":0.2243216049973853,"p50_s_adjusted":0.5986432099947706,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.06498050561797752,"kind":"estimate"},"per_item":null,"sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L17600"],"public_tiers":null,"vs_ref":{"test":"Wilson 95% intervals (unpaired)","n":231,"on":"official public accuracy","diff":-0.30303030303030304,"ref_wilson95":[0.47671740174484795,0.6041882445175688],"system_wilson95":[0.7917966716362469,0.8852558418229238],"verdict":"worse"},"correct_per_usd":12990.909136944292,"correct_per_second":1.4101151237699967},{"id":"djev","display":"djev (Maisa, diffusion-gemma)","author":"Maisa (David Villalón)","kind":"open weights","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.3.0 public","comparable":true,"public":{"correct":194,"n":231,"acc":0.8398268398268398,"wilson95":[0.7870314654792803,0.8815046641627942]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.9791666666666666,"n":96,"correct":94,"wilson95":[0.9271928642112758,0.9942680301991424]},"judge":{"acc":0.9315068493150684,"n":146,"correct":136,"wilson95":[0.8785152238371042,0.9623735458045264]},"hard":{"acc":0.6954545454545454,"n":220,"correct":153,"wilson95":[0.6317179839304611,0.7524825141801509]}},"latency":{"p50_s_raw":0.2370578795671463,"p50_s_adjusted":0.2370578795671463,"basis":"none (production API)"},"cost":{"usd_per_1000":0.025951254681647943,"kind":"announced"},"per_item":"official","sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L2106","https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.2/jevbench-v1.2-per-task.json#L26372"],"answered":231,"public_tiers":{"easy":{"correct":48,"n":48,"acc":1,"wilson95":[0.9258998692515198,1.0000000000000002]},"original":{"correct":71,"n":72,"acc":0.9861111111111112,"wilson95":[0.9254339777605758,0.997544048144242]},"hard":{"correct":75,"n":111,"acc":0.6756756756756757,"wilson95":[0.5839813427909794,0.755617268894298]}},"vs_ref":{"test":"exact McNemar (paired)","n":231,"on":"all 231 public items","diff":-0.2987012987012987,"ci95":[-0.3722943722943723,-0.22510822510822512],"only_ref":14,"only_system":83,"p":4.2224144708722075e-13,"p_holm":1.731189933057605e-11,"verdict":"worse"},"correct_per_usd":32361.70467013079,"correct_per_second":3.54270797224844},{"id":"decider-4b-v2","display":"decider-4b v2 (Mapika)","author":"Mapika","kind":"open weights","class":"system-one-open","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.4.2 addition (official offline measurement on an evaluator-owned Lium pod, 24 Sep 2026, jevbench-add-requests run 12)","comparable":true,"public":{"correct":193,"n":231,"acc":0.8354978354978355,"wilson95":[0.7822779606733993,0.8777417851515786]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.96875,"n":96,"correct":93,"wilson95":[0.9121132061243772,0.9893159297119012]},"judge":{"acc":0.8767123287671232,"n":146,"correct":128,"wilson95":[0.8135354700073181,0.9205737733306134]},"hard":{"acc":0.6727272727272727,"n":220,"correct":148,"wilson95":[0.6082231287280518,0.7313028928580704]}},"latency":{"p50_s_raw":0.01749592460691929,"p50_s_adjusted":0.18499184921383857,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.02006786516853933,"kind":"estimate"},"per_item":null,"sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L892"],"public_tiers":null,"vs_ref":{"test":"Wilson 95% intervals (unpaired)","n":231,"on":"official public accuracy","diff":-0.2943722943722944,"ref_wilson95":[0.47671740174484795,0.6041882445175688],"system_wilson95":[0.7822779606733993,0.8777417851515786],"verdict":"worse"},"correct_per_usd":41633.61814926169,"correct_per_second":4.516403501281045},{"id":"decider-35b-a3b","display":"decider-35b-a3b (Mapika)","author":"Mapika","kind":"open weights","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.3.0 public","comparable":true,"public":{"correct":192,"n":231,"acc":0.8311688311688312,"wilson95":[0.7775357965997839,0.8739675654080977]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.96875,"n":96,"correct":93,"wilson95":[0.9121132061243772,0.9893159297119012]},"judge":{"acc":0.910958904109589,"n":146,"correct":133,"wilson95":[0.8536207185523637,0.94722572872538]},"hard":{"acc":0.6545454545454545,"n":220,"correct":144,"wilson95":[0.589543218600646,0.7142432217658845]}},"latency":{"p50_s_raw":0.29191894084215164,"p50_s_adjusted":0.7338378816843033,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.0665503745318352,"kind":"estimate"},"per_item":"official","sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L4246","https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.2/jevbench-v1.2-per-task.json#L23000"],"answered":231,"public_tiers":{"easy":{"correct":48,"n":48,"acc":1,"wilson95":[0.9258998692515198,1.0000000000000002]},"original":{"correct":70,"n":72,"acc":0.9722222222222222,"wilson95":[0.9042582470003044,0.9923489781643758]},"hard":{"correct":74,"n":111,"acc":0.6666666666666666,"wilson95":[0.5746946727251381,0.7474886255403816]}},"vs_ref":{"test":"exact McNemar (paired)","n":231,"on":"all 231 public items","diff":-0.2900432900432901,"ci95":[-0.3593073593073593,-0.22077922077922077],"only_ref":10,"only_system":77,"p":5.91755837353286e-14,"p_holm":2.4853745168838013e-12,"verdict":"worse"},"correct_per_usd":12489.31861038936,"correct_per_second":1.1326327679638644},{"id":"ninfer-qwen3.8-27b","display":"NInfer Qwen3.8-27B NVFP4","author":"Igor L. / NInfer contributors","kind":"open weights","class":"native-logit","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"round 4 unpublished","comparable":true,"public":{"correct":192,"n":231,"acc":0.8311688311688312,"wilson95":[0.7775357965997839,0.8739675654080977]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.9791666666666666,"n":96,"correct":94,"wilson95":[0.9271928642112758,0.9942680301991424]},"judge":{"acc":0.9657534246575342,"n":146,"correct":141,"wilson95":[0.9223410969879317,0.9852848765935112]},"hard":{"acc":0.7045454545454546,"n":220,"correct":155,"wilson95":[0.6411629187420985,0.7609073699783094]}},"latency":{"p50_s_raw":0.368209820240736,"p50_s_adjusted":0.886419640481472,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.14492808988764044,"kind":"estimate"},"per_item":null,"sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L9596"],"public_tiers":null,"vs_ref":{"test":"Wilson 95% intervals (unpaired)","n":231,"on":"official public accuracy","diff":-0.29004329004329005,"ref_wilson95":[0.47671740174484795,0.6041882445175688],"system_wilson95":[0.7775357965997839,0.8739675654080977],"verdict":"worse"},"correct_per_usd":5735.043025911803,"correct_per_second":0.9376696918824694},{"id":"ninfer-qwen3.8-27b-t1.5","display":"NInfer Qwen3.8-27B NVFP4 (T=1.5)","author":"Igor L. / NInfer contributors","kind":"open weights","class":"native-logit","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"round 4 unpublished","comparable":true,"public":{"correct":192,"n":231,"acc":0.8311688311688312,"wilson95":[0.7775357965997839,0.8739675654080977]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.9791666666666666,"n":96,"correct":94,"wilson95":[0.9271928642112758,0.9942680301991424]},"judge":{"acc":0.9657534246575342,"n":146,"correct":141,"wilson95":[0.9223410969879317,0.9852848765935112]},"hard":{"acc":0.7045454545454546,"n":220,"correct":155,"wilson95":[0.6411629187420985,0.7609073699783094]}},"latency":{"p50_s_raw":0.368209820240736,"p50_s_adjusted":0.886419640481472,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.14492808988764044,"kind":"estimate"},"per_item":null,"sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L9397"],"public_tiers":null,"vs_ref":{"test":"Wilson 95% intervals (unpaired)","n":231,"on":"official public accuracy","diff":-0.29004329004329005,"ref_wilson95":[0.47671740174484795,0.6041882445175688],"system_wilson95":[0.7775357965997839,0.8739675654080977],"verdict":"worse"},"correct_per_usd":5735.043025911803,"correct_per_second":0.9376696918824694},{"id":"hopper","display":"Hopper","author":"HopitAI","kind":"open weights","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"round 4 unpublished","comparable":true,"public":{"correct":190,"n":231,"acc":0.8225108225108225,"wilson95":[0.76808415846789,0.8663864359057986]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.96875,"n":96,"correct":93,"wilson95":[0.9121132061243772,0.9893159297119012]},"judge":{"acc":0.8356164383561644,"n":146,"correct":122,"wilson95":[0.7670494247488154,0.886975173861342]},"hard":{"acc":0.65,"n":220,"correct":143,"wilson95":[0.5848888103245193,0.7099627347371131]}},"latency":{"p50_s_raw":0.12881201645359397,"p50_s_adjusted":0.407624032907188,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.02375376404494382,"kind":"estimate"},"per_item":null,"sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L1509"],"public_tiers":null,"vs_ref":{"test":"Wilson 95% intervals (unpaired)","n":231,"on":"official public accuracy","diff":-0.2813852813852814,"ref_wilson95":[0.47671740174484795,0.6041882445175688],"system_wilson95":[0.76808415846789,0.8663864359057986],"verdict":"worse"},"correct_per_usd":34626.54680557462,"correct_per_second":2.0178172926769022},{"id":"openjev-razorback16","display":"OpenJev (DiffusionGemma 26B-A4B NVFP4, razorback16)","author":"razorback16 / Codiv","kind":"open weights","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.3.0 public","comparable":true,"public":{"correct":189,"n":231,"acc":0.8181818181818182,"wilson95":[0.7633740680427609,0.8625801425138312]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.9583333333333334,"n":96,"correct":92,"wilson95":[0.8977180846850661,0.983679292577073]},"judge":{"acc":0.910958904109589,"n":146,"correct":133,"wilson95":[0.8536207185523637,0.94722572872538]},"hard":{"acc":0.6545454545454545,"n":220,"correct":144,"wilson95":[0.589543218600646,0.7142432217658845]}},"latency":{"p50_s_raw":0.24127069488167763,"p50_s_adjusted":0.6325413897633553,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.06560455056179774,"kind":"estimate"},"per_item":"official","sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L5734","https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.2/jevbench-v1.2-per-task.json#L2768"],"answered":231,"public_tiers":{"easy":{"correct":48,"n":48,"acc":1,"wilson95":[0.9258998692515198,1.0000000000000002]},"original":{"correct":70,"n":72,"acc":0.9722222222222222,"wilson95":[0.9042582470003044,0.9923489781643758]},"hard":{"correct":71,"n":111,"acc":0.6396396396396397,"wilson95":[0.5470363589264071,0.7229009990798392]}},"vs_ref":{"test":"exact McNemar (paired)","n":231,"on":"all 231 public items","diff":-0.2770562770562771,"ci95":[-0.35064935064935066,-0.1991341991341991],"only_ref":18,"only_system":82,"p":6.147806615047977e-11,"p_holm":2.2132103814172715e-9,"verdict":"worse"},"correct_per_usd":12471.41869238343,"correct_per_second":1.293483448550164},{"id":"simplejev-qwen3.6-35b-a3b","display":"SimpleJev Qwen3.6-35B-A3B","author":"Featherless AI","kind":"open weights","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.3.0 public","comparable":true,"public":{"correct":188,"n":231,"acc":0.8138528138528138,"wilson95":[0.7586740824392884,0.8587637443002072]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.9375,"n":96,"correct":90,"wilson95":[0.8702905342555145,0.9710433258583456]},"judge":{"acc":0.9315068493150684,"n":146,"correct":136,"wilson95":[0.8785152238371042,0.9623735458045264]},"hard":{"acc":0.6636363636363637,"n":220,"correct":146,"wilson95":[0.5988705989319933,0.7227856320443331]}},"latency":{"p50_s_raw":0.8519573211669922,"p50_s_adjusted":1.7039146423339844,"basis":"x2 (assumption, not measured)"},"cost":{"usd_per_1000":0.11555168539325844,"kind":"estimate"},"per_item":"official","sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L10317","https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.2/jevbench-v1.2-per-task.json#L54472"],"answered":231,"public_tiers":{"easy":{"correct":48,"n":48,"acc":1,"wilson95":[0.9258998692515198,1.0000000000000002]},"original":{"correct":67,"n":72,"acc":0.9305555555555556,"wilson95":[0.8475202043199935,0.9699746186242738]},"hard":{"correct":73,"n":111,"acc":0.6576576576576577,"wilson95":[0.5654420339811526,0.7393259508646095]}},"vs_ref":{"test":"exact McNemar (paired)","n":231,"on":"all 231 public items","diff":-0.2727272727272727,"ci95":[-0.341991341991342,-0.1991341991341991],"only_ref":13,"only_system":76,"p":5.446803038068455e-12,"p_holm":2.1787212152273818e-10,"verdict":"worse"},"correct_per_usd":7043.192932089383,"correct_per_second":0.4776370797178058},{"id":"jobe-qwen3.5-4b","display":"Jobe Qwen3.5-4B (frozen)","author":"MantisShrimpdev","kind":"open weights","class":"native-logit","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"round 4 unpublished","comparable":true,"public":{"correct":187,"n":231,"acc":0.8095238095238095,"wilson95":[0.7539839363320431,0.854937506590356]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.9791666666666666,"n":96,"correct":94,"wilson95":[0.9271928642112758,0.9942680301991424]},"judge":{"acc":0.9657534246575342,"n":146,"correct":141,"wilson95":[0.9223410969879317,0.9852848765935112]},"hard":{"acc":0.5863636363636363,"n":220,"correct":129,"wilson95":[0.5203479977619418,0.6494150130311191]}},"latency":{"p50_s_raw":0.13457110384479165,"p50_s_adjusted":0.4191422076895833,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.022356404494382014,"kind":"estimate"},"per_item":null,"sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L2915"],"public_tiers":null,"vs_ref":{"test":"Wilson 95% intervals (unpaired)","n":231,"on":"official public accuracy","diff":-0.2683982683982684,"ref_wilson95":[0.47671740174484795,0.6041882445175688],"system_wilson95":[0.7539839363320431,0.854937506590356],"verdict":"worse"},"correct_per_usd":36209.92855659041,"correct_per_second":1.9313822246299344},{"id":"semif-qwen3.5-4b","display":"SemIf, formerly OpenJev (Qwen3.5-4B, TheoLeeCJ)","author":"Theodore Lee (TheoLeeCJ)","kind":"open weights","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.3.0 public","comparable":true,"public":{"correct":187,"n":231,"acc":0.8095238095238095,"wilson95":[0.7539839363320431,0.854937506590356]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.9791666666666666,"n":96,"correct":94,"wilson95":[0.9271928642112758,0.9942680301991424]},"judge":{"acc":0.952054794520548,"n":146,"correct":139,"wilson95":[0.9043471165859538,0.9765839754195642]},"hard":{"acc":0.5954545454545455,"n":220,"correct":131,"wilson95":[0.5294992624664222,0.6581335389364348]}},"latency":{"p50_s_raw":0.1979611478745937,"p50_s_adjusted":0.5459222957491875,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.02244460674157303,"kind":"estimate"},"per_item":"official","sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L2707","https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.2/jevbench-v1.2-per-task.json#L1644"],"answered":231,"public_tiers":{"easy":{"correct":48,"n":48,"acc":1,"wilson95":[0.9258998692515198,1.0000000000000002]},"original":{"correct":71,"n":72,"acc":0.9861111111111112,"wilson95":[0.9254339777605758,0.997544048144242]},"hard":{"correct":68,"n":111,"acc":0.6126126126126126,"wilson95":[0.5196688341070039,0.6980225836399687]}},"vs_ref":{"test":"exact McNemar (paired)","n":231,"on":"all 231 public items","diff":-0.2683982683982684,"ci95":[-0.341991341991342,-0.1948051948051948],"only_ref":15,"only_system":77,"p":3.262400923439496e-11,"p_holm":1.2397123509070083e-9,"verdict":"worse"},"correct_per_usd":36067.63169632056,"correct_per_second":1.482855373057942},{"id":"localjev-qwen3.5-4b","display":"local-jev Qwen3.5-4B","author":"Amith Chandrappa (amithgc)","kind":"open weights","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"round 4 unpublished","comparable":true,"public":{"correct":186,"n":231,"acc":0.8051948051948052,"wilson95":[0.7493033792125899,0.8511016798927126]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.9583333333333334,"n":96,"correct":92,"wilson95":[0.8977180846850661,0.983679292577073]},"judge":{"acc":0.958904109589041,"n":146,"correct":140,"wilson95":[0.9132470509865872,0.9810314818068931]},"hard":{"acc":0.6045454545454545,"n":220,"correct":133,"wilson95":[0.5386730079504642,0.6668295840621885]}},"latency":{"p50_s_raw":0.7075801230967045,"p50_s_adjusted":1.5651602461934089,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.02974554307116105,"kind":"estimate"},"per_item":null,"sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L3114"],"public_tiers":null,"vs_ref":{"test":"Wilson 95% intervals (unpaired)","n":231,"on":"official public accuracy","diff":-0.26406926406926406,"ref_wilson95":[0.47671740174484795,0.6041882445175688],"system_wilson95":[0.7493033792125899,0.8511016798927126],"verdict":"worse"},"correct_per_usd":27069.42694804786,"correct_per_second":0.5144487966347864},{"id":"jqv","display":"jqv (Qwen3-32B zero-shot)","author":"hjmurmur (Octalab)","kind":"open weights","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.3.0 public","comparable":true,"public":{"correct":185,"n":231,"acc":0.8008658008658008,"wilson95":[0.744632174291213,0.847256500996993]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.9583333333333334,"n":96,"correct":92,"wilson95":[0.8977180846850661,0.983679292577073]},"judge":{"acc":0.9246575342465754,"n":146,"correct":135,"wilson95":[0.8701296486178092,0.9574116802358594]},"hard":{"acc":0.6454545454545455,"n":220,"correct":142,"wilson95":[0.5802405155798203,0.7056761341769144]}},"latency":{"p50_s_raw":0.7473905384540558,"p50_s_adjusted":1.6447810769081117,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.05641543071161049,"kind":"estimate"},"per_item":"official","sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L3848","https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.2/jevbench-v1.2-per-task.json#L35364"],"answered":231,"public_tiers":{"easy":{"correct":48,"n":48,"acc":1,"wilson95":[0.9258998692515198,1.0000000000000002]},"original":{"correct":69,"n":72,"acc":0.9583333333333334,"wilson95":[0.8845072317804198,0.9857291926441228]},"hard":{"correct":68,"n":111,"acc":0.6126126126126126,"wilson95":[0.5196688341070039,0.6980225836399687]}},"vs_ref":{"test":"exact McNemar (paired)","n":231,"on":"all 231 public items","diff":-0.2597402597402597,"ci95":[-0.329004329004329,-0.18614718614718617],"only_ref":13,"only_system":73,"p":2.718821367078067e-11,"p_holm":1.060340333160446e-9,"verdict":"worse"},"correct_per_usd":14195.864336474522,"correct_per_second":0.48691331150968886},{"id":"nimble-9b","display":"Bespoke Nimble 9B (Bespoke Labs)","author":"Bespoke Labs","kind":"open weights","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.3.0 public","comparable":true,"public":{"correct":184,"n":231,"acc":0.7965367965367965,"wilson95":[0.7399700975027714,0.843402193968338]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.9479166666666666,"n":96,"correct":91,"wilson95":[0.8838152190201324,0.977550399667867]},"judge":{"acc":0.8904109589041096,"n":146,"correct":130,"wilson95":[0.8293942123918577,0.9314099125219988]},"hard":{"acc":0.6545454545454545,"n":220,"correct":144,"wilson95":[0.589543218600646,0.7142432217658845]}},"latency":{"p50_s_raw":0.3889440931379795,"p50_s_adjusted":0.927888186275959,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.16583014981273408,"kind":"estimate"},"per_item":"official","sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L11918","https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.2/jevbench-v1.2-per-task.json#L7264"],"answered":231,"public_tiers":{"easy":{"correct":48,"n":48,"acc":1,"wilson95":[0.9258998692515198,1.0000000000000002]},"original":{"correct":67,"n":72,"acc":0.9305555555555556,"wilson95":[0.8475202043199935,0.9699746186242738]},"hard":{"correct":69,"n":111,"acc":0.6216216216216216,"wilson95":[0.5287598460478249,0.7063468851189056]}},"vs_ref":{"test":"exact McNemar (paired)","n":231,"on":"all 231 public items","diff":-0.2554112554112554,"ci95":[-0.3246753246753247,-0.18614718614718617],"only_ref":13,"only_system":72,"p":4.628573788074828e-11,"p_holm":1.712572301587686e-9,"verdict":"worse"},"correct_per_usd":4803.329174075379,"correct_per_second":0.8584404978078922},{"id":"metask-jev-4b","display":"metask-jev-4b","author":"Wayfind (metask-ai)","kind":"open weights","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"round 4 unpublished","comparable":true,"public":{"correct":184,"n":231,"acc":0.7965367965367965,"wilson95":[0.7399700975027714,0.843402193968338]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.9791666666666666,"n":96,"correct":94,"wilson95":[0.9271928642112758,0.9942680301991424]},"judge":{"acc":0.8972602739726028,"n":146,"correct":131,"wilson95":[0.8374065192584347,0.9367450464433844]},"hard":{"acc":0.5909090909090909,"n":220,"correct":130,"wilson95":[0.5249208347240336,0.6537770713739256]}},"latency":{"p50_s_raw":0.06571843556594104,"p50_s_adjusted":0.2814368711318821,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.032758801498127314,"kind":"estimate"},"per_item":null,"sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L2508"],"public_tiers":null,"vs_ref":{"test":"Wilson 95% intervals (unpaired)","n":231,"on":"official public accuracy","diff":-0.2554112554112554,"ref_wilson95":[0.47671740174484795,0.6041882445175688],"system_wilson95":[0.7399700975027714,0.843402193968338],"verdict":"worse"},"correct_per_usd":24315.19958330378,"correct_per_second":2.830250326950008},{"id":"reflex-4b","display":"reflex 4B (kshetrajna12)","author":"kshetrajna12","kind":"open weights","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.3.0 public","comparable":true,"public":{"correct":183,"n":231,"acc":0.7922077922077922,"wilson95":[0.7353169366048699,0.8395389710491433]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.9479166666666666,"n":96,"correct":91,"wilson95":[0.8838152190201324,0.977550399667867]},"judge":{"acc":0.9726027397260274,"n":146,"correct":142,"wilson95":[0.9316781374425416,0.9892952769268636]},"hard":{"acc":0.6318181818181818,"n":220,"correct":139,"wilson95":[0.5663317967833068,0.6927801670587339]}},"latency":{"p50_s_raw":1.7996779419481754,"p50_s_adjusted":3.7493558838963503,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.022087303370786515,"kind":"estimate"},"per_item":"official","sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L1907","https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.2/jevbench-v1.2-per-task.json#L53348"],"answered":231,"public_tiers":{"easy":{"correct":48,"n":48,"acc":1,"wilson95":[0.9258998692515198,1.0000000000000002]},"original":{"correct":68,"n":72,"acc":0.9444444444444444,"wilson95":[0.8656798395141685,0.9781857841702364]},"hard":{"correct":67,"n":111,"acc":0.6036036036036037,"wilson95":[0.5106087679601343,0.6896673363670807]}},"vs_ref":{"test":"exact McNemar (paired)","n":231,"on":"all 231 public items","diff":-0.25108225108225113,"ci95":[-0.3246753246753247,-0.1774891774891775],"only_ref":14,"only_system":72,"p":1.445015766748674e-10,"p_holm":5.057555183620359e-9,"verdict":"worse"},"correct_per_usd":35867.11238165885,"correct_per_second":0.21129170362577737},{"id":"spark-s1-4b-v6","display":"spark-s1-4b-v6 (Open Spark Jev, abhishek085)","author":"Abhishek Rai (abhishek085)","kind":"open weights","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"round 8 new (add-requests run 8, 23 Sep 2026)","comparable":true,"public":{"correct":183,"n":231,"acc":0.7922077922077922,"wilson95":[0.7353169366048699,0.8395389710491433]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.9791666666666666,"n":96,"correct":94,"wilson95":[0.9271928642112758,0.9942680301991424]},"judge":{"acc":0.9178082191780822,"n":146,"correct":134,"wilson95":[0.861834617077995,0.952359270987711]},"hard":{"acc":0.6,"n":220,"correct":132,"wilson95":[0.5340833099825639,0.662484386725191]}},"latency":{"p50_s_raw":0.31425347179174423,"p50_s_adjusted":0.7785069435834885,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.025333314606741573,"kind":"estimate"},"per_item":null,"sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L3521"],"public_tiers":null,"vs_ref":{"test":"Wilson 95% intervals (unpaired)","n":231,"on":"official public accuracy","diff":-0.2510822510822511,"ref_wilson95":[0.47671740174484795,0.6041882445175688],"system_wilson95":[0.7353169366048699,0.8395389710491433],"verdict":"worse"},"correct_per_usd":31271.383334771912,"correct_per_second":1.0175988778741505},{"id":"opensourcejev-qwen35-4b-q4km","display":"OpenSourceJev (Qwen3.5-4B Q4_K_M, native llama.cpp)","author":"sabeel111","kind":"not stated","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"round 5 unpublished","comparable":true,"public":{"correct":181,"n":231,"acc":0.7835497835497836,"wilson95":[0.7260365677792575,0.8317865722405626]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"hard":{"acc":0.5636363636363636,"n":220,"correct":124,"wilson95":[0.49756677303098945,0.6275217612375819]},"judge":{"acc":0.8561643835616438,"n":146,"correct":125,"wilson95":[0.7901083360896212,0.9039585848844232]},"standard":{"acc":0.9270833333333334,"n":96,"correct":89,"wilson95":[0.8570696826547382,0.9642324188849823]}},"latency":null,"cost":{"usd_per_1000":0.015903033707865177,"kind":"estimate"},"per_item":null,"sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L4644"],"public_tiers":null,"vs_ref":{"test":"Wilson 95% intervals (unpaired)","n":231,"on":"official public accuracy","diff":-0.24242424242424243,"ref_wilson95":[0.47671740174484795,0.6041882445175688],"system_wilson95":[0.7260365677792575,0.8317865722405626],"verdict":"worse"},"correct_per_usd":49270.45983448194,"correct_per_second":null},{"id":"qwen35-9b-jev-data-mix-v2","display":"Qwen3.5-9B Jev-like data-mix v2","author":"jsaurabh","kind":"not stated","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"round 5 new","comparable":true,"public":{"correct":181,"n":231,"acc":0.7835497835497836,"wilson95":[0.7260365677792575,0.8317865722405626]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.9479166666666666,"n":96,"correct":91,"wilson95":[0.8838152190201324,0.977550399667867]},"judge":{"acc":0.952054794520548,"n":146,"correct":139,"wilson95":[0.9043471165859538,0.9765839754195642]},"hard":{"acc":0.6045454545454545,"n":220,"correct":133,"wilson95":[0.5386730079504642,0.6668295840621885]}},"latency":null,"cost":{"usd_per_1000":0.08344007490636705,"kind":"estimate"},"per_item":null,"sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L6344"],"public_tiers":null,"vs_ref":{"test":"Wilson 95% intervals (unpaired)","n":231,"on":"official public accuracy","diff":-0.24242424242424243,"ref_wilson95":[0.47671740174484795,0.6041882445175688],"system_wilson95":[0.7260365677792575,0.8317865722405626],"verdict":"worse"},"correct_per_usd":9390.56903327388,"correct_per_second":null},{"id":"typecastlm","display":"typecastlm (Mikhail Gribov, Qwen3.5-4B computed head)","author":"Mikhail Gribov","kind":"open weights","class":"system-one-open","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.4.2 addition (official offline measurement on an evaluator-owned Lium pod, 24 Sep 2026, jevbench-add-requests run 12)","comparable":true,"public":{"correct":180,"n":231,"acc":0.7792207792207793,"wilson95":[0.7214089874602162,0.8278977687425074]},"published_tiers":{"easy":{"acc":0.9861111111111112,"n":72,"correct":71,"wilson95":[0.9254339777605758,0.997544048144242]},"standard":{"acc":0.9375,"n":96,"correct":90,"wilson95":[0.8702905342555145,0.9710433258583456]},"judge":{"acc":0.410958904109589,"n":146,"correct":60,"wilson95":[0.3344307242745428,0.4920525454819461]},"hard":{"acc":0.5818181818181818,"n":220,"correct":128,"wilson95":[0.5157807242316306,0.6450473912565327]}},"latency":{"p50_s_raw":0.027890041936188936,"p50_s_adjusted":0.20578008387237787,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.02126808988764045,"kind":"estimate"},"per_item":null,"sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L10193"],"public_tiers":null,"vs_ref":{"test":"Wilson 95% intervals (unpaired)","n":231,"on":"official public accuracy","diff":-0.23809523809523808,"ref_wilson95":[0.47671740174484795,0.6041882445175688],"system_wilson95":[0.7214089874602162,0.8278977687425074],"verdict":"worse"},"correct_per_usd":36638.02359955271,"correct_per_second":3.7866676140730986},{"id":"open-jev-zefan-9b","display":"Open-Jev 9B (Zefan Cai)","author":"Zefan Cai (@Zefan_Cai)","kind":"open weights","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.3.0 public","comparable":true,"public":{"correct":179,"n":231,"acc":0.7748917748917749,"wilson95":[0.7167895769448002,0.8240007954408269]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.90625,"n":96,"correct":87,"wilson95":[0.8313466923598775,0.949891892031564]},"judge":{"acc":0.815068493150685,"n":146,"correct":119,"wilson95":[0.7443051811091901,0.86967709513708]},"hard":{"acc":0.6090909090909091,"n":220,"correct":134,"wilson95":[0.5432683887750728,0.6711690985424782]}},"latency":{"p50_s_raw":0.754859171807766,"p50_s_adjusted":1.6597183436155318,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.24882153558052428,"kind":"estimate"},"per_item":"official","sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L14889","https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.2/jevbench-v1.2-per-task.json#L45480"],"answered":231,"public_tiers":{"easy":{"correct":48,"n":48,"acc":1,"wilson95":[0.9258998692515198,1.0000000000000002]},"original":{"correct":65,"n":72,"acc":0.9027777777777778,"wilson95":[0.8126495411763508,0.9521036802876413]},"hard":{"correct":66,"n":111,"acc":0.5945945945945946,"wilson95":[0.5015792834844461,0.681281507423011]}},"vs_ref":{"test":"exact McNemar (paired)","n":231,"on":"all 231 public items","diff":-0.23376623376623373,"ci95":[-0.30303030303030304,-0.16017316017316016],"only_ref":15,"only_system":69,"p":1.938192405161976e-9,"p_holm":6.422696572991168e-8,"verdict":"worse"},"correct_per_usd":3114.247217725261,"correct_per_second":0.4668814909906641},{"id":"standardone-8b","display":"Standard One 8B (Standard Thinking)","author":"Standard Thinking (myeongho12)","kind":"open weights","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.4.2 addition (official offline measurement on an evaluator-owned Lium pod, 24 Sep 2026, jevbench-add-requests run 12)","comparable":true,"public":{"correct":177,"n":231,"acc":0.7662337662337663,"wilson95":[0.7075746168850627,0.8161829878663716]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.9479166666666666,"n":96,"correct":91,"wilson95":[0.8838152190201324,0.977550399667867]},"judge":{"acc":0.9657534246575342,"n":146,"correct":141,"wilson95":[0.9223410969879317,0.9852848765935112]},"hard":{"acc":0.5681818181818182,"n":220,"correct":125,"wilson95":[0.5021120346617607,0.6319113949117088]}},"latency":{"p50_s_raw":0.02383658150210977,"p50_s_adjusted":0.19767316300421953,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.10444269662921347,"kind":"estimate"},"per_item":null,"sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L8692"],"public_tiers":null,"vs_ref":{"test":"Wilson 95% intervals (unpaired)","n":231,"on":"official public accuracy","diff":-0.22510822510822512,"ref_wilson95":[0.47671740174484795,0.6041882445175688],"system_wilson95":[0.7075746168850627,0.8161829878663716],"verdict":"worse"},"correct_per_usd":7336.403510855391,"correct_per_second":3.8762660271562015},{"id":"decision-2b","display":"Decision 2B (FlyMy.AI, v59)","author":"FlyMy.AI (@denti)","kind":"open weights","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"round 8 new (add-requests run 8, 23 Sep 2026)","comparable":true,"public":{"correct":174,"n":231,"acc":0.7532467532467533,"wilson95":[0.6938095940254198,0.804398859274725]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.875,"n":96,"correct":84,"wilson95":[0.7941153179593371,0.9270279907096858]},"judge":{"acc":0.6986301369863014,"n":146,"correct":102,"wilson95":[0.6198923677921786,0.7671834150587309]},"hard":{"acc":0.5818181818181818,"n":220,"correct":128,"wilson95":[0.5157807242316306,0.6450473912565327]}},"latency":{"p50_s_raw":0.1890188939869404,"p50_s_adjusted":0.5280377879738808,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.017769999999999998,"kind":"estimate"},"per_item":null,"sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L6141"],"public_tiers":null,"vs_ref":{"test":"Wilson 95% intervals (unpaired)","n":231,"on":"official public accuracy","diff":-0.21212121212121213,"ref_wilson95":[0.47671740174484795,0.6041882445175688],"system_wilson95":[0.6938095940254198,0.804398859274725],"verdict":"worse"},"correct_per_usd":42388.67491540537,"correct_per_second":1.426501607275145},{"id":"jev-qwen3.5-9b-base-nvfp4","display":"JEV Qwen3.5-9B Base NVFP4","author":"WilfLin","kind":"open weights","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"round 4 unpublished","comparable":true,"public":{"correct":174,"n":231,"acc":0.7532467532467533,"wilson95":[0.6938095940254198,0.804398859274725]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.9166666666666666,"n":96,"correct":88,"wilson95":[0.84410096689647,0.957169376069111]},"judge":{"acc":0.8972602739726028,"n":146,"correct":131,"wilson95":[0.8374065192584347,0.9367450464433844]},"hard":{"acc":0.5863636363636363,"n":220,"correct":129,"wilson95":[0.5203479977619418,0.6494150130311191]}},"latency":{"p50_s_raw":0.019635691001894884,"p50_s_adjusted":0.18927138200378976,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.07745842696629213,"kind":"estimate"},"per_item":null,"sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L5535"],"public_tiers":null,"vs_ref":{"test":"Wilson 95% intervals (unpaired)","n":231,"on":"official public accuracy","diff":-0.21212121212121213,"ref_wilson95":[0.47671740174484795,0.6041882445175688],"system_wilson95":[0.6938095940254198,0.804398859274725],"verdict":"worse"},"correct_per_usd":9724.529437895071,"correct_per_second":3.979718144773049},{"id":"jev-local","display":"jev-local (Qwen3.5-9B)","author":"us (GitHub)","kind":"open weights","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.3.0 public","comparable":true,"public":{"correct":173,"n":231,"acc":0.7489177489177489,"wilson95":[0.6892360157639211,0.8004560537191269]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.84375,"n":96,"correct":81,"wilson95":[0.7580851324863729,0.9029629004602312]},"judge":{"acc":0.8904109589041096,"n":146,"correct":130,"wilson95":[0.8293942123918577,0.9314099125219988]},"hard":{"acc":0.5909090909090909,"n":220,"correct":130,"wilson95":[0.5249208347240336,0.6537770713739256]}},"latency":{"p50_s_raw":1.0450106970965862,"p50_s_adjusted":2.2400213941931724,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.07745842696629213,"kind":"estimate"},"per_item":"official","sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L7585","https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.2/jevbench-v1.2-per-task.json#L34240"],"answered":231,"public_tiers":{"easy":{"correct":48,"n":48,"acc":1,"wilson95":[0.9258998692515198,1.0000000000000002]},"original":{"correct":60,"n":72,"acc":0.8333333333333334,"wilson95":[0.7308929297761552,0.9020062879871484]},"hard":{"correct":65,"n":111,"acc":0.5855855855855856,"wilson95":[0.492580053130941,0.6728654243567582]}},"vs_ref":{"test":"exact McNemar (paired)","n":231,"on":"all 231 public items","diff":-0.20779220779220775,"ci95":[-0.2813852813852814,-0.1341991341991342],"only_ref":19,"only_system":67,"p":1.9409574271727468e-7,"p_holm":0.000006016968024235515,"verdict":"worse"},"correct_per_usd":9668.641337677283,"correct_per_second":0.33433508753942043},{"id":"malkuth-4b","display":"Malkuth-4B (newfull5, Kev post-train)","author":"newfull5 (dhtocks)","kind":"open weights","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.4.2 addition (official offline measurement on an evaluator-owned Lium pod, 24 Sep 2026, jevbench-add-requests run 12)","comparable":true,"public":{"correct":173,"n":231,"acc":0.7489177489177489,"wilson95":[0.6892360157639211,0.8004560537191269]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.9583333333333334,"n":96,"correct":92,"wilson95":[0.8977180846850661,0.983679292577073]},"judge":{"acc":0.910958904109589,"n":146,"correct":133,"wilson95":[0.8536207185523637,0.94722572872538]},"hard":{"acc":0.5272727272727272,"n":220,"correct":116,"wilson95":[0.46139953769447273,0.592209834134915]}},"latency":{"p50_s_raw":0.08862553583458066,"p50_s_adjusted":0.32725107166916134,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.019154662921348316,"kind":"estimate"},"per_item":null,"sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L3724"],"public_tiers":null,"vs_ref":{"test":"Wilson 95% intervals (unpaired)","n":231,"on":"official public accuracy","diff":-0.2077922077922078,"ref_wilson95":[0.47671740174484795,0.6041882445175688],"system_wilson95":[0.6892360157639211,0.8004560537191269],"verdict":"worse"},"correct_per_usd":39098.45618233577,"correct_per_second":2.2885112189178005},{"id":"open-alternative-jev","display":"open-alternative-jev (Qwen3.5-4B, IkerMoel)","author":"IkerMoel","kind":"open weights","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.3.0 public","comparable":true,"public":{"correct":171,"n":231,"acc":0.7402597402597403,"wilson95":[0.6801102535546354,0.7925490482942198]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.84375,"n":96,"correct":81,"wilson95":[0.7580851324863729,0.9029629004602312]},"judge":{"acc":0.7465753424657534,"n":146,"correct":109,"wilson95":[0.6703220930074081,0.8101857753592381]},"hard":{"acc":0.5681818181818182,"n":220,"correct":125,"wilson95":[0.5021120346617607,0.6319113949117088]}},"latency":{"p50_s_raw":0.20686038956046104,"p50_s_adjusted":0.5637207791209221,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.022171404494382024,"kind":"estimate"},"per_item":"official","sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L7376","https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.2/jevbench-v1.2-per-task.json#L17380"],"answered":231,"public_tiers":{"easy":{"correct":48,"n":48,"acc":1,"wilson95":[0.9258998692515198,1.0000000000000002]},"original":{"correct":60,"n":72,"acc":0.8333333333333334,"wilson95":[0.7308929297761552,0.9020062879871484]},"hard":{"correct":63,"n":111,"acc":0.5675675675675675,"wilson95":[0.47467121716161054,0.6559436334865731]}},"vs_ref":{"test":"exact McNemar (paired)","n":231,"on":"all 231 public items","diff":-0.19913419913419916,"ci95":[-0.2727272727272727,-0.12554112554112554],"only_ref":18,"only_system":64,"p":3.3155201688474993e-7,"p_holm":0.0000099465605065425,"verdict":"worse"},"correct_per_usd":33388.040006545976,"correct_per_second":1.3131673829978676},{"id":"system-one-open","display":"system-one-open (Gemma 4 E2B LoRA on an L4)","author":"mithalouni","kind":"open weights","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.3.0 public","comparable":true,"public":{"correct":169,"n":231,"acc":0.7316017316017316,"wilson95":[0.671012256486486,0.7846142777281762]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.9375,"n":96,"correct":90,"wilson95":[0.8702905342555145,0.9710433258583456]},"judge":{"acc":0.8767123287671232,"n":146,"correct":128,"wilson95":[0.8135354700073181,0.9205737733306134]},"hard":{"acc":0.4909090909090909,"n":220,"correct":108,"wilson95":[0.42557476076210576,0.5565554486280984]}},"latency":{"p50_s_raw":0.6517308317124844,"p50_s_adjusted":1.3034616634249687,"basis":"x2 (assumption, not measured)"},"cost":{"usd_per_1000":0.014880936329588014,"kind":"estimate"},"per_item":"official","sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L3313","https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.2/jevbench-v1.2-per-task.json#L6140"],"answered":231,"public_tiers":{"easy":{"correct":48,"n":48,"acc":1,"wilson95":[0.9258998692515198,1.0000000000000002]},"original":{"correct":67,"n":72,"acc":0.9305555555555556,"wilson95":[0.8475202043199935,0.9699746186242738]},"hard":{"correct":54,"n":111,"acc":0.4864864864864865,"wilson95":[0.3955239529768768,0.5783530768934865]}},"vs_ref":{"test":"exact McNemar (paired)","n":231,"on":"all 231 public items","diff":-0.1904761904761905,"ci95":[-0.2597402597402597,-0.11688311688311688],"only_ref":18,"only_system":62,"p":8.142666624995286e-7,"p_holm":0.00002361373321248633,"verdict":"worse"},"correct_per_usd":49163.68939413279,"correct_per_second":0.561275986958741},{"id":"qwen3.8-27b","display":"Qwen3.8 27B (Chutes TEE)","author":"Qwen / Chutes","kind":"open weights","class":"llm-baseline","listing":"partial","ranked":false,"partial":true,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.3.0 public","comparable":true,"public":{"correct":166,"n":231,"acc":0.7186147186147186,"wilson95":[0.6574154923659918,0.7726618903973808]},"published_tiers":{"easy":{"acc":0.9861111111111112,"n":72,"correct":71,"wilson95":[0.9254339777605758,0.997544048144242]},"standard":{"acc":0.9895833333333334,"n":96,"correct":95,"wilson95":[0.9433338231810272,0.9981588298035302]},"judge":{"acc":0.952755905511811,"n":146,"correct":null,"wilson95":null},"hard":{"acc":0.2136363636363636,"n":220,"correct":47,"wilson95":[0.16463219707727686,0.27246939871415204]}},"latency":{"p50_s_raw":5.753906108438969,"p50_s_adjusted":5.753906108438969,"basis":"none (production API)"},"cost":{"usd_per_1000":2.669088141927965,"kind":"estimate"},"per_item":"official","sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L17991","https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.2/jevbench-v1.2-per-task.json#L15132"],"answered":171,"public_tiers":{"easy":{"correct":48,"n":48,"acc":1,"wilson95":[0.9258998692515198,1.0000000000000002]},"original":{"correct":71,"n":72,"acc":0.9861111111111112,"wilson95":[0.9254339777605758,0.997544048144242]},"hard":{"correct":47,"n":111,"acc":0.42342342342342343,"wilson95":[0.3355806201027102,0.5163892158293484]}},"vs_ref":{"test":"exact McNemar (paired)","n":171,"on":"the 171 items it answered","diff":-0.3508771929824561,"ci95":[-0.4269005847953216,-0.27485380116959063],"only_ref":2,"only_system":62,"p":2.2562247209423347e-16,"p_holm":1.1281123604711674e-14,"verdict":"worse"},"correct_per_usd":269.2360388277178,"correct_per_second":0.12489163101927613},{"id":"system-one-sg","display":"system-one (Qwen3-8B, Sean Goedecke)","author":"Sean Goedecke","kind":"open weights","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.3.0 public","comparable":true,"public":{"correct":166,"n":231,"acc":0.7186147186147186,"wilson95":[0.6574154923659918,0.7726618903973808]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.90625,"n":96,"correct":87,"wilson95":[0.8313466923598775,0.949891892031564]},"judge":{"acc":0.9178082191780822,"n":146,"correct":134,"wilson95":[0.861834617077995,0.952359270987711]},"hard":{"acc":0.5,"n":220,"correct":110,"wilson95":[0.4344990143943904,0.5655009856056096]}},"latency":{"p50_s_raw":0.1661309413611889,"p50_s_adjusted":0.4822618827223778,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.08944116853932584,"kind":"estimate"},"per_item":"official","sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L10914","https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.2/jevbench-v1.2-per-task.json#L3892"],"answered":231,"public_tiers":{"easy":{"correct":48,"n":48,"acc":1,"wilson95":[0.9258998692515198,1.0000000000000002]},"original":{"correct":64,"n":72,"acc":0.8888888888888888,"wilson95":[0.7957706167789822,0.942611803944872]},"hard":{"correct":54,"n":111,"acc":0.4864864864864865,"wilson95":[0.3955239529768768,0.5783530768934865]}},"vs_ref":{"test":"exact McNemar (paired)","n":231,"on":"all 231 public items","diff":-0.17748917748917747,"ci95":[-0.24675324675324675,-0.10822510822510822],"only_ref":16,"only_system":57,"p":0.0000015256583158538735,"p_holm":0.00004119277452805459,"verdict":"worse"},"correct_per_usd":8034.496086651142,"correct_per_second":1.4900923012163525},{"id":"kev-8b","display":"kev 8B (research preview)","author":"Jared Palmer","kind":"open weights","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.3.0 public","comparable":true,"public":{"correct":165,"n":231,"acc":0.7142857142857143,"wilson95":[0.6528962498677109,0.7686647490785654]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.9270833333333334,"n":96,"correct":89,"wilson95":[0.8570696826547382,0.9642324188849823]},"judge":{"acc":0.9041095890410958,"n":146,"correct":132,"wilson95":[0.8454801860355489,0.9420188204542322]},"hard":{"acc":0.4727272727272727,"n":220,"correct":104,"wilson95":[0.407790165865085,0.5386004623055273]}},"latency":{"p50_s_raw":0.5904169715940952,"p50_s_adjusted":1.3308339431881904,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.07333117415730338,"kind":"estimate"},"per_item":"official","sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L9795","https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.2/jevbench-v1.2-per-task.json#L39860"],"answered":231,"public_tiers":{"easy":{"correct":48,"n":48,"acc":1,"wilson95":[0.9258998692515198,1.0000000000000002]},"original":{"correct":67,"n":72,"acc":0.9305555555555556,"wilson95":[0.8475202043199935,0.9699746186242738]},"hard":{"correct":50,"n":111,"acc":0.45045045045045046,"wilson95":[0.36109608604955146,0.5431196901417806]}},"vs_ref":{"test":"exact McNemar (paired)","n":231,"on":"all 231 public items","diff":-0.17316017316017318,"ci95":[-0.23809523809523808,-0.1038961038961039],"only_ref":14,"only_system":54,"p":0.000001109725672785106,"p_holm":0.00003107231883798297,"verdict":"worse"},"correct_per_usd":9740.54653418604,"correct_per_second":0.5367203909561756},{"id":"decider-2b","display":"decider-2b (Mapika)","author":"Mapika","kind":"open weights","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.3.0 public","comparable":true,"public":{"correct":164,"n":231,"acc":0.70995670995671,"wilson95":[0.648383365587522,0.7646612495416578]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.8541666666666666,"n":96,"correct":82,"wilson95":[0.7699818489737323,0.9110979425470116]},"judge":{"acc":0.773972602739726,"n":146,"correct":113,"wilson95":[0.699612243206864,0.8342853883116318]},"hard":{"acc":0.4727272727272727,"n":220,"correct":104,"wilson95":[0.407790165865085,0.5386004623055273]}},"latency":{"p50_s_raw":0.26079703494906425,"p50_s_adjusted":0.6715940698981285,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.01996511235955056,"kind":"estimate"},"per_item":"official","sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L7987","https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.2/jevbench-v1.2-per-task.json#L21876"],"answered":231,"public_tiers":{"easy":{"correct":48,"n":48,"acc":1,"wilson95":[0.9258998692515198,1.0000000000000002]},"original":{"correct":61,"n":72,"acc":0.8472222222222222,"wilson95":[0.7467771054131739,0.9124929130902674]},"hard":{"correct":55,"n":111,"acc":0.4954954954954955,"wilson95":[0.4042029090022181,0.5870894342879031]}},"vs_ref":{"test":"exact McNemar (paired)","n":231,"on":"all 231 public items","diff":-0.1688311688311689,"ci95":[-0.24242424242424243,-0.09090909090909093],"only_ref":24,"only_system":63,"p":0.000034799669459257334,"p_holm":0.000835192067022176,"verdict":"worse"},"correct_per_usd":35559.86548791414,"correct_per_second":1.0571217671180455},{"id":"zerank-2","display":"ZeroEntropy zerank-2","author":"ZeroEntropy","kind":"open weights","class":"reranker","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.3.0 public","comparable":true,"public":{"correct":162,"n":231,"acc":0.7012987012987013,"wilson95":[0.6393763423007789,0.7566355051942079]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.7916666666666666,"n":96,"correct":76,"wilson95":[0.6999975286829019,0.8608917113930048]},"judge":{"acc":0.8835616438356164,"n":146,"correct":129,"wilson95":[0.8214385528954438,0.9260181312304504]},"hard":{"acc":0.4727272727272727,"n":220,"correct":104,"wilson95":[0.407790165865085,0.5386004623055273]}},"latency":{"p50_s_raw":0.12663885252550244,"p50_s_adjusted":0.4032777050510049,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.04729792435014604,"kind":"measured"},"per_item":"official","sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L4814","https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.2/jevbench-v1.2-per-task.json#L58968"],"answered":231,"public_tiers":{"easy":{"correct":48,"n":48,"acc":1,"wilson95":[0.9258998692515198,1.0000000000000002]},"original":{"correct":57,"n":72,"acc":0.7916666666666666,"wilson95":[0.6843073648741842,0.8694794506687065]},"hard":{"correct":57,"n":111,"acc":0.5135135135135135,"wilson95":[0.4216469231065134,0.6044760470231232]}},"vs_ref":{"test":"exact McNemar (paired)","n":231,"on":"all 231 public items","diff":-0.1601731601731602,"ci95":[-0.22510822510822512,-0.09090909090909093],"only_ref":17,"only_system":54,"p":0.000012526823964854767,"p_holm":0.00031317059912136925,"verdict":"worse"},"correct_per_usd":14827.261680808537,"correct_per_second":1.7389969554850644},{"id":"malkuth-2b","display":"Malkuth-2B (newfull5, Kev post-train)","author":"newfull5 (dhtocks)","kind":"open weights","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.4.2 addition (official offline measurement on an evaluator-owned Lium pod, 24 Sep 2026, jevbench-add-requests run 12)","comparable":true,"public":{"correct":161,"n":231,"acc":0.696969696969697,"wilson95":[0.6348820470540766,0.7526134166238139]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.8854166666666666,"n":96,"correct":85,"wilson95":[0.8063737868892716,0.9348012803538908]},"judge":{"acc":0.863013698630137,"n":146,"correct":126,"wilson95":[0.7978732891769384,0.9095410725850686]},"hard":{"acc":0.44545454545454544,"n":220,"correct":98,"wilson95":[0.38127383840854256,0.511507417932682]}},"latency":{"p50_s_raw":0.04497810127213597,"p50_s_adjusted":0.2399562025442719,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.019154662921348316,"kind":"estimate"},"per_item":null,"sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L5212"],"public_tiers":null,"vs_ref":{"test":"Wilson 95% intervals (unpaired)","n":231,"on":"official public accuracy","diff":-0.15584415584415584,"ref_wilson95":[0.47671740174484795,0.6041882445175688],"system_wilson95":[0.6348820470540766,0.7526134166238139],"verdict":"worse"},"correct_per_usd":36386.4245396304,"correct_per_second":2.904570457357134},{"id":"raw-qwen3-4b-instruct-2507","display":"Raw Qwen3 4B Instruct 2507 direct logits","author":"Alibaba Qwen / neutral reproduction","kind":"open weights","class":"raw-logit-control","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"round 4 unpublished","comparable":true,"public":{"correct":161,"n":231,"acc":0.696969696969697,"wilson95":[0.6348820470540766,0.7526134166238139]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.8645833333333334,"n":96,"correct":83,"wilson95":[0.7819886142239704,0.9191229358709132]},"judge":{"acc":0.9383561643835616,"n":146,"correct":137,"wilson95":[0.8870029325088038,0.9672332779207892]},"hard":{"acc":0.5181818181818182,"n":220,"correct":114,"wilson95":[0.45241138207483567,0.5833281991447561]}},"latency":{"p50_s_raw":0.08244980592280626,"p50_s_adjusted":0.31489961184561255,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.022073820224719105,"kind":"estimate"},"per_item":null,"sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L4445"],"public_tiers":null,"vs_ref":{"test":"Wilson 95% intervals (unpaired)","n":231,"on":"official public accuracy","diff":-0.15584415584415584,"ref_wilson95":[0.47671740174484795,0.6041882445175688],"system_wilson95":[0.6348820470540766,0.7526134166238139],"verdict":"worse"},"correct_per_usd":31574.493670524866,"correct_per_second":2.213307577245932},{"id":"raw-qwen3-8b","display":"Raw Qwen3 8B direct logits","author":"Alibaba Qwen / neutral reproduction","kind":"open weights","class":"raw-logit-control","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"round 4 unpublished","comparable":true,"public":{"correct":158,"n":231,"acc":0.683982683982684,"wilson95":[0.6214351554380546,0.7405111567885463]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.8645833333333334,"n":96,"correct":83,"wilson95":[0.7819886142239704,0.9191229358709132]},"judge":{"acc":0.9657534246575342,"n":146,"correct":141,"wilson95":[0.9223410969879317,0.9852848765935112]},"hard":{"acc":0.4636363636363636,"n":220,"correct":102,"wilson95":[0.398929907946647,0.5295909296141694]}},"latency":{"p50_s_raw":0.08492440730333328,"p50_s_adjusted":0.3198488146066666,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.0865558988764045,"kind":"estimate"},"per_item":null,"sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L10715"],"public_tiers":null,"vs_ref":{"test":"Wilson 95% intervals (unpaired)","n":231,"on":"official public accuracy","diff":-0.14285714285714285,"ref_wilson95":[0.47671740174484795,0.6041882445175688],"system_wilson95":[0.6214351554380546,0.7405111567885463],"verdict":"worse"},"correct_per_usd":7902.207623761859,"correct_per_second":2.1384562103936835},{"id":"qwen3-reranker-4b","display":"Qwen3-Reranker-4B","author":"Qwen","kind":"open weights","class":"reranker","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.3.0 public","comparable":true,"public":{"correct":157,"n":231,"acc":0.6796536796536796,"wilson95":[0.6169646312686614,0.7364652971408427]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.7916666666666666,"n":96,"correct":76,"wilson95":[0.6999975286829019,0.8608917113930048]},"judge":{"acc":0.8767123287671232,"n":146,"correct":128,"wilson95":[0.8135354700073181,0.9205737733306134]},"hard":{"acc":0.5,"n":220,"correct":110,"wilson95":[0.4344990143943904,0.5655009856056096]}},"latency":{"p50_s_raw":0.1302479188889265,"p50_s_adjusted":0.41049583777785303,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.04953623422454652,"kind":"measured"},"per_item":"official","sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L4047","https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.2/jevbench-v1.2-per-task.json#L51100"],"answered":231,"public_tiers":{"easy":{"correct":48,"n":48,"acc":1,"wilson95":[0.9258998692515198,1.0000000000000002]},"original":{"correct":54,"n":72,"acc":0.75,"wilson95":[0.6390649250187991,0.8356094883036788]},"hard":{"correct":55,"n":111,"acc":0.4954954954954955,"wilson95":[0.4042029090022181,0.5870894342879031]}},"vs_ref":{"test":"exact McNemar (paired)","n":231,"on":"all 231 public items","diff":-0.1385281385281385,"ci95":[-0.21645021645021645,-0.06060606060606061],"only_ref":28,"only_system":60,"p":0.0008464960856166748,"p_holm":0.016083425626716823,"verdict":"worse"},"correct_per_usd":13720.334019998903,"correct_per_second":1.6556895761303334},{"id":"decision-machine-1","display":"decision-machine-1 (milliseconds.ai)","author":"milliseconds.ai (Baptiste Laget)","kind":"hosted API","class":"decision-api","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.3.0 public","comparable":true,"public":{"correct":156,"n":231,"acc":0.6753246753246753,"wilson95":[0.6124998879392748,0.7324136566531331]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.7604166666666666,"n":96,"correct":73,"wilson95":[0.6660761760154771,0.8347177883380109]},"judge":{"acc":0.8972602739726028,"n":146,"correct":131,"wilson95":[0.8374065192584347,0.9367450464433844]},"hard":{"acc":0.4681818181818182,"n":220,"correct":103,"wilson95":[0.40335736068702654,0.5340983721786878]}},"latency":{"p50_s_raw":0.17223640158772469,"p50_s_adjusted":0.17223640158772469,"basis":"none (production API)"},"cost":{"usd_per_1000":0.035026591760299625,"kind":"measured"},"per_item":"official","sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L5013","https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.2/jevbench-v1.2-per-task.json#L24124"],"answered":231,"public_tiers":{"easy":{"correct":48,"n":48,"acc":1,"wilson95":[0.9258998692515198,1.0000000000000002]},"original":{"correct":54,"n":72,"acc":0.75,"wilson95":[0.6390649250187991,0.8356094883036788]},"hard":{"correct":54,"n":111,"acc":0.4864864864864865,"wilson95":[0.3955239529768768,0.5783530768934865]}},"vs_ref":{"test":"exact McNemar (paired)","n":231,"on":"all 231 public items","diff":-0.1341991341991342,"ci95":[-0.20346320346320343,-0.06060606060606061],"only_ref":22,"only_system":53,"p":0.0004496479716438124,"p_holm":0.008992959432876249,"verdict":"worse"},"correct_per_usd":19280.342202466647,"correct_per_second":3.9209172340999823},{"id":"kev-0.6b","display":"kev 0.6B (research preview)","author":"Jared Palmer","kind":"open weights","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.3.0 public","comparable":true,"public":{"correct":154,"n":231,"acc":0.6666666666666666,"wilson95":[0.6035875080493527,0.7242932689088621]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.8125,"n":96,"correct":78,"wilson95":[0.7229774566977561,0.8779753005264296]},"judge":{"acc":0.6643835616438356,"n":146,"correct":97,"wilson95":[0.5844447317631057,0.7358938471479918]},"hard":{"acc":0.4,"n":220,"correct":88,"wilson95":[0.3375156132748089,0.4659166900174362]}},"latency":{"p50_s_raw":0.5904003903269768,"p50_s_adjusted":1.3308007806539537,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.0062676217228464426,"kind":"estimate"},"per_item":"official","sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L10516","https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.2/jevbench-v1.2-per-task.json#L37612"],"answered":231,"public_tiers":{"easy":{"correct":48,"n":48,"acc":1,"wilson95":[0.9258998692515198,1.0000000000000002]},"original":{"correct":58,"n":72,"acc":0.8055555555555556,"wilson95":[0.6996724096124901,0.8804852066705383]},"hard":{"correct":48,"n":111,"acc":0.43243243243243246,"wilson95":[0.34405636651342697,0.5253287828383896]}},"vs_ref":{"test":"exact McNemar (paired)","n":231,"on":"all 231 public items","diff":-0.1255411255411255,"ci95":[-0.1948051948051948,-0.05194805194805195],"only_ref":23,"only_system":52,"p":0.001079515506666282,"p_holm":0.017843324652350882,"verdict":"worse"},"correct_per_usd":106366.76815331154,"correct_per_second":0.5009515145753579},{"id":"kev-4b","display":"kev 4B (research preview)","author":"Jared Palmer","kind":"open weights","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.3.0 public","comparable":true,"public":{"correct":153,"n":231,"acc":0.6623376623376623,"wilson95":[0.5991397595967315,0.7202246335443868]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.9166666666666666,"n":96,"correct":88,"wilson95":[0.84410096689647,0.957169376069111]},"judge":{"acc":0.8561643835616438,"n":146,"correct":125,"wilson95":[0.7901083360896212,0.9039585848844232]},"hard":{"acc":0.42272727272727273,"n":220,"correct":93,"wilson95":[0.35932576820929785,0.4887810116074369]}},"latency":{"p50_s_raw":0.5502474755048752,"p50_s_adjusted":1.2504949510097505,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.018802865168539323,"kind":"estimate"},"per_item":"official","sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L5942","https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.2/jevbench-v1.2-per-task.json#L38736"],"answered":231,"public_tiers":{"easy":{"correct":48,"n":48,"acc":1,"wilson95":[0.9258998692515198,1.0000000000000002]},"original":{"correct":64,"n":72,"acc":0.8888888888888888,"wilson95":[0.7957706167789822,0.942611803944872]},"hard":{"correct":41,"n":111,"acc":0.36936936936936937,"wilson95":[0.28536016189006386,0.46211779352344784]}},"vs_ref":{"test":"exact McNemar (paired)","n":231,"on":"all 231 public items","diff":-0.12121212121212122,"ci95":[-0.1948051948051948,-0.047619047619047616],"only_ref":25,"only_system":53,"p":0.00203114037416273,"p_holm":0.030467105612440953,"verdict":"worse"},"correct_per_usd":35225.35828453825,"correct_per_second":0.5296604051082634},{"id":"raw-phi-4-mini","display":"Raw Phi-4 mini direct logits","author":"Microsoft / neutral reproduction","kind":"open weights","class":"raw-logit-control","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"round 4 unpublished","comparable":true,"public":{"correct":152,"n":231,"acc":0.658008658008658,"wilson95":[0.5946975680174993,0.7161504413065227]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.7395833333333334,"n":96,"correct":71,"wilson95":[0.6437894463839564,0.8169410008212528]},"judge":{"acc":0.7808219178082192,"n":146,"correct":114,"wilson95":[0.7069945198799458,0.8402505524265124]},"hard":{"acc":0.5227272727272727,"n":220,"correct":115,"wilson95":[0.45690279150420726,0.5877716850202825]}},"latency":{"p50_s_raw":0.06377661414444447,"p50_s_adjusted":0.27755322828888895,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.047970318352059935,"kind":"estimate"},"per_item":null,"sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L5336"],"public_tiers":null,"vs_ref":{"test":"Wilson 95% intervals (unpaired)","n":231,"on":"official public accuracy","diff":-0.11688311688311688,"ref_wilson95":[0.47671740174484795,0.6041882445175688],"system_wilson95":[0.5946975680174993,0.7161504413065227],"verdict":"not distinguishable"},"correct_per_usd":13716.995855217248,"correct_per_second":2.3707476294376777},{"id":"open-jev-zefan-2b","display":"Open-Jev 2B (Zefan Cai)","author":"Zefan Cai (@Zefan_Cai)","kind":"open weights","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.3.0 public","comparable":true,"public":{"correct":149,"n":231,"acc":0.645021645021645,"wilson95":[0.5814038412466381,0.7038950166260943]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.7916666666666666,"n":96,"correct":76,"wilson95":[0.6999975286829019,0.8608917113930048]},"judge":{"acc":0.8835616438356164,"n":146,"correct":129,"wilson95":[0.8214385528954438,0.9260181312304504]},"hard":{"acc":0.4272727272727273,"n":220,"correct":94,"wilson95":[0.3637044411993868,0.4933372339222459]}},"latency":{"p50_s_raw":0.664745207875967,"p50_s_adjusted":1.479490415751934,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.24882153558052428,"kind":"estimate"},"per_item":"official","sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L15088","https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.2/jevbench-v1.2-per-task.json#L44356"],"answered":231,"public_tiers":{"easy":{"correct":48,"n":48,"acc":1,"wilson95":[0.9258998692515198,1.0000000000000002]},"original":{"correct":55,"n":72,"acc":0.7638888888888888,"wilson95":[0.6540123093347168,0.8470329047278986]},"hard":{"correct":46,"n":111,"acc":0.4144144144144144,"wilson95":[0.3271345756432418,0.507419946869059]}},"vs_ref":{"test":"exact McNemar (paired)","n":231,"on":"all 231 public items","diff":-0.10389610389610392,"ci95":[-0.17316017316017315,-0.03463203463203463],"only_ref":24,"only_system":48,"p":0.006309827313666047,"p_holm":0.08833758239132465,"verdict":"not distinguishable"},"correct_per_usd":2592.3063432461677,"correct_per_second":0.4359755481713075},{"id":"decision-fast","display":"Decision Fast (FlyMy.AI, v53a)","author":"FlyMy.AI (@denti)","kind":"open weights","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"round 8 new (add-requests run 8, 23 Sep 2026)","comparable":true,"public":{"correct":146,"n":231,"acc":0.6320346320346321,"wilson95":[0.5681584346203994,0.6915912718010435]},"published_tiers":{"easy":{"acc":0.9861111111111112,"n":72,"correct":71,"wilson95":[0.9254339777605758,0.997544048144242]},"standard":{"acc":0.78125,"n":96,"correct":75,"wilson95":[0.6886209318433697,0.8522365496583973]},"judge":{"acc":0.7465753424657534,"n":146,"correct":109,"wilson95":[0.6703220930074081,0.8101857753592381]},"hard":{"acc":0.3863636363636363,"n":220,"correct":85,"wilson95":[0.32449710401115284,0.4521305133663983]}},"latency":{"p50_s_raw":0.24040156602859497,"p50_s_adjusted":0.63080313205719,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.00627247191011236,"kind":"estimate"},"per_item":null,"sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L7784"],"public_tiers":null,"vs_ref":{"test":"Wilson 95% intervals (unpaired)","n":231,"on":"official public accuracy","diff":-0.09090909090909093,"ref_wilson95":[0.47671740174484795,0.6041882445175688],"system_wilson95":[0.5681584346203994,0.6915912718010435],"verdict":"not distinguishable"},"correct_per_usd":100763.24630735736,"correct_per_second":1.0019522730862571},{"id":"jeff","display":"jeff (Logan Markewich, GLiFormer 400M)","author":"Logan Markewich","kind":"open weights","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.3.0 public","comparable":true,"public":{"correct":145,"n":231,"acc":0.6277056277056277,"wilson95":[0.5637538429363366,0.6874794796680098]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.7604166666666666,"n":96,"correct":73,"wilson95":[0.6660761760154771,0.8347177883380109]},"judge":{"acc":0.6164383561643836,"n":146,"correct":90,"wilson95":[0.5355396069486553,0.6913668864467055]},"hard":{"acc":0.37727272727272726,"n":220,"correct":83,"wilson95":[0.31584680608399707,0.4429110206837581]}},"latency":{"p50_s_raw":0.937930017709732,"p50_s_adjusted":2.025860035419464,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.006036722846441948,"kind":"estimate"},"per_item":"official","sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L8186","https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.2/jevbench-v1.2-per-task.json#L33116"],"answered":231,"public_tiers":{"easy":{"correct":48,"n":48,"acc":1,"wilson95":[0.9258998692515198,1.0000000000000002]},"original":{"correct":54,"n":72,"acc":0.75,"wilson95":[0.6390649250187991,0.8356094883036788]},"hard":{"correct":43,"n":111,"acc":0.38738738738738737,"wilson95":[0.3019774163600313,0.48033116589299607]}},"vs_ref":{"test":"exact McNemar (paired)","n":231,"on":"all 231 public items","diff":-0.08658008658008653,"ci95":[-0.15584415584415584,-0.017316017316017316],"only_ref":24,"only_system":44,"p":0.02052693371370707,"p_holm":0.24632320456448484,"verdict":"not distinguishable"},"correct_per_usd":103981.19040293495,"correct_per_second":0.30984649320833174},{"id":"jevact","display":"JevAct (einptein, jev1-2b-v2)","author":"einptein","kind":"hosted demo","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.4.2 addition (official full-protocol measurement, jevbench-add-requests run 11, 24 Sep 2026)","comparable":true,"public":{"correct":143,"n":231,"acc":0.6190476190476191,"wilson95":[0.5549602497734083,0.6792403051967453]},"published_tiers":{"easy":{"acc":0.9861111111111112,"n":72,"correct":71,"wilson95":[0.9254339777605758,0.997544048144242]},"standard":{"acc":0.6875,"n":96,"correct":66,"wilson95":[0.5890812001253843,0.7714904542091271]},"judge":{"acc":0.589041095890411,"n":146,"correct":86,"wilson95":[0.5079474545180539,0.6655692757254573]},"hard":{"acc":0.37272727272727274,"n":220,"correct":82,"wilson95":[0.3115303832709914,0.43829254819186586]}},"latency":{"p50_s_raw":0.3992511257529259,"p50_s_adjusted":0.7985022515058517,"basis":"x2 (not a production API; no +0.15 s)"},"cost":{"usd_per_1000":0.015491685393258427,"kind":"estimate"},"per_item":null,"sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L12913"],"public_tiers":null,"vs_ref":{"test":"Wilson 95% intervals (unpaired)","n":231,"on":"official public accuracy","diff":-0.07792207792207792,"ref_wilson95":[0.47671740174484795,0.6041882445175688],"system_wilson95":[0.5549602497734083,0.6792403051967453],"verdict":"not distinguishable"},"correct_per_usd":39959.99165571825,"correct_per_second":0.7752609562217151},{"id":"smalljev","display":"smalljev semantic-v9","author":"Aditya (isHeSatoshi)","kind":"open weights","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.3.0 public","comparable":true,"public":{"correct":140,"n":231,"acc":0.6060606060606061,"wilson95":[0.5418083267934802,0.6668430767253838]},"published_tiers":{"easy":{"acc":0.9722222222222222,"n":72,"correct":70,"wilson95":[0.9042582470003044,0.9923489781643758]},"standard":{"acc":0.6875,"n":96,"correct":66,"wilson95":[0.5890812001253843,0.7714904542091271]},"judge":{"acc":0.4041095890410959,"n":146,"correct":59,"wilson95":[0.32795799687423904,0.4851778320942876]},"hard":{"acc":0.38181818181818183,"n":220,"correct":84,"wilson95":[0.3201690595965808,0.4475236624760725]}},"latency":{"p50_s_raw":0.4138255603611469,"p50_s_adjusted":0.977651120722294,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.025365692883895133,"kind":"estimate"},"per_item":"official","sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L14289","https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.2/jevbench-v1.2-per-task.json#L56720"],"answered":231,"public_tiers":{"easy":{"correct":47,"n":48,"acc":0.9791666666666666,"wilson95":[0.8910078189694375,0.9963128890632688]},"original":{"correct":49,"n":72,"acc":0.6805555555555556,"wilson95":[0.5660765117466556,0.7767438978751339]},"hard":{"correct":44,"n":111,"acc":0.3963963963963964,"wilson95":[0.31033266363291945,0.4893912320398657]}},"vs_ref":{"test":"exact McNemar (paired)","n":231,"on":"all 231 public items","diff":-0.06493506493506496,"ci95":[-0.14285714285714285,0.012987012987012988],"only_ref":35,"only_system":50,"p":0.12839640292640084,"p_holm":1,"verdict":"not distinguishable"},"correct_per_usd":23892.92533165528,"correct_per_second":0.6199150118222595},{"id":"mghafiri-qwen3.5-0.8b-decision-model","display":"Qwen3.5-0.8B Decision Model (Mourad Ghafiri)","author":"Mourad Ghafiri","kind":"open weights","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.4.1 addition prepared from a local offline remeasurement","comparable":true,"public":{"correct":137,"n":231,"acc":0.5930735930735931,"wilson95":[0.5287018324061313,0.6544004196614431]},"published_tiers":{"easy":{"acc":0.9861111111111112,"n":72,"correct":71,"wilson95":[0.9254339777605758,0.997544048144242]},"standard":{"acc":0.5520833333333334,"n":96,"correct":53,"wilson95":[0.4525163852068095,0.6476424076638883]},"judge":{"acc":0.3424657534246575,"n":146,"correct":50,"wilson95":[0.2704170698562312,0.42259179202063385]},"hard":{"acc":0.509090909090909,"n":220,"correct":112,"wilson95":[0.4434445513719016,0.5744252392378941]}},"latency":{"p50_s_raw":7.150214396417141,"p50_s_adjusted":14.450428792834282,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.0064648501872659175,"kind":"estimate"},"per_item":null,"sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L13709"],"public_tiers":null,"vs_ref":{"test":"Wilson 95% intervals (unpaired)","n":231,"on":"official public accuracy","diff":-0.05194805194805195,"ref_wilson95":[0.47671740174484795,0.6041882445175688],"system_wilson95":[0.5287018324061313,0.6544004196614431],"verdict":"not distinguishable"},"correct_per_usd":91738.18045185247,"correct_per_second":0.041041937341519445},{"id":"laya","display":"Laya (Convai Innovations, ModernBERT-large 421M)","author":"Convai Innovations","kind":"open weights","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.3.0 public","comparable":true,"public":{"correct":135,"n":231,"acc":0.5844155844155844,"wilson95":[0.5199890492209611,0.6460804352124203]},"published_tiers":{"easy":{"acc":0.9444444444444444,"n":72,"correct":68,"wilson95":[0.8656798395141685,0.9781857841702364]},"standard":{"acc":0.7291666666666666,"n":96,"correct":70,"wilson95":[0.6327365261230176,0.807962162508052]},"judge":{"acc":0.6917808219178082,"n":146,"correct":101,"wilson95":[0.6127665345809076,0.7609618074820393]},"hard":{"acc":0.3409090909090909,"n":220,"correct":75,"wilson95":[0.281482459903116,0.4057962044254555]}},"latency":{"p50_s_raw":0.787067785859108,"p50_s_adjusted":1.724135571718216,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.0028831273408239703,"kind":"estimate"},"per_item":"official","sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L8385","https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.2/jevbench-v1.2-per-task.json#L40984"],"answered":231,"public_tiers":{"easy":{"correct":46,"n":48,"acc":0.9583333333333334,"wilson95":[0.8602434401996508,0.9884981066142424]},"original":{"correct":50,"n":72,"acc":0.6944444444444444,"wilson95":[0.580457519474586,0.788733690887341]},"hard":{"correct":39,"n":111,"acc":0.35135135135135137,"wilson95":[0.26887011891433643,0.4437772096596597]}},"vs_ref":{"test":"exact McNemar (paired)","n":231,"on":"all 231 public items","diff":-0.04329004329004327,"ci95":[-0.11688311688311688,0.025974025974025976],"only_ref":30,"only_system":40,"p":0.2819789217936561,"p_holm":1,"verdict":"not distinguishable"},"correct_per_usd":202701.96745751923,"correct_per_second":0.3389615027971236},{"id":"lev-350m","display":"lev-350m (Franck Verrot, LFM2.5-350M)","author":"Franck Verrot (franckverrot)","kind":"open weights","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"round 8 new (add-requests run 8, 23 Sep 2026)","comparable":true,"public":{"correct":135,"n":231,"acc":0.5844155844155844,"wilson95":[0.5199890492209611,0.6460804352124203]},"published_tiers":{"easy":{"acc":0.9861111111111112,"n":72,"correct":71,"wilson95":[0.9254339777605758,0.997544048144242]},"standard":{"acc":0.71875,"n":96,"correct":69,"wilson95":[0.6217410378903089,0.798925892166621]},"judge":{"acc":0.6917808219178082,"n":146,"correct":101,"wilson95":[0.6127665345809076,0.7609618074820393]},"hard":{"acc":0.36818181818181817,"n":220,"correct":81,"wilson95":[0.30721983294126604,0.4336682032166933]}},"latency":{"p50_s_raw":0.17216148599982262,"p50_s_adjusted":0.49432297199964526,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.006263501872659176,"kind":"estimate"},"per_item":null,"sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L8816"],"public_tiers":null,"vs_ref":{"test":"Wilson 95% intervals (unpaired)","n":231,"on":"official public accuracy","diff":-0.04329004329004329,"ref_wilson95":[0.47671740174484795,0.6041882445175688],"system_wilson95":[0.5199890492209611,0.6460804352124203],"verdict":"not distinguishable"},"correct_per_usd":93304.92690784016,"correct_per_second":1.1822545532357007},{"id":"gliner2","display":"GLiNER2 (Fastino, gliner2.5-base)","author":"Fastino","kind":"open weights","class":"classifier","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.3.0 public","comparable":true,"public":{"correct":134,"n":231,"acc":0.5800865800865801,"wilson95":[0.5156400501312037,0.6419130504850814]},"published_tiers":{"easy":{"acc":0.9722222222222222,"n":72,"correct":70,"wilson95":[0.9042582470003044,0.9923489781643758]},"standard":{"acc":0.6666666666666666,"n":96,"correct":64,"wilson95":[0.5675650607292554,0.752943076456977]},"judge":{"acc":0.4589041095890411,"n":146,"correct":67,"wilson95":[0.3801640794698866,0.5397512758023392]},"hard":{"acc":0.3636363636363637,"n":220,"correct":80,"wilson95":[0.3029151989271971,0.4290379419258643]}},"latency":{"p50_s_raw":0.31302378326654434,"p50_s_adjusted":0.7760475665330887,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.00367125468164794,"kind":"estimate"},"per_item":"official","sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L14488","https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.2/jevbench-v1.2-per-task.json#L30868"],"answered":231,"public_tiers":{"easy":{"correct":47,"n":48,"acc":0.9791666666666666,"wilson95":[0.8910078189694375,0.9963128890632688]},"original":{"correct":46,"n":72,"acc":0.6388888888888888,"wilson95":[0.5235248037538494,0.7401832036475271]},"hard":{"correct":41,"n":111,"acc":0.36936936936936937,"wilson95":[0.28536016189006386,0.46211779352344784]}},"vs_ref":{"test":"exact McNemar (paired)","n":231,"on":"all 231 public items","diff":-0.038961038961038974,"ci95":[-0.11688311688311688,0.03896103896103896],"only_ref":37,"only_system":46,"p":0.379999396411918,"p_holm":1,"verdict":"not distinguishable"},"correct_per_usd":158007.71953696027,"correct_per_second":0.7474884338315191},{"id":"openjev-verdict-1.4","display":"openJev Verdict 1.4","author":"Hemant (heman10x)","kind":"open weights","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.3.0 public","comparable":true,"public":{"correct":133,"n":231,"acc":0.5757575757575758,"wilson95":[0.5112959506779503,0.6377407661212383]},"published_tiers":{"easy":{"acc":0.8611111111111112,"n":72,"correct":62,"wilson95":[0.7628681265904589,0.92277269265312]},"standard":{"acc":0.6770833333333334,"n":96,"correct":65,"wilson95":[0.5782983091847127,0.7622415865756593]},"judge":{"acc":0.5616438356164384,"n":146,"correct":82,"wilson95":[0.48060101335331307,0.6395259537383485]},"hard":{"acc":0.37727272727272726,"n":220,"correct":83,"wilson95":[0.31584680608399707,0.4429110206837581]}},"latency":{"p50_s_raw":0.31351133808493614,"p50_s_adjusted":0.7770226761698723,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.003872921348314607,"kind":"estimate"},"per_item":"official","sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L11520","https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.2/jevbench-v1.2-per-task.json#L48852"],"answered":231,"public_tiers":{"easy":{"correct":42,"n":48,"acc":0.875,"wilson95":[0.7529954159822886,0.9414294859563512]},"original":{"correct":50,"n":72,"acc":0.6944444444444444,"wilson95":[0.580457519474586,0.788733690887341]},"hard":{"correct":41,"n":111,"acc":0.36936936936936937,"wilson95":[0.28536016189006386,0.46211779352344784]}},"vs_ref":{"test":"exact McNemar (paired)","n":231,"on":"all 231 public items","diff":-0.03463203463203468,"ci95":[-0.1038961038961039,0.03463203463203463],"only_ref":31,"only_system":39,"p":0.40296307504607976,"p_holm":1,"verdict":"not distinguishable"},"correct_per_usd":148662.34657931543,"correct_per_second":0.7409791160737038},{"id":"von-395m","display":"Von (wfzyx, Option-Marker 395M)","author":"wfzyx (Victor Hugo)","kind":"not stated","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"round 5 new","comparable":true,"public":{"correct":132,"n":231,"acc":0.5714285714285714,"wilson95":[0.5069567309136671,0.633563602068425]},"published_tiers":{"easy":{"acc":0.9305555555555556,"n":72,"correct":67,"wilson95":[0.8475202043199935,0.9699746186242738]},"standard":{"acc":0.6875,"n":96,"correct":66,"wilson95":[0.5890812001253843,0.7714904542091271]},"judge":{"acc":0.684931506849315,"n":146,"correct":100,"wilson95":[0.605659164828241,0.7547217364467436]},"hard":{"acc":0.37272727272727274,"n":220,"correct":82,"wilson95":[0.3115303832709914,0.43829254819186586]}},"latency":null,"cost":{"usd_per_1000":0.005513183520599252,"kind":"estimate"},"per_item":null,"sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L9227"],"public_tiers":null,"vs_ref":{"test":"Wilson 95% intervals (unpaired)","n":231,"on":"official public accuracy","diff":-0.030303030303030304,"ref_wilson95":[0.47671740174484795,0.6041882445175688],"system_wilson95":[0.5069567309136671,0.633563602068425],"verdict":"not distinguishable"},"correct_per_usd":103647.66006673044,"correct_per_second":null},{"id":"gliner2-large","display":"GLiNER2 large (Fastino)","author":"Fastino","kind":"open weights","class":"classifier","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.3.0 public","comparable":true,"public":{"correct":131,"n":231,"acc":0.5670995670995671,"wilson95":[0.5026223721229895,0.629381577042006]},"published_tiers":{"easy":{"acc":0.9861111111111112,"n":72,"correct":71,"wilson95":[0.9254339777605758,0.997544048144242]},"standard":{"acc":0.625,"n":96,"correct":60,"wilson95":[0.5251071836230403,0.715273919266634]},"judge":{"acc":0.6095890410958904,"n":146,"correct":89,"wilson95":[0.5286181102403185,0.6849409423670798]},"hard":{"acc":0.3636363636363637,"n":220,"correct":80,"wilson95":[0.3029151989271971,0.4290379419258643]}},"latency":{"p50_s_raw":1.0967330671846869,"p50_s_adjusted":2.3434661343693732,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.007745842696629214,"kind":"estimate"},"per_item":"official","sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L13311","https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.2/jevbench-v1.2-per-task.json#L27496"],"answered":231,"public_tiers":{"easy":{"correct":48,"n":48,"acc":1,"wilson95":[0.9258998692515198,1.0000000000000002]},"original":{"correct":42,"n":72,"acc":0.5833333333333334,"wilson95":[0.4680769784187672,0.6901478260220589]},"hard":{"correct":41,"n":111,"acc":0.36936936936936937,"wilson95":[0.28536016189006386,0.46211779352344784]}},"vs_ref":{"test":"exact McNemar (paired)","n":231,"on":"all 231 public items","diff":-0.025974025974025983,"ci95":[-0.09956709956709955,0.047619047619047616],"only_ref":36,"only_system":42,"p":0.5715867180030725,"p_holm":1,"verdict":"not distinguishable"},"correct_per_usd":73213.41128530198,"correct_per_second":0.24199179104082663},{"id":"openjev-verdict","display":"openJev Verdict (heman10x, ModernBERT-base 151M)","author":"Hemant (heman10x)","kind":"open weights","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.3.0 public","comparable":true,"public":{"correct":128,"n":231,"acc":0.5541125541125541,"wilson95":[0.48964829246573255,0.6168065052479734]},"published_tiers":{"easy":{"acc":0.8611111111111112,"n":72,"correct":62,"wilson95":[0.7628681265904589,0.92277269265312]},"standard":{"acc":0.65625,"n":96,"correct":63,"wilson95":[0.556880310381901,0.7435960682301918]},"judge":{"acc":0.6095890410958904,"n":146,"correct":89,"wilson95":[0.5286181102403185,0.6849409423670798]},"hard":{"acc":0.38181818181818183,"n":220,"correct":84,"wilson95":[0.3201690595965808,0.4475236624760725]}},"latency":{"p50_s_raw":0.27805980294942856,"p50_s_adjusted":0.7061196058988571,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.00367125468164794,"kind":"estimate"},"per_item":"official","sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L12316","https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.2/jevbench-v1.2-per-task.json#L49976"],"answered":231,"public_tiers":{"easy":{"correct":41,"n":48,"acc":0.8541666666666666,"wilson95":[0.728328233591317,0.9275175071285092]},"original":{"correct":45,"n":72,"acc":0.625,"wilson95":[0.5095289893544211,0.7278082173068178]},"hard":{"correct":42,"n":111,"acc":0.3783783783783784,"wilson95":[0.29365311488109447,0.4712401539521751]}},"vs_ref":{"test":"exact McNemar (paired)","n":231,"on":"all 231 public items","diff":-0.012987012987012991,"ci95":[-0.08658008658008658,0.06060606060606061],"only_ref":36,"only_system":39,"p":0.8175538505509844,"p_holm":1,"verdict":"not distinguishable"},"correct_per_usd":150932.74702037996,"correct_per_second":0.784729031007707},{"id":"simplejev-qwen3.5-0.8b","display":"SimpleJev (Qwen3.5-0.8B, CPU)","author":"sabeel111 / Featherless AI","kind":"open weights","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"round 4 unpublished","comparable":true,"public":{"correct":126,"n":231,"acc":0.5454545454545454,"wilson95":[0.4810229213190247,0.6083991087604883]},"published_tiers":{"easy":{"acc":0.875,"n":72,"correct":63,"wilson95":[0.7791894618635244,0.932822158120192]},"standard":{"acc":0.5729166666666666,"n":96,"correct":55,"wilson95":[0.4730431404647973,0.6671791695541792]},"judge":{"acc":0.1917808219178082,"n":146,"correct":28,"wilson95":[0.13615172870387948,0.26321343583781287]},"hard":{"acc":0.4,"n":220,"correct":88,"wilson95":[0.3375156132748089,0.4659166900174362]}},"latency":{"p50_s_raw":3.993786856532097,"p50_s_adjusted":8.137573713064194,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.011435224719101123,"kind":"estimate"},"per_item":null,"sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L15610"],"public_tiers":null,"vs_ref":{"test":"Wilson 95% intervals (unpaired)","n":231,"on":"official public accuracy","diff":-0.004329004329004329,"ref_wilson95":[0.47671740174484795,0.6041882445175688],"system_wilson95":[0.4810229213190247,0.6083991087604883],"verdict":"not distinguishable"},"correct_per_usd":47699.50384476758,"correct_per_second":0.06702913726960946},{"id":"raw-qwen3-1.7b","display":"Raw Qwen3 1.7B direct logits","author":"Alibaba Qwen / neutral reproduction","kind":"open weights","class":"raw-logit-control","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"round 4 unpublished","comparable":true,"public":{"correct":125,"n":231,"acc":0.5411255411255411,"wilson95":[0.47671740174484795,0.6041882445175688]},"published_tiers":{"easy":{"acc":0.8194444444444444,"n":72,"correct":59,"wilson95":[0.7151965816839501,0.8913318353392159]},"standard":{"acc":0.5520833333333334,"n":96,"correct":53,"wilson95":[0.4525163852068095,0.6476424076638883]},"judge":{"acc":0.7534246575342466,"n":146,"correct":110,"wilson95":[0.6776101537730574,0.8162451553815511]},"hard":{"acc":0.43636363636363634,"n":220,"correct":96,"wilson95":[0.37247823876241815,0.5024332269690105]}},"latency":{"p50_s_raw":0.06664170743897557,"p50_s_adjusted":0.28328341487795117,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.014795880149812733,"kind":"estimate"},"per_item":null,"sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L12515"],"public_tiers":null,"vs_ref":{"test":"Wilson 95% intervals (unpaired)","n":231,"on":"official public accuracy","diff":0,"ref_wilson95":[0.47671740174484795,0.6041882445175688],"system_wilson95":[0.47671740174484795,0.6041882445175688],"verdict":"not distinguishable"},"correct_per_usd":36572.71724604974,"correct_per_second":1.9101913938684965},{"id":"tia-1.5","display":"TIA 1.5","author":"TIA","kind":"local evaluation","ours":true,"listing":"not on the official board","measured_by":"TIA","measured_on":"2026-10-03","public":{"correct":125,"n":231,"acc":0.5411255411255411,"wilson95":[0.47671740174484795,0.6041882445175688]},"answered":231,"published_tiers":null,"latency":{"p50_s_raw":0.0322,"p50_s_adjusted":0.2144,"basis":"TIA measurement through the unmodified harness, local CPU worker over loopback; JevBench's self-host rule applied (x2 + 0.15 s). Not measured by JevBench."},"cost":{"usd_per_1000":0.010945637695637696,"kind":"estimated list price","basis":"Hypothetical API price, not a deployed-model tariff: 500 VND per 1M input tokens x 569 measured input tokens per decision, at 26,000 VND/USD"},"per_item":"TIA run","sources":[],"comparable":true,"public_tiers":{"easy":{"correct":44,"n":48,"acc":0.9166666666666666,"wilson95":[0.8044669809657894,0.967116243410477]},"original":{"correct":44,"n":72,"acc":0.6111111111111112,"wilson95":[0.49562354903304096,0.7153428568880604]},"hard":{"correct":37,"n":111,"acc":0.3333333333333333,"wilson95":[0.2525113744596184,0.425305327274862]}},"vs_ref":null,"correct_per_usd":49437.55276508035,"correct_per_second":2.52390644181689},{"id":"verdict-small","display":"verdict-small (Manavarya09, multilingual-e5-small 118M)","author":"Manavarya09 (Manav)","kind":"open weights","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.4.2 addition (official full-protocol measurement, jevbench-add-requests run 11, 24 Sep 2026)","comparable":true,"public":{"correct":124,"n":231,"acc":0.5367965367965368,"wilson95":[0.4724166449651627,0.5999726174801575]},"published_tiers":{"easy":{"acc":0.8888888888888888,"n":72,"correct":64,"wilson95":[0.7957706167789822,0.942611803944872]},"standard":{"acc":0.5104166666666666,"n":96,"correct":49,"wilson95":[0.4119603701490724,0.608071388425067]},"judge":{"acc":0.2808219178082192,"n":146,"correct":41,"wilson95":[0.2142660834504186,0.3586158113347849]},"hard":{"acc":0.4,"n":220,"correct":88,"wilson95":[0.3375156132748089,0.4659166900174362]}},"latency":{"p50_s_raw":0.0534759983420372,"p50_s_adjusted":0.2569519966840744,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.0012943970037453184,"kind":"estimate"},"per_item":null,"sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L16207"],"public_tiers":null,"vs_ref":{"test":"Wilson 95% intervals (unpaired)","n":231,"on":"official public accuracy","diff":0.004329004329004329,"ref_wilson95":[0.47671740174484795,0.6041882445175688],"system_wilson95":[0.4724166449651627,0.5999726174801575],"verdict":"not distinguishable"},"correct_per_usd":414707.80235377874,"correct_per_second":2.0890926854969516},{"id":"opendecision","display":"OpenDecision (ModernBERT-large zero-shot)","author":"Deepan Wadhwa","kind":"open weights","class":"classifier","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.3.0 public","comparable":true,"public":{"correct":123,"n":231,"acc":0.5324675324675324,"wilson95":[0.4681206414496763,0.5957522371785474]},"published_tiers":{"easy":{"acc":0.875,"n":72,"correct":63,"wilson95":[0.7791894618635244,0.932822158120192]},"standard":{"acc":0.625,"n":96,"correct":60,"wilson95":[0.5251071836230403,0.715273919266634]},"judge":{"acc":0.7123287671232876,"n":146,"correct":104,"wilson95":[0.6342008168062436,0.7795698476205906]},"hard":{"acc":0.33181818181818185,"n":220,"correct":73,"wilson95":[0.2729525626942851,0.3964563110244907]}},"latency":{"p50_s_raw":0.3378134034574032,"p50_s_adjusted":0.8256268069148064,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.006635318352059925,"kind":"estimate"},"per_item":"official","sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L11122","https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.2/jevbench-v1.2-per-task.json#L46604"],"answered":231,"public_tiers":{"easy":{"correct":42,"n":48,"acc":0.875,"wilson95":[0.7529954159822886,0.9414294859563512]},"original":{"correct":43,"n":72,"acc":0.5972222222222222,"wilson95":[0.4818067159467256,0.702788889234238]},"hard":{"correct":38,"n":111,"acc":0.34234234234234234,"wilson95":[0.2606740491353907,0.4345579660188475]}},"vs_ref":{"test":"exact McNemar (paired)","n":231,"on":"all 231 public items","diff":0.008658008658008698,"ci95":[-0.06060606060606061,0.08225108225108226],"only_ref":38,"only_system":36,"p":0.9075606176124988,"p_holm":1,"verdict":"not distinguishable"},"correct_per_usd":80247.47332616357,"correct_per_second":0.644925198658764},{"id":"open-jev-deberta-v3-large","display":"open-jev-deberta-v3-large (local CPU)","author":"Kotoba Labs","kind":"open weights","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.3.0 public","comparable":true,"public":{"correct":121,"n":231,"acc":0.5238095238095238,"wilson95":[0.45954286150850665,0.5872972494855242]},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.4895833333333333,"n":96,"correct":47,"wilson95":[0.39192861157493286,0.5880396298509275]},"judge":{"acc":0.5342465753424658,"n":146,"correct":78,"wilson95":[0.45349427020007393,0.613242933739738]},"hard":{"acc":0.3636363636363637,"n":220,"correct":80,"wilson95":[0.3029151989271971,0.4290379419258643]}},"latency":{"p50_s_raw":1.767673410475254,"p50_s_adjusted":3.685346820950508,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.007340468164794008,"kind":"estimate"},"per_item":"official","sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L14081","https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.2/jevbench-v1.2-per-task.json#L11760"],"answered":231,"public_tiers":{"easy":{"correct":48,"n":48,"acc":1,"wilson95":[0.9258998692515198,1.0000000000000002]},"original":{"correct":31,"n":72,"acc":0.4305555555555556,"wilson95":[0.32257905708400253,0.5455669392153093]},"hard":{"correct":42,"n":111,"acc":0.3783783783783784,"wilson95":[0.29365311488109447,0.4712401539521751]}},"vs_ref":{"test":"exact McNemar (paired)","n":231,"on":"all 231 public items","diff":0.017316017316017285,"ci95":[-0.05627705627705628,0.09090909090909093],"only_ref":38,"only_system":34,"p":0.723948145746914,"p_holm":1,"verdict":"not distinguishable"},"correct_per_usd":71359.14386520922,"correct_per_second":0.14213303367589844},{"id":"kev-0.5b","display":"kev 0.5B","author":"Jared Palmer","kind":"open weights","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.3.0 public","comparable":true,"public":{"correct":114,"n":231,"acc":0.4935064935064935,"wilson95":[0.42966946517263654,0.5575559591017185]},"published_tiers":{"easy":{"acc":0.9583333333333334,"n":72,"correct":69,"wilson95":[0.8845072317804198,0.9857291926441228]},"standard":{"acc":0.5208333333333334,"n":96,"correct":50,"wilson95":[0.422037676312656,0.6180258408356231]},"judge":{"acc":0.7123287671232876,"n":146,"correct":104,"wilson95":[0.6342008168062436,0.7795698476205906]},"hard":{"acc":0.3090909090909091,"n":220,"correct":68,"wilson95":[0.25174014409962653,0.3729942530946594]}},"latency":{"p50_s_raw":0.43036095052957535,"p50_s_adjusted":1.0107219010591506,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.0062676404494382025,"kind":"estimate"},"per_item":"official","sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L11719","https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.2/jevbench-v1.2-per-task.json#L36488"],"answered":231,"public_tiers":{"easy":{"correct":46,"n":48,"acc":0.9583333333333334,"wilson95":[0.8602434401996508,0.9884981066142424]},"original":{"correct":35,"n":72,"acc":0.4861111111111111,"wilson95":[0.374326744458874,0.5993024548009882]},"hard":{"correct":33,"n":111,"acc":0.2972972972972973,"wilson95":[0.22020824409746803,0.3879472039579811]}},"vs_ref":{"test":"exact McNemar (paired)","n":231,"on":"all 231 public items","diff":0.047619047619047616,"ci95":[-0.025974025974025976,0.12121212121212122],"only_ref":45,"only_system":34,"p":0.26042552589724327,"p_holm":1,"verdict":"not distinguishable"},"correct_per_usd":78738.80090724233,"correct_per_second":0.4882712969703542},{"id":"gliner2.5-multi","display":"GLiNER2.5 multi (Fastino, 287M)","author":"Fastino","kind":"open weights","class":"classifier","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.3.0 public","comparable":true,"public":{"correct":113,"n":231,"acc":0.4891774891774891,"wilson95":[0.4254207052254166,0.5532883352318422]},"published_tiers":{"easy":{"acc":0.9027777777777778,"n":72,"correct":65,"wilson95":[0.8126495411763508,0.9521036802876413]},"standard":{"acc":0.5104166666666666,"n":96,"correct":49,"wilson95":[0.4119603701490724,0.608071388425067]},"judge":{"acc":0.4383561643835616,"n":146,"correct":64,"wilson95":[0.3604740462616515,0.519398986646687]},"hard":{"acc":0.37727272727272726,"n":220,"correct":83,"wilson95":[0.31584680608399707,0.4429110206837581]}},"latency":{"p50_s_raw":0.42792757973074913,"p50_s_adjusted":1.0058551594614982,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.003872921348314607,"kind":"estimate"},"per_item":"official","sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L15287","https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.2/jevbench-v1.2-per-task.json#L28620"],"answered":231,"public_tiers":{"easy":{"correct":44,"n":48,"acc":0.9166666666666666,"wilson95":[0.8044669809657894,0.967116243410477]},"original":{"correct":32,"n":72,"acc":0.4444444444444444,"wilson95":[0.33539070481712396,0.5591260922223253]},"hard":{"correct":37,"n":111,"acc":0.3333333333333333,"wilson95":[0.2525113744596184,0.425305327274862]}},"vs_ref":{"test":"exact McNemar (paired)","n":231,"on":"all 231 public items","diff":0.051948051948051965,"ci95":[-0.017316017316017316,0.12554112554112554],"only_ref":43,"only_system":31,"p":0.20067820312938575,"p_holm":1,"verdict":"not distinguishable"},"correct_per_usd":126307.10649220034,"correct_per_second":0.48632994977067945},{"id":"raw-qwen3-0.6b","display":"Raw Qwen3 0.6B direct logits","author":"Alibaba Qwen / neutral reproduction","kind":"open weights","class":"raw-logit-control","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"round 4 unpublished","comparable":true,"public":{"correct":111,"n":231,"acc":0.4805194805194805,"wilson95":[0.4169373442364457,0.5447389285866201]},"published_tiers":{"easy":{"acc":0.8888888888888888,"n":72,"correct":64,"wilson95":[0.7957706167789822,0.942611803944872]},"standard":{"acc":0.5416666666666666,"n":96,"correct":52,"wilson95":[0.44231552841140265,0.6378115058851554]},"judge":{"acc":0.4657534246575342,"n":146,"correct":68,"wilson95":[0.3867570662602619,0.5465057297999262]},"hard":{"acc":0.35,"n":220,"correct":77,"wilson95":[0.29003726526288676,0.41511118967548066]}},"latency":{"p50_s_raw":0.0677520353347063,"p50_s_adjusted":0.28550407066941264,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.007397940074906366,"kind":"estimate"},"per_item":null,"sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L16008"],"public_tiers":null,"vs_ref":{"test":"Wilson 95% intervals (unpaired)","n":231,"on":"official public accuracy","diff":0.06060606060606061,"ref_wilson95":[0.47671740174484795,0.6041882445175688],"system_wilson95":[0.4169373442364457,0.5447389285866201],"verdict":"not distinguishable"},"correct_per_usd":64953.145828984336,"correct_per_second":1.6830564951064313},{"id":"gliner2.5-small","display":"GLiNER2.5 small (Fastino, 74M)","author":"Fastino","kind":"open weights","class":"classifier","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.3.0 public","comparable":true,"public":{"correct":106,"n":231,"acc":0.4588744588744589,"wilson95":[0.3958117554824313,0.5232825982551521]},"published_tiers":{"easy":{"acc":0.8333333333333334,"n":72,"correct":60,"wilson95":[0.7308929297761552,0.9020062879871484]},"standard":{"acc":0.4791666666666667,"n":96,"correct":46,"wilson95":[0.3819741591643769,0.5779623236873439]},"judge":{"acc":0.5,"n":146,"correct":73,"wilson95":[0.4199424855514092,0.5800575144485908]},"hard":{"acc":0.33181818181818185,"n":220,"correct":73,"wilson95":[0.2729525626942851,0.3964563110244907]}},"latency":{"p50_s_raw":0.11413825303316116,"p50_s_adjusted":0.3782765060663223,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.003872921348314607,"kind":"estimate"},"per_item":"official","sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L15809","https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.2/jevbench-v1.2-per-task.json#L29744"],"answered":231,"public_tiers":{"easy":{"correct":41,"n":48,"acc":0.8541666666666666,"wilson95":[0.728328233591317,0.9275175071285092]},"original":{"correct":30,"n":72,"acc":0.4166666666666667,"wilson95":[0.30985217397794124,0.531923021581233]},"hard":{"correct":35,"n":111,"acc":0.3153153153153153,"wilson95":[0.2362888128600028,0.4066972620349619]}},"vs_ref":{"test":"exact McNemar (paired)","n":231,"on":"all 231 public items","diff":0.08225108225108224,"ci95":[0.012987012987012988,0.15151515151515152],"only_ref":44,"only_system":25,"p":0.029493493531317288,"p_holm":0.3244284288444902,"verdict":"not distinguishable"},"correct_per_usd":118482.77246171005,"correct_per_second":1.2130662399477845},{"id":"tia-1.4-mini","display":"TIA 1.4 mini","author":"TIA","kind":"local evaluation","ours":true,"listing":"not on the official board","measured_by":"TIA","measured_on":"2026-10-02","public":{"correct":102,"n":231,"acc":0.44155844155844154,"wilson95":[0.37899698709923224,0.5060318313699651]},"answered":231,"published_tiers":null,"latency":{"p50_s_raw":0.058561925000503834,"p50_s_adjusted":0.2671238500010077,"basis":"TIA measurement through the unmodified harness, local CPU worker over loopback; JevBench's self-host rule applied (x2 + 0.15 s). Not measured by JevBench."},"cost":{"usd_per_1000":0.010945637695637696,"kind":"estimated list price","basis":"Hypothetical API price, not a deployed-model tariff: 500 VND per 1M input tokens x 569 measured input tokens per decision, at 26,000 VND/USD"},"per_item":"TIA run","sources":[],"comparable":true,"public_tiers":{"easy":{"correct":29,"n":48,"acc":0.6041666666666666,"wilson95":[0.4631040838356026,0.729791722258464]},"original":{"correct":30,"n":72,"acc":0.4166666666666667,"wilson95":[0.30985217397794124,0.531923021581233]},"hard":{"correct":43,"n":111,"acc":0.38738738738738737,"wilson95":[0.3019774163600313,0.48033116589299607]}},"vs_ref":{"test":"exact McNemar (paired)","n":231,"on":"all 231 public items","diff":0.09956709956709958,"ci95":[0.021645021645021644,0.1774891774891775],"only_ref":55,"only_system":32,"p":0.017827552929735955,"p_holm":0.23175818808656745,"verdict":"not distinguishable"},"correct_per_usd":40341.04305630556,"correct_per_second":1.6530101732090785},{"id":"mirror","display":"Mirror","author":"Bluusun","kind":"open weights","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"round 4 unpublished","comparable":true,"public":{"correct":95,"n":231,"acc":0.41125541125541126,"wilson95":[0.3497571009003516,0.4756570308491703]},"published_tiers":{"easy":{"acc":0.875,"n":72,"correct":63,"wilson95":[0.7791894618635244,0.932822158120192]},"standard":{"acc":0.4583333333333333,"n":96,"correct":44,"wilson95":[0.3621884941148445,0.5576844715885974]},"judge":{"acc":0.3150684931506849,"n":146,"correct":46,"wilson95":[0.24527826355325644,0.394340835171759]},"hard":{"acc":0.17272727272727273,"n":220,"correct":38,"wilson95":[0.12850605152664688,0.22818148652070044]}},"latency":{"p50_s_raw":0.8995787426829338,"p50_s_adjusted":1.7991574853658676,"basis":"x2 (assumption, not measured)"},"cost":{"usd_per_1000":0.007745842696629214,"kind":"estimate"},"per_item":null,"sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L16605"],"public_tiers":null,"vs_ref":{"test":"Wilson 95% intervals (unpaired)","n":231,"on":"official public accuracy","diff":0.12987012987012986,"ref_wilson95":[0.47671740174484795,0.6041882445175688],"system_wilson95":[0.3497571009003516,0.4756570308491703],"verdict":"better"},"correct_per_usd":53093.695206898374,"correct_per_second":0.22858221951136223},{"id":"clm-8b","display":"CLM-8B (Contrastive-LM, clm-latest)","author":"Contrastive-LM (Kwok, Kang, Suresh, Saad-Falcon, Pavone, Ré, Mirhoseini)","kind":"open weights","class":"system-one-open","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.4.2 addition (official offline measurement on an evaluator-owned Lium pod, 24 Sep 2026)","comparable":true,"public":{"correct":94,"n":231,"acc":0.4069264069264069,"wilson95":[0.3455995803385568,0.4712981675938686]},"published_tiers":{"easy":{"acc":0.6666666666666666,"n":72,"correct":48,"wilson95":[0.5517960426376316,0.7646535662440201]},"standard":{"acc":0.3854166666666667,"n":96,"correct":37,"wilson95":[0.2942576852499145,0.4853929704345506]},"judge":{"acc":0.7123287671232876,"n":146,"correct":104,"wilson95":[0.6342008168062436,0.7795698476205906]},"hard":{"acc":0.35909090909090907,"n":220,"correct":79,"wilson95":[0.29861652717647597,0.42440171837168744]}},"latency":{"p50_s_raw":0.016774784307926893,"p50_s_adjusted":0.18354956861585375,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.005234400749063671,"kind":"estimate"},"per_item":null,"sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L15486"],"public_tiers":null,"vs_ref":{"test":"Wilson 95% intervals (unpaired)","n":231,"on":"official public accuracy","diff":0.1341991341991342,"ref_wilson95":[0.47671740174484795,0.6041882445175688],"system_wilson95":[0.3455995803385568,0.4712981675938686],"verdict":"better"},"correct_per_usd":77740.78188400036,"correct_per_second":2.2169837281287914},{"id":"tia-1.3-mini","display":"TIA 1.3 mini","author":"TIA","kind":"local evaluation","ours":true,"listing":"not on the official board","measured_by":"TIA","measured_on":"2026-09-30","public":{"correct":92,"n":231,"acc":0.3982683982683982,"wilson95":[0.3372994657077536,0.4625655145904789]},"answered":231,"published_tiers":null,"latency":{"p50_s_raw":0.058537225999998554,"p50_s_adjusted":0.26707445199999713,"basis":"TIA measurement through the unmodified harness, local CPU worker over loopback; JevBench's self-host rule applied (x2 + 0.15 s). Not measured by JevBench."},"cost":null,"per_item":"TIA run","sources":[],"comparable":true,"public_tiers":{"easy":{"correct":27,"n":48,"acc":0.5625,"wilson95":[0.42275017314264846,0.6929873105137916]},"original":{"correct":29,"n":72,"acc":0.4027777777777778,"wilson95":[0.297211110765762,0.5181932840532745]},"hard":{"correct":36,"n":111,"acc":0.32432432432432434,"wilson95":[0.24438273110570197,0.41601865720902065]}},"vs_ref":{"test":"exact McNemar (paired)","n":231,"on":"all 231 public items","diff":0.14285714285714285,"ci95":[0.06060606060606061,0.22510822510822512],"only_ref":65,"only_system":32,"p":0.0010496073324912285,"p_holm":0.017843324652350882,"verdict":"better"},"correct_per_usd":null,"correct_per_second":1.4912261179830206},{"id":"bge-reranker-v2-m3","display":"BAAI bge-reranker-v2-m3","author":"BAAI","kind":"open weights","class":"reranker","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.3.0 public","comparable":true,"public":{"correct":91,"n":231,"acc":0.3939393939393939,"wilson95":[0.3331569232746162,0.45819167320651977]},"published_tiers":{"easy":{"acc":0.4305555555555556,"n":72,"correct":31,"wilson95":[0.32257905708400253,0.5455669392153093]},"standard":{"acc":0.3645833333333333,"n":96,"correct":35,"wilson95":[0.2752394002114644,0.4643477383247217]},"judge":{"acc":0.08904109589041095,"n":146,"correct":13,"wilson95":[0.05277427127462007,0.14637928144763634]},"hard":{"acc":0.36818181818181817,"n":220,"correct":81,"wilson95":[0.30721983294126604,0.4336682032166933]}},"latency":{"p50_s_raw":0.034560746513307095,"p50_s_adjusted":0.21912149302661416,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.007718915951068509,"kind":"measured"},"per_item":"official","sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L17003","https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.2/jevbench-v1.2-per-task.json#L18504"],"answered":231,"public_tiers":{"easy":{"correct":23,"n":48,"acc":0.4791666666666667,"wilson95":[0.34471316079869785,0.6167076779824889]},"original":{"correct":26,"n":72,"acc":0.3611111111111111,"wilson95":[0.25981679635247285,0.4764751962461506]},"hard":{"correct":42,"n":111,"acc":0.3783783783783784,"wilson95":[0.29365311488109447,0.4712401539521751]}},"vs_ref":{"test":"exact McNemar (paired)","n":231,"on":"all 231 public items","diff":0.1471861471861472,"ci95":[0.06493506493506493,0.22943722943722944],"only_ref":68,"only_system":34,"p":0.000986536808184603,"p_holm":0.017757662547322854,"verdict":"better"},"correct_per_usd":51035.58536414196,"correct_per_second":1.7978126586219756},{"id":"tia-1.2-mini","display":"TIA 1.2 mini","author":"TIA","kind":"local evaluation","ours":true,"listing":"not on the official board","measured_by":"TIA","measured_on":"2026-09-30","public":{"correct":89,"n":231,"acc":0.3852813852813853,"wilson95":[0.3248870077390437,0.44942882110789933]},"answered":231,"published_tiers":null,"latency":{"p50_s_raw":0.06379614299999048,"p50_s_adjusted":0.277592285999981,"basis":"TIA measurement through the unmodified harness, local CPU worker over loopback; JevBench's self-host rule applied (x2 + 0.15 s). Not measured by JevBench."},"cost":null,"per_item":"TIA run","sources":[],"comparable":true,"public_tiers":{"easy":{"correct":26,"n":48,"acc":0.5416666666666666,"wilson95":[0.4029107444200485,0.6742475780175782]},"original":{"correct":32,"n":72,"acc":0.4444444444444444,"wilson95":[0.33539070481712396,0.5591260922223253]},"hard":{"correct":31,"n":111,"acc":0.27927927927927926,"wilson95":[0.20427656922103127,0.36904825199490215]}},"vs_ref":{"test":"exact McNemar (paired)","n":231,"on":"all 231 public items","diff":0.15584415584415584,"ci95":[0.07792207792207792,0.23376623376623376],"only_ref":63,"only_system":27,"p":0.000187755796179299,"p_holm":0.003942871719765279,"verdict":"better"},"correct_per_usd":null,"correct_per_second":1.3879398121366084},{"id":"mxbai-rerank-base-v2","display":"Mixedbread mxbai-rerank-base-v2","author":"Mixedbread","kind":"open weights","class":"reranker","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.3.0 public","comparable":true,"public":{"correct":86,"n":231,"acc":0.3722943722943723,"wilson95":[0.3125205203319901,0.4362461570636634]},"published_tiers":{"easy":{"acc":0.4444444444444444,"n":72,"correct":32,"wilson95":[0.33539070481712396,0.5591260922223253]},"standard":{"acc":0.3333333333333333,"n":96,"correct":32,"wilson95":[0.2470569235430229,0.4324349392707446]},"judge":{"acc":0.2671232876712329,"n":146,"correct":39,"wilson95":[0.2019983297168611,0.34418868349241766]},"hard":{"acc":0.4,"n":220,"correct":88,"wilson95":[0.3375156132748089,0.4659166900174362]}},"latency":{"p50_s_raw":0.06881788885220885,"p50_s_adjusted":0.2876357777044177,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.011722048450570632,"kind":"measured"},"per_item":"official","sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L16804","https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.2/jevbench-v1.2-per-task.json#L43232"],"answered":231,"public_tiers":{"easy":{"correct":22,"n":48,"acc":0.4583333333333333,"wilson95":[0.3257524219824218,0.5970892555799516]},"original":{"correct":24,"n":72,"acc":0.3333333333333333,"wilson95":[0.23534643375597977,0.4482039573623684]},"hard":{"correct":40,"n":111,"acc":0.36036036036036034,"wilson95":[0.2770990009201609,0.45296364107359294]}},"vs_ref":{"test":"exact McNemar (paired)","n":231,"on":"all 231 public items","diff":0.16883116883116883,"ci95":[0.08658008658008658,0.2510822510822511],"only_ref":70,"only_system":31,"p":0.00013082231066914517,"p_holm":0.002878090834721194,"verdict":"better"},"correct_per_usd":31760.180301613484,"correct_per_second":1.2943256755665216},{"id":"gte-reranker-modernbert-base","display":"Alibaba GTE Reranker ModernBERT-base","author":"Alibaba-NLP","kind":"open weights","class":"reranker","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.3.0 public","comparable":true,"public":{"correct":78,"n":231,"acc":0.33766233766233766,"wilson95":[0.2797753664556132,0.4008602404032685]},"published_tiers":{"easy":{"acc":0.3333333333333333,"n":72,"correct":24,"wilson95":[0.23534643375597977,0.4482039573623684]},"standard":{"acc":0.3958333333333333,"n":96,"correct":38,"wilson95":[0.3038335339518769,0.49584888030672775]},"judge":{"acc":0.3013698630136986,"n":146,"correct":44,"wilson95":[0.2328165849412692,0.3801076322078214]},"hard":{"acc":0.33636363636363636,"n":220,"correct":74,"wilson95":[0.27721436795566695,0.4011294010680067]}},"latency":{"p50_s_raw":0.048107756301760674,"p50_s_adjusted":0.24621551260352137,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.010280148864935354,"kind":"measured"},"per_item":"official","sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L17202","https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.2/jevbench-v1.2-per-task.json#L31992"],"answered":231,"public_tiers":{"easy":{"correct":17,"n":48,"acc":0.3541666666666667,"wilson95":[0.23433644045327673,0.4956094310150302]},"original":{"correct":26,"n":72,"acc":0.3611111111111111,"wilson95":[0.25981679635247285,0.4764751962461506]},"hard":{"correct":35,"n":111,"acc":0.3153153153153153,"wilson95":[0.2362888128600028,0.4066972620349619]}},"vs_ref":{"test":"exact McNemar (paired)","n":231,"on":"all 231 public items","diff":0.20346320346320343,"ci95":[0.11255411255411256,0.2943722943722944],"only_ref":87,"only_system":40,"p":0.00003692611090309835,"p_holm":0.000849300550771262,"verdict":"better"},"correct_per_usd":32846.05525646355,"correct_per_second":1.3714096812659904},{"id":"certo","display":"Certo v1 (AltSlate Labs)","author":"AltSlate Labs","kind":"open weights","class":"jev-rebuild","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.3.0 public","comparable":true,"public":{"correct":73,"n":231,"acc":0.31601731601731603,"wilson95":[0.25948884321145377,0.37856484456194545]},"published_tiers":{"easy":{"acc":0.2777777777777778,"n":72,"correct":20,"wilson95":[0.187602857967967,0.39046433018983057]},"standard":{"acc":0.3020833333333333,"n":96,"correct":29,"wilson95":[0.21931158441576212,0.4000850026755868]},"judge":{"acc":0.2191780821917808,"n":146,"correct":32,"wilson95":[0.1597494475734877,0.29300548012005423]},"hard":{"acc":0.3181818181818182,"n":220,"correct":70,"wilson95":[0.260205511529095,0.3823986762749868]}},"latency":{"p50_s_raw":0.019205978140234947,"p50_s_adjusted":0.1884119562804699,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.000965486891385768,"kind":"estimate"},"per_item":"official","sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L17401","https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.2/jevbench-v1.2-per-task.json#L19628"],"answered":231,"public_tiers":{"easy":{"correct":12,"n":48,"acc":0.25,"wilson95":[0.1492063361161345,0.3878437292581058]},"original":{"correct":24,"n":72,"acc":0.3333333333333333,"wilson95":[0.23534643375597977,0.4482039573623684]},"hard":{"correct":37,"n":111,"acc":0.3333333333333333,"wilson95":[0.2525113744596184,0.425305327274862]}},"vs_ref":{"test":"exact McNemar (paired)","n":231,"on":"all 231 public items","diff":0.2251082251082251,"ci95":[0.1341991341991342,0.31179653679653524],"only_ref":89,"only_system":37,"p":0.0000040992910972001675,"p_holm":0.00010658156852720436,"verdict":"better"},"correct_per_usd":327313.93749296264,"correct_per_second":1.6772678457140635},{"id":"needle-3","display":"Needle 3 (Cactus, 2-bit, local CPU)","author":"Cactus Compute","kind":"open weights","class":"small-tool-model","listing":"partial","ranked":false,"partial":true,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.3.0 public","comparable":true,"public":{"correct":52,"n":231,"acc":0.22510822510822512,"wilson95":[0.175999204559173,0.2832104230551999]},"published_tiers":{"easy":{"acc":0.4722222222222222,"n":72,"correct":34,"wilson95":[0.36126510255786615,0.5859932959618586]},"standard":{"acc":0.16666666666666666,"n":96,"correct":16,"wilson95":[0.10527382333433688,0.2537099022931983]},"judge":{"acc":0.3150684931506849,"n":146,"correct":46,"wilson95":[0.24527826355325644,0.394340835171759]},"hard":{"acc":0.07727272727272727,"n":220,"correct":17,"wilson95":[0.0488023867745157,0.1202523498699746]}},"latency":{"p50_s_raw":1.6870602630078793,"p50_s_adjusted":3.5241205260157584,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.023839264044943825,"kind":"estimate"},"per_item":"official","sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L18263","https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.2/jevbench-v1.2-per-task.json#L14008"],"answered":164,"public_tiers":{"easy":{"correct":23,"n":48,"acc":0.4791666666666667,"wilson95":[0.34471316079869785,0.6167076779824889]},"original":{"correct":12,"n":72,"acc":0.16666666666666666,"wilson95":[0.09799371201285148,0.26910707022384484]},"hard":{"correct":17,"n":111,"acc":0.15315315315315314,"wilson95":[0.09787476338857666,0.23163566995074736]}},"vs_ref":{"test":"exact McNemar (paired)","n":164,"on":"the 164 items it answered","diff":0.3231707317073171,"ci95":[0.22560975609756095,0.4207317073170731],"only_ref":67,"only_system":14,"p":1.889028403820931e-9,"p_holm":6.422696572991168e-8,"verdict":"better"},"correct_per_usd":9442.750610246683,"correct_per_second":0.06387642631585141},{"id":"needle-3-tools","display":"Needle 3, options as tools (post-hoc adapter mode)","author":"Cactus Compute","kind":"open weights","class":"small-tool-model","listing":"partial","ranked":false,"partial":true,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.3.0 public","comparable":true,"public":{"correct":51,"n":231,"acc":0.22077922077922077,"wilson95":[0.17210223125749252,0.27859101253978386]},"published_tiers":{"easy":{"acc":0.6666666666666666,"n":72,"correct":48,"wilson95":[0.5517960426376316,0.7646535662440201]},"standard":{"acc":0.3125,"n":96,"correct":30,"wilson95":[0.22850954579087288,0.4109187998746155]},"judge":{"acc":0.3424657534246575,"n":146,"correct":50,"wilson95":[0.2704170698562312,0.42259179202063385]}},"latency":{"p50_s_raw":3.778900783509016,"p50_s_adjusted":7.707801567018032,"basis":"x2 + 0.15 s (assumption, not measured)"},"cost":{"usd_per_1000":0.014372006369426754,"kind":"estimate"},"per_item":"official","sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L18192","https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.2/jevbench-v1.2-per-task.json#L16256"],"answered":120,"public_tiers":{"easy":{"correct":32,"n":48,"acc":0.6666666666666666,"wilson95":[0.5254010959289259,0.7832321938215805]},"original":{"correct":19,"n":72,"acc":0.2638888888888889,"wilson95":[0.17593776601789957,0.3757586213997603]},"hard":{"correct":0,"n":111,"acc":0,"wilson95":[-3.469446951953614e-18,0.03345010520344105]}},"vs_ref":{"test":"exact McNemar (paired)","n":120,"on":"the 120 items it answered","diff":0.3083333333333333,"ci95":[0.20833333333333337,0.4083333333333333],"only_ref":44,"only_system":7,"p":1.2115257419509362e-7,"p_holm":0.000003876882374242996,"verdict":"better"},"correct_per_usd":15361.753613530218,"correct_per_second":0.02864360464648483},{"id":"kushal-gemma4-31b-it-autoloops","display":"Autoloops – Gemma 4 31B IT","author":"Autoloops","kind":"hosted API","class":"jev","listing":"ranked","ranked":true,"partial":false,"ours":false,"measured_by":"JevBench maintainers","measured_in":"v1.4.2 addition (Autoloops operator API, official full-protocol measurement, 24 Sep 2026)","comparable":false,"public":{"correct":null,"n":null,"acc":0.9283819628647216,"wilson95":null,"published_rate":0.9283819628647216,"not_compared_because":"published public accuracy 0.9283819628647215 is not k/231: measured on a different item set"},"published_tiers":{"easy":{"acc":1,"n":72,"correct":72,"wilson95":[0.9493488266449552,1]},"standard":{"acc":0.9895833333333334,"n":96,"correct":95,"wilson95":[0.9433338231810272,0.9981588298035302]},"judge":{"acc":0.958904109589041,"n":146,"correct":140,"wilson95":[0.9132470509865872,0.9810314818068931]},"hard":{"acc":0.8272727272727273,"n":220,"correct":182,"wilson95":[0.7718185134792995,0.8714939484733532]}},"latency":{"p50_s_raw":0.6035970412194729,"p50_s_adjusted":0.6035970412194729,"basis":"none (production API)"},"cost":{"usd_per_1000":0.136347191011236,"kind":"measured"},"per_item":null,"sources":["https://github.com/fstandhartinger/jevbench/blob/fcc69279a933f882ccc4489eb14bf467647dbdb7/results/v1.4.2.2/jevbench-v1.4.2.2-results.json#L8584"],"public_tiers":null,"vs_ref":null,"correct_per_usd":null,"correct_per_second":null}]}