{"schema_version":1,"updated":"2026-10-10","notes":"Every row states its split and source. split=dev: internal dev set carved from the typed-decisions TRAIN split (186 cases / 930 decisions), measured by TIA. split=test: official TEST split (400 cases / 2,000 decisions), numbers quoted from the dataset card; TIA has not read the test split yet. Dev and test numbers are not comparable. measured_on for card rows is the date the card was read at the pinned revision. The tia-1.3-mini, tia-1.2-mini, tia-1.1-mini rows are recomputed from the per-decision predictions of those models on the same dev set. A row's `credits` name entries of the top-level `credits` whose data its text quotes.","dataset":{"id":"LocalLLaMA/typed-decisions","revision":"c76749ec58bd8c3d2ea706b31c333a9059c38f90","url":"https://huggingface.co/datasets/LocalLLaMA/typed-decisions/tree/c76749ec58bd8c3d2ea706b31c333a9059c38f90","splits":{"dev":{"cases":186,"decisions":930,"label":{"vi":"dev nội bộ (tách từ tập train)","en":"internal dev (carved from the train split)"}},"test":{"cases":400,"decisions":2000,"label":{"vi":"test (dataset card)","en":"test (dataset card)"}}}},"usd_ref_rate_vnd":26000,"rows":[{"id":"tia-1.3-mini","model":"TIA 1.3 mini","org":"TIA","kind":"ours","split":"dev","measured_on":"2026-09-30","measured_by":"TIA","source":"/models","detail":{"vi":"chạy System One API ngày 30/09/2026 tới khi TIA 1.4 mini và TIA 1.5 thay thế; hiệu chỉnh nhiệt độ T = 1,2941; chấm offline","en":"served the System One API on Sep 30, 2026 until succeeded by TIA 1.4 mini and TIA 1.5; temperature-calibrated, T = 1.2941; scored offline"},"params_m":null,"size_mb":null,"metrics":{"acc":0.76,"kl":0.138,"brier":0.077,"tv":0.171,"ece":0.163},"by_qtype":{"choice":{"n":280,"acc":0.736,"kl":0.178,"brier":0.091,"tv":0.195,"ece":0.182},"noul":{"n":278,"acc":0.82,"kl":0.078,"brier":0.061,"tv":0.124,"ece":0.099},"score":{"n":372,"acc":0.734,"kl":0.153,"brier":0.078,"tv":0.189,"ece":0.207}},"latency":null,"price_vnd_per_mtok":500,"access":{"vi":"System One API công khai, có gói miễn phí","en":"the public System One API, with a free tier"},"vietnamese":{"vi":"nhận đầu vào tiếng Việt; tập giữ lại tiếng Việt của TIA (1144 quyết định, dev): accuracy 0,658; bộ kiểm tra nhanh 25 câu: đúng 15/25 câu; tin nhắn SMS tiếng Việt thật (382 tin, dev, đo ngày 30/09/2026): accuracy 0,940, nhận ra 89/99 tin lừa đảo, báo nhầm 13/283 tin hợp lệ","en":"accepts Vietnamese input; TIA's Vietnamese held-out set (1,144 decisions, dev): accuracy 0.658; quick 25-case sanity set: 15 of 25 correct; real Vietnamese SMS (382 messages, dev, measured Sep 30, 2026): accuracy 0.940, caught 89 of 99 scam messages, flagged 13 of 283 legitimate ones"},"credits":["vi_sms"]},{"id":"tia-1.2-mini","model":"TIA 1.2 mini","org":"TIA","kind":"ours","split":"dev","measured_on":"2026-09-30","measured_by":"TIA","source":"/models","detail":{"vi":"chạy System One API ngày 30/09/2026 tới khi TIA 1.3 mini thay thế; hiệu chỉnh nhiệt độ T = 1,2722; chấm offline","en":"served the System One API on Sep 30, 2026 until TIA 1.3 mini replaced it; temperature-calibrated, T = 1.2722; scored offline"},"params_m":null,"size_mb":null,"metrics":{"acc":0.755,"kl":0.138,"brier":0.077,"tv":0.172,"ece":0.157},"by_qtype":{"choice":{"n":280,"acc":0.754,"kl":0.172,"brier":0.088,"tv":0.19,"ece":0.189},"noul":{"n":278,"acc":0.806,"kl":0.079,"brier":0.061,"tv":0.127,"ece":0.09},"score":{"n":372,"acc":0.718,"kl":0.157,"brier":0.081,"tv":0.192,"ece":0.188}},"latency":null,"price_vnd_per_mtok":null,"vietnamese":{"vi":"nhận đầu vào tiếng Việt; tập giữ lại tiếng Việt của TIA (1144 quyết định, dev): accuracy 0,629; bộ kiểm tra nhanh 25 câu: đúng 12/25 câu; tin nhắn SMS tiếng Việt thật (382 tin, dev, đo ngày 30/09/2026): accuracy 0,709, nhận ra 0/99 tin lừa đảo, báo nhầm 12/283 tin hợp lệ","en":"accepts Vietnamese input; TIA's Vietnamese held-out set (1,144 decisions, dev): accuracy 0.629; quick 25-case sanity set: 12 of 25 correct; real Vietnamese SMS (382 messages, dev, measured Sep 30, 2026): accuracy 0.709, caught 0 of 99 scam messages, flagged 12 of 283 legitimate ones"},"credits":["vi_sms"]},{"id":"tia-1.1-mini","model":"TIA 1.1 mini","org":"TIA","kind":"ours","split":"dev","measured_on":"2026-09-29","measured_by":"TIA","source":"/models","detail":{"vi":"chạy System One API từ 29/09/2026 tới 30/09/2026; hiệu chỉnh nhiệt độ T = 1,2230; chấm offline","en":"served the System One API from Sep 29 to Sep 30, 2026; temperature-calibrated, T = 1.2230; scored offline"},"params_m":null,"size_mb":null,"metrics":{"acc":0.759,"kl":0.134,"brier":0.075,"tv":0.168,"ece":0.158},"by_qtype":{"choice":{"n":280,"acc":0.732,"kl":0.175,"brier":0.088,"tv":0.191,"ece":0.167},"noul":{"n":278,"acc":0.813,"kl":0.072,"brier":0.057,"tv":0.118,"ece":0.091},"score":{"n":372,"acc":0.739,"kl":0.149,"brier":0.078,"tv":0.187,"ece":0.207}},"latency":null,"price_vnd_per_mtok":null,"vietnamese":{"vi":"nhận đầu vào tiếng Việt; tập giữ lại tiếng Việt của TIA (1144 quyết định, dev): accuracy 0,482; bộ kiểm tra nhanh 25 câu: đúng 8/25 câu","en":"accepts Vietnamese input; TIA's Vietnamese held-out set (1,144 decisions, dev): accuracy 0.482; quick 25-case sanity set: 8 of 25 correct"}},{"id":"tia-system-one","model":"TIA System One","org":"TIA","kind":"ours","split":"dev","measured_on":"2026-09-26","measured_by":"TIA","source":"/research#ket-qua","detail":{"vi":"bản của báo cáo kỹ thuật; hiệu chỉnh nhiệt độ T = 1,2015; đo qua bản đóng gói để phục vụ","en":"the build in the technical report; temperature-calibrated, T = 1.2015; measured through the packaged serving build"},"params_m":null,"size_mb":null,"metrics":{"acc":0.77,"kl":0.108,"brier":0.061,"tv":0.15,"ece":0.156},"by_qtype":{"choice":{"n":280,"acc":0.736,"kl":0.134,"brier":0.069,"tv":0.17,"ece":0.172},"noul":{"n":278,"acc":0.806,"kl":0.065,"brier":0.052,"tv":0.11,"ece":0.078},"score":{"n":372,"acc":0.769,"kl":0.121,"brier":0.061,"tv":0.165,"ece":0.222}},"latency":{"p50_ms":69,"kind":"server","hardware":"AMD Ryzen 9 9950X3D 16-Core Processor, 32 vCPU","note":{"vi":"phía server (thời gian mô hình), một request 3 câu hỏi, 98 token, tuần tự, n = 200; p95 81 ms","en":"server-side (model time), one request of 3 questions, 98 tokens, sequential, n = 200; p95 81 ms"}},"price_vnd_per_mtok":null,"access":{"vi":"chưa cung cấp công khai; là bản đo trong báo cáo kỹ thuật","en":"not publicly offered; the build measured in the technical report"},"vietnamese":{"vi":"nhận đầu vào tiếng Việt; chưa có số đo riêng cho tiếng Việt","en":"accepts Vietnamese input; no Vietnamese-specific measurement yet"}},{"id":"tia-system-one-prev","model":"TIA System One (previous)","org":"TIA","kind":"ours","split":"dev","measured_on":"2026-09-26","measured_by":"TIA","source":"/research#ket-qua","detail":{"vi":"phiên bản trước, hiệu chỉnh T = 5,12","en":"previous version, calibrated T = 5.12"},"params_m":null,"size_mb":null,"metrics":{"acc":0.744,"kl":0.187,"brier":0.096,"tv":0.209,"ece":0.152},"by_qtype":{"choice":{"n":280,"acc":0.714,"kl":0.224,"brier":0.099,"tv":0.235,"ece":0.207},"noul":{"n":278,"acc":0.752,"kl":0.136,"brier":0.104,"tv":0.172,"ece":0.062},"score":{"n":372,"acc":0.761,"kl":0.196,"brier":0.089,"tv":0.218,"ece":0.228}},"latency":null,"price_vnd_per_mtok":null},{"id":"meragpt-decider-1","model":"meraGPT Decider 1","org":"meraGPT","kind":"competitor","split":"test","measured_on":"2026-09-24","measured_by":"dataset card","source":"https://huggingface.co/datasets/LocalLLaMA/typed-decisions/blob/c76749ec58bd8c3d2ea706b31c333a9059c38f90/README.md","detail":null,"params_m":null,"size_mb":null,"metrics":{"acc":0.768,"kl":0.096,"brier":0.052,"tv":0.149,"ece":0.18},"latency":{"p50_ms":526,"kind":"e2e","hardware":null,"note":{"vi":"end-to-end từ client, theo dataset card","en":"end-to-end from the client, per the dataset card"}},"price_usd_per_mtok":0.03},{"id":"typesafe-jev-1.13.0","model":"TypeSafe Jev 1.13.0","org":"TypeSafe","kind":"competitor","split":"test","measured_on":"2026-09-24","measured_by":"dataset card","source":"https://huggingface.co/datasets/LocalLLaMA/typed-decisions/blob/c76749ec58bd8c3d2ea706b31c333a9059c38f90/README.md","detail":null,"params_m":null,"size_mb":null,"metrics":{"acc":0.727,"kl":1.442,"brier":0.148,"tv":0.251,"ece":0.144},"latency":{"p50_ms":710,"kind":"e2e","hardware":null,"note":{"vi":"end-to-end từ client, theo dataset card","en":"end-to-end from the client, per the dataset card"}},"price_usd_per_mtok":0.042},{"id":"card-encoder-149m","model":"Card baseline: 149M encoder specialist","org":"dataset card baseline","kind":"baseline","split":"test","measured_on":"2026-09-24","measured_by":"dataset card","source":"https://huggingface.co/datasets/LocalLLaMA/typed-decisions/blob/c76749ec58bd8c3d2ea706b31c333a9059c38f90/README.md","detail":null,"params_m":149,"size_mb":null,"metrics":{"acc":0.646,"kl":0.223,"brier":0.119,"tv":null,"ece":null},"latency":null},{"id":"minilm-l6","model":"MiniLM-L6","org":"dataset card baseline","kind":"baseline","split":"test","measured_on":"2026-09-24","measured_by":"dataset card","source":"https://huggingface.co/datasets/LocalLLaMA/typed-decisions/blob/c76749ec58bd8c3d2ea706b31c333a9059c38f90/README.md","detail":null,"params_m":null,"size_mb":null,"metrics":{"acc":0.587,"kl":0.262,"brier":0.143,"tv":null,"ece":null},"latency":null},{"id":"prior","model":"Prior","org":"dataset card baseline","kind":"baseline","split":"test","measured_on":"2026-09-24","measured_by":"dataset card","source":"https://huggingface.co/datasets/LocalLLaMA/typed-decisions/blob/c76749ec58bd8c3d2ea706b31c333a9059c38f90/README.md","detail":{"vi":"trả phân phối nhãn của tập train, không đọc đầu vào","en":"returns the train-split label distribution without reading the input"},"params_m":null,"size_mb":null,"metrics":{"acc":0.47,"kl":0.347,"brier":0.189,"tv":null,"ece":null},"latency":null},{"id":"uniform","model":"Uniform","org":"dataset card baseline","kind":"baseline","split":"test","measured_on":"2026-09-24","measured_by":"dataset card","source":"https://huggingface.co/datasets/LocalLLaMA/typed-decisions/blob/c76749ec58bd8c3d2ea706b31c333a9059c38f90/README.md","detail":{"vi":"chia đều xác suất cho mọi lựa chọn","en":"spreads probability evenly over every option"},"params_m":null,"size_mb":null,"metrics":{"acc":0.308,"kl":0.444,"brier":0.238,"tv":null,"ece":null},"latency":null}],"dev_references":{"uniform":{"acc":0.255,"kl":0.452,"brier":0.242,"tv":0.384,"ece":0.07},"prior":{"acc":0.495,"kl":0.44,"brier":0.204,"tv":0.316,"ece":0.036}},"not_measured":[{"model":"Laya","reason":{"vi":"Chúng tôi chưa chạy Laya theo cùng giao thức (cùng tập, cùng mã metric, cùng cách đo độ trễ). Không đặt con số tự báo cáo của bên khác vào bảng này.","en":"We have not run Laya under the same protocol (same split, same metric code, same latency method). We do not put another party's self-reported numbers in this table."}},{"model":"Verdict","reason":{"vi":"Chưa có số đo công khai trên typed-decisions theo cùng giao thức, và chúng tôi chưa tự đo.","en":"No public typed-decisions numbers under the same protocol, and we have not measured it ourselves."}}],"credits":{"vi_sms":{"title":"Vietnamese SMS Dataset with Quality Assurance","label":{"vi":"Dữ liệu SMS tiếng Việt","en":"Vietnamese SMS data"},"authors":"Tran Nguyen Thai Tuan, Le Hoang Khang, Nguyen Minh Tai, Nguyen Van Thang, Mai Hoang Dinh","url":"https://huggingface.co/datasets/trannguyenthaituan/vietnamese_sms_dataset","revision":"a90e2bd","license":"CC BY 4.0","license_url":"https://creativecommons.org/licenses/by/4.0/","changes":{"vi":"chuyển sang dạng quyết định có kiểu của TIA, bỏ trùng lặp, chia lại tập","en":"mapped to TIA typed decisions, deduplicated, re-split"}}},"promote_gate":{"challenger":"TIA System One","champion":"TIA System One (previous)","split":"dev","decisions":930,"method":"paired bootstrap over decisions, 2,000 resamples, 95% CI of challenger − champion; Brier here is averaged per option, so it is smaller than the Brier column","deltas":[{"metric":"kl","delta":-0.078,"ci":[-0.09,-0.065]},{"metric":"brier","delta":-0.013,"ci":[-0.016,-0.01]},{"metric":"acc","delta":0.026,"ci":[-0.001,0.053]}],"verdict":"PROMOTE"}}