{"schema_version":1,"updated":"2026-10-03","notes":"TIA's own Vietnamese sets. Every number is on a DEV split; the sealed test splits have never been read. Claude Opus 5.5 answered blind (protocol v2: each session reads its whole chunk, opens nothing else, writes explicit per-case probabilities; transcripts audited). Opus answers are for evaluation only, never training data. Historical model labels and the served block refer to the recorded 2026-10-03 evaluation, not the currently deployed revision.","sets":[{"id":"vi-sms","split":"dev","cases":382,"decisions":382,"classes":{"scam":99,"legit":283},"majority_baseline":{"rule":"always legit","acc":0.741},"status":"measured","source":"vi-sms datasheet (real SMS messages, internal to TIA), Sep 30, 2026"},{"id":"vn-law","version":"v1.1","subset":"dev300","split":"dev","cases":152,"decisions":300,"decisive":291,"decisive_excl_insufficient":273,"built_from":"official legal texts on vbpl.vn","baselines":[{"id":"first-option","rule":"always the first option (opus_arm score with no arm answers: uniform arm, argmax = first option)","on":"vn-law v1.1 dev300, the same decisions as the rows","split":"dev","n":291,"acc":0.436,"vs_tia_1_3_mini":{"only_baseline_right":65,"only_tia_right":54,"mcnemar_exact_p":0.36},"source":"vn-law v1.1 datasheet (official legal texts from vbpl.vn, internal to TIA), Oct 1, 2026"},{"id":"majority-label","rule":"always the majority label per question type","on":"vn-law v1.1 gen-dev (all decisive decisions, not dev300)","split":"dev","n":1148,"acc":0.405,"tia_1_3_mini_acc":0.342,"source":"vn-law v1.1 datasheet (official legal texts from vbpl.vn, internal to TIA), Oct 1, 2026"}],"status":"in progress","gate":{"model":"TIA 1.4 mini","declared":"before training","on":{"set":"vn-law v1.1 gen-dev","decisions":1189,"decisive":1148},"acc_decisive_min":0.9,"wilson95_lo_min":0.88,"case_cluster_lo_min":0.88,"training":"scored","source":"TIA internal evaluation procedure, Oct 1, 2026"},"source":"vn-law v1.1 datasheet (official legal texts from vbpl.vn, internal to TIA), Oct 1, 2026"}],"rows":[{"set":"vi-sms","model":"claude-opus-5.5","display":"Claude Opus 5.5","kind":"reference","split":"dev","n":382,"acc":0.982,"kl":0.124,"brier":0.046,"ece":0.059,"source":"TIA internal evaluation report, Oct 1, 2026, section 8.3"},{"set":"vi-sms","model":"tia-1.4-mini","display":"TIA 1.4 mini","kind":"ours","split":"dev","n":382,"acc":0.953,"kl":0.224,"source":"TIA internal evaluation procedure, Oct 1, 2026"},{"set":"vi-sms","model":"tia-1.3-mini","display":"TIA 1.3 mini","kind":"ours","split":"dev","n":382,"acc":0.94,"kl":0.232,"brier":0.094,"ece":0.048,"source":"TIA internal evaluation report, Oct 1, 2026, section 8.3"},{"set":"vi-sms","model":"tia-1.2-mini","display":"TIA 1.2 mini","kind":"ours","split":"dev","n":382,"acc":0.709,"kl":0.65,"brier":0.457,"ece":0.15,"source":"vi-sms datasheet (real SMS messages, internal to TIA), Sep 30, 2026"},{"set":"vn-law","model":"claude-opus-5.5","display":"Claude Opus 5.5","kind":"reference","split":"dev","decisive":{"n":291,"acc":0.986},"excl_insufficient":{"n":273,"acc":1},"source":"vn-law v1.1 datasheet (official legal texts from vbpl.vn, internal to TIA), Oct 1, 2026"},{"set":"vn-law","model":"tia-1.3-mini","display":"TIA 1.3 mini","kind":"ours","split":"dev","decisive":{"n":291,"acc":0.399},"excl_insufficient":{"n":273,"acc":0.388},"source":"vn-law v1.1 datasheet (official legal texts from vbpl.vn, internal to TIA), Oct 1, 2026"}],"paired":[{"set":"vi-sms","split":"dev","a":"claude-opus-5.5","b":"tia-1.3-mini","n":382,"only_a_right":19,"only_b_right":3,"mcnemar_exact_p":0.00086,"acc_diff_pts":4.2,"acc_diff_ci95_pts":[1.8,6.5],"kl_diff":-0.108,"kl_diff_ci95":[-0.221,-0.012],"brier_diff":-0.048,"brier_diff_ci95":[-0.087,-0.012],"ci_method":"paired bootstrap, 10,000 resamples, seed 0; difference is a minus b","source":"TIA internal evaluation report, Oct 1, 2026, section 8.3"},{"set":"vn-law","split":"dev","a":"claude-opus-5.5","b":"tia-1.3-mini","decisive":{"n":291,"only_a_right":172,"only_b_right":1},"excl_insufficient":{"n":273,"only_a_right":167,"only_b_right":0},"source":"Pre-registered evaluation criteria (PREREG-v2, appendix A1), Oct 1, 2026"}],"progress":[{"set":"vi-sms","split":"dev","from":"tia-1.2-mini","to":"tia-1.3-mini","n":382,"acc":[0.709,0.94],"acc_diff_pts":23,"acc_diff_ci95_pts":[18.1,27.8],"kl":[0.65,0.232],"kl_diff":-0.418,"kl_diff_ci95":[-0.522,-0.3],"scam_auroc":[0.457,0.983],"source":"TIA internal evaluation procedure, Oct 1, 2026"}],"detection":[{"set":"vi-sms","split":"dev","model":"tia-1.4-mini","scam_caught":89,"scams":99,"legit_flagged":8,"legit":283,"legit_fpr":0.028,"source":"TIA internal evaluation procedure, Oct 1, 2026"},{"set":"vi-sms","split":"dev","model":"tia-1.3-mini","scam_caught":89,"scams":99,"legit_flagged":13,"legit":283,"legit_fpr":0.046,"scam_auroc":0.983,"source":"TIA internal evaluation procedure, Oct 1, 2026"},{"set":"vi-sms","split":"dev","model":"tia-1.2-mini","scam_caught":0,"scams":99,"scam_auroc":0.457,"source":"TIA internal evaluation procedure, Oct 1, 2026"}],"cascade":{"set":"vi-sms","split":"dev","descriptive_only":true,"rule":"escalate a decision to Claude Opus 5.5 when the TIA 1.3 mini max probability < tau","why_descriptive":"tau was not chosen on separate data, so no row is a validated operating point","tau":0.99,"escalated":46,"n":382,"rate":0.12,"acc":0.982,"source":"TIA internal evaluation report, Oct 1, 2026, section 8.4"},"served":{"model":"TIA 1.4 mini","against":"TIA 1.3 mini","split":"dev","measured_on":"2026-10-01","cases":600,"gate_passed":false,"source":"TIA internal evaluation procedure, Oct 1, 2026","decisive":{"n":1148,"from":0.342,"to":0.922,"wilson_lo":0.905,"case_cluster_lo":0.905,"lowest_family":0.766,"excl_n":1078,"excl_acc":0.917},"arcade":{"k9_lo":0.993,"k9_hi":1,"k20_lo":0.996,"k20_hi":1},"sms_recall":0.899,"sms_fpr":0.028,"rows":[{"id":"vn-law","n":1189,"from":0.347,"to":0.92,"diff_pts":57.4,"ci":[54.2,60.3],"result":"pass"},{"id":"sms-aug","n":1576,"from":0.69,"to":0.873,"diff_pts":18.3,"ci":[15.9,20.6],"result":"pass"},{"id":"gen-dev","n":8875,"from":0.894,"to":0.9,"diff_pts":0.66,"ci":[0.28,1.08],"result":"pass"},{"id":"rule-dev","n":12914,"from":0.508,"to":0.501,"diff_pts":-0.71,"ci":[-1.49,0.02],"result":"pass"},{"id":"td-dev","n":930,"from":0.76,"to":0.753,"diff_pts":-0.75,"ci":[-2.15,0.75],"result":"fail"},{"id":"vi-dev","n":1144,"from":0.658,"to":0.666,"diff_pts":0.79,"ci":[-1.4,3.06],"result":"fail"},{"id":"vi-sms","n":382,"from":0.94,"to":0.953,"diff_pts":1.31,"ci":[-1.05,3.66],"result":"fail"}]}}