{ "status": "COMPLETE_26_DECISION_PLUS_3_LLM_REFERENCES_1342", "question_count": 1342, "model_count": 29, "primary_cases": 38918, "family_count": 31, "ranked_model_ids": [ "reference_sol", "mammoth1", "drex_max", "respan02", "pplx_decider12", "nimble_v4_max", "d3_turbo", "liquid_ultra", "amos_large", "reference_sonnet", "kettei_fat", "jev114", "jev113", "mercury_decide", "elephant1", "reference_qwen", "liquid_d1", "clef", "pplx_flash12", "microsoft_decision", "drex20", "clef_flash", "clef_omni", "drex15", "kev4b", "luna_decisions", "tev1_4b_native", "classify1", "solar_flash" ], "models": { "respan02": { "n": 1342, "correct": 1127, "answered": 1342, "wrong_valid": 215, "failed": 0, "accuracy": 0.8397913561847988, "accuracy_among_usable": 0.8397913561847988, "eventual": { "n": 1342, "correct": 1127, "answered": 1342, "wrong_valid": 215, "failed": 0, "accuracy": 0.8397913561847988, "accuracy_among_usable": 0.8397913561847988 }, "median_ms": 205.52456245059147, "p95_ms": 2157.423888955964, "attempts": 1342, "recoveries": 0, "first_statuses": { "ok": 1342 }, "model_output_failures": 0, "probability_warnings": 0, "cost": { "known_usd": 0.00474628, "known_decimal": "0.00474628", "unknown_attempts": 0, "exact_usd": 0.00474628, "cost_sources": { "api_reported": 1342 }, "known_usd_per_1000": 0.00353672131147541 }, "confidence_diagnostics": { "0.8": { "n": 796, "correct": 755, "answered": 796, "wrong_valid": 41, "failed": 0, "accuracy": 0.9484924623115578, "accuracy_among_usable": 0.9484924623115578 }, "0.9": { "n": 712, "correct": 684, "answered": 712, "wrong_valid": 28, "failed": 0, "accuracy": 0.9606741573033708, "accuracy_among_usable": 0.9606741573033708 }, "0.95": { "n": 629, "correct": 612, "answered": 629, "wrong_valid": 17, "failed": 0, "accuracy": 0.972972972972973, "accuracy_among_usable": 0.972972972972973 } }, "latency_first_usable": { "n": 1342, "median_ms": 205.52456245059147, "p95_ms": 2157.423888955964 }, "provider_failures": 0, "dates_toronto": { "2026-10-18": 1136, "2026-10-19": 206 }, "returned_identities": { "respan/span-02-20261015": 1342 }, "providers": { "Respan": 1342 }, "model_id": "respan02", "label": "Respan Span-02", "route": "OpenRouter", "families": { "F01": { "n": 43, "correct": 39, "answered": 43, "wrong_valid": 4, "failed": 0, "accuracy": 0.9069767441860465, "accuracy_among_usable": 0.9069767441860465 }, "F02": { "n": 46, "correct": 32, "answered": 46, "wrong_valid": 14, "failed": 0, "accuracy": 0.6956521739130435, "accuracy_among_usable": 0.6956521739130435 }, "F03": { "n": 52, "correct": 44, "answered": 52, "wrong_valid": 8, "failed": 0, "accuracy": 0.8461538461538461, "accuracy_among_usable": 0.8461538461538461 }, "F04": { "n": 54, "correct": 50, "answered": 54, "wrong_valid": 4, "failed": 0, "accuracy": 0.9259259259259259, "accuracy_among_usable": 0.9259259259259259 }, "F05": { "n": 49, "correct": 43, "answered": 49, "wrong_valid": 6, "failed": 0, "accuracy": 0.8775510204081632, "accuracy_among_usable": 0.8775510204081632 }, "F06": { "n": 40, "correct": 38, "answered": 40, "wrong_valid": 2, "failed": 0, "accuracy": 0.95, "accuracy_among_usable": 0.95 }, "F07": { "n": 76, "correct": 58, "answered": 76, "wrong_valid": 18, "failed": 0, "accuracy": 0.7631578947368421, "accuracy_among_usable": 0.7631578947368421 }, "F08": { "n": 67, "correct": 63, "answered": 67, "wrong_valid": 4, "failed": 0, "accuracy": 0.9402985074626866, "accuracy_among_usable": 0.9402985074626866 }, "F09": { "n": 71, "correct": 46, "answered": 71, "wrong_valid": 25, "failed": 0, "accuracy": 0.647887323943662, "accuracy_among_usable": 0.647887323943662 }, "F10": { "n": 74, "correct": 55, "answered": 74, "wrong_valid": 19, "failed": 0, "accuracy": 0.7432432432432432, "accuracy_among_usable": 0.7432432432432432 }, "F11": { "n": 41, "correct": 36, "answered": 41, "wrong_valid": 5, "failed": 0, "accuracy": 0.8780487804878049, "accuracy_among_usable": 0.8780487804878049 }, "F12": { "n": 52, "correct": 28, "answered": 52, "wrong_valid": 24, "failed": 0, "accuracy": 0.5384615384615384, "accuracy_among_usable": 0.5384615384615384 }, "F13": { "n": 48, "correct": 33, "answered": 48, "wrong_valid": 15, "failed": 0, "accuracy": 0.6875, "accuracy_among_usable": 0.6875 }, "F14": { "n": 42, "correct": 40, "answered": 42, "wrong_valid": 2, "failed": 0, "accuracy": 0.9523809523809523, "accuracy_among_usable": 0.9523809523809523 }, "F15": { "n": 49, "correct": 42, "answered": 49, "wrong_valid": 7, "failed": 0, "accuracy": 0.8571428571428571, "accuracy_among_usable": 0.8571428571428571 }, "F16": { "n": 50, "correct": 42, "answered": 50, "wrong_valid": 8, "failed": 0, "accuracy": 0.84, "accuracy_among_usable": 0.84 }, "F17": { "n": 50, "correct": 41, "answered": 50, "wrong_valid": 9, "failed": 0, "accuracy": 0.82, "accuracy_among_usable": 0.82 }, "F18": { "n": 50, "correct": 47, "answered": 50, "wrong_valid": 3, "failed": 0, "accuracy": 0.94, "accuracy_among_usable": 0.94 }, "F19": { "n": 58, "correct": 53, "answered": 58, "wrong_valid": 5, "failed": 0, "accuracy": 0.9137931034482759, "accuracy_among_usable": 0.9137931034482759 }, "F20": { "n": 43, "correct": 39, "answered": 43, "wrong_valid": 4, "failed": 0, "accuracy": 0.9069767441860465, "accuracy_among_usable": 0.9069767441860465 }, "F21": { "n": 43, "correct": 40, "answered": 43, "wrong_valid": 3, "failed": 0, "accuracy": 0.9302325581395349, "accuracy_among_usable": 0.9302325581395349 }, "F22": { "n": 51, "correct": 49, "answered": 51, "wrong_valid": 2, "failed": 0, "accuracy": 0.9607843137254902, "accuracy_among_usable": 0.9607843137254902 }, "F23": { "n": 52, "correct": 51, "answered": 52, "wrong_valid": 1, "failed": 0, "accuracy": 0.9807692307692307, "accuracy_among_usable": 0.9807692307692307 }, "F24": { "n": 43, "correct": 38, "answered": 43, "wrong_valid": 5, "failed": 0, "accuracy": 0.8837209302325582, "accuracy_among_usable": 0.8837209302325582 }, "F25": { "n": 50, "correct": 46, "answered": 50, "wrong_valid": 4, "failed": 0, "accuracy": 0.92, "accuracy_among_usable": 0.92 }, "F26": { "n": 8, "correct": 6, "answered": 8, "wrong_valid": 2, "failed": 0, "accuracy": 0.75, "accuracy_among_usable": 0.75 }, "F27": { "n": 8, "correct": 5, "answered": 8, "wrong_valid": 3, "failed": 0, "accuracy": 0.625, "accuracy_among_usable": 0.625 }, "F28": { "n": 8, "correct": 8, "answered": 8, "wrong_valid": 0, "failed": 0, "accuracy": 1.0, "accuracy_among_usable": 1.0 }, "F29": { "n": 8, "correct": 5, "answered": 8, "wrong_valid": 3, "failed": 0, "accuracy": 0.625, "accuracy_among_usable": 0.625 }, "F30": { "n": 8, "correct": 5, "answered": 8, "wrong_valid": 3, "failed": 0, "accuracy": 0.625, "accuracy_among_usable": 0.625 }, "F31": { "n": 8, "correct": 5, "answered": 8, "wrong_valid": 3, "failed": 0, "accuracy": 0.625, "accuracy_among_usable": 0.625 } }, "macro_family_accuracy": 0.8244083125467018, "release_exclusion_diagnostic": { "n": 1305, "correct": 1112, "answered": 1305, "wrong_valid": 193, "failed": 0, "accuracy": 0.8521072796934865, "accuracy_among_usable": 0.8521072796934865 }, "new_domains": { "n": 48, "correct": 34, "answered": 48, "wrong_valid": 14, "failed": 0, "accuracy": 0.7083333333333334, "accuracy_among_usable": 0.7083333333333334 } }, "pplx_decider12": { "n": 1342, "correct": 1120, "answered": 1341, "wrong_valid": 221, "failed": 1, "accuracy": 0.834575260804769, "accuracy_among_usable": 0.8351976137211037, "eventual": { "n": 1342, "correct": 1121, "answered": 1342, "wrong_valid": 221, "failed": 0, "accuracy": 0.8353204172876304, "accuracy_among_usable": 0.8353204172876304 }, "median_ms": 249.11764601711184, "p95_ms": 332.19568336498935, "attempts": 1343, "recoveries": 1, "first_statuses": { "ok": 1341, "provider_failure": 1 }, "model_output_failures": 0, "probability_warnings": 0, "cost": { "known_usd": 0.00939382, "known_decimal": "0.00939382", "unknown_attempts": 1, "exact_usd": null, "cost_sources": { "api_reported": 1342, "unreported": 1 }, "known_usd_per_1000": 0.006999865871833086 }, "confidence_diagnostics": { "0.8": { "n": 940, "correct": 931, "answered": 940, "wrong_valid": 9, "failed": 0, "accuracy": 0.9904255319148936, "accuracy_among_usable": 0.9904255319148936 }, "0.9": { "n": 886, "correct": 884, "answered": 886, "wrong_valid": 2, "failed": 0, "accuracy": 0.9977426636568849, "accuracy_among_usable": 0.9977426636568849 }, "0.95": { "n": 855, "correct": 854, "answered": 855, "wrong_valid": 1, "failed": 0, "accuracy": 0.9988304093567252, "accuracy_among_usable": 0.9988304093567252 } }, "latency_first_usable": { "n": 1341, "median_ms": 249.14037506096065, "p95_ms": 332.33487501274794 }, "provider_failures": 1, "dates_toronto": { "2026-10-18": 1136, "2026-10-19": 206 }, "returned_identities": { "perplexity/pplx-decider-v1.2-20261016": 1341, "unreported (error)": 1 }, "providers": { "Perplexity": 1341, "unreported": 1 }, "model_id": "pplx_decider12", "label": "Perplexity Decider v1.2", "route": "OpenRouter", "families": { "F01": { "n": 43, "correct": 40, "answered": 43, "wrong_valid": 3, "failed": 0, "accuracy": 0.9302325581395349, "accuracy_among_usable": 0.9302325581395349 }, "F02": { "n": 46, "correct": 45, "answered": 46, "wrong_valid": 1, "failed": 0, "accuracy": 0.9782608695652174, "accuracy_among_usable": 0.9782608695652174 }, "F03": { "n": 52, "correct": 50, "answered": 52, "wrong_valid": 2, "failed": 0, "accuracy": 0.9615384615384616, "accuracy_among_usable": 0.9615384615384616 }, "F04": { "n": 54, "correct": 48, "answered": 54, "wrong_valid": 6, "failed": 0, "accuracy": 0.8888888888888888, "accuracy_among_usable": 0.8888888888888888 }, "F05": { "n": 49, "correct": 44, "answered": 49, "wrong_valid": 5, "failed": 0, "accuracy": 0.8979591836734694, "accuracy_among_usable": 0.8979591836734694 }, "F06": { "n": 40, "correct": 38, "answered": 40, "wrong_valid": 2, "failed": 0, "accuracy": 0.95, "accuracy_among_usable": 0.95 }, "F07": { "n": 76, "correct": 54, "answered": 76, "wrong_valid": 22, "failed": 0, "accuracy": 0.7105263157894737, "accuracy_among_usable": 0.7105263157894737 }, "F08": { "n": 67, "correct": 65, "answered": 67, "wrong_valid": 2, "failed": 0, "accuracy": 0.9701492537313433, "accuracy_among_usable": 0.9701492537313433 }, "F09": { "n": 71, "correct": 43, "answered": 71, "wrong_valid": 28, "failed": 0, "accuracy": 0.6056338028169014, "accuracy_among_usable": 0.6056338028169014 }, "F10": { "n": 74, "correct": 58, "answered": 74, "wrong_valid": 16, "failed": 0, "accuracy": 0.7837837837837838, "accuracy_among_usable": 0.7837837837837838 }, "F11": { "n": 41, "correct": 37, "answered": 41, "wrong_valid": 4, "failed": 0, "accuracy": 0.9024390243902439, "accuracy_among_usable": 0.9024390243902439 }, "F12": { "n": 52, "correct": 33, "answered": 52, "wrong_valid": 19, "failed": 0, "accuracy": 0.6346153846153846, "accuracy_among_usable": 0.6346153846153846 }, "F13": { "n": 48, "correct": 14, "answered": 48, "wrong_valid": 34, "failed": 0, "accuracy": 0.2916666666666667, "accuracy_among_usable": 0.2916666666666667 }, "F14": { "n": 42, "correct": 39, "answered": 42, "wrong_valid": 3, "failed": 0, "accuracy": 0.9285714285714286, "accuracy_among_usable": 0.9285714285714286 }, "F15": { "n": 49, "correct": 35, "answered": 49, "wrong_valid": 14, "failed": 0, "accuracy": 0.7142857142857143, "accuracy_among_usable": 0.7142857142857143 }, "F16": { "n": 50, "correct": 36, "answered": 50, "wrong_valid": 14, "failed": 0, "accuracy": 0.72, "accuracy_among_usable": 0.72 }, "F17": { "n": 50, "correct": 46, "answered": 50, "wrong_valid": 4, "failed": 0, "accuracy": 0.92, "accuracy_among_usable": 0.92 }, "F18": { "n": 50, "correct": 46, "answered": 50, "wrong_valid": 4, "failed": 0, "accuracy": 0.92, "accuracy_among_usable": 0.92 }, "F19": { "n": 58, "correct": 54, "answered": 58, "wrong_valid": 4, "failed": 0, "accuracy": 0.9310344827586207, "accuracy_among_usable": 0.9310344827586207 }, "F20": { "n": 43, "correct": 38, "answered": 43, "wrong_valid": 5, "failed": 0, "accuracy": 0.8837209302325582, "accuracy_among_usable": 0.8837209302325582 }, "F21": { "n": 43, "correct": 42, "answered": 43, "wrong_valid": 1, "failed": 0, "accuracy": 0.9767441860465116, "accuracy_among_usable": 0.9767441860465116 }, "F22": { "n": 51, "correct": 47, "answered": 50, "wrong_valid": 3, "failed": 1, "accuracy": 0.9215686274509803, "accuracy_among_usable": 0.94 }, "F23": { "n": 52, "correct": 52, "answered": 52, "wrong_valid": 0, "failed": 0, "accuracy": 1.0, "accuracy_among_usable": 1.0 }, "F24": { "n": 43, "correct": 42, "answered": 43, "wrong_valid": 1, "failed": 0, "accuracy": 0.9767441860465116, "accuracy_among_usable": 0.9767441860465116 }, "F25": { "n": 50, "correct": 48, "answered": 50, "wrong_valid": 2, "failed": 0, "accuracy": 0.96, "accuracy_among_usable": 0.96 }, "F26": { "n": 8, "correct": 5, "answered": 8, "wrong_valid": 3, "failed": 0, "accuracy": 0.625, "accuracy_among_usable": 0.625 }, "F27": { "n": 8, "correct": 4, "answered": 8, "wrong_valid": 4, "failed": 0, "accuracy": 0.5, "accuracy_among_usable": 0.5 }, "F28": { "n": 8, "correct": 3, "answered": 8, "wrong_valid": 5, "failed": 0, "accuracy": 0.375, "accuracy_among_usable": 0.375 }, "F29": { "n": 8, "correct": 4, "answered": 8, "wrong_valid": 4, "failed": 0, "accuracy": 0.5, "accuracy_among_usable": 0.5 }, "F30": { "n": 8, "correct": 4, "answered": 8, "wrong_valid": 4, "failed": 0, "accuracy": 0.5, "accuracy_among_usable": 0.5 }, "F31": { "n": 8, "correct": 6, "answered": 8, "wrong_valid": 2, "failed": 0, "accuracy": 0.75, "accuracy_among_usable": 0.75 } }, "macro_family_accuracy": 0.793818185451345, "release_exclusion_diagnostic": { "n": 1305, "correct": 1105, "answered": 1304, "wrong_valid": 199, "failed": 1, "accuracy": 0.8467432950191571, "accuracy_among_usable": 0.8473926380368099 }, "new_domains": { "n": 48, "correct": 26, "answered": 48, "wrong_valid": 22, "failed": 0, "accuracy": 0.5416666666666666, "accuracy_among_usable": 0.5416666666666666 } }, "nimble_v4_max": { "n": 1342, "correct": 1115, "answered": 1342, "wrong_valid": 227, "failed": 0, "accuracy": 0.830849478390462, "accuracy_among_usable": 0.830849478390462, "eventual": { "n": 1342, "correct": 1115, "answered": 1342, "wrong_valid": 227, "failed": 0, "accuracy": 0.830849478390462, "accuracy_among_usable": 0.830849478390462 }, "median_ms": 235.37189594935626, "p95_ms": 328.7085750023835, "attempts": 1342, "recoveries": 0, "first_statuses": { "ok": 1342 }, "model_output_failures": 0, "probability_warnings": 0, "cost": { "known_usd": 0.03708378, "known_decimal": "0.03708378", "unknown_attempts": 0, "exact_usd": 0.03708378, "cost_sources": { "usage_times_published_tariff": 1134, "usage_times_current_published_tariff": 2, "usage_times_captured_tariff": 206 }, "known_usd_per_1000": 0.02763321907600596 }, "confidence_diagnostics": { "0.8": { "n": 837, "correct": 836, "answered": 837, "wrong_valid": 1, "failed": 0, "accuracy": 0.998805256869773, "accuracy_among_usable": 0.998805256869773 }, "0.9": { "n": 760, "correct": 760, "answered": 760, "wrong_valid": 0, "failed": 0, "accuracy": 1.0, "accuracy_among_usable": 1.0 }, "0.95": { "n": 685, "correct": 685, "answered": 685, "wrong_valid": 0, "failed": 0, "accuracy": 1.0, "accuracy_among_usable": 1.0 } }, "latency_first_usable": { "n": 1342, "median_ms": 235.37189594935626, "p95_ms": 328.7085750023835 }, "provider_failures": 0, "dates_toronto": { "2026-10-16": 1134, "2026-10-18": 2, "2026-10-19": 206 }, "returned_identities": { "nimble-v4-max": 1342 }, "providers": { "unreported": 1342 }, "model_id": "nimble_v4_max", "label": "Nimble v4 Max", "route": "Bespoke direct", "families": { "F01": { "n": 43, "correct": 41, "answered": 43, "wrong_valid": 2, "failed": 0, "accuracy": 0.9534883720930233, "accuracy_among_usable": 0.9534883720930233 }, "F02": { "n": 46, "correct": 45, "answered": 46, "wrong_valid": 1, "failed": 0, "accuracy": 0.9782608695652174, "accuracy_among_usable": 0.9782608695652174 }, "F03": { "n": 52, "correct": 49, "answered": 52, "wrong_valid": 3, "failed": 0, "accuracy": 0.9423076923076923, "accuracy_among_usable": 0.9423076923076923 }, "F04": { "n": 54, "correct": 50, "answered": 54, "wrong_valid": 4, "failed": 0, "accuracy": 0.9259259259259259, "accuracy_among_usable": 0.9259259259259259 }, "F05": { "n": 49, "correct": 45, "answered": 49, "wrong_valid": 4, "failed": 0, "accuracy": 0.9183673469387755, "accuracy_among_usable": 0.9183673469387755 }, "F06": { "n": 40, "correct": 38, "answered": 40, "wrong_valid": 2, "failed": 0, "accuracy": 0.95, "accuracy_among_usable": 0.95 }, "F07": { "n": 76, "correct": 51, "answered": 76, "wrong_valid": 25, "failed": 0, "accuracy": 0.6710526315789473, "accuracy_among_usable": 0.6710526315789473 }, "F08": { "n": 67, "correct": 60, "answered": 67, "wrong_valid": 7, "failed": 0, "accuracy": 0.8955223880597015, "accuracy_among_usable": 0.8955223880597015 }, "F09": { "n": 71, "correct": 39, "answered": 71, "wrong_valid": 32, "failed": 0, "accuracy": 0.5492957746478874, "accuracy_among_usable": 0.5492957746478874 }, "F10": { "n": 74, "correct": 53, "answered": 74, "wrong_valid": 21, "failed": 0, "accuracy": 0.7162162162162162, "accuracy_among_usable": 0.7162162162162162 }, "F11": { "n": 41, "correct": 32, "answered": 41, "wrong_valid": 9, "failed": 0, "accuracy": 0.7804878048780488, "accuracy_among_usable": 0.7804878048780488 }, "F12": { "n": 52, "correct": 35, "answered": 52, "wrong_valid": 17, "failed": 0, "accuracy": 0.6730769230769231, "accuracy_among_usable": 0.6730769230769231 }, "F13": { "n": 48, "correct": 19, "answered": 48, "wrong_valid": 29, "failed": 0, "accuracy": 0.3958333333333333, "accuracy_among_usable": 0.3958333333333333 }, "F14": { "n": 42, "correct": 41, "answered": 42, "wrong_valid": 1, "failed": 0, "accuracy": 0.9761904761904762, "accuracy_among_usable": 0.9761904761904762 }, "F15": { "n": 49, "correct": 36, "answered": 49, "wrong_valid": 13, "failed": 0, "accuracy": 0.7346938775510204, "accuracy_among_usable": 0.7346938775510204 }, "F16": { "n": 50, "correct": 38, "answered": 50, "wrong_valid": 12, "failed": 0, "accuracy": 0.76, "accuracy_among_usable": 0.76 }, "F17": { "n": 50, "correct": 46, "answered": 50, "wrong_valid": 4, "failed": 0, "accuracy": 0.92, "accuracy_among_usable": 0.92 }, "F18": { "n": 50, "correct": 45, "answered": 50, "wrong_valid": 5, "failed": 0, "accuracy": 0.9, "accuracy_among_usable": 0.9 }, "F19": { "n": 58, "correct": 57, "answered": 58, "wrong_valid": 1, "failed": 0, "accuracy": 0.9827586206896551, "accuracy_among_usable": 0.9827586206896551 }, "F20": { "n": 43, "correct": 40, "answered": 43, "wrong_valid": 3, "failed": 0, "accuracy": 0.9302325581395349, "accuracy_among_usable": 0.9302325581395349 }, "F21": { "n": 43, "correct": 40, "answered": 43, "wrong_valid": 3, "failed": 0, "accuracy": 0.9302325581395349, "accuracy_among_usable": 0.9302325581395349 }, "F22": { "n": 51, "correct": 49, "answered": 51, "wrong_valid": 2, "failed": 0, "accuracy": 0.9607843137254902, "accuracy_among_usable": 0.9607843137254902 }, "F23": { "n": 52, "correct": 52, "answered": 52, "wrong_valid": 0, "failed": 0, "accuracy": 1.0, "accuracy_among_usable": 1.0 }, "F24": { "n": 43, "correct": 42, "answered": 43, "wrong_valid": 1, "failed": 0, "accuracy": 0.9767441860465116, "accuracy_among_usable": 0.9767441860465116 }, "F25": { "n": 50, "correct": 47, "answered": 50, "wrong_valid": 3, "failed": 0, "accuracy": 0.94, "accuracy_among_usable": 0.94 }, "F26": { "n": 8, "correct": 4, "answered": 8, "wrong_valid": 4, "failed": 0, "accuracy": 0.5, "accuracy_among_usable": 0.5 }, "F27": { "n": 8, "correct": 3, "answered": 8, "wrong_valid": 5, "failed": 0, "accuracy": 0.375, "accuracy_among_usable": 0.375 }, "F28": { "n": 8, "correct": 4, "answered": 8, "wrong_valid": 4, "failed": 0, "accuracy": 0.5, "accuracy_among_usable": 0.5 }, "F29": { "n": 8, "correct": 5, "answered": 8, "wrong_valid": 3, "failed": 0, "accuracy": 0.625, "accuracy_among_usable": 0.625 }, "F30": { "n": 8, "correct": 3, "answered": 8, "wrong_valid": 5, "failed": 0, "accuracy": 0.375, "accuracy_among_usable": 0.375 }, "F31": { "n": 8, "correct": 6, "answered": 8, "wrong_valid": 2, "failed": 0, "accuracy": 0.75, "accuracy_among_usable": 0.75 } }, "macro_family_accuracy": 0.7898861893259329, "release_exclusion_diagnostic": { "n": 1305, "correct": 1099, "answered": 1305, "wrong_valid": 206, "failed": 0, "accuracy": 0.8421455938697318, "accuracy_among_usable": 0.8421455938697318 }, "new_domains": { "n": 48, "correct": 25, "answered": 48, "wrong_valid": 23, "failed": 0, "accuracy": 0.5208333333333334, "accuracy_among_usable": 0.5208333333333334 } }, "jev114": { "n": 1342, "correct": 1030, "answered": 1342, "wrong_valid": 312, "failed": 0, "accuracy": 0.767511177347243, "accuracy_among_usable": 0.767511177347243, "eventual": { "n": 1342, "correct": 1030, "answered": 1342, "wrong_valid": 312, "failed": 0, "accuracy": 0.767511177347243, "accuracy_among_usable": 0.767511177347243 }, "median_ms": 179.4867084827274, "p95_ms": 268.8569815829396, "attempts": 1342, "recoveries": 0, "first_statuses": { "ok": 1342 }, "model_output_failures": 0, "probability_warnings": 17, "cost": { "known_usd": 0.03297441, "known_decimal": "0.032974410", "unknown_attempts": 0, "exact_usd": 0.03297441, "cost_sources": { "recorded_accounted_or_reported": 1134, "api_reported": 208 }, "known_usd_per_1000": 0.024571095380029807 }, "confidence_diagnostics": { "0.8": { "n": 740, "correct": 728, "answered": 740, "wrong_valid": 12, "failed": 0, "accuracy": 0.9837837837837838, "accuracy_among_usable": 0.9837837837837838 }, "0.9": { "n": 668, "correct": 664, "answered": 668, "wrong_valid": 4, "failed": 0, "accuracy": 0.9940119760479041, "accuracy_among_usable": 0.9940119760479041 }, "0.95": { "n": 614, "correct": 613, "answered": 614, "wrong_valid": 1, "failed": 0, "accuracy": 0.998371335504886, "accuracy_among_usable": 0.998371335504886 } }, "latency_first_usable": { "n": 1342, "median_ms": 179.4867084827274, "p95_ms": 268.8569815829396 }, "provider_failures": 0, "dates_toronto": { "2026-10-15": 1134, "2026-10-18": 2, "2026-10-19": 206 }, "returned_identities": { "typesafe/jev-1.14-20261014": 1342 }, "providers": { "TypeSafe": 1342 }, "model_id": "jev114", "label": "Jev 1.14", "route": "OpenRouter", "families": { "F01": { "n": 43, "correct": 39, "answered": 43, "wrong_valid": 4, "failed": 0, "accuracy": 0.9069767441860465, "accuracy_among_usable": 0.9069767441860465 }, "F02": { "n": 46, "correct": 46, "answered": 46, "wrong_valid": 0, "failed": 0, "accuracy": 1.0, "accuracy_among_usable": 1.0 }, "F03": { "n": 52, "correct": 48, "answered": 52, "wrong_valid": 4, "failed": 0, "accuracy": 0.9230769230769231, "accuracy_among_usable": 0.9230769230769231 }, "F04": { "n": 54, "correct": 49, "answered": 54, "wrong_valid": 5, "failed": 0, "accuracy": 0.9074074074074074, "accuracy_among_usable": 0.9074074074074074 }, "F05": { "n": 49, "correct": 46, "answered": 49, "wrong_valid": 3, "failed": 0, "accuracy": 0.9387755102040817, "accuracy_among_usable": 0.9387755102040817 }, "F06": { "n": 40, "correct": 38, "answered": 40, "wrong_valid": 2, "failed": 0, "accuracy": 0.95, "accuracy_among_usable": 0.95 }, "F07": { "n": 76, "correct": 48, "answered": 76, "wrong_valid": 28, "failed": 0, "accuracy": 0.631578947368421, "accuracy_among_usable": 0.631578947368421 }, "F08": { "n": 67, "correct": 57, "answered": 67, "wrong_valid": 10, "failed": 0, "accuracy": 0.8507462686567164, "accuracy_among_usable": 0.8507462686567164 }, "F09": { "n": 71, "correct": 28, "answered": 71, "wrong_valid": 43, "failed": 0, "accuracy": 0.39436619718309857, "accuracy_among_usable": 0.39436619718309857 }, "F10": { "n": 74, "correct": 39, "answered": 74, "wrong_valid": 35, "failed": 0, "accuracy": 0.527027027027027, "accuracy_among_usable": 0.527027027027027 }, "F11": { "n": 41, "correct": 32, "answered": 41, "wrong_valid": 9, "failed": 0, "accuracy": 0.7804878048780488, "accuracy_among_usable": 0.7804878048780488 }, "F12": { "n": 52, "correct": 30, "answered": 52, "wrong_valid": 22, "failed": 0, "accuracy": 0.5769230769230769, "accuracy_among_usable": 0.5769230769230769 }, "F13": { "n": 48, "correct": 7, "answered": 48, "wrong_valid": 41, "failed": 0, "accuracy": 0.14583333333333334, "accuracy_among_usable": 0.14583333333333334 }, "F14": { "n": 42, "correct": 40, "answered": 42, "wrong_valid": 2, "failed": 0, "accuracy": 0.9523809523809523, "accuracy_among_usable": 0.9523809523809523 }, "F15": { "n": 49, "correct": 34, "answered": 49, "wrong_valid": 15, "failed": 0, "accuracy": 0.6938775510204082, "accuracy_among_usable": 0.6938775510204082 }, "F16": { "n": 50, "correct": 34, "answered": 50, "wrong_valid": 16, "failed": 0, "accuracy": 0.68, "accuracy_among_usable": 0.68 }, "F17": { "n": 50, "correct": 44, "answered": 50, "wrong_valid": 6, "failed": 0, "accuracy": 0.88, "accuracy_among_usable": 0.88 }, "F18": { "n": 50, "correct": 38, "answered": 50, "wrong_valid": 12, "failed": 0, "accuracy": 0.76, "accuracy_among_usable": 0.76 }, "F19": { "n": 58, "correct": 48, "answered": 58, "wrong_valid": 10, "failed": 0, "accuracy": 0.8275862068965517, "accuracy_among_usable": 0.8275862068965517 }, "F20": { "n": 43, "correct": 38, "answered": 43, "wrong_valid": 5, "failed": 0, "accuracy": 0.8837209302325582, "accuracy_among_usable": 0.8837209302325582 }, "F21": { "n": 43, "correct": 42, "answered": 43, "wrong_valid": 1, "failed": 0, "accuracy": 0.9767441860465116, "accuracy_among_usable": 0.9767441860465116 }, "F22": { "n": 51, "correct": 48, "answered": 51, "wrong_valid": 3, "failed": 0, "accuracy": 0.9411764705882353, "accuracy_among_usable": 0.9411764705882353 }, "F23": { "n": 52, "correct": 49, "answered": 52, "wrong_valid": 3, "failed": 0, "accuracy": 0.9423076923076923, "accuracy_among_usable": 0.9423076923076923 }, "F24": { "n": 43, "correct": 37, "answered": 43, "wrong_valid": 6, "failed": 0, "accuracy": 0.8604651162790697, "accuracy_among_usable": 0.8604651162790697 }, "F25": { "n": 50, "correct": 45, "answered": 50, "wrong_valid": 5, "failed": 0, "accuracy": 0.9, "accuracy_among_usable": 0.9 }, "F26": { "n": 8, "correct": 5, "answered": 8, "wrong_valid": 3, "failed": 0, "accuracy": 0.625, "accuracy_among_usable": 0.625 }, "F27": { "n": 8, "correct": 4, "answered": 8, "wrong_valid": 4, "failed": 0, "accuracy": 0.5, "accuracy_among_usable": 0.5 }, "F28": { "n": 8, "correct": 2, "answered": 8, "wrong_valid": 6, "failed": 0, "accuracy": 0.25, "accuracy_among_usable": 0.25 }, "F29": { "n": 8, "correct": 5, "answered": 8, "wrong_valid": 3, "failed": 0, "accuracy": 0.625, "accuracy_among_usable": 0.625 }, "F30": { "n": 8, "correct": 4, "answered": 8, "wrong_valid": 4, "failed": 0, "accuracy": 0.5, "accuracy_among_usable": 0.5 }, "F31": { "n": 8, "correct": 6, "answered": 8, "wrong_valid": 2, "failed": 0, "accuracy": 0.75, "accuracy_among_usable": 0.75 } }, "macro_family_accuracy": 0.7445631724514891, "release_exclusion_diagnostic": { "n": 1305, "correct": 1014, "answered": 1305, "wrong_valid": 291, "failed": 0, "accuracy": 0.7770114942528735, "accuracy_among_usable": 0.7770114942528735 }, "new_domains": { "n": 48, "correct": 26, "answered": 48, "wrong_valid": 22, "failed": 0, "accuracy": 0.5416666666666666, "accuracy_among_usable": 0.5416666666666666 } }, "liquid_ultra": { "n": 1342, "correct": 1084, "answered": 1341, "wrong_valid": 257, "failed": 1, "accuracy": 0.8077496274217586, "accuracy_among_usable": 0.808351976137211, "eventual": { "n": 1342, "correct": 1085, "answered": 1342, "wrong_valid": 257, "failed": 0, "accuracy": 0.8084947839046199, "accuracy_among_usable": 0.8084947839046199 }, "median_ms": 200.17650042427704, "p95_ms": 393.4226100856908, "attempts": 1343, "recoveries": 1, "first_statuses": { "ok": 1341, "provider_failure": 1 }, "model_output_failures": 0, "probability_warnings": 0, "cost": { "known_usd": 0.016270002, "known_decimal": "0.016270002", "unknown_attempts": 1, "exact_usd": null, "cost_sources": { "api_reported": 1342, "unreported": 1 }, "known_usd_per_1000": 0.012123697466467957 }, "confidence_diagnostics": { "0.8": { "n": 839, "correct": 835, "answered": 839, "wrong_valid": 4, "failed": 0, "accuracy": 0.9952324195470799, "accuracy_among_usable": 0.9952324195470799 }, "0.9": { "n": 774, "correct": 772, "answered": 774, "wrong_valid": 2, "failed": 0, "accuracy": 0.9974160206718347, "accuracy_among_usable": 0.9974160206718347 }, "0.95": { "n": 707, "correct": 706, "answered": 707, "wrong_valid": 1, "failed": 0, "accuracy": 0.9985855728429985, "accuracy_among_usable": 0.9985855728429985 } }, "latency_first_usable": { "n": 1341, "median_ms": 200.2075839554891, "p95_ms": 393.87520798482 }, "provider_failures": 1, "dates_toronto": { "2026-10-18": 1136, "2026-10-19": 206 }, "returned_identities": { "liquid/d1-ultra-20261015": 1341, "unreported (error)": 1 }, "providers": { "Liquid": 1341, "unreported": 1 }, "model_id": "liquid_ultra", "label": "Liquid d1-ultra", "route": "OpenRouter", "families": { "F01": { "n": 43, "correct": 41, "answered": 43, "wrong_valid": 2, "failed": 0, "accuracy": 0.9534883720930233, "accuracy_among_usable": 0.9534883720930233 }, "F02": { "n": 46, "correct": 44, "answered": 46, "wrong_valid": 2, "failed": 0, "accuracy": 0.9565217391304348, "accuracy_among_usable": 0.9565217391304348 }, "F03": { "n": 52, "correct": 50, "answered": 52, "wrong_valid": 2, "failed": 0, "accuracy": 0.9615384615384616, "accuracy_among_usable": 0.9615384615384616 }, "F04": { "n": 54, "correct": 51, "answered": 54, "wrong_valid": 3, "failed": 0, "accuracy": 0.9444444444444444, "accuracy_among_usable": 0.9444444444444444 }, "F05": { "n": 49, "correct": 45, "answered": 49, "wrong_valid": 4, "failed": 0, "accuracy": 0.9183673469387755, "accuracy_among_usable": 0.9183673469387755 }, "F06": { "n": 40, "correct": 39, "answered": 40, "wrong_valid": 1, "failed": 0, "accuracy": 0.975, "accuracy_among_usable": 0.975 }, "F07": { "n": 76, "correct": 57, "answered": 76, "wrong_valid": 19, "failed": 0, "accuracy": 0.75, "accuracy_among_usable": 0.75 }, "F08": { "n": 67, "correct": 55, "answered": 67, "wrong_valid": 12, "failed": 0, "accuracy": 0.8208955223880597, "accuracy_among_usable": 0.8208955223880597 }, "F09": { "n": 71, "correct": 42, "answered": 71, "wrong_valid": 29, "failed": 0, "accuracy": 0.5915492957746479, "accuracy_among_usable": 0.5915492957746479 }, "F10": { "n": 74, "correct": 46, "answered": 73, "wrong_valid": 27, "failed": 1, "accuracy": 0.6216216216216216, "accuracy_among_usable": 0.6301369863013698 }, "F11": { "n": 41, "correct": 35, "answered": 41, "wrong_valid": 6, "failed": 0, "accuracy": 0.8536585365853658, "accuracy_among_usable": 0.8536585365853658 }, "F12": { "n": 52, "correct": 27, "answered": 52, "wrong_valid": 25, "failed": 0, "accuracy": 0.5192307692307693, "accuracy_among_usable": 0.5192307692307693 }, "F13": { "n": 48, "correct": 11, "answered": 48, "wrong_valid": 37, "failed": 0, "accuracy": 0.22916666666666666, "accuracy_among_usable": 0.22916666666666666 }, "F14": { "n": 42, "correct": 38, "answered": 42, "wrong_valid": 4, "failed": 0, "accuracy": 0.9047619047619048, "accuracy_among_usable": 0.9047619047619048 }, "F15": { "n": 49, "correct": 38, "answered": 49, "wrong_valid": 11, "failed": 0, "accuracy": 0.7755102040816326, "accuracy_among_usable": 0.7755102040816326 }, "F16": { "n": 50, "correct": 34, "answered": 50, "wrong_valid": 16, "failed": 0, "accuracy": 0.68, "accuracy_among_usable": 0.68 }, "F17": { "n": 50, "correct": 45, "answered": 50, "wrong_valid": 5, "failed": 0, "accuracy": 0.9, "accuracy_among_usable": 0.9 }, "F18": { "n": 50, "correct": 40, "answered": 50, "wrong_valid": 10, "failed": 0, "accuracy": 0.8, "accuracy_among_usable": 0.8 }, "F19": { "n": 58, "correct": 57, "answered": 58, "wrong_valid": 1, "failed": 0, "accuracy": 0.9827586206896551, "accuracy_among_usable": 0.9827586206896551 }, "F20": { "n": 43, "correct": 39, "answered": 43, "wrong_valid": 4, "failed": 0, "accuracy": 0.9069767441860465, "accuracy_among_usable": 0.9069767441860465 }, "F21": { "n": 43, "correct": 40, "answered": 43, "wrong_valid": 3, "failed": 0, "accuracy": 0.9302325581395349, "accuracy_among_usable": 0.9302325581395349 }, "F22": { "n": 51, "correct": 47, "answered": 51, "wrong_valid": 4, "failed": 0, "accuracy": 0.9215686274509803, "accuracy_among_usable": 0.9215686274509803 }, "F23": { "n": 52, "correct": 51, "answered": 52, "wrong_valid": 1, "failed": 0, "accuracy": 0.9807692307692307, "accuracy_among_usable": 0.9807692307692307 }, "F24": { "n": 43, "correct": 42, "answered": 43, "wrong_valid": 1, "failed": 0, "accuracy": 0.9767441860465116, "accuracy_among_usable": 0.9767441860465116 }, "F25": { "n": 50, "correct": 45, "answered": 50, "wrong_valid": 5, "failed": 0, "accuracy": 0.9, "accuracy_among_usable": 0.9 }, "F26": { "n": 8, "correct": 5, "answered": 8, "wrong_valid": 3, "failed": 0, "accuracy": 0.625, "accuracy_among_usable": 0.625 }, "F27": { "n": 8, "correct": 5, "answered": 8, "wrong_valid": 3, "failed": 0, "accuracy": 0.625, "accuracy_among_usable": 0.625 }, "F28": { "n": 8, "correct": 4, "answered": 8, "wrong_valid": 4, "failed": 0, "accuracy": 0.5, "accuracy_among_usable": 0.5 }, "F29": { "n": 8, "correct": 3, "answered": 8, "wrong_valid": 5, "failed": 0, "accuracy": 0.375, "accuracy_among_usable": 0.375 }, "F30": { "n": 8, "correct": 3, "answered": 8, "wrong_valid": 5, "failed": 0, "accuracy": 0.375, "accuracy_among_usable": 0.375 }, "F31": { "n": 8, "correct": 5, "answered": 8, "wrong_valid": 3, "failed": 0, "accuracy": 0.625, "accuracy_among_usable": 0.625 } }, "macro_family_accuracy": 0.7703162855657344, "release_exclusion_diagnostic": { "n": 1305, "correct": 1071, "answered": 1304, "wrong_valid": 233, "failed": 1, "accuracy": 0.8206896551724138, "accuracy_among_usable": 0.821319018404908 }, "new_domains": { "n": 48, "correct": 25, "answered": 48, "wrong_valid": 23, "failed": 0, "accuracy": 0.5208333333333334, "accuracy_among_usable": 0.5208333333333334 } }, "mercury_decide": { "n": 1342, "correct": 1017, "answered": 1342, "wrong_valid": 325, "failed": 0, "accuracy": 0.7578241430700448, "accuracy_among_usable": 0.7578241430700448, "eventual": { "n": 1342, "correct": 1017, "answered": 1342, "wrong_valid": 325, "failed": 0, "accuracy": 0.7578241430700448, "accuracy_among_usable": 0.7578241430700448 }, "median_ms": 238.4314374648966, "p95_ms": 388.3725829655305, "attempts": 1342, "recoveries": 0, "first_statuses": { "ok": 1342 }, "model_output_failures": 0, "probability_warnings": 0, "cost": { "known_usd": 0.00933446, "known_decimal": "0.00933446", "unknown_attempts": 0, "exact_usd": 0.00933446, "cost_sources": { "api_reported": 1342 }, "known_usd_per_1000": 0.006955633383010432 }, "confidence_diagnostics": { "0.8": { "n": 972, "correct": 873, "answered": 972, "wrong_valid": 99, "failed": 0, "accuracy": 0.8981481481481481, "accuracy_among_usable": 0.8981481481481481 }, "0.9": { "n": 897, "correct": 831, "answered": 897, "wrong_valid": 66, "failed": 0, "accuracy": 0.9264214046822743, "accuracy_among_usable": 0.9264214046822743 }, "0.95": { "n": 842, "correct": 796, "answered": 842, "wrong_valid": 46, "failed": 0, "accuracy": 0.9453681710213777, "accuracy_among_usable": 0.9453681710213777 } }, "latency_first_usable": { "n": 1342, "median_ms": 238.4314374648966, "p95_ms": 388.3725829655305 }, "provider_failures": 0, "dates_toronto": { "2026-10-18": 1136, "2026-10-19": 206 }, "returned_identities": { "inception/mercury-decide-20260930": 1342 }, "providers": { "Inception": 1342 }, "model_id": "mercury_decide", "label": "Mercury Decide", "route": "OpenRouter", "families": { "F01": { "n": 43, "correct": 38, "answered": 43, "wrong_valid": 5, "failed": 0, "accuracy": 0.8837209302325582, "accuracy_among_usable": 0.8837209302325582 }, "F02": { "n": 46, "correct": 43, "answered": 46, "wrong_valid": 3, "failed": 0, "accuracy": 0.9347826086956522, "accuracy_among_usable": 0.9347826086956522 }, "F03": { "n": 52, "correct": 28, "answered": 52, "wrong_valid": 24, "failed": 0, "accuracy": 0.5384615384615384, "accuracy_among_usable": 0.5384615384615384 }, "F04": { "n": 54, "correct": 49, "answered": 54, "wrong_valid": 5, "failed": 0, "accuracy": 0.9074074074074074, "accuracy_among_usable": 0.9074074074074074 }, "F05": { "n": 49, "correct": 46, "answered": 49, "wrong_valid": 3, "failed": 0, "accuracy": 0.9387755102040817, "accuracy_among_usable": 0.9387755102040817 }, "F06": { "n": 40, "correct": 37, "answered": 40, "wrong_valid": 3, "failed": 0, "accuracy": 0.925, "accuracy_among_usable": 0.925 }, "F07": { "n": 76, "correct": 50, "answered": 76, "wrong_valid": 26, "failed": 0, "accuracy": 0.6578947368421053, "accuracy_among_usable": 0.6578947368421053 }, "F08": { "n": 67, "correct": 52, "answered": 67, "wrong_valid": 15, "failed": 0, "accuracy": 0.7761194029850746, "accuracy_among_usable": 0.7761194029850746 }, "F09": { "n": 71, "correct": 37, "answered": 71, "wrong_valid": 34, "failed": 0, "accuracy": 0.5211267605633803, "accuracy_among_usable": 0.5211267605633803 }, "F10": { "n": 74, "correct": 39, "answered": 74, "wrong_valid": 35, "failed": 0, "accuracy": 0.527027027027027, "accuracy_among_usable": 0.527027027027027 }, "F11": { "n": 41, "correct": 30, "answered": 41, "wrong_valid": 11, "failed": 0, "accuracy": 0.7317073170731707, "accuracy_among_usable": 0.7317073170731707 }, "F12": { "n": 52, "correct": 34, "answered": 52, "wrong_valid": 18, "failed": 0, "accuracy": 0.6538461538461539, "accuracy_among_usable": 0.6538461538461539 }, "F13": { "n": 48, "correct": 12, "answered": 48, "wrong_valid": 36, "failed": 0, "accuracy": 0.25, "accuracy_among_usable": 0.25 }, "F14": { "n": 42, "correct": 35, "answered": 42, "wrong_valid": 7, "failed": 0, "accuracy": 0.8333333333333334, "accuracy_among_usable": 0.8333333333333334 }, "F15": { "n": 49, "correct": 31, "answered": 49, "wrong_valid": 18, "failed": 0, "accuracy": 0.6326530612244898, "accuracy_among_usable": 0.6326530612244898 }, "F16": { "n": 50, "correct": 35, "answered": 50, "wrong_valid": 15, "failed": 0, "accuracy": 0.7, "accuracy_among_usable": 0.7 }, "F17": { "n": 50, "correct": 43, "answered": 50, "wrong_valid": 7, "failed": 0, "accuracy": 0.86, "accuracy_among_usable": 0.86 }, "F18": { "n": 50, "correct": 35, "answered": 50, "wrong_valid": 15, "failed": 0, "accuracy": 0.7, "accuracy_among_usable": 0.7 }, "F19": { "n": 58, "correct": 55, "answered": 58, "wrong_valid": 3, "failed": 0, "accuracy": 0.9482758620689655, "accuracy_among_usable": 0.9482758620689655 }, "F20": { "n": 43, "correct": 38, "answered": 43, "wrong_valid": 5, "failed": 0, "accuracy": 0.8837209302325582, "accuracy_among_usable": 0.8837209302325582 }, "F21": { "n": 43, "correct": 40, "answered": 43, "wrong_valid": 3, "failed": 0, "accuracy": 0.9302325581395349, "accuracy_among_usable": 0.9302325581395349 }, "F22": { "n": 51, "correct": 47, "answered": 51, "wrong_valid": 4, "failed": 0, "accuracy": 0.9215686274509803, "accuracy_among_usable": 0.9215686274509803 }, "F23": { "n": 52, "correct": 51, "answered": 52, "wrong_valid": 1, "failed": 0, "accuracy": 0.9807692307692307, "accuracy_among_usable": 0.9807692307692307 }, "F24": { "n": 43, "correct": 40, "answered": 43, "wrong_valid": 3, "failed": 0, "accuracy": 0.9302325581395349, "accuracy_among_usable": 0.9302325581395349 }, "F25": { "n": 50, "correct": 46, "answered": 50, "wrong_valid": 4, "failed": 0, "accuracy": 0.92, "accuracy_among_usable": 0.92 }, "F26": { "n": 8, "correct": 5, "answered": 8, "wrong_valid": 3, "failed": 0, "accuracy": 0.625, "accuracy_among_usable": 0.625 }, "F27": { "n": 8, "correct": 3, "answered": 8, "wrong_valid": 5, "failed": 0, "accuracy": 0.375, "accuracy_among_usable": 0.375 }, "F28": { "n": 8, "correct": 3, "answered": 8, "wrong_valid": 5, "failed": 0, "accuracy": 0.375, "accuracy_among_usable": 0.375 }, "F29": { "n": 8, "correct": 5, "answered": 8, "wrong_valid": 3, "failed": 0, "accuracy": 0.625, "accuracy_among_usable": 0.625 }, "F30": { "n": 8, "correct": 5, "answered": 8, "wrong_valid": 3, "failed": 0, "accuracy": 0.625, "accuracy_among_usable": 0.625 }, "F31": { "n": 8, "correct": 5, "answered": 8, "wrong_valid": 3, "failed": 0, "accuracy": 0.625, "accuracy_among_usable": 0.625 } }, "macro_family_accuracy": 0.7334405017644122, "release_exclusion_diagnostic": { "n": 1305, "correct": 1008, "answered": 1305, "wrong_valid": 297, "failed": 0, "accuracy": 0.7724137931034483, "accuracy_among_usable": 0.7724137931034483 }, "new_domains": { "n": 48, "correct": 26, "answered": 48, "wrong_valid": 22, "failed": 0, "accuracy": 0.5416666666666666, "accuracy_among_usable": 0.5416666666666666 } }, "jev113": { "n": 1342, "correct": 1026, "answered": 1342, "wrong_valid": 316, "failed": 0, "accuracy": 0.7645305514157973, "accuracy_among_usable": 0.7645305514157973, "eventual": { "n": 1342, "correct": 1026, "answered": 1342, "wrong_valid": 316, "failed": 0, "accuracy": 0.7645305514157973, "accuracy_among_usable": 0.7645305514157973 }, "median_ms": 189.37308349995874, "p95_ms": 322.841158945812, "attempts": 1342, "recoveries": 0, "first_statuses": { "ok": 1342 }, "model_output_failures": 0, "probability_warnings": 11, "cost": { "known_usd": 0.03297441, "known_decimal": "0.032974410", "unknown_attempts": 0, "exact_usd": 0.03297441, "cost_sources": { "recorded_accounted_or_reported": 1134, "api_reported": 208 }, "known_usd_per_1000": 0.024571095380029807 }, "confidence_diagnostics": { "0.8": { "n": 719, "correct": 702, "answered": 719, "wrong_valid": 17, "failed": 0, "accuracy": 0.9763560500695411, "accuracy_among_usable": 0.9763560500695411 }, "0.9": { "n": 633, "correct": 629, "answered": 633, "wrong_valid": 4, "failed": 0, "accuracy": 0.9936808846761453, "accuracy_among_usable": 0.9936808846761453 }, "0.95": { "n": 562, "correct": 561, "answered": 562, "wrong_valid": 1, "failed": 0, "accuracy": 0.998220640569395, "accuracy_among_usable": 0.998220640569395 } }, "latency_first_usable": { "n": 1342, "median_ms": 189.37308349995874, "p95_ms": 322.841158945812 }, "provider_failures": 0, "dates_toronto": { "2026-10-15": 1134, "2026-10-18": 2, "2026-10-19": 206 }, "returned_identities": { "typesafe/jev-1.13-20260917": 1342 }, "providers": { "TypeSafe": 1342 }, "model_id": "jev113", "label": "Jev 1.13", "route": "OpenRouter", "families": { "F01": { "n": 43, "correct": 41, "answered": 43, "wrong_valid": 2, "failed": 0, "accuracy": 0.9534883720930233, "accuracy_among_usable": 0.9534883720930233 }, "F02": { "n": 46, "correct": 45, "answered": 46, "wrong_valid": 1, "failed": 0, "accuracy": 0.9782608695652174, "accuracy_among_usable": 0.9782608695652174 }, "F03": { "n": 52, "correct": 49, "answered": 52, "wrong_valid": 3, "failed": 0, "accuracy": 0.9423076923076923, "accuracy_among_usable": 0.9423076923076923 }, "F04": { "n": 54, "correct": 51, "answered": 54, "wrong_valid": 3, "failed": 0, "accuracy": 0.9444444444444444, "accuracy_among_usable": 0.9444444444444444 }, "F05": { "n": 49, "correct": 44, "answered": 49, "wrong_valid": 5, "failed": 0, "accuracy": 0.8979591836734694, "accuracy_among_usable": 0.8979591836734694 }, "F06": { "n": 40, "correct": 40, "answered": 40, "wrong_valid": 0, "failed": 0, "accuracy": 1.0, "accuracy_among_usable": 1.0 }, "F07": { "n": 76, "correct": 50, "answered": 76, "wrong_valid": 26, "failed": 0, "accuracy": 0.6578947368421053, "accuracy_among_usable": 0.6578947368421053 }, "F08": { "n": 67, "correct": 54, "answered": 67, "wrong_valid": 13, "failed": 0, "accuracy": 0.8059701492537313, "accuracy_among_usable": 0.8059701492537313 }, "F09": { "n": 71, "correct": 25, "answered": 71, "wrong_valid": 46, "failed": 0, "accuracy": 0.352112676056338, "accuracy_among_usable": 0.352112676056338 }, "F10": { "n": 74, "correct": 38, "answered": 74, "wrong_valid": 36, "failed": 0, "accuracy": 0.5135135135135135, "accuracy_among_usable": 0.5135135135135135 }, "F11": { "n": 41, "correct": 29, "answered": 41, "wrong_valid": 12, "failed": 0, "accuracy": 0.7073170731707317, "accuracy_among_usable": 0.7073170731707317 }, "F12": { "n": 52, "correct": 29, "answered": 52, "wrong_valid": 23, "failed": 0, "accuracy": 0.5576923076923077, "accuracy_among_usable": 0.5576923076923077 }, "F13": { "n": 48, "correct": 10, "answered": 48, "wrong_valid": 38, "failed": 0, "accuracy": 0.20833333333333334, "accuracy_among_usable": 0.20833333333333334 }, "F14": { "n": 42, "correct": 41, "answered": 42, "wrong_valid": 1, "failed": 0, "accuracy": 0.9761904761904762, "accuracy_among_usable": 0.9761904761904762 }, "F15": { "n": 49, "correct": 30, "answered": 49, "wrong_valid": 19, "failed": 0, "accuracy": 0.6122448979591837, "accuracy_among_usable": 0.6122448979591837 }, "F16": { "n": 50, "correct": 33, "answered": 50, "wrong_valid": 17, "failed": 0, "accuracy": 0.66, "accuracy_among_usable": 0.66 }, "F17": { "n": 50, "correct": 47, "answered": 50, "wrong_valid": 3, "failed": 0, "accuracy": 0.94, "accuracy_among_usable": 0.94 }, "F18": { "n": 50, "correct": 42, "answered": 50, "wrong_valid": 8, "failed": 0, "accuracy": 0.84, "accuracy_among_usable": 0.84 }, "F19": { "n": 58, "correct": 51, "answered": 58, "wrong_valid": 7, "failed": 0, "accuracy": 0.8793103448275862, "accuracy_among_usable": 0.8793103448275862 }, "F20": { "n": 43, "correct": 38, "answered": 43, "wrong_valid": 5, "failed": 0, "accuracy": 0.8837209302325582, "accuracy_among_usable": 0.8837209302325582 }, "F21": { "n": 43, "correct": 41, "answered": 43, "wrong_valid": 2, "failed": 0, "accuracy": 0.9534883720930233, "accuracy_among_usable": 0.9534883720930233 }, "F22": { "n": 51, "correct": 46, "answered": 51, "wrong_valid": 5, "failed": 0, "accuracy": 0.9019607843137255, "accuracy_among_usable": 0.9019607843137255 }, "F23": { "n": 52, "correct": 48, "answered": 52, "wrong_valid": 4, "failed": 0, "accuracy": 0.9230769230769231, "accuracy_among_usable": 0.9230769230769231 }, "F24": { "n": 43, "correct": 39, "answered": 43, "wrong_valid": 4, "failed": 0, "accuracy": 0.9069767441860465, "accuracy_among_usable": 0.9069767441860465 }, "F25": { "n": 50, "correct": 44, "answered": 50, "wrong_valid": 6, "failed": 0, "accuracy": 0.88, "accuracy_among_usable": 0.88 }, "F26": { "n": 8, "correct": 5, "answered": 8, "wrong_valid": 3, "failed": 0, "accuracy": 0.625, "accuracy_among_usable": 0.625 }, "F27": { "n": 8, "correct": 3, "answered": 8, "wrong_valid": 5, "failed": 0, "accuracy": 0.375, "accuracy_among_usable": 0.375 }, "F28": { "n": 8, "correct": 2, "answered": 8, "wrong_valid": 6, "failed": 0, "accuracy": 0.25, "accuracy_among_usable": 0.25 }, "F29": { "n": 8, "correct": 3, "answered": 8, "wrong_valid": 5, "failed": 0, "accuracy": 0.375, "accuracy_among_usable": 0.375 }, "F30": { "n": 8, "correct": 4, "answered": 8, "wrong_valid": 4, "failed": 0, "accuracy": 0.5, "accuracy_among_usable": 0.5 }, "F31": { "n": 8, "correct": 4, "answered": 8, "wrong_valid": 4, "failed": 0, "accuracy": 0.5, "accuracy_among_usable": 0.5 } }, "macro_family_accuracy": 0.7258472201556591, "release_exclusion_diagnostic": { "n": 1305, "correct": 1009, "answered": 1305, "wrong_valid": 296, "failed": 0, "accuracy": 0.7731800766283525, "accuracy_among_usable": 0.7731800766283525 }, "new_domains": { "n": 48, "correct": 21, "answered": 48, "wrong_valid": 27, "failed": 0, "accuracy": 0.4375, "accuracy_among_usable": 0.4375 } }, "luna_decisions": { "n": 1342, "correct": 758, "answered": 1300, "wrong_valid": 542, "failed": 42, "accuracy": 0.5648286140089419, "accuracy_among_usable": 0.583076923076923, "eventual": { "n": 1342, "correct": 761, "answered": 1306, "wrong_valid": 545, "failed": 36, "accuracy": 0.5670640834575261, "accuracy_among_usable": 0.5826952526799387 }, "median_ms": 253.59799998113886, "p95_ms": 527.3192601103807, "attempts": 1384, "recoveries": 42, "first_statuses": { "ok": 1300, "provider_failure": 42 }, "model_output_failures": 0, "probability_warnings": 0, "cost": { "known_usd": 0.0480967, "known_decimal": "0.0480967", "unknown_attempts": 78, "exact_usd": null, "cost_sources": { "recorded_accounted_or_reported": 1106, "unreported": 78, "api_reported": 200 }, "known_usd_per_1000": 0.03583956780923994 }, "confidence_diagnostics": { "0.8": { "n": 222, "correct": 218, "answered": 222, "wrong_valid": 4, "failed": 0, "accuracy": 0.9819819819819819, "accuracy_among_usable": 0.9819819819819819 }, "0.9": { "n": 144, "correct": 144, "answered": 144, "wrong_valid": 0, "failed": 0, "accuracy": 1.0, "accuracy_among_usable": 1.0 }, "0.95": { "n": 90, "correct": 90, "answered": 90, "wrong_valid": 0, "failed": 0, "accuracy": 1.0, "accuracy_among_usable": 1.0 } }, "latency_first_usable": { "n": 1300, "median_ms": 253.08333351858892, "p95_ms": 495.9701889660206 }, "provider_failures": 42, "dates_toronto": { "2026-10-15": 1134, "2026-10-18": 2, "2026-10-19": 206 }, "returned_identities": { "openai/gpt-6-luna-decisions-20261006": 1300, "unreported (error)": 42 }, "providers": { "OpenAI": 1300, "unreported": 42 }, "model_id": "luna_decisions", "label": "Luna 6 Decisions", "route": "OpenRouter", "families": { "F01": { "n": 43, "correct": 39, "answered": 42, "wrong_valid": 3, "failed": 1, "accuracy": 0.9069767441860465, "accuracy_among_usable": 0.9285714285714286 }, "F02": { "n": 46, "correct": 27, "answered": 42, "wrong_valid": 15, "failed": 4, "accuracy": 0.5869565217391305, "accuracy_among_usable": 0.6428571428571429 }, "F03": { "n": 52, "correct": 15, "answered": 52, "wrong_valid": 37, "failed": 0, "accuracy": 0.28846153846153844, "accuracy_among_usable": 0.28846153846153844 }, "F04": { "n": 54, "correct": 34, "answered": 53, "wrong_valid": 19, "failed": 1, "accuracy": 0.6296296296296297, "accuracy_among_usable": 0.6415094339622641 }, "F05": { "n": 49, "correct": 38, "answered": 48, "wrong_valid": 10, "failed": 1, "accuracy": 0.7755102040816326, "accuracy_among_usable": 0.7916666666666666 }, "F06": { "n": 40, "correct": 35, "answered": 39, "wrong_valid": 4, "failed": 1, "accuracy": 0.875, "accuracy_among_usable": 0.8974358974358975 }, "F07": { "n": 76, "correct": 28, "answered": 73, "wrong_valid": 45, "failed": 3, "accuracy": 0.3684210526315789, "accuracy_among_usable": 0.3835616438356164 }, "F08": { "n": 67, "correct": 36, "answered": 54, "wrong_valid": 18, "failed": 13, "accuracy": 0.5373134328358209, "accuracy_among_usable": 0.6666666666666666 }, "F09": { "n": 71, "correct": 19, "answered": 70, "wrong_valid": 51, "failed": 1, "accuracy": 0.2676056338028169, "accuracy_among_usable": 0.2714285714285714 }, "F10": { "n": 74, "correct": 23, "answered": 72, "wrong_valid": 49, "failed": 2, "accuracy": 0.3108108108108108, "accuracy_among_usable": 0.3194444444444444 }, "F11": { "n": 41, "correct": 22, "answered": 41, "wrong_valid": 19, "failed": 0, "accuracy": 0.5365853658536586, "accuracy_among_usable": 0.5365853658536586 }, "F12": { "n": 52, "correct": 23, "answered": 50, "wrong_valid": 27, "failed": 2, "accuracy": 0.4423076923076923, "accuracy_among_usable": 0.46 }, "F13": { "n": 48, "correct": 11, "answered": 48, "wrong_valid": 37, "failed": 0, "accuracy": 0.22916666666666666, "accuracy_among_usable": 0.22916666666666666 }, "F14": { "n": 42, "correct": 25, "answered": 41, "wrong_valid": 16, "failed": 1, "accuracy": 0.5952380952380952, "accuracy_among_usable": 0.6097560975609756 }, "F15": { "n": 49, "correct": 22, "answered": 45, "wrong_valid": 23, "failed": 4, "accuracy": 0.4489795918367347, "accuracy_among_usable": 0.4888888888888889 }, "F16": { "n": 50, "correct": 17, "answered": 50, "wrong_valid": 33, "failed": 0, "accuracy": 0.34, "accuracy_among_usable": 0.34 }, "F17": { "n": 50, "correct": 32, "answered": 48, "wrong_valid": 16, "failed": 2, "accuracy": 0.64, "accuracy_among_usable": 0.6666666666666666 }, "F18": { "n": 50, "correct": 21, "answered": 48, "wrong_valid": 27, "failed": 2, "accuracy": 0.42, "accuracy_among_usable": 0.4375 }, "F19": { "n": 58, "correct": 47, "answered": 57, "wrong_valid": 10, "failed": 1, "accuracy": 0.8103448275862069, "accuracy_among_usable": 0.8245614035087719 }, "F20": { "n": 43, "correct": 25, "answered": 42, "wrong_valid": 17, "failed": 1, "accuracy": 0.5813953488372093, "accuracy_among_usable": 0.5952380952380952 }, "F21": { "n": 43, "correct": 39, "answered": 42, "wrong_valid": 3, "failed": 1, "accuracy": 0.9069767441860465, "accuracy_among_usable": 0.9285714285714286 }, "F22": { "n": 51, "correct": 44, "answered": 51, "wrong_valid": 7, "failed": 0, "accuracy": 0.8627450980392157, "accuracy_among_usable": 0.8627450980392157 }, "F23": { "n": 52, "correct": 44, "answered": 51, "wrong_valid": 7, "failed": 1, "accuracy": 0.8461538461538461, "accuracy_among_usable": 0.8627450980392157 }, "F24": { "n": 43, "correct": 36, "answered": 43, "wrong_valid": 7, "failed": 0, "accuracy": 0.8372093023255814, "accuracy_among_usable": 0.8372093023255814 }, "F25": { "n": 50, "correct": 34, "answered": 50, "wrong_valid": 16, "failed": 0, "accuracy": 0.68, "accuracy_among_usable": 0.68 }, "F26": { "n": 8, "correct": 5, "answered": 8, "wrong_valid": 3, "failed": 0, "accuracy": 0.625, "accuracy_among_usable": 0.625 }, "F27": { "n": 8, "correct": 4, "answered": 8, "wrong_valid": 4, "failed": 0, "accuracy": 0.5, "accuracy_among_usable": 0.5 }, "F28": { "n": 8, "correct": 4, "answered": 8, "wrong_valid": 4, "failed": 0, "accuracy": 0.5, "accuracy_among_usable": 0.5 }, "F29": { "n": 8, "correct": 2, "answered": 8, "wrong_valid": 6, "failed": 0, "accuracy": 0.25, "accuracy_among_usable": 0.25 }, "F30": { "n": 8, "correct": 3, "answered": 8, "wrong_valid": 5, "failed": 0, "accuracy": 0.375, "accuracy_among_usable": 0.375 }, "F31": { "n": 8, "correct": 4, "answered": 8, "wrong_valid": 4, "failed": 0, "accuracy": 0.5, "accuracy_among_usable": 0.5 } }, "macro_family_accuracy": 0.5636705853938696, "release_exclusion_diagnostic": { "n": 1305, "correct": 749, "answered": 1267, "wrong_valid": 518, "failed": 38, "accuracy": 0.5739463601532567, "accuracy_among_usable": 0.5911602209944752 }, "new_domains": { "n": 48, "correct": 22, "answered": 48, "wrong_valid": 26, "failed": 0, "accuracy": 0.4583333333333333, "accuracy_among_usable": 0.4583333333333333 } }, "microsoft_decision": { "n": 1342, "correct": 921, "answered": 1342, "wrong_valid": 421, "failed": 0, "accuracy": 0.6862891207153502, "accuracy_among_usable": 0.6862891207153502, "eventual": { "n": 1342, "correct": 921, "answered": 1342, "wrong_valid": 421, "failed": 0, "accuracy": 0.6862891207153502, "accuracy_among_usable": 0.6862891207153502 }, "median_ms": 283.31577096832916, "p95_ms": 565.7315687392838, "attempts": 1342, "recoveries": 0, "first_statuses": { "ok": 1342 }, "model_output_failures": 0, "probability_warnings": 0, "cost": { "known_usd": 0.016421832, "known_decimal": "0.016421832", "unknown_attempts": 0, "exact_usd": 0.016421832, "cost_sources": { "api_reported": 1342 }, "known_usd_per_1000": 0.012236834575260806 }, "confidence_diagnostics": { "0.8": { "n": 715, "correct": 654, "answered": 715, "wrong_valid": 61, "failed": 0, "accuracy": 0.9146853146853147, "accuracy_among_usable": 0.9146853146853147 }, "0.9": { "n": 529, "correct": 512, "answered": 529, "wrong_valid": 17, "failed": 0, "accuracy": 0.9678638941398866, "accuracy_among_usable": 0.9678638941398866 }, "0.95": { "n": 389, "correct": 389, "answered": 389, "wrong_valid": 0, "failed": 0, "accuracy": 1.0, "accuracy_among_usable": 1.0 } }, "latency_first_usable": { "n": 1342, "median_ms": 283.31577096832916, "p95_ms": 565.7315687392838 }, "provider_failures": 0, "dates_toronto": { "2026-10-16": 1134, "2026-10-18": 2, "2026-10-19": 206 }, "returned_identities": { "microsoft/microsoft-decision-1-20261009": 1342 }, "providers": { "Azure": 1342 }, "model_id": "microsoft_decision", "label": "Microsoft-Decision-1", "route": "OpenRouter", "families": { "F01": { "n": 43, "correct": 38, "answered": 43, "wrong_valid": 5, "failed": 0, "accuracy": 0.8837209302325582, "accuracy_among_usable": 0.8837209302325582 }, "F02": { "n": 46, "correct": 38, "answered": 46, "wrong_valid": 8, "failed": 0, "accuracy": 0.8260869565217391, "accuracy_among_usable": 0.8260869565217391 }, "F03": { "n": 52, "correct": 25, "answered": 52, "wrong_valid": 27, "failed": 0, "accuracy": 0.4807692307692308, "accuracy_among_usable": 0.4807692307692308 }, "F04": { "n": 54, "correct": 37, "answered": 54, "wrong_valid": 17, "failed": 0, "accuracy": 0.6851851851851852, "accuracy_among_usable": 0.6851851851851852 }, "F05": { "n": 49, "correct": 41, "answered": 49, "wrong_valid": 8, "failed": 0, "accuracy": 0.8367346938775511, "accuracy_among_usable": 0.8367346938775511 }, "F06": { "n": 40, "correct": 39, "answered": 40, "wrong_valid": 1, "failed": 0, "accuracy": 0.975, "accuracy_among_usable": 0.975 }, "F07": { "n": 76, "correct": 41, "answered": 76, "wrong_valid": 35, "failed": 0, "accuracy": 0.5394736842105263, "accuracy_among_usable": 0.5394736842105263 }, "F08": { "n": 67, "correct": 44, "answered": 67, "wrong_valid": 23, "failed": 0, "accuracy": 0.6567164179104478, "accuracy_among_usable": 0.6567164179104478 }, "F09": { "n": 71, "correct": 40, "answered": 71, "wrong_valid": 31, "failed": 0, "accuracy": 0.5633802816901409, "accuracy_among_usable": 0.5633802816901409 }, "F10": { "n": 74, "correct": 36, "answered": 74, "wrong_valid": 38, "failed": 0, "accuracy": 0.4864864864864865, "accuracy_among_usable": 0.4864864864864865 }, "F11": { "n": 41, "correct": 28, "answered": 41, "wrong_valid": 13, "failed": 0, "accuracy": 0.6829268292682927, "accuracy_among_usable": 0.6829268292682927 }, "F12": { "n": 52, "correct": 24, "answered": 52, "wrong_valid": 28, "failed": 0, "accuracy": 0.46153846153846156, "accuracy_among_usable": 0.46153846153846156 }, "F13": { "n": 48, "correct": 7, "answered": 48, "wrong_valid": 41, "failed": 0, "accuracy": 0.14583333333333334, "accuracy_among_usable": 0.14583333333333334 }, "F14": { "n": 42, "correct": 33, "answered": 42, "wrong_valid": 9, "failed": 0, "accuracy": 0.7857142857142857, "accuracy_among_usable": 0.7857142857142857 }, "F15": { "n": 49, "correct": 24, "answered": 49, "wrong_valid": 25, "failed": 0, "accuracy": 0.4897959183673469, "accuracy_among_usable": 0.4897959183673469 }, "F16": { "n": 50, "correct": 32, "answered": 50, "wrong_valid": 18, "failed": 0, "accuracy": 0.64, "accuracy_among_usable": 0.64 }, "F17": { "n": 50, "correct": 42, "answered": 50, "wrong_valid": 8, "failed": 0, "accuracy": 0.84, "accuracy_among_usable": 0.84 }, "F18": { "n": 50, "correct": 24, "answered": 50, "wrong_valid": 26, "failed": 0, "accuracy": 0.48, "accuracy_among_usable": 0.48 }, "F19": { "n": 58, "correct": 52, "answered": 58, "wrong_valid": 6, "failed": 0, "accuracy": 0.896551724137931, "accuracy_among_usable": 0.896551724137931 }, "F20": { "n": 43, "correct": 37, "answered": 43, "wrong_valid": 6, "failed": 0, "accuracy": 0.8604651162790697, "accuracy_among_usable": 0.8604651162790697 }, "F21": { "n": 43, "correct": 40, "answered": 43, "wrong_valid": 3, "failed": 0, "accuracy": 0.9302325581395349, "accuracy_among_usable": 0.9302325581395349 }, "F22": { "n": 51, "correct": 44, "answered": 51, "wrong_valid": 7, "failed": 0, "accuracy": 0.8627450980392157, "accuracy_among_usable": 0.8627450980392157 }, "F23": { "n": 52, "correct": 47, "answered": 52, "wrong_valid": 5, "failed": 0, "accuracy": 0.9038461538461539, "accuracy_among_usable": 0.9038461538461539 }, "F24": { "n": 43, "correct": 39, "answered": 43, "wrong_valid": 4, "failed": 0, "accuracy": 0.9069767441860465, "accuracy_among_usable": 0.9069767441860465 }, "F25": { "n": 50, "correct": 46, "answered": 50, "wrong_valid": 4, "failed": 0, "accuracy": 0.92, "accuracy_among_usable": 0.92 }, "F26": { "n": 8, "correct": 4, "answered": 8, "wrong_valid": 4, "failed": 0, "accuracy": 0.5, "accuracy_among_usable": 0.5 }, "F27": { "n": 8, "correct": 4, "answered": 8, "wrong_valid": 4, "failed": 0, "accuracy": 0.5, "accuracy_among_usable": 0.5 }, "F28": { "n": 8, "correct": 2, "answered": 8, "wrong_valid": 6, "failed": 0, "accuracy": 0.25, "accuracy_among_usable": 0.25 }, "F29": { "n": 8, "correct": 3, "answered": 8, "wrong_valid": 5, "failed": 0, "accuracy": 0.375, "accuracy_among_usable": 0.375 }, "F30": { "n": 8, "correct": 4, "answered": 8, "wrong_valid": 4, "failed": 0, "accuracy": 0.5, "accuracy_among_usable": 0.5 }, "F31": { "n": 8, "correct": 6, "answered": 8, "wrong_valid": 2, "failed": 0, "accuracy": 0.75, "accuracy_among_usable": 0.75 } }, "macro_family_accuracy": 0.6650058093462433, "release_exclusion_diagnostic": { "n": 1305, "correct": 909, "answered": 1305, "wrong_valid": 396, "failed": 0, "accuracy": 0.696551724137931, "accuracy_among_usable": 0.696551724137931 }, "new_domains": { "n": 48, "correct": 23, "answered": 48, "wrong_valid": 25, "failed": 0, "accuracy": 0.4791666666666667, "accuracy_among_usable": 0.4791666666666667 } }, "liquid_d1": { "n": 1342, "correct": 965, "answered": 1342, "wrong_valid": 377, "failed": 0, "accuracy": 0.7190760059612519, "accuracy_among_usable": 0.7190760059612519, "eventual": { "n": 1342, "correct": 965, "answered": 1342, "wrong_valid": 377, "failed": 0, "accuracy": 0.7190760059612519, "accuracy_among_usable": 0.7190760059612519 }, "median_ms": 206.22245804406703, "p95_ms": 826.7250909528241, "attempts": 1342, "recoveries": 0, "first_statuses": { "ok": 1342 }, "model_output_failures": 0, "probability_warnings": 0, "cost": { "known_usd": 0.00774762, "known_decimal": "0.00774762", "unknown_attempts": 0, "exact_usd": 0.00774762, "cost_sources": { "api_reported": 1342 }, "known_usd_per_1000": 0.005773189269746647 }, "confidence_diagnostics": { "0.8": { "n": 701, "correct": 674, "answered": 701, "wrong_valid": 27, "failed": 0, "accuracy": 0.9614835948644793, "accuracy_among_usable": 0.9614835948644793 }, "0.9": { "n": 617, "correct": 610, "answered": 617, "wrong_valid": 7, "failed": 0, "accuracy": 0.9886547811993517, "accuracy_among_usable": 0.9886547811993517 }, "0.95": { "n": 567, "correct": 565, "answered": 567, "wrong_valid": 2, "failed": 0, "accuracy": 0.9964726631393298, "accuracy_among_usable": 0.9964726631393298 } }, "latency_first_usable": { "n": 1342, "median_ms": 206.22245804406703, "p95_ms": 826.7250909528241 }, "provider_failures": 0, "dates_toronto": { "2026-10-18": 1136, "2026-10-19": 206 }, "returned_identities": { "liquid/d1-20260930": 1342 }, "providers": { "Liquid": 1342 }, "model_id": "liquid_d1", "label": "Liquid d1", "route": "OpenRouter", "families": { "F01": { "n": 43, "correct": 40, "answered": 43, "wrong_valid": 3, "failed": 0, "accuracy": 0.9302325581395349, "accuracy_among_usable": 0.9302325581395349 }, "F02": { "n": 46, "correct": 41, "answered": 46, "wrong_valid": 5, "failed": 0, "accuracy": 0.8913043478260869, "accuracy_among_usable": 0.8913043478260869 }, "F03": { "n": 52, "correct": 35, "answered": 52, "wrong_valid": 17, "failed": 0, "accuracy": 0.6730769230769231, "accuracy_among_usable": 0.6730769230769231 }, "F04": { "n": 54, "correct": 50, "answered": 54, "wrong_valid": 4, "failed": 0, "accuracy": 0.9259259259259259, "accuracy_among_usable": 0.9259259259259259 }, "F05": { "n": 49, "correct": 42, "answered": 49, "wrong_valid": 7, "failed": 0, "accuracy": 0.8571428571428571, "accuracy_among_usable": 0.8571428571428571 }, "F06": { "n": 40, "correct": 38, "answered": 40, "wrong_valid": 2, "failed": 0, "accuracy": 0.95, "accuracy_among_usable": 0.95 }, "F07": { "n": 76, "correct": 47, "answered": 76, "wrong_valid": 29, "failed": 0, "accuracy": 0.618421052631579, "accuracy_among_usable": 0.618421052631579 }, "F08": { "n": 67, "correct": 48, "answered": 67, "wrong_valid": 19, "failed": 0, "accuracy": 0.7164179104477612, "accuracy_among_usable": 0.7164179104477612 }, "F09": { "n": 71, "correct": 23, "answered": 71, "wrong_valid": 48, "failed": 0, "accuracy": 0.323943661971831, "accuracy_among_usable": 0.323943661971831 }, "F10": { "n": 74, "correct": 40, "answered": 74, "wrong_valid": 34, "failed": 0, "accuracy": 0.5405405405405406, "accuracy_among_usable": 0.5405405405405406 }, "F11": { "n": 41, "correct": 32, "answered": 41, "wrong_valid": 9, "failed": 0, "accuracy": 0.7804878048780488, "accuracy_among_usable": 0.7804878048780488 }, "F12": { "n": 52, "correct": 33, "answered": 52, "wrong_valid": 19, "failed": 0, "accuracy": 0.6346153846153846, "accuracy_among_usable": 0.6346153846153846 }, "F13": { "n": 48, "correct": 9, "answered": 48, "wrong_valid": 39, "failed": 0, "accuracy": 0.1875, "accuracy_among_usable": 0.1875 }, "F14": { "n": 42, "correct": 37, "answered": 42, "wrong_valid": 5, "failed": 0, "accuracy": 0.8809523809523809, "accuracy_among_usable": 0.8809523809523809 }, "F15": { "n": 49, "correct": 24, "answered": 49, "wrong_valid": 25, "failed": 0, "accuracy": 0.4897959183673469, "accuracy_among_usable": 0.4897959183673469 }, "F16": { "n": 50, "correct": 29, "answered": 50, "wrong_valid": 21, "failed": 0, "accuracy": 0.58, "accuracy_among_usable": 0.58 }, "F17": { "n": 50, "correct": 41, "answered": 50, "wrong_valid": 9, "failed": 0, "accuracy": 0.82, "accuracy_among_usable": 0.82 }, "F18": { "n": 50, "correct": 28, "answered": 50, "wrong_valid": 22, "failed": 0, "accuracy": 0.56, "accuracy_among_usable": 0.56 }, "F19": { "n": 58, "correct": 51, "answered": 58, "wrong_valid": 7, "failed": 0, "accuracy": 0.8793103448275862, "accuracy_among_usable": 0.8793103448275862 }, "F20": { "n": 43, "correct": 39, "answered": 43, "wrong_valid": 4, "failed": 0, "accuracy": 0.9069767441860465, "accuracy_among_usable": 0.9069767441860465 }, "F21": { "n": 43, "correct": 40, "answered": 43, "wrong_valid": 3, "failed": 0, "accuracy": 0.9302325581395349, "accuracy_among_usable": 0.9302325581395349 }, "F22": { "n": 51, "correct": 46, "answered": 51, "wrong_valid": 5, "failed": 0, "accuracy": 0.9019607843137255, "accuracy_among_usable": 0.9019607843137255 }, "F23": { "n": 52, "correct": 49, "answered": 52, "wrong_valid": 3, "failed": 0, "accuracy": 0.9423076923076923, "accuracy_among_usable": 0.9423076923076923 }, "F24": { "n": 43, "correct": 38, "answered": 43, "wrong_valid": 5, "failed": 0, "accuracy": 0.8837209302325582, "accuracy_among_usable": 0.8837209302325582 }, "F25": { "n": 50, "correct": 46, "answered": 50, "wrong_valid": 4, "failed": 0, "accuracy": 0.92, "accuracy_among_usable": 0.92 }, "F26": { "n": 8, "correct": 3, "answered": 8, "wrong_valid": 5, "failed": 0, "accuracy": 0.375, "accuracy_among_usable": 0.375 }, "F27": { "n": 8, "correct": 3, "answered": 8, "wrong_valid": 5, "failed": 0, "accuracy": 0.375, "accuracy_among_usable": 0.375 }, "F28": { "n": 8, "correct": 2, "answered": 8, "wrong_valid": 6, "failed": 0, "accuracy": 0.25, "accuracy_among_usable": 0.25 }, "F29": { "n": 8, "correct": 3, "answered": 8, "wrong_valid": 5, "failed": 0, "accuracy": 0.375, "accuracy_among_usable": 0.375 }, "F30": { "n": 8, "correct": 3, "answered": 8, "wrong_valid": 5, "failed": 0, "accuracy": 0.375, "accuracy_among_usable": 0.375 }, "F31": { "n": 8, "correct": 5, "answered": 8, "wrong_valid": 3, "failed": 0, "accuracy": 0.625, "accuracy_among_usable": 0.625 } }, "macro_family_accuracy": 0.6806408490491404, "release_exclusion_diagnostic": { "n": 1305, "correct": 948, "answered": 1305, "wrong_valid": 357, "failed": 0, "accuracy": 0.7264367816091954, "accuracy_among_usable": 0.7264367816091954 }, "new_domains": { "n": 48, "correct": 19, "answered": 48, "wrong_valid": 29, "failed": 0, "accuracy": 0.3958333333333333, "accuracy_among_usable": 0.3958333333333333 } }, "pplx_flash12": { "n": 1342, "correct": 923, "answered": 1286, "wrong_valid": 363, "failed": 56, "accuracy": 0.687779433681073, "accuracy_among_usable": 0.7177293934681181, "eventual": { "n": 1342, "correct": 940, "answered": 1308, "wrong_valid": 368, "failed": 34, "accuracy": 0.7004470938897168, "accuracy_among_usable": 0.7186544342507645 }, "median_ms": 158.6297499598004, "p95_ms": 230.8821223326959, "attempts": 1398, "recoveries": 56, "first_statuses": { "ok": 1286, "provider_failure": 56 }, "model_output_failures": 0, "probability_warnings": 0, "cost": { "known_usd": 0.00457042, "known_decimal": "0.00457042", "unknown_attempts": 90, "exact_usd": null, "cost_sources": { "api_reported": 1308, "unreported": 90 }, "known_usd_per_1000": 0.003405678092399404 }, "confidence_diagnostics": { "0.8": { "n": 808, "correct": 748, "answered": 808, "wrong_valid": 60, "failed": 0, "accuracy": 0.9257425742574258, "accuracy_among_usable": 0.9257425742574258 }, "0.9": { "n": 713, "correct": 674, "answered": 713, "wrong_valid": 39, "failed": 0, "accuracy": 0.9453015427769986, "accuracy_among_usable": 0.9453015427769986 }, "0.95": { "n": 648, "correct": 629, "answered": 648, "wrong_valid": 19, "failed": 0, "accuracy": 0.970679012345679, "accuracy_among_usable": 0.970679012345679 } }, "latency_first_usable": { "n": 1286, "median_ms": 159.40052102087066, "p95_ms": 232.63112452696078 }, "provider_failures": 56, "dates_toronto": { "2026-10-18": 1136, "2026-10-19": 206 }, "returned_identities": { "perplexity/pplx-decider-flash-v1.2-20261016": 1286, "unreported (error)": 56 }, "providers": { "Perplexity": 1286, "unreported": 56 }, "model_id": "pplx_flash12", "label": "Perplexity Decider Flash v1.2", "route": "OpenRouter", "families": { "F01": { "n": 43, "correct": 36, "answered": 40, "wrong_valid": 4, "failed": 3, "accuracy": 0.8372093023255814, "accuracy_among_usable": 0.9 }, "F02": { "n": 46, "correct": 43, "answered": 46, "wrong_valid": 3, "failed": 0, "accuracy": 0.9347826086956522, "accuracy_among_usable": 0.9347826086956522 }, "F03": { "n": 52, "correct": 41, "answered": 51, "wrong_valid": 10, "failed": 1, "accuracy": 0.7884615384615384, "accuracy_among_usable": 0.803921568627451 }, "F04": { "n": 54, "correct": 38, "answered": 51, "wrong_valid": 13, "failed": 3, "accuracy": 0.7037037037037037, "accuracy_among_usable": 0.7450980392156863 }, "F05": { "n": 49, "correct": 38, "answered": 45, "wrong_valid": 7, "failed": 4, "accuracy": 0.7755102040816326, "accuracy_among_usable": 0.8444444444444444 }, "F06": { "n": 40, "correct": 37, "answered": 39, "wrong_valid": 2, "failed": 1, "accuracy": 0.925, "accuracy_among_usable": 0.9487179487179487 }, "F07": { "n": 76, "correct": 40, "answered": 73, "wrong_valid": 33, "failed": 3, "accuracy": 0.5263157894736842, "accuracy_among_usable": 0.547945205479452 }, "F08": { "n": 67, "correct": 47, "answered": 65, "wrong_valid": 18, "failed": 2, "accuracy": 0.7014925373134329, "accuracy_among_usable": 0.7230769230769231 }, "F09": { "n": 71, "correct": 38, "answered": 69, "wrong_valid": 31, "failed": 2, "accuracy": 0.5352112676056338, "accuracy_among_usable": 0.5507246376811594 }, "F10": { "n": 74, "correct": 30, "answered": 67, "wrong_valid": 37, "failed": 7, "accuracy": 0.40540540540540543, "accuracy_among_usable": 0.44776119402985076 }, "F11": { "n": 41, "correct": 30, "answered": 40, "wrong_valid": 10, "failed": 1, "accuracy": 0.7317073170731707, "accuracy_among_usable": 0.75 }, "F12": { "n": 52, "correct": 23, "answered": 50, "wrong_valid": 27, "failed": 2, "accuracy": 0.4423076923076923, "accuracy_among_usable": 0.46 }, "F13": { "n": 48, "correct": 12, "answered": 45, "wrong_valid": 33, "failed": 3, "accuracy": 0.25, "accuracy_among_usable": 0.26666666666666666 }, "F14": { "n": 42, "correct": 28, "answered": 41, "wrong_valid": 13, "failed": 1, "accuracy": 0.6666666666666666, "accuracy_among_usable": 0.6829268292682927 }, "F15": { "n": 49, "correct": 25, "answered": 47, "wrong_valid": 22, "failed": 2, "accuracy": 0.5102040816326531, "accuracy_among_usable": 0.5319148936170213 }, "F16": { "n": 50, "correct": 34, "answered": 49, "wrong_valid": 15, "failed": 1, "accuracy": 0.68, "accuracy_among_usable": 0.6938775510204082 }, "F17": { "n": 50, "correct": 42, "answered": 48, "wrong_valid": 6, "failed": 2, "accuracy": 0.84, "accuracy_among_usable": 0.875 }, "F18": { "n": 50, "correct": 29, "answered": 47, "wrong_valid": 18, "failed": 3, "accuracy": 0.58, "accuracy_among_usable": 0.6170212765957447 }, "F19": { "n": 58, "correct": 47, "answered": 52, "wrong_valid": 5, "failed": 6, "accuracy": 0.8103448275862069, "accuracy_among_usable": 0.9038461538461539 }, "F20": { "n": 43, "correct": 37, "answered": 42, "wrong_valid": 5, "failed": 1, "accuracy": 0.8604651162790697, "accuracy_among_usable": 0.8809523809523809 }, "F21": { "n": 43, "correct": 40, "answered": 42, "wrong_valid": 2, "failed": 1, "accuracy": 0.9302325581395349, "accuracy_among_usable": 0.9523809523809523 }, "F22": { "n": 51, "correct": 45, "answered": 49, "wrong_valid": 4, "failed": 2, "accuracy": 0.8823529411764706, "accuracy_among_usable": 0.9183673469387755 }, "F23": { "n": 52, "correct": 47, "answered": 50, "wrong_valid": 3, "failed": 2, "accuracy": 0.9038461538461539, "accuracy_among_usable": 0.94 }, "F24": { "n": 43, "correct": 39, "answered": 42, "wrong_valid": 3, "failed": 1, "accuracy": 0.9069767441860465, "accuracy_among_usable": 0.9285714285714286 }, "F25": { "n": 50, "correct": 39, "answered": 48, "wrong_valid": 9, "failed": 2, "accuracy": 0.78, "accuracy_among_usable": 0.8125 }, "F26": { "n": 8, "correct": 2, "answered": 8, "wrong_valid": 6, "failed": 0, "accuracy": 0.25, "accuracy_among_usable": 0.25 }, "F27": { "n": 8, "correct": 4, "answered": 8, "wrong_valid": 4, "failed": 0, "accuracy": 0.5, "accuracy_among_usable": 0.5 }, "F28": { "n": 8, "correct": 2, "answered": 8, "wrong_valid": 6, "failed": 0, "accuracy": 0.25, "accuracy_among_usable": 0.25 }, "F29": { "n": 8, "correct": 3, "answered": 8, "wrong_valid": 5, "failed": 0, "accuracy": 0.375, "accuracy_among_usable": 0.375 }, "F30": { "n": 8, "correct": 3, "answered": 8, "wrong_valid": 5, "failed": 0, "accuracy": 0.375, "accuracy_among_usable": 0.375 }, "F31": { "n": 8, "correct": 4, "answered": 8, "wrong_valid": 4, "failed": 0, "accuracy": 0.5, "accuracy_among_usable": 0.5 } }, "macro_family_accuracy": 0.650264401805159, "release_exclusion_diagnostic": { "n": 1305, "correct": 915, "answered": 1249, "wrong_valid": 334, "failed": 56, "accuracy": 0.7011494252873564, "accuracy_among_usable": 0.732586068855084 }, "new_domains": { "n": 48, "correct": 18, "answered": 48, "wrong_valid": 30, "failed": 0, "accuracy": 0.375, "accuracy_among_usable": 0.375 } }, "clef": { "n": 1342, "correct": 936, "answered": 1342, "wrong_valid": 406, "failed": 0, "accuracy": 0.6974664679582713, "accuracy_among_usable": 0.6974664679582713, "eventual": { "n": 1342, "correct": 936, "answered": 1342, "wrong_valid": 406, "failed": 0, "accuracy": 0.6974664679582713, "accuracy_among_usable": 0.6974664679582713 }, "median_ms": 267.92083302279934, "p95_ms": 416.5649816102814, "attempts": 1342, "recoveries": 0, "first_statuses": { "ok": 1342 }, "model_output_failures": 0, "probability_warnings": 562, "cost": { "known_usd": 0.023611182, "known_decimal": "0.023611182", "unknown_attempts": 0, "exact_usd": 0.023611182, "cost_sources": { "api_reported": 1342 }, "known_usd_per_1000": 0.017594025335320417 }, "confidence_diagnostics": { "0.8": { "n": 669, "correct": 645, "answered": 669, "wrong_valid": 24, "failed": 0, "accuracy": 0.9641255605381166, "accuracy_among_usable": 0.9641255605381166 }, "0.9": { "n": 566, "correct": 558, "answered": 566, "wrong_valid": 8, "failed": 0, "accuracy": 0.9858657243816255, "accuracy_among_usable": 0.9858657243816255 }, "0.95": { "n": 455, "correct": 453, "answered": 455, "wrong_valid": 2, "failed": 0, "accuracy": 0.9956043956043956, "accuracy_among_usable": 0.9956043956043956 } }, "latency_first_usable": { "n": 1342, "median_ms": 267.92083302279934, "p95_ms": 416.5649816102814 }, "provider_failures": 0, "dates_toronto": { "2026-10-18": 1136, "2026-10-19": 206 }, "returned_identities": { "cloudflare/clef": 1342 }, "providers": { "PrimeIntellect": 1342 }, "model_id": "clef", "label": "Cloudflare Clef", "route": "OpenRouter", "families": { "F01": { "n": 43, "correct": 40, "answered": 43, "wrong_valid": 3, "failed": 0, "accuracy": 0.9302325581395349, "accuracy_among_usable": 0.9302325581395349 }, "F02": { "n": 46, "correct": 39, "answered": 46, "wrong_valid": 7, "failed": 0, "accuracy": 0.8478260869565217, "accuracy_among_usable": 0.8478260869565217 }, "F03": { "n": 52, "correct": 39, "answered": 52, "wrong_valid": 13, "failed": 0, "accuracy": 0.75, "accuracy_among_usable": 0.75 }, "F04": { "n": 54, "correct": 46, "answered": 54, "wrong_valid": 8, "failed": 0, "accuracy": 0.8518518518518519, "accuracy_among_usable": 0.8518518518518519 }, "F05": { "n": 49, "correct": 44, "answered": 49, "wrong_valid": 5, "failed": 0, "accuracy": 0.8979591836734694, "accuracy_among_usable": 0.8979591836734694 }, "F06": { "n": 40, "correct": 37, "answered": 40, "wrong_valid": 3, "failed": 0, "accuracy": 0.925, "accuracy_among_usable": 0.925 }, "F07": { "n": 76, "correct": 43, "answered": 76, "wrong_valid": 33, "failed": 0, "accuracy": 0.5657894736842105, "accuracy_among_usable": 0.5657894736842105 }, "F08": { "n": 67, "correct": 42, "answered": 67, "wrong_valid": 25, "failed": 0, "accuracy": 0.6268656716417911, "accuracy_among_usable": 0.6268656716417911 }, "F09": { "n": 71, "correct": 27, "answered": 71, "wrong_valid": 44, "failed": 0, "accuracy": 0.38028169014084506, "accuracy_among_usable": 0.38028169014084506 }, "F10": { "n": 74, "correct": 33, "answered": 74, "wrong_valid": 41, "failed": 0, "accuracy": 0.44594594594594594, "accuracy_among_usable": 0.44594594594594594 }, "F11": { "n": 41, "correct": 29, "answered": 41, "wrong_valid": 12, "failed": 0, "accuracy": 0.7073170731707317, "accuracy_among_usable": 0.7073170731707317 }, "F12": { "n": 52, "correct": 27, "answered": 52, "wrong_valid": 25, "failed": 0, "accuracy": 0.5192307692307693, "accuracy_among_usable": 0.5192307692307693 }, "F13": { "n": 48, "correct": 11, "answered": 48, "wrong_valid": 37, "failed": 0, "accuracy": 0.22916666666666666, "accuracy_among_usable": 0.22916666666666666 }, "F14": { "n": 42, "correct": 31, "answered": 42, "wrong_valid": 11, "failed": 0, "accuracy": 0.7380952380952381, "accuracy_among_usable": 0.7380952380952381 }, "F15": { "n": 49, "correct": 23, "answered": 49, "wrong_valid": 26, "failed": 0, "accuracy": 0.46938775510204084, "accuracy_among_usable": 0.46938775510204084 }, "F16": { "n": 50, "correct": 31, "answered": 50, "wrong_valid": 19, "failed": 0, "accuracy": 0.62, "accuracy_among_usable": 0.62 }, "F17": { "n": 50, "correct": 42, "answered": 50, "wrong_valid": 8, "failed": 0, "accuracy": 0.84, "accuracy_among_usable": 0.84 }, "F18": { "n": 50, "correct": 27, "answered": 50, "wrong_valid": 23, "failed": 0, "accuracy": 0.54, "accuracy_among_usable": 0.54 }, "F19": { "n": 58, "correct": 51, "answered": 58, "wrong_valid": 7, "failed": 0, "accuracy": 0.8793103448275862, "accuracy_among_usable": 0.8793103448275862 }, "F20": { "n": 43, "correct": 38, "answered": 43, "wrong_valid": 5, "failed": 0, "accuracy": 0.8837209302325582, "accuracy_among_usable": 0.8837209302325582 }, "F21": { "n": 43, "correct": 40, "answered": 43, "wrong_valid": 3, "failed": 0, "accuracy": 0.9302325581395349, "accuracy_among_usable": 0.9302325581395349 }, "F22": { "n": 51, "correct": 45, "answered": 51, "wrong_valid": 6, "failed": 0, "accuracy": 0.8823529411764706, "accuracy_among_usable": 0.8823529411764706 }, "F23": { "n": 52, "correct": 49, "answered": 52, "wrong_valid": 3, "failed": 0, "accuracy": 0.9423076923076923, "accuracy_among_usable": 0.9423076923076923 }, "F24": { "n": 43, "correct": 39, "answered": 43, "wrong_valid": 4, "failed": 0, "accuracy": 0.9069767441860465, "accuracy_among_usable": 0.9069767441860465 }, "F25": { "n": 50, "correct": 41, "answered": 50, "wrong_valid": 9, "failed": 0, "accuracy": 0.82, "accuracy_among_usable": 0.82 }, "F26": { "n": 8, "correct": 2, "answered": 8, "wrong_valid": 6, "failed": 0, "accuracy": 0.25, "accuracy_among_usable": 0.25 }, "F27": { "n": 8, "correct": 3, "answered": 8, "wrong_valid": 5, "failed": 0, "accuracy": 0.375, "accuracy_among_usable": 0.375 }, "F28": { "n": 8, "correct": 3, "answered": 8, "wrong_valid": 5, "failed": 0, "accuracy": 0.375, "accuracy_among_usable": 0.375 }, "F29": { "n": 8, "correct": 3, "answered": 8, "wrong_valid": 5, "failed": 0, "accuracy": 0.375, "accuracy_among_usable": 0.375 }, "F30": { "n": 8, "correct": 5, "answered": 8, "wrong_valid": 3, "failed": 0, "accuracy": 0.625, "accuracy_among_usable": 0.625 }, "F31": { "n": 8, "correct": 6, "answered": 8, "wrong_valid": 2, "failed": 0, "accuracy": 0.75, "accuracy_among_usable": 0.75 } }, "macro_family_accuracy": 0.6735435862957904, "release_exclusion_diagnostic": { "n": 1305, "correct": 926, "answered": 1305, "wrong_valid": 379, "failed": 0, "accuracy": 0.7095785440613027, "accuracy_among_usable": 0.7095785440613027 }, "new_domains": { "n": 48, "correct": 22, "answered": 48, "wrong_valid": 26, "failed": 0, "accuracy": 0.4583333333333333, "accuracy_among_usable": 0.4583333333333333 } }, "clef_flash": { "n": 1342, "correct": 860, "answered": 1342, "wrong_valid": 482, "failed": 0, "accuracy": 0.6408345752608048, "accuracy_among_usable": 0.6408345752608048, "eventual": { "n": 1342, "correct": 860, "answered": 1342, "wrong_valid": 482, "failed": 0, "accuracy": 0.6408345752608048, "accuracy_among_usable": 0.6408345752608048 }, "median_ms": 244.66049950569868, "p95_ms": 352.7314375038259, "attempts": 1342, "recoveries": 0, "first_statuses": { "ok": 1342 }, "model_output_failures": 0, "probability_warnings": 555, "cost": { "known_usd": 0.011805591, "known_decimal": "0.011805591", "unknown_attempts": 0, "exact_usd": 0.011805591, "cost_sources": { "api_reported": 1342 }, "known_usd_per_1000": 0.008797012667660208 }, "confidence_diagnostics": { "0.8": { "n": 529, "correct": 498, "answered": 529, "wrong_valid": 31, "failed": 0, "accuracy": 0.941398865784499, "accuracy_among_usable": 0.941398865784499 }, "0.9": { "n": 344, "correct": 334, "answered": 344, "wrong_valid": 10, "failed": 0, "accuracy": 0.9709302325581395, "accuracy_among_usable": 0.9709302325581395 }, "0.95": { "n": 178, "correct": 177, "answered": 178, "wrong_valid": 1, "failed": 0, "accuracy": 0.9943820224719101, "accuracy_among_usable": 0.9943820224719101 } }, "latency_first_usable": { "n": 1342, "median_ms": 244.66049950569868, "p95_ms": 352.7314375038259 }, "provider_failures": 0, "dates_toronto": { "2026-10-18": 1136, "2026-10-19": 206 }, "returned_identities": { "cloudflare/clef-flash": 1342 }, "providers": { "PrimeIntellect": 1342 }, "model_id": "clef_flash", "label": "Cloudflare Clef Flash", "route": "OpenRouter", "families": { "F01": { "n": 43, "correct": 38, "answered": 43, "wrong_valid": 5, "failed": 0, "accuracy": 0.8837209302325582, "accuracy_among_usable": 0.8837209302325582 }, "F02": { "n": 46, "correct": 34, "answered": 46, "wrong_valid": 12, "failed": 0, "accuracy": 0.7391304347826086, "accuracy_among_usable": 0.7391304347826086 }, "F03": { "n": 52, "correct": 35, "answered": 52, "wrong_valid": 17, "failed": 0, "accuracy": 0.6730769230769231, "accuracy_among_usable": 0.6730769230769231 }, "F04": { "n": 54, "correct": 37, "answered": 54, "wrong_valid": 17, "failed": 0, "accuracy": 0.6851851851851852, "accuracy_among_usable": 0.6851851851851852 }, "F05": { "n": 49, "correct": 42, "answered": 49, "wrong_valid": 7, "failed": 0, "accuracy": 0.8571428571428571, "accuracy_among_usable": 0.8571428571428571 }, "F06": { "n": 40, "correct": 32, "answered": 40, "wrong_valid": 8, "failed": 0, "accuracy": 0.8, "accuracy_among_usable": 0.8 }, "F07": { "n": 76, "correct": 35, "answered": 76, "wrong_valid": 41, "failed": 0, "accuracy": 0.4605263157894737, "accuracy_among_usable": 0.4605263157894737 }, "F08": { "n": 67, "correct": 48, "answered": 67, "wrong_valid": 19, "failed": 0, "accuracy": 0.7164179104477612, "accuracy_among_usable": 0.7164179104477612 }, "F09": { "n": 71, "correct": 28, "answered": 71, "wrong_valid": 43, "failed": 0, "accuracy": 0.39436619718309857, "accuracy_among_usable": 0.39436619718309857 }, "F10": { "n": 74, "correct": 34, "answered": 74, "wrong_valid": 40, "failed": 0, "accuracy": 0.4594594594594595, "accuracy_among_usable": 0.4594594594594595 }, "F11": { "n": 41, "correct": 27, "answered": 41, "wrong_valid": 14, "failed": 0, "accuracy": 0.6585365853658537, "accuracy_among_usable": 0.6585365853658537 }, "F12": { "n": 52, "correct": 28, "answered": 52, "wrong_valid": 24, "failed": 0, "accuracy": 0.5384615384615384, "accuracy_among_usable": 0.5384615384615384 }, "F13": { "n": 48, "correct": 9, "answered": 48, "wrong_valid": 39, "failed": 0, "accuracy": 0.1875, "accuracy_among_usable": 0.1875 }, "F14": { "n": 42, "correct": 22, "answered": 42, "wrong_valid": 20, "failed": 0, "accuracy": 0.5238095238095238, "accuracy_among_usable": 0.5238095238095238 }, "F15": { "n": 49, "correct": 16, "answered": 49, "wrong_valid": 33, "failed": 0, "accuracy": 0.32653061224489793, "accuracy_among_usable": 0.32653061224489793 }, "F16": { "n": 50, "correct": 25, "answered": 50, "wrong_valid": 25, "failed": 0, "accuracy": 0.5, "accuracy_among_usable": 0.5 }, "F17": { "n": 50, "correct": 41, "answered": 50, "wrong_valid": 9, "failed": 0, "accuracy": 0.82, "accuracy_among_usable": 0.82 }, "F18": { "n": 50, "correct": 29, "answered": 50, "wrong_valid": 21, "failed": 0, "accuracy": 0.58, "accuracy_among_usable": 0.58 }, "F19": { "n": 58, "correct": 43, "answered": 58, "wrong_valid": 15, "failed": 0, "accuracy": 0.7413793103448276, "accuracy_among_usable": 0.7413793103448276 }, "F20": { "n": 43, "correct": 33, "answered": 43, "wrong_valid": 10, "failed": 0, "accuracy": 0.7674418604651163, "accuracy_among_usable": 0.7674418604651163 }, "F21": { "n": 43, "correct": 36, "answered": 43, "wrong_valid": 7, "failed": 0, "accuracy": 0.8372093023255814, "accuracy_among_usable": 0.8372093023255814 }, "F22": { "n": 51, "correct": 45, "answered": 51, "wrong_valid": 6, "failed": 0, "accuracy": 0.8823529411764706, "accuracy_among_usable": 0.8823529411764706 }, "F23": { "n": 52, "correct": 44, "answered": 52, "wrong_valid": 8, "failed": 0, "accuracy": 0.8461538461538461, "accuracy_among_usable": 0.8461538461538461 }, "F24": { "n": 43, "correct": 38, "answered": 43, "wrong_valid": 5, "failed": 0, "accuracy": 0.8837209302325582, "accuracy_among_usable": 0.8837209302325582 }, "F25": { "n": 50, "correct": 40, "answered": 50, "wrong_valid": 10, "failed": 0, "accuracy": 0.8, "accuracy_among_usable": 0.8 }, "F26": { "n": 8, "correct": 3, "answered": 8, "wrong_valid": 5, "failed": 0, "accuracy": 0.375, "accuracy_among_usable": 0.375 }, "F27": { "n": 8, "correct": 4, "answered": 8, "wrong_valid": 4, "failed": 0, "accuracy": 0.5, "accuracy_among_usable": 0.5 }, "F28": { "n": 8, "correct": 5, "answered": 8, "wrong_valid": 3, "failed": 0, "accuracy": 0.625, "accuracy_among_usable": 0.625 }, "F29": { "n": 8, "correct": 3, "answered": 8, "wrong_valid": 5, "failed": 0, "accuracy": 0.375, "accuracy_among_usable": 0.375 }, "F30": { "n": 8, "correct": 2, "answered": 8, "wrong_valid": 6, "failed": 0, "accuracy": 0.25, "accuracy_among_usable": 0.25 }, "F31": { "n": 8, "correct": 4, "answered": 8, "wrong_valid": 4, "failed": 0, "accuracy": 0.5, "accuracy_among_usable": 0.5 } }, "macro_family_accuracy": 0.618939440770327, "release_exclusion_diagnostic": { "n": 1305, "correct": 850, "answered": 1305, "wrong_valid": 455, "failed": 0, "accuracy": 0.6513409961685823, "accuracy_among_usable": 0.6513409961685823 }, "new_domains": { "n": 48, "correct": 21, "answered": 48, "wrong_valid": 27, "failed": 0, "accuracy": 0.4375, "accuracy_among_usable": 0.4375 } }, "clef_omni": { "n": 1342, "correct": 860, "answered": 1341, "wrong_valid": 481, "failed": 1, "accuracy": 0.6408345752608048, "accuracy_among_usable": 0.6413124533929903, "eventual": { "n": 1342, "correct": 861, "answered": 1342, "wrong_valid": 481, "failed": 0, "accuracy": 0.6415797317436661, "accuracy_among_usable": 0.6415797317436661 }, "median_ms": 240.51106302067637, "p95_ms": 399.6966604376212, "attempts": 1343, "recoveries": 1, "first_statuses": { "ok": 1341, "provider_failure": 1 }, "model_output_failures": 0, "probability_warnings": 0, "cost": { "known_usd": 0.0825471, "known_decimal": "0.08254710", "unknown_attempts": 1, "exact_usd": null, "cost_sources": { "api_reported": 1342, "unreported": 1 }, "known_usd_per_1000": 0.06151050670640834 }, "confidence_diagnostics": { "0.8": { "n": 145, "correct": 132, "answered": 145, "wrong_valid": 13, "failed": 0, "accuracy": 0.9103448275862069, "accuracy_among_usable": 0.9103448275862069 }, "0.9": { "n": 5, "correct": 5, "answered": 5, "wrong_valid": 0, "failed": 0, "accuracy": 1.0, "accuracy_among_usable": 1.0 }, "0.95": { "n": 0, "correct": 0, "answered": 0, "wrong_valid": 0, "failed": 0, "accuracy": null, "accuracy_among_usable": null } }, "latency_first_usable": { "n": 1341, "median_ms": 240.54758402053267, "p95_ms": 399.72724998369813 }, "provider_failures": 1, "dates_toronto": { "2026-10-18": 1136, "2026-10-19": 206 }, "returned_identities": { "cloudflare/clef-omni-20261009": 1341, "unreported (error)": 1 }, "providers": { "Cloudflare": 1341, "unreported": 1 }, "model_id": "clef_omni", "label": "Cloudflare Clef Omni", "route": "OpenRouter", "families": { "F01": { "n": 43, "correct": 40, "answered": 43, "wrong_valid": 3, "failed": 0, "accuracy": 0.9302325581395349, "accuracy_among_usable": 0.9302325581395349 }, "F02": { "n": 46, "correct": 26, "answered": 46, "wrong_valid": 20, "failed": 0, "accuracy": 0.5652173913043478, "accuracy_among_usable": 0.5652173913043478 }, "F03": { "n": 52, "correct": 41, "answered": 52, "wrong_valid": 11, "failed": 0, "accuracy": 0.7884615384615384, "accuracy_among_usable": 0.7884615384615384 }, "F04": { "n": 54, "correct": 41, "answered": 54, "wrong_valid": 13, "failed": 0, "accuracy": 0.7592592592592593, "accuracy_among_usable": 0.7592592592592593 }, "F05": { "n": 49, "correct": 41, "answered": 49, "wrong_valid": 8, "failed": 0, "accuracy": 0.8367346938775511, "accuracy_among_usable": 0.8367346938775511 }, "F06": { "n": 40, "correct": 33, "answered": 40, "wrong_valid": 7, "failed": 0, "accuracy": 0.825, "accuracy_among_usable": 0.825 }, "F07": { "n": 76, "correct": 36, "answered": 76, "wrong_valid": 40, "failed": 0, "accuracy": 0.47368421052631576, "accuracy_among_usable": 0.47368421052631576 }, "F08": { "n": 67, "correct": 41, "answered": 66, "wrong_valid": 25, "failed": 1, "accuracy": 0.6119402985074627, "accuracy_among_usable": 0.6212121212121212 }, "F09": { "n": 71, "correct": 20, "answered": 71, "wrong_valid": 51, "failed": 0, "accuracy": 0.28169014084507044, "accuracy_among_usable": 0.28169014084507044 }, "F10": { "n": 74, "correct": 37, "answered": 74, "wrong_valid": 37, "failed": 0, "accuracy": 0.5, "accuracy_among_usable": 0.5 }, "F11": { "n": 41, "correct": 26, "answered": 41, "wrong_valid": 15, "failed": 0, "accuracy": 0.6341463414634146, "accuracy_among_usable": 0.6341463414634146 }, "F12": { "n": 52, "correct": 30, "answered": 52, "wrong_valid": 22, "failed": 0, "accuracy": 0.5769230769230769, "accuracy_among_usable": 0.5769230769230769 }, "F13": { "n": 48, "correct": 11, "answered": 48, "wrong_valid": 37, "failed": 0, "accuracy": 0.22916666666666666, "accuracy_among_usable": 0.22916666666666666 }, "F14": { "n": 42, "correct": 32, "answered": 42, "wrong_valid": 10, "failed": 0, "accuracy": 0.7619047619047619, "accuracy_among_usable": 0.7619047619047619 }, "F15": { "n": 49, "correct": 20, "answered": 49, "wrong_valid": 29, "failed": 0, "accuracy": 0.40816326530612246, "accuracy_among_usable": 0.40816326530612246 }, "F16": { "n": 50, "correct": 21, "answered": 50, "wrong_valid": 29, "failed": 0, "accuracy": 0.42, "accuracy_among_usable": 0.42 }, "F17": { "n": 50, "correct": 38, "answered": 50, "wrong_valid": 12, "failed": 0, "accuracy": 0.76, "accuracy_among_usable": 0.76 }, "F18": { "n": 50, "correct": 20, "answered": 50, "wrong_valid": 30, "failed": 0, "accuracy": 0.4, "accuracy_among_usable": 0.4 }, "F19": { "n": 58, "correct": 50, "answered": 58, "wrong_valid": 8, "failed": 0, "accuracy": 0.8620689655172413, "accuracy_among_usable": 0.8620689655172413 }, "F20": { "n": 43, "correct": 30, "answered": 43, "wrong_valid": 13, "failed": 0, "accuracy": 0.6976744186046512, "accuracy_among_usable": 0.6976744186046512 }, "F21": { "n": 43, "correct": 37, "answered": 43, "wrong_valid": 6, "failed": 0, "accuracy": 0.8604651162790697, "accuracy_among_usable": 0.8604651162790697 }, "F22": { "n": 51, "correct": 43, "answered": 51, "wrong_valid": 8, "failed": 0, "accuracy": 0.8431372549019608, "accuracy_among_usable": 0.8431372549019608 }, "F23": { "n": 52, "correct": 46, "answered": 52, "wrong_valid": 6, "failed": 0, "accuracy": 0.8846153846153846, "accuracy_among_usable": 0.8846153846153846 }, "F24": { "n": 43, "correct": 38, "answered": 43, "wrong_valid": 5, "failed": 0, "accuracy": 0.8837209302325582, "accuracy_among_usable": 0.8837209302325582 }, "F25": { "n": 50, "correct": 42, "answered": 50, "wrong_valid": 8, "failed": 0, "accuracy": 0.84, "accuracy_among_usable": 0.84 }, "F26": { "n": 8, "correct": 5, "answered": 8, "wrong_valid": 3, "failed": 0, "accuracy": 0.625, "accuracy_among_usable": 0.625 }, "F27": { "n": 8, "correct": 3, "answered": 8, "wrong_valid": 5, "failed": 0, "accuracy": 0.375, "accuracy_among_usable": 0.375 }, "F28": { "n": 8, "correct": 3, "answered": 8, "wrong_valid": 5, "failed": 0, "accuracy": 0.375, "accuracy_among_usable": 0.375 }, "F29": { "n": 8, "correct": 3, "answered": 8, "wrong_valid": 5, "failed": 0, "accuracy": 0.375, "accuracy_among_usable": 0.375 }, "F30": { "n": 8, "correct": 2, "answered": 8, "wrong_valid": 6, "failed": 0, "accuracy": 0.25, "accuracy_among_usable": 0.25 }, "F31": { "n": 8, "correct": 4, "answered": 8, "wrong_valid": 4, "failed": 0, "accuracy": 0.5, "accuracy_among_usable": 0.5 } }, "macro_family_accuracy": 0.6172324604301932, "release_exclusion_diagnostic": { "n": 1305, "correct": 849, "answered": 1304, "wrong_valid": 455, "failed": 1, "accuracy": 0.6505747126436782, "accuracy_among_usable": 0.6510736196319018 }, "new_domains": { "n": 48, "correct": 20, "answered": 48, "wrong_valid": 28, "failed": 0, "accuracy": 0.4166666666666667, "accuracy_among_usable": 0.4166666666666667 } }, "solar_flash": { "n": 1342, "correct": 538, "answered": 1340, "wrong_valid": 802, "failed": 2, "accuracy": 0.4008941877794337, "accuracy_among_usable": 0.4014925373134328, "eventual": { "n": 1342, "correct": 539, "answered": 1342, "wrong_valid": 803, "failed": 0, "accuracy": 0.4016393442622951, "accuracy_among_usable": 0.4016393442622951 }, "median_ms": 539.9097080226056, "p95_ms": 746.6297103848774, "attempts": 1344, "recoveries": 2, "first_statuses": { "ok": 1340, "provider_failure": 2 }, "model_output_failures": 0, "probability_warnings": 285, "cost": { "known_usd": 0.04053205, "known_decimal": "0.04053205", "unknown_attempts": 2, "exact_usd": null, "cost_sources": { "api_reported": 1342, "unreported": 2 }, "known_usd_per_1000": 0.030202719821162443 }, "confidence_diagnostics": { "0.8": { "n": 56, "correct": 51, "answered": 56, "wrong_valid": 5, "failed": 0, "accuracy": 0.9107142857142857, "accuracy_among_usable": 0.9107142857142857 }, "0.9": { "n": 32, "correct": 32, "answered": 32, "wrong_valid": 0, "failed": 0, "accuracy": 1.0, "accuracy_among_usable": 1.0 }, "0.95": { "n": 17, "correct": 17, "answered": 17, "wrong_valid": 0, "failed": 0, "accuracy": 1.0, "accuracy_among_usable": 1.0 } }, "latency_first_usable": { "n": 1340, "median_ms": 539.9481870117597, "p95_ms": 746.8044103821738 }, "provider_failures": 2, "dates_toronto": { "2026-10-18": 1136, "2026-10-19": 206 }, "returned_identities": { "upstage/solar-decide-flash-20261008": 1340, "unreported (error)": 2 }, "providers": { "Upstage": 1340, "unreported": 2 }, "model_id": "solar_flash", "label": "Solar Decide Flash", "route": "OpenRouter", "families": { "F01": { "n": 43, "correct": 27, "answered": 43, "wrong_valid": 16, "failed": 0, "accuracy": 0.627906976744186, "accuracy_among_usable": 0.627906976744186 }, "F02": { "n": 46, "correct": 22, "answered": 46, "wrong_valid": 24, "failed": 0, "accuracy": 0.4782608695652174, "accuracy_among_usable": 0.4782608695652174 }, "F03": { "n": 52, "correct": 15, "answered": 52, "wrong_valid": 37, "failed": 0, "accuracy": 0.28846153846153844, "accuracy_among_usable": 0.28846153846153844 }, "F04": { "n": 54, "correct": 14, "answered": 54, "wrong_valid": 40, "failed": 0, "accuracy": 0.25925925925925924, "accuracy_among_usable": 0.25925925925925924 }, "F05": { "n": 49, "correct": 36, "answered": 49, "wrong_valid": 13, "failed": 0, "accuracy": 0.7346938775510204, "accuracy_among_usable": 0.7346938775510204 }, "F06": { "n": 40, "correct": 34, "answered": 40, "wrong_valid": 6, "failed": 0, "accuracy": 0.85, "accuracy_among_usable": 0.85 }, "F07": { "n": 76, "correct": 19, "answered": 75, "wrong_valid": 56, "failed": 1, "accuracy": 0.25, "accuracy_among_usable": 0.25333333333333335 }, "F08": { "n": 67, "correct": 26, "answered": 67, "wrong_valid": 41, "failed": 0, "accuracy": 0.3880597014925373, "accuracy_among_usable": 0.3880597014925373 }, "F09": { "n": 71, "correct": 8, "answered": 71, "wrong_valid": 63, "failed": 0, "accuracy": 0.11267605633802817, "accuracy_among_usable": 0.11267605633802817 }, "F10": { "n": 74, "correct": 23, "answered": 74, "wrong_valid": 51, "failed": 0, "accuracy": 0.3108108108108108, "accuracy_among_usable": 0.3108108108108108 }, "F11": { "n": 41, "correct": 8, "answered": 41, "wrong_valid": 33, "failed": 0, "accuracy": 0.1951219512195122, "accuracy_among_usable": 0.1951219512195122 }, "F12": { "n": 52, "correct": 14, "answered": 52, "wrong_valid": 38, "failed": 0, "accuracy": 0.2692307692307692, "accuracy_among_usable": 0.2692307692307692 }, "F13": { "n": 48, "correct": 10, "answered": 48, "wrong_valid": 38, "failed": 0, "accuracy": 0.20833333333333334, "accuracy_among_usable": 0.20833333333333334 }, "F14": { "n": 42, "correct": 17, "answered": 42, "wrong_valid": 25, "failed": 0, "accuracy": 0.40476190476190477, "accuracy_among_usable": 0.40476190476190477 }, "F15": { "n": 49, "correct": 13, "answered": 49, "wrong_valid": 36, "failed": 0, "accuracy": 0.2653061224489796, "accuracy_among_usable": 0.2653061224489796 }, "F16": { "n": 50, "correct": 21, "answered": 50, "wrong_valid": 29, "failed": 0, "accuracy": 0.42, "accuracy_among_usable": 0.42 }, "F17": { "n": 50, "correct": 19, "answered": 50, "wrong_valid": 31, "failed": 0, "accuracy": 0.38, "accuracy_among_usable": 0.38 }, "F18": { "n": 50, "correct": 10, "answered": 50, "wrong_valid": 40, "failed": 0, "accuracy": 0.2, "accuracy_among_usable": 0.2 }, "F19": { "n": 58, "correct": 28, "answered": 58, "wrong_valid": 30, "failed": 0, "accuracy": 0.4827586206896552, "accuracy_among_usable": 0.4827586206896552 }, "F20": { "n": 43, "correct": 16, "answered": 43, "wrong_valid": 27, "failed": 0, "accuracy": 0.37209302325581395, "accuracy_among_usable": 0.37209302325581395 }, "F21": { "n": 43, "correct": 33, "answered": 43, "wrong_valid": 10, "failed": 0, "accuracy": 0.7674418604651163, "accuracy_among_usable": 0.7674418604651163 }, "F22": { "n": 51, "correct": 32, "answered": 51, "wrong_valid": 19, "failed": 0, "accuracy": 0.6274509803921569, "accuracy_among_usable": 0.6274509803921569 }, "F23": { "n": 52, "correct": 29, "answered": 52, "wrong_valid": 23, "failed": 0, "accuracy": 0.5576923076923077, "accuracy_among_usable": 0.5576923076923077 }, "F24": { "n": 43, "correct": 23, "answered": 42, "wrong_valid": 19, "failed": 1, "accuracy": 0.5348837209302325, "accuracy_among_usable": 0.5476190476190477 }, "F25": { "n": 50, "correct": 28, "answered": 50, "wrong_valid": 22, "failed": 0, "accuracy": 0.56, "accuracy_among_usable": 0.56 }, "F26": { "n": 8, "correct": 1, "answered": 8, "wrong_valid": 7, "failed": 0, "accuracy": 0.125, "accuracy_among_usable": 0.125 }, "F27": { "n": 8, "correct": 2, "answered": 8, "wrong_valid": 6, "failed": 0, "accuracy": 0.25, "accuracy_among_usable": 0.25 }, "F28": { "n": 8, "correct": 0, "answered": 8, "wrong_valid": 8, "failed": 0, "accuracy": 0.0, "accuracy_among_usable": 0.0 }, "F29": { "n": 8, "correct": 2, "answered": 8, "wrong_valid": 6, "failed": 0, "accuracy": 0.25, "accuracy_among_usable": 0.25 }, "F30": { "n": 8, "correct": 4, "answered": 8, "wrong_valid": 4, "failed": 0, "accuracy": 0.5, "accuracy_among_usable": 0.5 }, "F31": { "n": 8, "correct": 4, "answered": 8, "wrong_valid": 4, "failed": 0, "accuracy": 0.5, "accuracy_among_usable": 0.5 } }, "macro_family_accuracy": 0.39258721563362514, "release_exclusion_diagnostic": { "n": 1305, "correct": 531, "answered": 1304, "wrong_valid": 773, "failed": 1, "accuracy": 0.4068965517241379, "accuracy_among_usable": 0.4072085889570552 }, "new_domains": { "n": 48, "correct": 13, "answered": 48, "wrong_valid": 35, "failed": 0, "accuracy": 0.2708333333333333, "accuracy_among_usable": 0.2708333333333333 } }, "drex20": { "n": 1342, "correct": 915, "answered": 1302, "wrong_valid": 387, "failed": 40, "accuracy": 0.6818181818181818, "accuracy_among_usable": 0.7027649769585254, "eventual": { "n": 1342, "correct": 933, "answered": 1337, "wrong_valid": 404, "failed": 5, "accuracy": 0.6952309985096871, "accuracy_among_usable": 0.6978309648466716 }, "median_ms": 231.8915204377845, "p95_ms": 754.8576059227344, "attempts": 1382, "recoveries": 40, "first_statuses": { "ok": 1302, "provider_failure": 40 }, "model_output_failures": 0, "probability_warnings": 540, "cost": { "known_usd": 0.02235882, "known_decimal": "0.02235882", "unknown_attempts": 45, "exact_usd": null, "cost_sources": { "api_reported": 1337, "unreported": 45 }, "known_usd_per_1000": 0.016660819672131146 }, "confidence_diagnostics": { "0.8": { "n": 558, "correct": 550, "answered": 558, "wrong_valid": 8, "failed": 0, "accuracy": 0.985663082437276, "accuracy_among_usable": 0.985663082437276 }, "0.9": { "n": 488, "correct": 486, "answered": 488, "wrong_valid": 2, "failed": 0, "accuracy": 0.9959016393442623, "accuracy_among_usable": 0.9959016393442623 }, "0.95": { "n": 434, "correct": 434, "answered": 434, "wrong_valid": 0, "failed": 0, "accuracy": 1.0, "accuracy_among_usable": 1.0 } }, "latency_first_usable": { "n": 1302, "median_ms": 233.71191648766398, "p95_ms": 761.6356104030277 }, "provider_failures": 40, "dates_toronto": { "2026-10-18": 1136, "2026-10-19": 206 }, "returned_identities": { "nace-ai/drex-v2.0-20261009": 1302, "unreported (error)": 40 }, "providers": { "DeepInfra": 1302, "unreported": 40 }, "model_id": "drex20", "label": "Drex v2.0", "route": "OpenRouter", "families": { "F01": { "n": 43, "correct": 39, "answered": 43, "wrong_valid": 4, "failed": 0, "accuracy": 0.9069767441860465, "accuracy_among_usable": 0.9069767441860465 }, "F02": { "n": 46, "correct": 33, "answered": 43, "wrong_valid": 10, "failed": 3, "accuracy": 0.717391304347826, "accuracy_among_usable": 0.7674418604651163 }, "F03": { "n": 52, "correct": 27, "answered": 50, "wrong_valid": 23, "failed": 2, "accuracy": 0.5192307692307693, "accuracy_among_usable": 0.54 }, "F04": { "n": 54, "correct": 44, "answered": 52, "wrong_valid": 8, "failed": 2, "accuracy": 0.8148148148148148, "accuracy_among_usable": 0.8461538461538461 }, "F05": { "n": 49, "correct": 43, "answered": 48, "wrong_valid": 5, "failed": 1, "accuracy": 0.8775510204081632, "accuracy_among_usable": 0.8958333333333334 }, "F06": { "n": 40, "correct": 37, "answered": 38, "wrong_valid": 1, "failed": 2, "accuracy": 0.925, "accuracy_among_usable": 0.9736842105263158 }, "F07": { "n": 76, "correct": 50, "answered": 75, "wrong_valid": 25, "failed": 1, "accuracy": 0.6578947368421053, "accuracy_among_usable": 0.6666666666666666 }, "F08": { "n": 67, "correct": 43, "answered": 64, "wrong_valid": 21, "failed": 3, "accuracy": 0.6417910447761194, "accuracy_among_usable": 0.671875 }, "F09": { "n": 71, "correct": 27, "answered": 68, "wrong_valid": 41, "failed": 3, "accuracy": 0.38028169014084506, "accuracy_among_usable": 0.39705882352941174 }, "F10": { "n": 74, "correct": 37, "answered": 69, "wrong_valid": 32, "failed": 5, "accuracy": 0.5, "accuracy_among_usable": 0.5362318840579711 }, "F11": { "n": 41, "correct": 33, "answered": 41, "wrong_valid": 8, "failed": 0, "accuracy": 0.8048780487804879, "accuracy_among_usable": 0.8048780487804879 }, "F12": { "n": 52, "correct": 28, "answered": 50, "wrong_valid": 22, "failed": 2, "accuracy": 0.5384615384615384, "accuracy_among_usable": 0.56 }, "F13": { "n": 48, "correct": 12, "answered": 46, "wrong_valid": 34, "failed": 2, "accuracy": 0.25, "accuracy_among_usable": 0.2608695652173913 }, "F14": { "n": 42, "correct": 33, "answered": 41, "wrong_valid": 8, "failed": 1, "accuracy": 0.7857142857142857, "accuracy_among_usable": 0.8048780487804879 }, "F15": { "n": 49, "correct": 26, "answered": 48, "wrong_valid": 22, "failed": 1, "accuracy": 0.5306122448979592, "accuracy_among_usable": 0.5416666666666666 }, "F16": { "n": 50, "correct": 26, "answered": 49, "wrong_valid": 23, "failed": 1, "accuracy": 0.52, "accuracy_among_usable": 0.5306122448979592 }, "F17": { "n": 50, "correct": 25, "answered": 48, "wrong_valid": 23, "failed": 2, "accuracy": 0.5, "accuracy_among_usable": 0.5208333333333334 }, "F18": { "n": 50, "correct": 36, "answered": 50, "wrong_valid": 14, "failed": 0, "accuracy": 0.72, "accuracy_among_usable": 0.72 }, "F19": { "n": 58, "correct": 48, "answered": 58, "wrong_valid": 10, "failed": 0, "accuracy": 0.8275862068965517, "accuracy_among_usable": 0.8275862068965517 }, "F20": { "n": 43, "correct": 33, "answered": 43, "wrong_valid": 10, "failed": 0, "accuracy": 0.7674418604651163, "accuracy_among_usable": 0.7674418604651163 }, "F21": { "n": 43, "correct": 42, "answered": 43, "wrong_valid": 1, "failed": 0, "accuracy": 0.9767441860465116, "accuracy_among_usable": 0.9767441860465116 }, "F22": { "n": 51, "correct": 42, "answered": 47, "wrong_valid": 5, "failed": 4, "accuracy": 0.8235294117647058, "accuracy_among_usable": 0.8936170212765957 }, "F23": { "n": 52, "correct": 47, "answered": 50, "wrong_valid": 3, "failed": 2, "accuracy": 0.9038461538461539, "accuracy_among_usable": 0.94 }, "F24": { "n": 43, "correct": 37, "answered": 42, "wrong_valid": 5, "failed": 1, "accuracy": 0.8604651162790697, "accuracy_among_usable": 0.8809523809523809 }, "F25": { "n": 50, "correct": 44, "answered": 48, "wrong_valid": 4, "failed": 2, "accuracy": 0.88, "accuracy_among_usable": 0.9166666666666666 }, "F26": { "n": 8, "correct": 5, "answered": 8, "wrong_valid": 3, "failed": 0, "accuracy": 0.625, "accuracy_among_usable": 0.625 }, "F27": { "n": 8, "correct": 4, "answered": 8, "wrong_valid": 4, "failed": 0, "accuracy": 0.5, "accuracy_among_usable": 0.5 }, "F28": { "n": 8, "correct": 3, "answered": 8, "wrong_valid": 5, "failed": 0, "accuracy": 0.375, "accuracy_among_usable": 0.375 }, "F29": { "n": 8, "correct": 3, "answered": 8, "wrong_valid": 5, "failed": 0, "accuracy": 0.375, "accuracy_among_usable": 0.375 }, "F30": { "n": 8, "correct": 4, "answered": 8, "wrong_valid": 4, "failed": 0, "accuracy": 0.5, "accuracy_among_usable": 0.5 }, "F31": { "n": 8, "correct": 4, "answered": 8, "wrong_valid": 4, "failed": 0, "accuracy": 0.5, "accuracy_among_usable": 0.5 } }, "macro_family_accuracy": 0.6614584250935184, "release_exclusion_diagnostic": { "n": 1305, "correct": 900, "answered": 1265, "wrong_valid": 365, "failed": 40, "accuracy": 0.6896551724137931, "accuracy_among_usable": 0.7114624505928854 }, "new_domains": { "n": 48, "correct": 23, "answered": 48, "wrong_valid": 25, "failed": 0, "accuracy": 0.4791666666666667, "accuracy_among_usable": 0.4791666666666667 } }, "drex15": { "n": 1342, "correct": 853, "answered": 1342, "wrong_valid": 489, "failed": 0, "accuracy": 0.6356184798807749, "accuracy_among_usable": 0.6356184798807749, "eventual": { "n": 1342, "correct": 853, "answered": 1342, "wrong_valid": 489, "failed": 0, "accuracy": 0.6356184798807749, "accuracy_among_usable": 0.6356184798807749 }, "median_ms": 203.44412501435727, "p95_ms": 648.9552770508458, "attempts": 1342, "recoveries": 0, "first_statuses": { "ok": 1342 }, "model_output_failures": 0, "probability_warnings": 569, "cost": { "known_usd": 0.01497636, "known_decimal": "0.01497636", "unknown_attempts": 0, "exact_usd": 0.01497636, "cost_sources": { "api_reported": 1342 }, "known_usd_per_1000": 0.01115973174366617 }, "confidence_diagnostics": { "0.8": { "n": 700, "correct": 612, "answered": 700, "wrong_valid": 88, "failed": 0, "accuracy": 0.8742857142857143, "accuracy_among_usable": 0.8742857142857143 }, "0.9": { "n": 606, "correct": 554, "answered": 606, "wrong_valid": 52, "failed": 0, "accuracy": 0.9141914191419142, "accuracy_among_usable": 0.9141914191419142 }, "0.95": { "n": 538, "correct": 503, "answered": 538, "wrong_valid": 35, "failed": 0, "accuracy": 0.9349442379182156, "accuracy_among_usable": 0.9349442379182156 } }, "latency_first_usable": { "n": 1342, "median_ms": 203.44412501435727, "p95_ms": 648.9552770508458 }, "provider_failures": 0, "dates_toronto": { "2026-10-18": 1136, "2026-10-19": 206 }, "returned_identities": { "nace-ai/drex-v1.5-20261009": 1342 }, "providers": { "DeepInfra": 1342 }, "model_id": "drex15", "label": "Drex v1.5", "route": "OpenRouter", "families": { "F01": { "n": 43, "correct": 38, "answered": 43, "wrong_valid": 5, "failed": 0, "accuracy": 0.8837209302325582, "accuracy_among_usable": 0.8837209302325582 }, "F02": { "n": 46, "correct": 26, "answered": 46, "wrong_valid": 20, "failed": 0, "accuracy": 0.5652173913043478, "accuracy_among_usable": 0.5652173913043478 }, "F03": { "n": 52, "correct": 26, "answered": 52, "wrong_valid": 26, "failed": 0, "accuracy": 0.5, "accuracy_among_usable": 0.5 }, "F04": { "n": 54, "correct": 36, "answered": 54, "wrong_valid": 18, "failed": 0, "accuracy": 0.6666666666666666, "accuracy_among_usable": 0.6666666666666666 }, "F05": { "n": 49, "correct": 41, "answered": 49, "wrong_valid": 8, "failed": 0, "accuracy": 0.8367346938775511, "accuracy_among_usable": 0.8367346938775511 }, "F06": { "n": 40, "correct": 37, "answered": 40, "wrong_valid": 3, "failed": 0, "accuracy": 0.925, "accuracy_among_usable": 0.925 }, "F07": { "n": 76, "correct": 51, "answered": 76, "wrong_valid": 25, "failed": 0, "accuracy": 0.6710526315789473, "accuracy_among_usable": 0.6710526315789473 }, "F08": { "n": 67, "correct": 48, "answered": 67, "wrong_valid": 19, "failed": 0, "accuracy": 0.7164179104477612, "accuracy_among_usable": 0.7164179104477612 }, "F09": { "n": 71, "correct": 33, "answered": 71, "wrong_valid": 38, "failed": 0, "accuracy": 0.4647887323943662, "accuracy_among_usable": 0.4647887323943662 }, "F10": { "n": 74, "correct": 30, "answered": 74, "wrong_valid": 44, "failed": 0, "accuracy": 0.40540540540540543, "accuracy_among_usable": 0.40540540540540543 }, "F11": { "n": 41, "correct": 21, "answered": 41, "wrong_valid": 20, "failed": 0, "accuracy": 0.5121951219512195, "accuracy_among_usable": 0.5121951219512195 }, "F12": { "n": 52, "correct": 23, "answered": 52, "wrong_valid": 29, "failed": 0, "accuracy": 0.4423076923076923, "accuracy_among_usable": 0.4423076923076923 }, "F13": { "n": 48, "correct": 13, "answered": 48, "wrong_valid": 35, "failed": 0, "accuracy": 0.2708333333333333, "accuracy_among_usable": 0.2708333333333333 }, "F14": { "n": 42, "correct": 28, "answered": 42, "wrong_valid": 14, "failed": 0, "accuracy": 0.6666666666666666, "accuracy_among_usable": 0.6666666666666666 }, "F15": { "n": 49, "correct": 26, "answered": 49, "wrong_valid": 23, "failed": 0, "accuracy": 0.5306122448979592, "accuracy_among_usable": 0.5306122448979592 }, "F16": { "n": 50, "correct": 23, "answered": 50, "wrong_valid": 27, "failed": 0, "accuracy": 0.46, "accuracy_among_usable": 0.46 }, "F17": { "n": 50, "correct": 29, "answered": 50, "wrong_valid": 21, "failed": 0, "accuracy": 0.58, "accuracy_among_usable": 0.58 }, "F18": { "n": 50, "correct": 22, "answered": 50, "wrong_valid": 28, "failed": 0, "accuracy": 0.44, "accuracy_among_usable": 0.44 }, "F19": { "n": 58, "correct": 48, "answered": 58, "wrong_valid": 10, "failed": 0, "accuracy": 0.8275862068965517, "accuracy_among_usable": 0.8275862068965517 }, "F20": { "n": 43, "correct": 27, "answered": 43, "wrong_valid": 16, "failed": 0, "accuracy": 0.627906976744186, "accuracy_among_usable": 0.627906976744186 }, "F21": { "n": 43, "correct": 39, "answered": 43, "wrong_valid": 4, "failed": 0, "accuracy": 0.9069767441860465, "accuracy_among_usable": 0.9069767441860465 }, "F22": { "n": 51, "correct": 44, "answered": 51, "wrong_valid": 7, "failed": 0, "accuracy": 0.8627450980392157, "accuracy_among_usable": 0.8627450980392157 }, "F23": { "n": 52, "correct": 46, "answered": 52, "wrong_valid": 6, "failed": 0, "accuracy": 0.8846153846153846, "accuracy_among_usable": 0.8846153846153846 }, "F24": { "n": 43, "correct": 39, "answered": 43, "wrong_valid": 4, "failed": 0, "accuracy": 0.9069767441860465, "accuracy_among_usable": 0.9069767441860465 }, "F25": { "n": 50, "correct": 41, "answered": 50, "wrong_valid": 9, "failed": 0, "accuracy": 0.82, "accuracy_among_usable": 0.82 }, "F26": { "n": 8, "correct": 1, "answered": 8, "wrong_valid": 7, "failed": 0, "accuracy": 0.125, "accuracy_among_usable": 0.125 }, "F27": { "n": 8, "correct": 5, "answered": 8, "wrong_valid": 3, "failed": 0, "accuracy": 0.625, "accuracy_among_usable": 0.625 }, "F28": { "n": 8, "correct": 2, "answered": 8, "wrong_valid": 6, "failed": 0, "accuracy": 0.25, "accuracy_among_usable": 0.25 }, "F29": { "n": 8, "correct": 2, "answered": 8, "wrong_valid": 6, "failed": 0, "accuracy": 0.25, "accuracy_among_usable": 0.25 }, "F30": { "n": 8, "correct": 4, "answered": 8, "wrong_valid": 4, "failed": 0, "accuracy": 0.5, "accuracy_among_usable": 0.5 }, "F31": { "n": 8, "correct": 4, "answered": 8, "wrong_valid": 4, "failed": 0, "accuracy": 0.5, "accuracy_among_usable": 0.5 } }, "macro_family_accuracy": 0.6007879540558678, "release_exclusion_diagnostic": { "n": 1305, "correct": 843, "answered": 1305, "wrong_valid": 462, "failed": 0, "accuracy": 0.6459770114942529, "accuracy_among_usable": 0.6459770114942529 }, "new_domains": { "n": 48, "correct": 18, "answered": 48, "wrong_valid": 30, "failed": 0, "accuracy": 0.375, "accuracy_among_usable": 0.375 } }, "classify1": { "n": 1342, "correct": 550, "answered": 1342, "wrong_valid": 792, "failed": 0, "accuracy": 0.4098360655737705, "accuracy_among_usable": 0.4098360655737705, "eventual": { "n": 1342, "correct": 550, "answered": 1342, "wrong_valid": 792, "failed": 0, "accuracy": 0.4098360655737705, "accuracy_among_usable": 0.4098360655737705 }, "median_ms": 268.4367289766669, "p95_ms": 361.68582041864283, "attempts": 1342, "recoveries": 0, "first_statuses": { "ok": 1342 }, "model_output_failures": 0, "probability_warnings": 0, "cost": { "known_usd": 0.0287026, "known_decimal": "0.02870260", "unknown_attempts": 0, "exact_usd": 0.0287026, "cost_sources": { "api_reported": 1342 }, "known_usd_per_1000": 0.021387928464977647 }, "confidence_diagnostics": { "0.8": { "n": 68, "correct": 67, "answered": 68, "wrong_valid": 1, "failed": 0, "accuracy": 0.9852941176470589, "accuracy_among_usable": 0.9852941176470589 }, "0.9": { "n": 24, "correct": 24, "answered": 24, "wrong_valid": 0, "failed": 0, "accuracy": 1.0, "accuracy_among_usable": 1.0 }, "0.95": { "n": 6, "correct": 6, "answered": 6, "wrong_valid": 0, "failed": 0, "accuracy": 1.0, "accuracy_among_usable": 1.0 } }, "latency_first_usable": { "n": 1342, "median_ms": 268.4367289766669, "p95_ms": 361.68582041864283 }, "provider_failures": 0, "dates_toronto": { "2026-10-18": 1136, "2026-10-19": 206 }, "returned_identities": { "scaledown/classify-1-20260928": 1342 }, "providers": { "ScaleDown": 1342 }, "model_id": "classify1", "label": "ScaleDown Classify-1", "route": "OpenRouter", "families": { "F01": { "n": 43, "correct": 31, "answered": 43, "wrong_valid": 12, "failed": 0, "accuracy": 0.7209302325581395, "accuracy_among_usable": 0.7209302325581395 }, "F02": { "n": 46, "correct": 18, "answered": 46, "wrong_valid": 28, "failed": 0, "accuracy": 0.391304347826087, "accuracy_among_usable": 0.391304347826087 }, "F03": { "n": 52, "correct": 13, "answered": 52, "wrong_valid": 39, "failed": 0, "accuracy": 0.25, "accuracy_among_usable": 0.25 }, "F04": { "n": 54, "correct": 17, "answered": 54, "wrong_valid": 37, "failed": 0, "accuracy": 0.3148148148148148, "accuracy_among_usable": 0.3148148148148148 }, "F05": { "n": 49, "correct": 28, "answered": 49, "wrong_valid": 21, "failed": 0, "accuracy": 0.5714285714285714, "accuracy_among_usable": 0.5714285714285714 }, "F06": { "n": 40, "correct": 34, "answered": 40, "wrong_valid": 6, "failed": 0, "accuracy": 0.85, "accuracy_among_usable": 0.85 }, "F07": { "n": 76, "correct": 26, "answered": 76, "wrong_valid": 50, "failed": 0, "accuracy": 0.34210526315789475, "accuracy_among_usable": 0.34210526315789475 }, "F08": { "n": 67, "correct": 29, "answered": 67, "wrong_valid": 38, "failed": 0, "accuracy": 0.43283582089552236, "accuracy_among_usable": 0.43283582089552236 }, "F09": { "n": 71, "correct": 22, "answered": 71, "wrong_valid": 49, "failed": 0, "accuracy": 0.30985915492957744, "accuracy_among_usable": 0.30985915492957744 }, "F10": { "n": 74, "correct": 15, "answered": 74, "wrong_valid": 59, "failed": 0, "accuracy": 0.20270270270270271, "accuracy_among_usable": 0.20270270270270271 }, "F11": { "n": 41, "correct": 17, "answered": 41, "wrong_valid": 24, "failed": 0, "accuracy": 0.4146341463414634, "accuracy_among_usable": 0.4146341463414634 }, "F12": { "n": 52, "correct": 8, "answered": 52, "wrong_valid": 44, "failed": 0, "accuracy": 0.15384615384615385, "accuracy_among_usable": 0.15384615384615385 }, "F13": { "n": 48, "correct": 11, "answered": 48, "wrong_valid": 37, "failed": 0, "accuracy": 0.22916666666666666, "accuracy_among_usable": 0.22916666666666666 }, "F14": { "n": 42, "correct": 13, "answered": 42, "wrong_valid": 29, "failed": 0, "accuracy": 0.30952380952380953, "accuracy_among_usable": 0.30952380952380953 }, "F15": { "n": 49, "correct": 17, "answered": 49, "wrong_valid": 32, "failed": 0, "accuracy": 0.3469387755102041, "accuracy_among_usable": 0.3469387755102041 }, "F16": { "n": 50, "correct": 19, "answered": 50, "wrong_valid": 31, "failed": 0, "accuracy": 0.38, "accuracy_among_usable": 0.38 }, "F17": { "n": 50, "correct": 21, "answered": 50, "wrong_valid": 29, "failed": 0, "accuracy": 0.42, "accuracy_among_usable": 0.42 }, "F18": { "n": 50, "correct": 12, "answered": 50, "wrong_valid": 38, "failed": 0, "accuracy": 0.24, "accuracy_among_usable": 0.24 }, "F19": { "n": 58, "correct": 28, "answered": 58, "wrong_valid": 30, "failed": 0, "accuracy": 0.4827586206896552, "accuracy_among_usable": 0.4827586206896552 }, "F20": { "n": 43, "correct": 14, "answered": 43, "wrong_valid": 29, "failed": 0, "accuracy": 0.32558139534883723, "accuracy_among_usable": 0.32558139534883723 }, "F21": { "n": 43, "correct": 31, "answered": 43, "wrong_valid": 12, "failed": 0, "accuracy": 0.7209302325581395, "accuracy_among_usable": 0.7209302325581395 }, "F22": { "n": 51, "correct": 29, "answered": 51, "wrong_valid": 22, "failed": 0, "accuracy": 0.5686274509803921, "accuracy_among_usable": 0.5686274509803921 }, "F23": { "n": 52, "correct": 30, "answered": 52, "wrong_valid": 22, "failed": 0, "accuracy": 0.5769230769230769, "accuracy_among_usable": 0.5769230769230769 }, "F24": { "n": 43, "correct": 33, "answered": 43, "wrong_valid": 10, "failed": 0, "accuracy": 0.7674418604651163, "accuracy_among_usable": 0.7674418604651163 }, "F25": { "n": 50, "correct": 25, "answered": 50, "wrong_valid": 25, "failed": 0, "accuracy": 0.5, "accuracy_among_usable": 0.5 }, "F26": { "n": 8, "correct": 1, "answered": 8, "wrong_valid": 7, "failed": 0, "accuracy": 0.125, "accuracy_among_usable": 0.125 }, "F27": { "n": 8, "correct": 2, "answered": 8, "wrong_valid": 6, "failed": 0, "accuracy": 0.25, "accuracy_among_usable": 0.25 }, "F28": { "n": 8, "correct": 1, "answered": 8, "wrong_valid": 7, "failed": 0, "accuracy": 0.125, "accuracy_among_usable": 0.125 }, "F29": { "n": 8, "correct": 1, "answered": 8, "wrong_valid": 7, "failed": 0, "accuracy": 0.125, "accuracy_among_usable": 0.125 }, "F30": { "n": 8, "correct": 1, "answered": 8, "wrong_valid": 7, "failed": 0, "accuracy": 0.125, "accuracy_among_usable": 0.125 }, "F31": { "n": 8, "correct": 3, "answered": 8, "wrong_valid": 5, "failed": 0, "accuracy": 0.375, "accuracy_among_usable": 0.375 } }, "macro_family_accuracy": 0.38539848700538143, "release_exclusion_diagnostic": { "n": 1305, "correct": 542, "answered": 1305, "wrong_valid": 763, "failed": 0, "accuracy": 0.4153256704980843, "accuracy_among_usable": 0.4153256704980843 }, "new_domains": { "n": 48, "correct": 9, "answered": 48, "wrong_valid": 39, "failed": 0, "accuracy": 0.1875, "accuracy_among_usable": 0.1875 } }, "kev4b": { "n": 1342, "correct": 822, "answered": 1342, "wrong_valid": 520, "failed": 0, "accuracy": 0.6125186289120715, "accuracy_among_usable": 0.6125186289120715, "eventual": { "n": 1342, "correct": 822, "answered": 1342, "wrong_valid": 520, "failed": 0, "accuracy": 0.6125186289120715, "accuracy_among_usable": 0.6125186289120715 }, "median_ms": 224.8376039788127, "p95_ms": 377.38076279056264, "attempts": 1342, "recoveries": 0, "first_statuses": { "ok": 1342 }, "model_output_failures": 0, "probability_warnings": 536, "cost": { "known_usd": 0.015725178, "known_decimal": "0.015725178", "unknown_attempts": 0, "exact_usd": 0.015725178, "cost_sources": { "api_reported": 1342 }, "known_usd_per_1000": 0.011717718330849477 }, "confidence_diagnostics": { "0.8": { "n": 230, "correct": 228, "answered": 230, "wrong_valid": 2, "failed": 0, "accuracy": 0.991304347826087, "accuracy_among_usable": 0.991304347826087 }, "0.9": { "n": 144, "correct": 144, "answered": 144, "wrong_valid": 0, "failed": 0, "accuracy": 1.0, "accuracy_among_usable": 1.0 }, "0.95": { "n": 96, "correct": 96, "answered": 96, "wrong_valid": 0, "failed": 0, "accuracy": 1.0, "accuracy_among_usable": 1.0 } }, "latency_first_usable": { "n": 1342, "median_ms": 224.8376039788127, "p95_ms": 377.38076279056264 }, "provider_failures": 0, "dates_toronto": { "2026-10-18": 1136, "2026-10-19": 206 }, "returned_identities": { "jaredpalmer/kev-4b-20260924": 1342 }, "providers": { "Phala": 1342 }, "model_id": "kev4b", "label": "Kev 4B", "route": "OpenRouter", "families": { "F01": { "n": 43, "correct": 39, "answered": 43, "wrong_valid": 4, "failed": 0, "accuracy": 0.9069767441860465, "accuracy_among_usable": 0.9069767441860465 }, "F02": { "n": 46, "correct": 36, "answered": 46, "wrong_valid": 10, "failed": 0, "accuracy": 0.782608695652174, "accuracy_among_usable": 0.782608695652174 }, "F03": { "n": 52, "correct": 33, "answered": 52, "wrong_valid": 19, "failed": 0, "accuracy": 0.6346153846153846, "accuracy_among_usable": 0.6346153846153846 }, "F04": { "n": 54, "correct": 34, "answered": 54, "wrong_valid": 20, "failed": 0, "accuracy": 0.6296296296296297, "accuracy_among_usable": 0.6296296296296297 }, "F05": { "n": 49, "correct": 42, "answered": 49, "wrong_valid": 7, "failed": 0, "accuracy": 0.8571428571428571, "accuracy_among_usable": 0.8571428571428571 }, "F06": { "n": 40, "correct": 35, "answered": 40, "wrong_valid": 5, "failed": 0, "accuracy": 0.875, "accuracy_among_usable": 0.875 }, "F07": { "n": 76, "correct": 37, "answered": 76, "wrong_valid": 39, "failed": 0, "accuracy": 0.4868421052631579, "accuracy_among_usable": 0.4868421052631579 }, "F08": { "n": 67, "correct": 45, "answered": 67, "wrong_valid": 22, "failed": 0, "accuracy": 0.6716417910447762, "accuracy_among_usable": 0.6716417910447762 }, "F09": { "n": 71, "correct": 21, "answered": 71, "wrong_valid": 50, "failed": 0, "accuracy": 0.29577464788732394, "accuracy_among_usable": 0.29577464788732394 }, "F10": { "n": 74, "correct": 33, "answered": 74, "wrong_valid": 41, "failed": 0, "accuracy": 0.44594594594594594, "accuracy_among_usable": 0.44594594594594594 }, "F11": { "n": 41, "correct": 30, "answered": 41, "wrong_valid": 11, "failed": 0, "accuracy": 0.7317073170731707, "accuracy_among_usable": 0.7317073170731707 }, "F12": { "n": 52, "correct": 25, "answered": 52, "wrong_valid": 27, "failed": 0, "accuracy": 0.4807692307692308, "accuracy_among_usable": 0.4807692307692308 }, "F13": { "n": 48, "correct": 11, "answered": 48, "wrong_valid": 37, "failed": 0, "accuracy": 0.22916666666666666, "accuracy_among_usable": 0.22916666666666666 }, "F14": { "n": 42, "correct": 31, "answered": 42, "wrong_valid": 11, "failed": 0, "accuracy": 0.7380952380952381, "accuracy_among_usable": 0.7380952380952381 }, "F15": { "n": 49, "correct": 26, "answered": 49, "wrong_valid": 23, "failed": 0, "accuracy": 0.5306122448979592, "accuracy_among_usable": 0.5306122448979592 }, "F16": { "n": 50, "correct": 19, "answered": 50, "wrong_valid": 31, "failed": 0, "accuracy": 0.38, "accuracy_among_usable": 0.38 }, "F17": { "n": 50, "correct": 34, "answered": 50, "wrong_valid": 16, "failed": 0, "accuracy": 0.68, "accuracy_among_usable": 0.68 }, "F18": { "n": 50, "correct": 29, "answered": 50, "wrong_valid": 21, "failed": 0, "accuracy": 0.58, "accuracy_among_usable": 0.58 }, "F19": { "n": 58, "correct": 39, "answered": 58, "wrong_valid": 19, "failed": 0, "accuracy": 0.6724137931034483, "accuracy_among_usable": 0.6724137931034483 }, "F20": { "n": 43, "correct": 24, "answered": 43, "wrong_valid": 19, "failed": 0, "accuracy": 0.5581395348837209, "accuracy_among_usable": 0.5581395348837209 }, "F21": { "n": 43, "correct": 40, "answered": 43, "wrong_valid": 3, "failed": 0, "accuracy": 0.9302325581395349, "accuracy_among_usable": 0.9302325581395349 }, "F22": { "n": 51, "correct": 48, "answered": 51, "wrong_valid": 3, "failed": 0, "accuracy": 0.9411764705882353, "accuracy_among_usable": 0.9411764705882353 }, "F23": { "n": 52, "correct": 37, "answered": 52, "wrong_valid": 15, "failed": 0, "accuracy": 0.7115384615384616, "accuracy_among_usable": 0.7115384615384616 }, "F24": { "n": 43, "correct": 20, "answered": 43, "wrong_valid": 23, "failed": 0, "accuracy": 0.46511627906976744, "accuracy_among_usable": 0.46511627906976744 }, "F25": { "n": 50, "correct": 36, "answered": 50, "wrong_valid": 14, "failed": 0, "accuracy": 0.72, "accuracy_among_usable": 0.72 }, "F26": { "n": 8, "correct": 3, "answered": 8, "wrong_valid": 5, "failed": 0, "accuracy": 0.375, "accuracy_among_usable": 0.375 }, "F27": { "n": 8, "correct": 1, "answered": 8, "wrong_valid": 7, "failed": 0, "accuracy": 0.125, "accuracy_among_usable": 0.125 }, "F28": { "n": 8, "correct": 4, "answered": 8, "wrong_valid": 4, "failed": 0, "accuracy": 0.5, "accuracy_among_usable": 0.5 }, "F29": { "n": 8, "correct": 3, "answered": 8, "wrong_valid": 5, "failed": 0, "accuracy": 0.375, "accuracy_among_usable": 0.375 }, "F30": { "n": 8, "correct": 3, "answered": 8, "wrong_valid": 5, "failed": 0, "accuracy": 0.375, "accuracy_among_usable": 0.375 }, "F31": { "n": 8, "correct": 4, "answered": 8, "wrong_valid": 4, "failed": 0, "accuracy": 0.5, "accuracy_among_usable": 0.5 } }, "macro_family_accuracy": 0.586617599877185, "release_exclusion_diagnostic": { "n": 1305, "correct": 807, "answered": 1305, "wrong_valid": 498, "failed": 0, "accuracy": 0.6183908045977011, "accuracy_among_usable": 0.6183908045977011 }, "new_domains": { "n": 48, "correct": 18, "answered": 48, "wrong_valid": 30, "failed": 0, "accuracy": 0.375, "accuracy_among_usable": 0.375 } }, "tev1_4b_native": { "n": 1342, "correct": 751, "answered": 1342, "wrong_valid": 591, "failed": 0, "accuracy": 0.559612518628912, "accuracy_among_usable": 0.559612518628912, "eventual": { "n": 1342, "correct": 751, "answered": 1342, "wrong_valid": 591, "failed": 0, "accuracy": 0.559612518628912, "accuracy_among_usable": 0.559612518628912 }, "median_ms": 197.5683750351891, "p95_ms": 252.86549585871398, "attempts": 1342, "recoveries": 0, "first_statuses": { "ok": 1342 }, "model_output_failures": 0, "probability_warnings": 0, "cost": { "known_usd": 0.02101113, "known_decimal": "0.021011130", "unknown_attempts": 0, "exact_usd": 0.02101113, "cost_sources": { "api_reported": 1342 }, "known_usd_per_1000": 0.015656579731743666 }, "confidence_diagnostics": { "0.8": { "n": 425, "correct": 390, "answered": 425, "wrong_valid": 35, "failed": 0, "accuracy": 0.9176470588235294, "accuracy_among_usable": 0.9176470588235294 }, "0.9": { "n": 317, "correct": 302, "answered": 317, "wrong_valid": 15, "failed": 0, "accuracy": 0.9526813880126183, "accuracy_among_usable": 0.9526813880126183 }, "0.95": { "n": 259, "correct": 248, "answered": 259, "wrong_valid": 11, "failed": 0, "accuracy": 0.9575289575289575, "accuracy_among_usable": 0.9575289575289575 } }, "latency_first_usable": { "n": 1342, "median_ms": 197.5683750351891, "p95_ms": 252.86549585871398 }, "provider_failures": 0, "dates_toronto": { "2026-10-18": 1136, "2026-10-19": 206 }, "returned_identities": { "togethercomputer/tev1-4b-experimental-20260923": 1342 }, "providers": { "Together": 1342 }, "model_id": "tev1_4b_native", "label": "Tev1 4B experimental (OpenRouter native adapter)", "route": "OpenRouter", "families": { "F01": { "n": 43, "correct": 38, "answered": 43, "wrong_valid": 5, "failed": 0, "accuracy": 0.8837209302325582, "accuracy_among_usable": 0.8837209302325582 }, "F02": { "n": 46, "correct": 25, "answered": 46, "wrong_valid": 21, "failed": 0, "accuracy": 0.5434782608695652, "accuracy_among_usable": 0.5434782608695652 }, "F03": { "n": 52, "correct": 29, "answered": 52, "wrong_valid": 23, "failed": 0, "accuracy": 0.5576923076923077, "accuracy_among_usable": 0.5576923076923077 }, "F04": { "n": 54, "correct": 33, "answered": 54, "wrong_valid": 21, "failed": 0, "accuracy": 0.6111111111111112, "accuracy_among_usable": 0.6111111111111112 }, "F05": { "n": 49, "correct": 37, "answered": 49, "wrong_valid": 12, "failed": 0, "accuracy": 0.7551020408163265, "accuracy_among_usable": 0.7551020408163265 }, "F06": { "n": 40, "correct": 32, "answered": 40, "wrong_valid": 8, "failed": 0, "accuracy": 0.8, "accuracy_among_usable": 0.8 }, "F07": { "n": 76, "correct": 28, "answered": 76, "wrong_valid": 48, "failed": 0, "accuracy": 0.3684210526315789, "accuracy_among_usable": 0.3684210526315789 }, "F08": { "n": 67, "correct": 40, "answered": 67, "wrong_valid": 27, "failed": 0, "accuracy": 0.5970149253731343, "accuracy_among_usable": 0.5970149253731343 }, "F09": { "n": 71, "correct": 16, "answered": 71, "wrong_valid": 55, "failed": 0, "accuracy": 0.22535211267605634, "accuracy_among_usable": 0.22535211267605634 }, "F10": { "n": 74, "correct": 25, "answered": 74, "wrong_valid": 49, "failed": 0, "accuracy": 0.33783783783783783, "accuracy_among_usable": 0.33783783783783783 }, "F11": { "n": 41, "correct": 23, "answered": 41, "wrong_valid": 18, "failed": 0, "accuracy": 0.5609756097560976, "accuracy_among_usable": 0.5609756097560976 }, "F12": { "n": 52, "correct": 20, "answered": 52, "wrong_valid": 32, "failed": 0, "accuracy": 0.38461538461538464, "accuracy_among_usable": 0.38461538461538464 }, "F13": { "n": 48, "correct": 15, "answered": 48, "wrong_valid": 33, "failed": 0, "accuracy": 0.3125, "accuracy_among_usable": 0.3125 }, "F14": { "n": 42, "correct": 27, "answered": 42, "wrong_valid": 15, "failed": 0, "accuracy": 0.6428571428571429, "accuracy_among_usable": 0.6428571428571429 }, "F15": { "n": 49, "correct": 19, "answered": 49, "wrong_valid": 30, "failed": 0, "accuracy": 0.3877551020408163, "accuracy_among_usable": 0.3877551020408163 }, "F16": { "n": 50, "correct": 19, "answered": 50, "wrong_valid": 31, "failed": 0, "accuracy": 0.38, "accuracy_among_usable": 0.38 }, "F17": { "n": 50, "correct": 31, "answered": 50, "wrong_valid": 19, "failed": 0, "accuracy": 0.62, "accuracy_among_usable": 0.62 }, "F18": { "n": 50, "correct": 20, "answered": 50, "wrong_valid": 30, "failed": 0, "accuracy": 0.4, "accuracy_among_usable": 0.4 }, "F19": { "n": 58, "correct": 42, "answered": 58, "wrong_valid": 16, "failed": 0, "accuracy": 0.7241379310344828, "accuracy_among_usable": 0.7241379310344828 }, "F20": { "n": 43, "correct": 28, "answered": 43, "wrong_valid": 15, "failed": 0, "accuracy": 0.6511627906976745, "accuracy_among_usable": 0.6511627906976745 }, "F21": { "n": 43, "correct": 35, "answered": 43, "wrong_valid": 8, "failed": 0, "accuracy": 0.813953488372093, "accuracy_among_usable": 0.813953488372093 }, "F22": { "n": 51, "correct": 47, "answered": 51, "wrong_valid": 4, "failed": 0, "accuracy": 0.9215686274509803, "accuracy_among_usable": 0.9215686274509803 }, "F23": { "n": 52, "correct": 39, "answered": 52, "wrong_valid": 13, "failed": 0, "accuracy": 0.75, "accuracy_among_usable": 0.75 }, "F24": { "n": 43, "correct": 29, "answered": 43, "wrong_valid": 14, "failed": 0, "accuracy": 0.6744186046511628, "accuracy_among_usable": 0.6744186046511628 }, "F25": { "n": 50, "correct": 39, "answered": 50, "wrong_valid": 11, "failed": 0, "accuracy": 0.78, "accuracy_among_usable": 0.78 }, "F26": { "n": 8, "correct": 3, "answered": 8, "wrong_valid": 5, "failed": 0, "accuracy": 0.375, "accuracy_among_usable": 0.375 }, "F27": { "n": 8, "correct": 0, "answered": 8, "wrong_valid": 8, "failed": 0, "accuracy": 0.0, "accuracy_among_usable": 0.0 }, "F28": { "n": 8, "correct": 2, "answered": 8, "wrong_valid": 6, "failed": 0, "accuracy": 0.25, "accuracy_among_usable": 0.25 }, "F29": { "n": 8, "correct": 3, "answered": 8, "wrong_valid": 5, "failed": 0, "accuracy": 0.375, "accuracy_among_usable": 0.375 }, "F30": { "n": 8, "correct": 3, "answered": 8, "wrong_valid": 5, "failed": 0, "accuracy": 0.375, "accuracy_among_usable": 0.375 }, "F31": { "n": 8, "correct": 4, "answered": 8, "wrong_valid": 4, "failed": 0, "accuracy": 0.5, "accuracy_among_usable": 0.5 } }, "macro_family_accuracy": 0.5341508148618164, "release_exclusion_diagnostic": { "n": 1305, "correct": 737, "answered": 1305, "wrong_valid": 568, "failed": 0, "accuracy": 0.5647509578544061, "accuracy_among_usable": 0.5647509578544061 }, "new_domains": { "n": 48, "correct": 15, "answered": 48, "wrong_valid": 33, "failed": 0, "accuracy": 0.3125, "accuracy_among_usable": 0.3125 } }, "reference_sol": { "n": 1342, "correct": 1143, "answered": 1342, "failed": 0, "wrong_valid": 199, "accuracy": 0.8517138599105812, "eventual": { "n": 1342, "correct": 1143, "answered": 1342, "failed": 0, "wrong_valid": 199, "accuracy": 0.8517138599105812 }, "latency_first_all": { "n": 1342, "median_ms": 1023.0274999630637, "p95_ms": 1546.3492952927484 }, "latency_first_usable": { "n": 1342, "median_ms": 1023.0274999630637, "p95_ms": 1546.3492952927484 }, "cost_all_attempts": { "known_usd": 0.8482945, "exact_usd": "0.8482945", "reported_attempts": 1342, "missing_attempts": 0, "known_usd_per_1000": 0.6321121460506706 }, "cost_first_attempts": { "known_usd": 0.8482945, "exact_usd": "0.8482945", "reported_attempts": 1342, "missing_attempts": 0 }, "cost_recovery_attempts": { "known_usd": 0.0, "exact_usd": "0", "reported_attempts": 0, "missing_attempts": 0 }, "attempts": 1342, "recoveries": 0, "statuses": { "ok": 1342 }, "reasoning_token_counts": { "0": 1342 }, "reasoning_text_responses": 0, "reasoning_verification_counts": { "explicit_reported_zero": 1342 }, "usage": { "prompt_tokens": 390314, "completion_tokens": 6710, "total_tokens": 397024 }, "returned_identities": { "openai/gpt-6-sol": 1342 }, "providers": { "OpenAI": 1342 }, "dates_toronto": { "2026-10-19": 1342 }, "families": { "F01": { "n": 43, "correct": 42, "answered": 43, "failed": 0, "wrong_valid": 1, "accuracy": 0.9767441860465116 }, "F02": { "n": 46, "correct": 45, "answered": 46, "failed": 0, "wrong_valid": 1, "accuracy": 0.9782608695652174 }, "F03": { "n": 52, "correct": 52, "answered": 52, "failed": 0, "wrong_valid": 0, "accuracy": 1.0 }, "F04": { "n": 54, "correct": 53, "answered": 54, "failed": 0, "wrong_valid": 1, "accuracy": 0.9814814814814815 }, "F05": { "n": 49, "correct": 46, "answered": 49, "failed": 0, "wrong_valid": 3, "accuracy": 0.9387755102040817 }, "F06": { "n": 40, "correct": 38, "answered": 40, "failed": 0, "wrong_valid": 2, "accuracy": 0.95 }, "F07": { "n": 76, "correct": 56, "answered": 76, "failed": 0, "wrong_valid": 20, "accuracy": 0.7368421052631579 }, "F08": { "n": 67, "correct": 61, "answered": 67, "failed": 0, "wrong_valid": 6, "accuracy": 0.9104477611940298 }, "F09": { "n": 71, "correct": 51, "answered": 71, "failed": 0, "wrong_valid": 20, "accuracy": 0.7183098591549296 }, "F10": { "n": 74, "correct": 38, "answered": 74, "failed": 0, "wrong_valid": 36, "accuracy": 0.5135135135135135 }, "F11": { "n": 41, "correct": 37, "answered": 41, "failed": 0, "wrong_valid": 4, "accuracy": 0.9024390243902439 }, "F12": { "n": 52, "correct": 32, "answered": 52, "failed": 0, "wrong_valid": 20, "accuracy": 0.6153846153846154 }, "F13": { "n": 48, "correct": 18, "answered": 48, "failed": 0, "wrong_valid": 30, "accuracy": 0.375 }, "F14": { "n": 42, "correct": 40, "answered": 42, "failed": 0, "wrong_valid": 2, "accuracy": 0.9523809523809523 }, "F15": { "n": 49, "correct": 43, "answered": 49, "failed": 0, "wrong_valid": 6, "accuracy": 0.8775510204081632 }, "F16": { "n": 50, "correct": 38, "answered": 50, "failed": 0, "wrong_valid": 12, "accuracy": 0.76 }, "F17": { "n": 50, "correct": 45, "answered": 50, "failed": 0, "wrong_valid": 5, "accuracy": 0.9 }, "F18": { "n": 50, "correct": 44, "answered": 50, "failed": 0, "wrong_valid": 6, "accuracy": 0.88 }, "F19": { "n": 58, "correct": 57, "answered": 58, "failed": 0, "wrong_valid": 1, "accuracy": 0.9827586206896551 }, "F20": { "n": 43, "correct": 39, "answered": 43, "failed": 0, "wrong_valid": 4, "accuracy": 0.9069767441860465 }, "F21": { "n": 43, "correct": 41, "answered": 43, "failed": 0, "wrong_valid": 2, "accuracy": 0.9534883720930233 }, "F22": { "n": 51, "correct": 48, "answered": 51, "failed": 0, "wrong_valid": 3, "accuracy": 0.9411764705882353 }, "F23": { "n": 52, "correct": 52, "answered": 52, "failed": 0, "wrong_valid": 0, "accuracy": 1.0 }, "F24": { "n": 43, "correct": 42, "answered": 43, "failed": 0, "wrong_valid": 1, "accuracy": 0.9767441860465116 }, "F25": { "n": 50, "correct": 49, "answered": 50, "failed": 0, "wrong_valid": 1, "accuracy": 0.98 }, "F26": { "n": 8, "correct": 5, "answered": 8, "failed": 0, "wrong_valid": 3, "accuracy": 0.625 }, "F27": { "n": 8, "correct": 6, "answered": 8, "failed": 0, "wrong_valid": 2, "accuracy": 0.75 }, "F28": { "n": 8, "correct": 6, "answered": 8, "failed": 0, "wrong_valid": 2, "accuracy": 0.75 }, "F29": { "n": 8, "correct": 4, "answered": 8, "failed": 0, "wrong_valid": 4, "accuracy": 0.5 }, "F30": { "n": 8, "correct": 7, "answered": 8, "failed": 0, "wrong_valid": 1, "accuracy": 0.875 }, "F31": { "n": 8, "correct": 8, "answered": 8, "failed": 0, "wrong_valid": 0, "accuracy": 1.0 } }, "macro_family_accuracy": 0.8454282352448506, "new_domains": { "n": 48, "correct": 36, "answered": 48, "failed": 0, "wrong_valid": 12, "accuracy": 0.75 }, "release_exclusion_diagnostic": { "n": 1305, "correct": 1123, "answered": 1305, "failed": 0, "wrong_valid": 182, "accuracy": 0.8605363984674329 }, "matched_original_1136": { "n": 1136, "correct": 1014, "answered": 1136, "failed": 0, "wrong_valid": 122, "accuracy": 0.8926056338028169 }, "added_206": { "n": 206, "correct": 129, "answered": 206, "failed": 0, "wrong_valid": 77, "accuracy": 0.6262135922330098 }, "initial_100": { "n": 100, "correct": 82, "answered": 100, "failed": 0, "wrong_valid": 18, "accuracy": 0.82 }, "model_id": "reference_sol", "label": "GPT-6 Sol \u2014 no reasoning", "route": "OpenRouter \u2192 OpenAI, standard service tier", "median_ms": 1023.0274999630637, "p95_ms": 1546.3492952927484, "accuracy_among_usable": 0.8517138599105812, "first_statuses": { "ok": 1342 }, "provider_failures": 0, "model_output_failures": 0, "cost": { "known_usd": 0.8482945, "known_decimal": "0.8482945", "unknown_attempts": 0, "exact_usd": 0.8482945, "known_usd_per_1000": 0.6321121460506706, "scope": "API-reported USD charge" }, "reasoning_setting": "reasoning.effort=none", "reasoning_verification": "Every successful response reports exactly zero reasoning/thinking tokens, with no reasoning content.", "compatibility_known_usd": 0.00018, "compatibility_unknown_attempts": 0 }, "reference_sonnet": { "n": 1342, "correct": 1070, "answered": 1342, "failed": 0, "wrong_valid": 272, "accuracy": 0.797317436661699, "eventual": { "n": 1342, "correct": 1070, "answered": 1342, "failed": 0, "wrong_valid": 272, "accuracy": 0.797317436661699 }, "latency_first_all": { "n": 1342, "median_ms": 1505.6673745275475, "p95_ms": 2423.8794767647055 }, "latency_first_usable": { "n": 1342, "median_ms": 1505.6673745275475, "p95_ms": 2423.8794767647055 }, "cost_all_attempts": { "known_usd": 1.923908, "exact_usd": "1.923908", "reported_attempts": 1342, "missing_attempts": 0, "known_usd_per_1000": 1.4336125186289121 }, "cost_first_attempts": { "known_usd": 1.923908, "exact_usd": "1.923908", "reported_attempts": 1342, "missing_attempts": 0 }, "cost_recovery_attempts": { "known_usd": 0.0, "exact_usd": "0", "reported_attempts": 0, "missing_attempts": 0 }, "attempts": 1342, "recoveries": 0, "statuses": { "ok": 1342 }, "reasoning_token_counts": { "0": 1342 }, "reasoning_text_responses": 0, "reasoning_verification_counts": { "explicit_reported_zero": 1342 }, "usage": { "prompt_tokens": 890684, "completion_tokens": 14254, "total_tokens": 904938 }, "returned_identities": { "anthropic/claude-sonnet-5.5": 1342 }, "providers": { "Anthropic": 1342 }, "dates_toronto": { "2026-10-19": 1342 }, "families": { "F01": { "n": 43, "correct": 42, "answered": 43, "failed": 0, "wrong_valid": 1, "accuracy": 0.9767441860465116 }, "F02": { "n": 46, "correct": 45, "answered": 46, "failed": 0, "wrong_valid": 1, "accuracy": 0.9782608695652174 }, "F03": { "n": 52, "correct": 48, "answered": 52, "failed": 0, "wrong_valid": 4, "accuracy": 0.9230769230769231 }, "F04": { "n": 54, "correct": 49, "answered": 54, "failed": 0, "wrong_valid": 5, "accuracy": 0.9074074074074074 }, "F05": { "n": 49, "correct": 45, "answered": 49, "failed": 0, "wrong_valid": 4, "accuracy": 0.9183673469387755 }, "F06": { "n": 40, "correct": 37, "answered": 40, "failed": 0, "wrong_valid": 3, "accuracy": 0.925 }, "F07": { "n": 76, "correct": 50, "answered": 76, "failed": 0, "wrong_valid": 26, "accuracy": 0.6578947368421053 }, "F08": { "n": 67, "correct": 62, "answered": 67, "failed": 0, "wrong_valid": 5, "accuracy": 0.9253731343283582 }, "F09": { "n": 71, "correct": 30, "answered": 71, "failed": 0, "wrong_valid": 41, "accuracy": 0.4225352112676056 }, "F10": { "n": 74, "correct": 41, "answered": 74, "failed": 0, "wrong_valid": 33, "accuracy": 0.5540540540540541 }, "F11": { "n": 41, "correct": 33, "answered": 41, "failed": 0, "wrong_valid": 8, "accuracy": 0.8048780487804879 }, "F12": { "n": 52, "correct": 29, "answered": 52, "failed": 0, "wrong_valid": 23, "accuracy": 0.5576923076923077 }, "F13": { "n": 48, "correct": 15, "answered": 48, "failed": 0, "wrong_valid": 33, "accuracy": 0.3125 }, "F14": { "n": 42, "correct": 38, "answered": 42, "failed": 0, "wrong_valid": 4, "accuracy": 0.9047619047619048 }, "F15": { "n": 49, "correct": 43, "answered": 49, "failed": 0, "wrong_valid": 6, "accuracy": 0.8775510204081632 }, "F16": { "n": 50, "correct": 36, "answered": 50, "failed": 0, "wrong_valid": 14, "accuracy": 0.72 }, "F17": { "n": 50, "correct": 45, "answered": 50, "failed": 0, "wrong_valid": 5, "accuracy": 0.9 }, "F18": { "n": 50, "correct": 41, "answered": 50, "failed": 0, "wrong_valid": 9, "accuracy": 0.82 }, "F19": { "n": 58, "correct": 53, "answered": 58, "failed": 0, "wrong_valid": 5, "accuracy": 0.9137931034482759 }, "F20": { "n": 43, "correct": 36, "answered": 43, "failed": 0, "wrong_valid": 7, "accuracy": 0.8372093023255814 }, "F21": { "n": 43, "correct": 40, "answered": 43, "failed": 0, "wrong_valid": 3, "accuracy": 0.9302325581395349 }, "F22": { "n": 51, "correct": 49, "answered": 51, "failed": 0, "wrong_valid": 2, "accuracy": 0.9607843137254902 }, "F23": { "n": 52, "correct": 52, "answered": 52, "failed": 0, "wrong_valid": 0, "accuracy": 1.0 }, "F24": { "n": 43, "correct": 39, "answered": 43, "failed": 0, "wrong_valid": 4, "accuracy": 0.9069767441860465 }, "F25": { "n": 50, "correct": 46, "answered": 50, "failed": 0, "wrong_valid": 4, "accuracy": 0.92 }, "F26": { "n": 8, "correct": 5, "answered": 8, "failed": 0, "wrong_valid": 3, "accuracy": 0.625 }, "F27": { "n": 8, "correct": 4, "answered": 8, "failed": 0, "wrong_valid": 4, "accuracy": 0.5 }, "F28": { "n": 8, "correct": 4, "answered": 8, "failed": 0, "wrong_valid": 4, "accuracy": 0.5 }, "F29": { "n": 8, "correct": 1, "answered": 8, "failed": 0, "wrong_valid": 7, "accuracy": 0.125 }, "F30": { "n": 8, "correct": 5, "answered": 8, "failed": 0, "wrong_valid": 3, "accuracy": 0.625 }, "F31": { "n": 8, "correct": 7, "answered": 8, "failed": 0, "wrong_valid": 1, "accuracy": 0.875 } }, "macro_family_accuracy": 0.7679062313869276, "new_domains": { "n": 48, "correct": 26, "answered": 48, "failed": 0, "wrong_valid": 22, "accuracy": 0.5416666666666666 }, "release_exclusion_diagnostic": { "n": 1305, "correct": 1056, "answered": 1305, "failed": 0, "wrong_valid": 249, "accuracy": 0.8091954022988506 }, "matched_original_1136": { "n": 1136, "correct": 971, "answered": 1136, "failed": 0, "wrong_valid": 165, "accuracy": 0.8547535211267606 }, "added_206": { "n": 206, "correct": 99, "answered": 206, "failed": 0, "wrong_valid": 107, "accuracy": 0.48058252427184467 }, "initial_100": { "n": 100, "correct": 80, "answered": 100, "failed": 0, "wrong_valid": 20, "accuracy": 0.8 }, "model_id": "reference_sonnet", "label": "Sonnet 5.5 \u2014 no thinking, JSON-choice", "route": "OpenRouter \u2192 Anthropic, standard service", "median_ms": 1505.6673745275475, "p95_ms": 2423.8794767647055, "accuracy_among_usable": 0.797317436661699, "first_statuses": { "ok": 1342 }, "provider_failures": 0, "model_output_failures": 0, "cost": { "known_usd": 1.923908, "known_decimal": "1.923908", "unknown_attempts": 0, "exact_usd": 1.923908, "known_usd_per_1000": 1.4336125186289121, "scope": "API-reported USD charge" }, "reasoning_setting": "thinking.type=between_tools; effort=high; no tools", "reasoning_verification": "Every successful response reports exactly zero reasoning/thinking tokens, with no reasoning content.", "compatibility_known_usd": 0.002182, "compatibility_unknown_attempts": 0 }, "reference_qwen": { "n": 1342, "correct": 967, "answered": 1317, "failed": 25, "wrong_valid": 350, "accuracy": 0.7205663189269746, "eventual": { "n": 1342, "correct": 971, "answered": 1322, "failed": 20, "wrong_valid": 351, "accuracy": 0.7235469448584203 }, "latency_first_all": { "n": 1342, "median_ms": 1190.6628959695809, "p95_ms": 1731.1974538024515 }, "latency_first_usable": { "n": 1317, "median_ms": 1194.47400001809, "p95_ms": 1735.4193163570026 }, "cost_all_attempts": { "known_usd": 0.82509, "exact_usd": "0.82509000000000002271", "reported_attempts": 1322, "missing_attempts": 43, "known_usd_per_1000": 0.6148211624441132 }, "cost_first_attempts": { "known_usd": 0.822154, "exact_usd": "0.82215400000000002274", "reported_attempts": 1317, "missing_attempts": 25 }, "cost_recovery_attempts": { "known_usd": 0.002936, "exact_usd": "0.00293599999999999997", "reported_attempts": 5, "missing_attempts": 18 }, "attempts": 1365, "recoveries": 23, "statuses": { "ok": 1317, "transport_failure": 1, "http_failure": 24 }, "reasoning_token_counts": { "None": 1342 }, "reasoning_text_responses": 0, "reasoning_verification_counts": { "requested_none_catalog_supported_no_token_telemetry": 1342 }, "hypercredits_all_attempts": { "known": 16.5018, "missing_attempts": 43, "scope": "Provider-reported quota debit, not a cash invoice" }, "usage": { "prompt_tokens": 408579, "completion_tokens": 1322, "total_tokens": 409901 }, "returned_identities": { "qwen3.8-max": 1322, "unreported": 43 }, "providers": { "Hyper Charm (request host)": 1365 }, "dates_toronto": { "2026-10-19": 1342 }, "families": { "F01": { "n": 43, "correct": 37, "answered": 40, "failed": 3, "wrong_valid": 3, "accuracy": 0.8604651162790697 }, "F02": { "n": 46, "correct": 38, "answered": 46, "failed": 0, "wrong_valid": 8, "accuracy": 0.8260869565217391 }, "F03": { "n": 52, "correct": 44, "answered": 52, "failed": 0, "wrong_valid": 8, "accuracy": 0.8461538461538461 }, "F04": { "n": 54, "correct": 45, "answered": 54, "failed": 0, "wrong_valid": 9, "accuracy": 0.8333333333333334 }, "F05": { "n": 49, "correct": 44, "answered": 48, "failed": 1, "wrong_valid": 4, "accuracy": 0.8979591836734694 }, "F06": { "n": 40, "correct": 40, "answered": 40, "failed": 0, "wrong_valid": 0, "accuracy": 1.0 }, "F07": { "n": 76, "correct": 42, "answered": 74, "failed": 2, "wrong_valid": 32, "accuracy": 0.5526315789473685 }, "F08": { "n": 67, "correct": 58, "answered": 67, "failed": 0, "wrong_valid": 9, "accuracy": 0.8656716417910447 }, "F09": { "n": 71, "correct": 24, "answered": 70, "failed": 1, "wrong_valid": 46, "accuracy": 0.3380281690140845 }, "F10": { "n": 74, "correct": 30, "answered": 70, "failed": 4, "wrong_valid": 40, "accuracy": 0.40540540540540543 }, "F11": { "n": 41, "correct": 25, "answered": 38, "failed": 3, "wrong_valid": 13, "accuracy": 0.6097560975609756 }, "F12": { "n": 52, "correct": 27, "answered": 51, "failed": 1, "wrong_valid": 24, "accuracy": 0.5192307692307693 }, "F13": { "n": 48, "correct": 11, "answered": 47, "failed": 1, "wrong_valid": 36, "accuracy": 0.22916666666666666 }, "F14": { "n": 42, "correct": 34, "answered": 42, "failed": 0, "wrong_valid": 8, "accuracy": 0.8095238095238095 }, "F15": { "n": 49, "correct": 28, "answered": 47, "failed": 2, "wrong_valid": 19, "accuracy": 0.5714285714285714 }, "F16": { "n": 50, "correct": 30, "answered": 50, "failed": 0, "wrong_valid": 20, "accuracy": 0.6 }, "F17": { "n": 50, "correct": 38, "answered": 50, "failed": 0, "wrong_valid": 12, "accuracy": 0.76 }, "F18": { "n": 50, "correct": 36, "answered": 50, "failed": 0, "wrong_valid": 14, "accuracy": 0.72 }, "F19": { "n": 58, "correct": 53, "answered": 57, "failed": 1, "wrong_valid": 4, "accuracy": 0.9137931034482759 }, "F20": { "n": 43, "correct": 35, "answered": 42, "failed": 1, "wrong_valid": 7, "accuracy": 0.813953488372093 }, "F21": { "n": 43, "correct": 40, "answered": 43, "failed": 0, "wrong_valid": 3, "accuracy": 0.9302325581395349 }, "F22": { "n": 51, "correct": 47, "answered": 50, "failed": 1, "wrong_valid": 3, "accuracy": 0.9215686274509803 }, "F23": { "n": 52, "correct": 49, "answered": 51, "failed": 1, "wrong_valid": 2, "accuracy": 0.9423076923076923 }, "F24": { "n": 43, "correct": 38, "answered": 41, "failed": 2, "wrong_valid": 3, "accuracy": 0.8837209302325582 }, "F25": { "n": 50, "correct": 46, "answered": 49, "failed": 1, "wrong_valid": 3, "accuracy": 0.92 }, "F26": { "n": 8, "correct": 6, "answered": 8, "failed": 0, "wrong_valid": 2, "accuracy": 0.75 }, "F27": { "n": 8, "correct": 4, "answered": 8, "failed": 0, "wrong_valid": 4, "accuracy": 0.5 }, "F28": { "n": 8, "correct": 4, "answered": 8, "failed": 0, "wrong_valid": 4, "accuracy": 0.5 }, "F29": { "n": 8, "correct": 3, "answered": 8, "failed": 0, "wrong_valid": 5, "accuracy": 0.375 }, "F30": { "n": 8, "correct": 4, "answered": 8, "failed": 0, "wrong_valid": 4, "accuracy": 0.5 }, "F31": { "n": 8, "correct": 7, "answered": 8, "failed": 0, "wrong_valid": 1, "accuracy": 0.875 } }, "macro_family_accuracy": 0.7119489530800417, "new_domains": { "n": 48, "correct": 28, "answered": 48, "failed": 0, "wrong_valid": 20, "accuracy": 0.5833333333333334 }, "release_exclusion_diagnostic": { "n": 1305, "correct": 948, "answered": 1281, "failed": 24, "wrong_valid": 333, "accuracy": 0.7264367816091954 }, "matched_original_1136": { "n": 1136, "correct": 855, "answered": 1113, "failed": 23, "wrong_valid": 258, "accuracy": 0.7526408450704225 }, "added_206": { "n": 206, "correct": 112, "answered": 204, "failed": 2, "wrong_valid": 92, "accuracy": 0.5436893203883495 }, "initial_100": { "n": 100, "correct": 70, "answered": 100, "failed": 0, "wrong_valid": 30, "accuracy": 0.7 }, "model_id": "reference_qwen", "label": "Qwen3.8-Max \u2014 none, Hyper Charm", "route": "Hyper Charm direct, sequential capacity1", "median_ms": 1190.6628959695809, "p95_ms": 1731.1974538024515, "accuracy_among_usable": 0.7342444950645406, "first_statuses": { "ok": 1317, "transport_failure": 1, "http_failure": 24 }, "provider_failures": 25, "model_output_failures": 0, "cost": { "known_usd": 0.82509, "known_decimal": "0.82509000000000002271", "unknown_attempts": 43, "exact_usd": null, "known_usd_per_1000": 0.6148211624441132, "scope": "USD-equivalent prepaid quota valuation, not cash invoice" }, "reasoning_setting": "reasoning_effort=none, documented/catalog-supported", "reasoning_verification": "Provider omits reasoning-token telemetry. The none setting is catalog-supported and requested explicitly; no reasoning text is allowed. Unknown token counts are not treated as verified zeros.", "compatibility_known_usd": 0.001112, "compatibility_unknown_attempts": 0 }, "d3_turbo": { "model_id": "d3_turbo", "label": "d3-turbo", "n": 1342, "correct": 1106, "answered": 1342, "wrong_valid": 236, "failed": 0, "accuracy": 0.8241430700447094, "accuracy_among_usable": 0.8241430700447094, "mean_response_ms": 181.9034249486399, "median_ms": 165.2547080302611, "p95_ms": 297.8397538594437, "attempts": 1342, "recoveries": 0, "first_statuses": { "ok": 1342 }, "provider_failures": 0, "model_output_failures": 0, "probability_warnings": 0, "eventual": { "n": 1342, "correct": 1106, "answered": 1342, "failed": 0, "wrong_valid": 236, "accuracy": 0.8241430700447094 }, "cost": { "known_usd": null, "known_decimal": null, "known_usd_per_1000": null, "unknown_attempts": 1342, "exact_usd": null, "public_free_access": true, "accounted_access_cost_usd": 0.0, "scope": "Currently free public endpoint; no API-reported monetary charge and no production-price guarantee" }, "families": { "F01": { "n": 43, "correct": 41, "answered": 43, "failed": 0, "wrong_valid": 2, "accuracy": 0.9534883720930233 }, "F02": { "n": 46, "correct": 46, "answered": 46, "failed": 0, "wrong_valid": 0, "accuracy": 1.0 }, "F03": { "n": 52, "correct": 51, "answered": 52, "failed": 0, "wrong_valid": 1, "accuracy": 0.9807692307692307 }, "F04": { "n": 54, "correct": 50, "answered": 54, "failed": 0, "wrong_valid": 4, "accuracy": 0.9259259259259259 }, "F05": { "n": 49, "correct": 45, "answered": 49, "failed": 0, "wrong_valid": 4, "accuracy": 0.9183673469387755 }, "F06": { "n": 40, "correct": 38, "answered": 40, "failed": 0, "wrong_valid": 2, "accuracy": 0.95 }, "F07": { "n": 76, "correct": 54, "answered": 76, "failed": 0, "wrong_valid": 22, "accuracy": 0.7105263157894737 }, "F08": { "n": 67, "correct": 63, "answered": 67, "failed": 0, "wrong_valid": 4, "accuracy": 0.9402985074626866 }, "F09": { "n": 71, "correct": 35, "answered": 71, "failed": 0, "wrong_valid": 36, "accuracy": 0.49295774647887325 }, "F10": { "n": 74, "correct": 49, "answered": 74, "failed": 0, "wrong_valid": 25, "accuracy": 0.6621621621621622 }, "F11": { "n": 41, "correct": 35, "answered": 41, "failed": 0, "wrong_valid": 6, "accuracy": 0.8536585365853658 }, "F12": { "n": 52, "correct": 31, "answered": 52, "failed": 0, "wrong_valid": 21, "accuracy": 0.5961538461538461 }, "F13": { "n": 48, "correct": 20, "answered": 48, "failed": 0, "wrong_valid": 28, "accuracy": 0.4166666666666667 }, "F14": { "n": 42, "correct": 41, "answered": 42, "failed": 0, "wrong_valid": 1, "accuracy": 0.9761904761904762 }, "F15": { "n": 49, "correct": 36, "answered": 49, "failed": 0, "wrong_valid": 13, "accuracy": 0.7346938775510204 }, "F16": { "n": 50, "correct": 34, "answered": 50, "failed": 0, "wrong_valid": 16, "accuracy": 0.68 }, "F17": { "n": 50, "correct": 47, "answered": 50, "failed": 0, "wrong_valid": 3, "accuracy": 0.94 }, "F18": { "n": 50, "correct": 38, "answered": 50, "failed": 0, "wrong_valid": 12, "accuracy": 0.76 }, "F19": { "n": 58, "correct": 56, "answered": 58, "failed": 0, "wrong_valid": 2, "accuracy": 0.9655172413793104 }, "F20": { "n": 43, "correct": 38, "answered": 43, "failed": 0, "wrong_valid": 5, "accuracy": 0.8837209302325582 }, "F21": { "n": 43, "correct": 41, "answered": 43, "failed": 0, "wrong_valid": 2, "accuracy": 0.9534883720930233 }, "F22": { "n": 51, "correct": 48, "answered": 51, "failed": 0, "wrong_valid": 3, "accuracy": 0.9411764705882353 }, "F23": { "n": 52, "correct": 52, "answered": 52, "failed": 0, "wrong_valid": 0, "accuracy": 1.0 }, "F24": { "n": 43, "correct": 42, "answered": 43, "failed": 0, "wrong_valid": 1, "accuracy": 0.9767441860465116 }, "F25": { "n": 50, "correct": 48, "answered": 50, "failed": 0, "wrong_valid": 2, "accuracy": 0.96 }, "F26": { "n": 8, "correct": 6, "answered": 8, "failed": 0, "wrong_valid": 2, "accuracy": 0.75 }, "F27": { "n": 8, "correct": 4, "answered": 8, "failed": 0, "wrong_valid": 4, "accuracy": 0.5 }, "F28": { "n": 8, "correct": 2, "answered": 8, "failed": 0, "wrong_valid": 6, "accuracy": 0.25 }, "F29": { "n": 8, "correct": 5, "answered": 8, "failed": 0, "wrong_valid": 3, "accuracy": 0.625 }, "F30": { "n": 8, "correct": 4, "answered": 8, "failed": 0, "wrong_valid": 4, "accuracy": 0.5 }, "F31": { "n": 8, "correct": 6, "answered": 8, "failed": 0, "wrong_valid": 2, "accuracy": 0.75 } }, "dates_toronto": { "2026-10-20": 1342 }, "returned_identities": { "vllm-sr/d3-turbo": 1342 }, "providers": { "direct": 1342 }, "route": "Direct \u2192 https://decision.vllm-sr.ai/v1/systemone \u2192 vllm-sr/d3-turbo", "output_token_values": { "0": 1342 }, "input_tokens": 480427, "macro_family_accuracy": 0.7918550390679729, "new_domains": { "n": 48, "correct": 27, "answered": 48, "failed": 0, "wrong_valid": 21, "accuracy": 0.5625 }, "release_exclusion_diagnostic": { "n": 1305, "correct": 1092, "answered": 1305, "failed": 0, "wrong_valid": 213, "accuracy": 0.8367816091954023 }, "compatibility": { "status": "ok", "accounted_cost_usd": 0.0, "reported_cost_usd": null, "cost_source": "public_free_endpoint_not_api_reported" } }, "amos_large": { "model_id": "amos_large", "label": "Amos 1 Large", "n": 1342, "correct": 1074, "answered": 1342, "wrong_valid": 268, "failed": 0, "accuracy": 0.8002980625931445, "accuracy_among_usable": 0.8002980625931445, "mean_response_ms": 306.1684600538026, "median_ms": 310.6975415139459, "p95_ms": 359.8499482031912, "attempts": 1342, "recoveries": 0, "first_statuses": { "ok": 1342 }, "provider_failures": 0, "model_output_failures": 0, "probability_warnings": 0, "eventual": { "n": 1342, "correct": 1074, "answered": 1342, "failed": 0, "wrong_valid": 268, "accuracy": 0.8002980625931445 }, "cost": { "known_usd": null, "known_decimal": null, "known_usd_per_1000": null, "unknown_attempts": 1342, "exact_usd": null, "public_free_access": true, "accounted_access_cost_usd": 0.0, "scope": "Currently free public endpoint; no API-reported monetary charge and no production-price guarantee" }, "families": { "F01": { "n": 43, "correct": 40, "answered": 43, "failed": 0, "wrong_valid": 3, "accuracy": 0.9302325581395349 }, "F02": { "n": 46, "correct": 44, "answered": 46, "failed": 0, "wrong_valid": 2, "accuracy": 0.9565217391304348 }, "F03": { "n": 52, "correct": 49, "answered": 52, "failed": 0, "wrong_valid": 3, "accuracy": 0.9423076923076923 }, "F04": { "n": 54, "correct": 51, "answered": 54, "failed": 0, "wrong_valid": 3, "accuracy": 0.9444444444444444 }, "F05": { "n": 49, "correct": 45, "answered": 49, "failed": 0, "wrong_valid": 4, "accuracy": 0.9183673469387755 }, "F06": { "n": 40, "correct": 38, "answered": 40, "failed": 0, "wrong_valid": 2, "accuracy": 0.95 }, "F07": { "n": 76, "correct": 54, "answered": 76, "failed": 0, "wrong_valid": 22, "accuracy": 0.7105263157894737 }, "F08": { "n": 67, "correct": 62, "answered": 67, "failed": 0, "wrong_valid": 5, "accuracy": 0.9253731343283582 }, "F09": { "n": 71, "correct": 37, "answered": 71, "failed": 0, "wrong_valid": 34, "accuracy": 0.5211267605633803 }, "F10": { "n": 74, "correct": 51, "answered": 74, "failed": 0, "wrong_valid": 23, "accuracy": 0.6891891891891891 }, "F11": { "n": 41, "correct": 35, "answered": 41, "failed": 0, "wrong_valid": 6, "accuracy": 0.8536585365853658 }, "F12": { "n": 52, "correct": 26, "answered": 52, "failed": 0, "wrong_valid": 26, "accuracy": 0.5 }, "F13": { "n": 48, "correct": 16, "answered": 48, "failed": 0, "wrong_valid": 32, "accuracy": 0.3333333333333333 }, "F14": { "n": 42, "correct": 41, "answered": 42, "failed": 0, "wrong_valid": 1, "accuracy": 0.9761904761904762 }, "F15": { "n": 49, "correct": 36, "answered": 49, "failed": 0, "wrong_valid": 13, "accuracy": 0.7346938775510204 }, "F16": { "n": 50, "correct": 30, "answered": 50, "failed": 0, "wrong_valid": 20, "accuracy": 0.6 }, "F17": { "n": 50, "correct": 46, "answered": 50, "failed": 0, "wrong_valid": 4, "accuracy": 0.92 }, "F18": { "n": 50, "correct": 36, "answered": 50, "failed": 0, "wrong_valid": 14, "accuracy": 0.72 }, "F19": { "n": 58, "correct": 55, "answered": 58, "failed": 0, "wrong_valid": 3, "accuracy": 0.9482758620689655 }, "F20": { "n": 43, "correct": 38, "answered": 43, "failed": 0, "wrong_valid": 5, "accuracy": 0.8837209302325582 }, "F21": { "n": 43, "correct": 41, "answered": 43, "failed": 0, "wrong_valid": 2, "accuracy": 0.9534883720930233 }, "F22": { "n": 51, "correct": 47, "answered": 51, "failed": 0, "wrong_valid": 4, "accuracy": 0.9215686274509803 }, "F23": { "n": 52, "correct": 50, "answered": 52, "failed": 0, "wrong_valid": 2, "accuracy": 0.9615384615384616 }, "F24": { "n": 43, "correct": 41, "answered": 43, "failed": 0, "wrong_valid": 2, "accuracy": 0.9534883720930233 }, "F25": { "n": 50, "correct": 44, "answered": 50, "failed": 0, "wrong_valid": 6, "accuracy": 0.88 }, "F26": { "n": 8, "correct": 3, "answered": 8, "failed": 0, "wrong_valid": 5, "accuracy": 0.375 }, "F27": { "n": 8, "correct": 3, "answered": 8, "failed": 0, "wrong_valid": 5, "accuracy": 0.375 }, "F28": { "n": 8, "correct": 3, "answered": 8, "failed": 0, "wrong_valid": 5, "accuracy": 0.375 }, "F29": { "n": 8, "correct": 3, "answered": 8, "failed": 0, "wrong_valid": 5, "accuracy": 0.375 }, "F30": { "n": 8, "correct": 3, "answered": 8, "failed": 0, "wrong_valid": 5, "accuracy": 0.375 }, "F31": { "n": 8, "correct": 6, "answered": 8, "failed": 0, "wrong_valid": 2, "accuracy": 0.75 } }, "dates_toronto": { "2026-10-20": 1342 }, "returned_identities": { "amos-1-large": 1342 }, "providers": { "direct": 1342 }, "route": "Direct \u2192 https://api.amos.scira.ai/v1/systemone \u2192 amos-1-large", "output_token_values": { "0": 1342 }, "input_tokens": 465665, "macro_family_accuracy": 0.7500982590312416, "new_domains": { "n": 48, "correct": 21, "answered": 48, "failed": 0, "wrong_valid": 27, "accuracy": 0.4375 }, "release_exclusion_diagnostic": { "n": 1305, "correct": 1059, "answered": 1305, "failed": 0, "wrong_valid": 246, "accuracy": 0.8114942528735632 }, "compatibility": { "status": "ok", "accounted_cost_usd": 0.0, "reported_cost_usd": null, "cost_source": "public_free_endpoint_not_api_reported" } }, "kettei_fat": { "model_id": "kettei_fat", "label": "kettei-fat", "n": 1342, "correct": 1066, "answered": 1342, "wrong_valid": 276, "failed": 0, "accuracy": 0.7943368107302533, "accuracy_among_usable": 0.7943368107302533, "mean_response_ms": 179.42590906099804, "median_ms": 173.85120794642717, "p95_ms": 213.5956204903778, "attempts": 1342, "recoveries": 0, "first_statuses": { "ok": 1342 }, "provider_failures": 0, "model_output_failures": 0, "probability_warnings": 0, "eventual": { "n": 1342, "correct": 1066, "answered": 1342, "failed": 0, "wrong_valid": 276, "accuracy": 0.7943368107302533 }, "cost": { "known_usd": 0.02071056, "known_decimal": "0.02071056", "known_usd_per_1000": 0.015432608047690016, "unknown_attempts": 0, "exact_usd": 0.02071056, "public_free_access": false, "accounted_access_cost_usd": 0.02071056, "scope": "API-reported USD charges; starter credit is payment credit, not a zero price" }, "families": { "F01": { "n": 43, "correct": 41, "answered": 43, "failed": 0, "wrong_valid": 2, "accuracy": 0.9534883720930233 }, "F02": { "n": 46, "correct": 46, "answered": 46, "failed": 0, "wrong_valid": 0, "accuracy": 1.0 }, "F03": { "n": 52, "correct": 50, "answered": 52, "failed": 0, "wrong_valid": 2, "accuracy": 0.9615384615384616 }, "F04": { "n": 54, "correct": 50, "answered": 54, "failed": 0, "wrong_valid": 4, "accuracy": 0.9259259259259259 }, "F05": { "n": 49, "correct": 47, "answered": 49, "failed": 0, "wrong_valid": 2, "accuracy": 0.9591836734693877 }, "F06": { "n": 40, "correct": 39, "answered": 40, "failed": 0, "wrong_valid": 1, "accuracy": 0.975 }, "F07": { "n": 76, "correct": 44, "answered": 76, "failed": 0, "wrong_valid": 32, "accuracy": 0.5789473684210527 }, "F08": { "n": 67, "correct": 53, "answered": 67, "failed": 0, "wrong_valid": 14, "accuracy": 0.7910447761194029 }, "F09": { "n": 71, "correct": 19, "answered": 71, "failed": 0, "wrong_valid": 52, "accuracy": 0.2676056338028169 }, "F10": { "n": 74, "correct": 43, "answered": 74, "failed": 0, "wrong_valid": 31, "accuracy": 0.581081081081081 }, "F11": { "n": 41, "correct": 36, "answered": 41, "failed": 0, "wrong_valid": 5, "accuracy": 0.8780487804878049 }, "F12": { "n": 52, "correct": 29, "answered": 52, "failed": 0, "wrong_valid": 23, "accuracy": 0.5576923076923077 }, "F13": { "n": 48, "correct": 16, "answered": 48, "failed": 0, "wrong_valid": 32, "accuracy": 0.3333333333333333 }, "F14": { "n": 42, "correct": 39, "answered": 42, "failed": 0, "wrong_valid": 3, "accuracy": 0.9285714285714286 }, "F15": { "n": 49, "correct": 38, "answered": 49, "failed": 0, "wrong_valid": 11, "accuracy": 0.7755102040816326 }, "F16": { "n": 50, "correct": 37, "answered": 50, "failed": 0, "wrong_valid": 13, "accuracy": 0.74 }, "F17": { "n": 50, "correct": 43, "answered": 50, "failed": 0, "wrong_valid": 7, "accuracy": 0.86 }, "F18": { "n": 50, "correct": 45, "answered": 50, "failed": 0, "wrong_valid": 5, "accuracy": 0.9 }, "F19": { "n": 58, "correct": 56, "answered": 58, "failed": 0, "wrong_valid": 2, "accuracy": 0.9655172413793104 }, "F20": { "n": 43, "correct": 38, "answered": 43, "failed": 0, "wrong_valid": 5, "accuracy": 0.8837209302325582 }, "F21": { "n": 43, "correct": 42, "answered": 43, "failed": 0, "wrong_valid": 1, "accuracy": 0.9767441860465116 }, "F22": { "n": 51, "correct": 48, "answered": 51, "failed": 0, "wrong_valid": 3, "accuracy": 0.9411764705882353 }, "F23": { "n": 52, "correct": 51, "answered": 52, "failed": 0, "wrong_valid": 1, "accuracy": 0.9807692307692307 }, "F24": { "n": 43, "correct": 43, "answered": 43, "failed": 0, "wrong_valid": 0, "accuracy": 1.0 }, "F25": { "n": 50, "correct": 46, "answered": 50, "failed": 0, "wrong_valid": 4, "accuracy": 0.92 }, "F26": { "n": 8, "correct": 4, "answered": 8, "failed": 0, "wrong_valid": 4, "accuracy": 0.5 }, "F27": { "n": 8, "correct": 4, "answered": 8, "failed": 0, "wrong_valid": 4, "accuracy": 0.5 }, "F28": { "n": 8, "correct": 3, "answered": 8, "failed": 0, "wrong_valid": 5, "accuracy": 0.375 }, "F29": { "n": 8, "correct": 5, "answered": 8, "failed": 0, "wrong_valid": 3, "accuracy": 0.625 }, "F30": { "n": 8, "correct": 5, "answered": 8, "failed": 0, "wrong_valid": 3, "accuracy": 0.625 }, "F31": { "n": 8, "correct": 6, "answered": 8, "failed": 0, "wrong_valid": 2, "accuracy": 0.75 } }, "dates_toronto": { "2026-10-20": 1342 }, "returned_identities": { "kettei-fat": 1342 }, "providers": { "direct": 1342 }, "route": "Direct \u2192 https://api.sqwish.ai/v1/systemone \u2192 kettei-fat", "output_token_values": { "0": 1342 }, "input_tokens": 345176, "macro_family_accuracy": 0.7745128840526937, "new_domains": { "n": 48, "correct": 27, "answered": 48, "failed": 0, "wrong_valid": 21, "accuracy": 0.5625 }, "release_exclusion_diagnostic": { "n": 1305, "correct": 1049, "answered": 1305, "failed": 0, "wrong_valid": 256, "accuracy": 0.803831417624521 }, "compatibility": { "status": "ok", "accounted_cost_usd": 1.92e-06, "reported_cost_usd": 1.92e-06, "cost_source": "api_reported_charge" } }, "mammoth1": { "model_id": "mammoth1", "label": "Mammoth-1", "n": 1342, "correct": 1139, "answered": 1342, "wrong_valid": 203, "failed": 0, "accuracy": 0.8487332339791356, "accuracy_among_usable": 0.8487332339791356, "mean_response_ms": 301.3734981484365, "median_ms": 239.56312499649357, "p95_ms": 665.6302211500588, "attempts": 1342, "recoveries": 0, "first_statuses": { "ok": 1342 }, "provider_failures": 0, "model_output_failures": 0, "probability_warnings": 0, "eventual": { "n": 1342, "correct": 1139, "answered": 1342, "failed": 0, "wrong_valid": 203, "accuracy": 0.8487332339791356 }, "cost": { "known_usd": null, "known_decimal": null, "known_usd_per_1000": null, "unknown_attempts": 1342, "exact_usd": null, "public_free_access": true, "accounted_access_cost_usd": 0.0, "scope": "Captured promotional free access; no monetary receipts, not a permanent production price" }, "families": { "F01": { "n": 43, "correct": 40, "answered": 43, "failed": 0, "wrong_valid": 3, "accuracy": 0.9302325581395349 }, "F02": { "n": 46, "correct": 45, "answered": 46, "failed": 0, "wrong_valid": 1, "accuracy": 0.9782608695652174 }, "F03": { "n": 52, "correct": 50, "answered": 52, "failed": 0, "wrong_valid": 2, "accuracy": 0.9615384615384616 }, "F04": { "n": 54, "correct": 51, "answered": 54, "failed": 0, "wrong_valid": 3, "accuracy": 0.9444444444444444 }, "F05": { "n": 49, "correct": 43, "answered": 49, "failed": 0, "wrong_valid": 6, "accuracy": 0.8775510204081632 }, "F06": { "n": 40, "correct": 39, "answered": 40, "failed": 0, "wrong_valid": 1, "accuracy": 0.975 }, "F07": { "n": 76, "correct": 59, "answered": 76, "failed": 0, "wrong_valid": 17, "accuracy": 0.7763157894736842 }, "F08": { "n": 67, "correct": 64, "answered": 67, "failed": 0, "wrong_valid": 3, "accuracy": 0.9552238805970149 }, "F09": { "n": 71, "correct": 44, "answered": 71, "failed": 0, "wrong_valid": 27, "accuracy": 0.6197183098591549 }, "F10": { "n": 74, "correct": 48, "answered": 74, "failed": 0, "wrong_valid": 26, "accuracy": 0.6486486486486487 }, "F11": { "n": 41, "correct": 38, "answered": 41, "failed": 0, "wrong_valid": 3, "accuracy": 0.926829268292683 }, "F12": { "n": 52, "correct": 38, "answered": 52, "failed": 0, "wrong_valid": 14, "accuracy": 0.7307692307692307 }, "F13": { "n": 48, "correct": 18, "answered": 48, "failed": 0, "wrong_valid": 30, "accuracy": 0.375 }, "F14": { "n": 42, "correct": 40, "answered": 42, "failed": 0, "wrong_valid": 2, "accuracy": 0.9523809523809523 }, "F15": { "n": 49, "correct": 38, "answered": 49, "failed": 0, "wrong_valid": 11, "accuracy": 0.7755102040816326 }, "F16": { "n": 50, "correct": 37, "answered": 50, "failed": 0, "wrong_valid": 13, "accuracy": 0.74 }, "F17": { "n": 50, "correct": 47, "answered": 50, "failed": 0, "wrong_valid": 3, "accuracy": 0.94 }, "F18": { "n": 50, "correct": 44, "answered": 50, "failed": 0, "wrong_valid": 6, "accuracy": 0.88 }, "F19": { "n": 58, "correct": 57, "answered": 58, "failed": 0, "wrong_valid": 1, "accuracy": 0.9827586206896551 }, "F20": { "n": 43, "correct": 39, "answered": 43, "failed": 0, "wrong_valid": 4, "accuracy": 0.9069767441860465 }, "F21": { "n": 43, "correct": 41, "answered": 43, "failed": 0, "wrong_valid": 2, "accuracy": 0.9534883720930233 }, "F22": { "n": 51, "correct": 48, "answered": 51, "failed": 0, "wrong_valid": 3, "accuracy": 0.9411764705882353 }, "F23": { "n": 52, "correct": 52, "answered": 52, "failed": 0, "wrong_valid": 0, "accuracy": 1.0 }, "F24": { "n": 43, "correct": 42, "answered": 43, "failed": 0, "wrong_valid": 1, "accuracy": 0.9767441860465116 }, "F25": { "n": 50, "correct": 47, "answered": 50, "failed": 0, "wrong_valid": 3, "accuracy": 0.94 }, "F26": { "n": 8, "correct": 5, "answered": 8, "failed": 0, "wrong_valid": 3, "accuracy": 0.625 }, "F27": { "n": 8, "correct": 4, "answered": 8, "failed": 0, "wrong_valid": 4, "accuracy": 0.5 }, "F28": { "n": 8, "correct": 4, "answered": 8, "failed": 0, "wrong_valid": 4, "accuracy": 0.5 }, "F29": { "n": 8, "correct": 6, "answered": 8, "failed": 0, "wrong_valid": 2, "accuracy": 0.75 }, "F30": { "n": 8, "correct": 5, "answered": 8, "failed": 0, "wrong_valid": 3, "accuracy": 0.625 }, "F31": { "n": 8, "correct": 6, "answered": 8, "failed": 0, "wrong_valid": 2, "accuracy": 0.75 } }, "dates_toronto": { "2026-10-21": 1342 }, "returned_identities": { "decision-models/baseten-mammoth-27b-latest": 1342 }, "providers": { "Baseten direct": 1342 }, "route": "Direct \u2192 Baseten \u2192 decision-models/baseten-mammoth-27b-latest", "output_token_values": { "1": 1342 }, "input_tokens": 484453, "macro_family_accuracy": 0.8205989687678159, "new_domains": { "n": 48, "correct": 30, "answered": 48, "failed": 0, "wrong_valid": 18, "accuracy": 0.625 }, "release_exclusion_diagnostic": { "n": 1305, "correct": 1128, "answered": 1305, "failed": 0, "wrong_valid": 177, "accuracy": 0.864367816091954 }, "original_1136": { "n": 1136, "correct": 1019, "answered": 1136, "failed": 0, "wrong_valid": 117, "accuracy": 0.8970070422535211 }, "added_206": { "n": 206, "correct": 120, "answered": 206, "failed": 0, "wrong_valid": 86, "accuracy": 0.5825242718446602 }, "compatibility": { "status": "ok", "reported_cost_usd": null, "accounted_cost_usd": 0.0, "cost_source": "captured_promotional_zero_tariff_not_receipt" } }, "elephant1": { "model_id": "elephant1", "label": "Elephant-1", "n": 1342, "correct": 1004, "answered": 1342, "wrong_valid": 338, "failed": 0, "accuracy": 0.7481371087928465, "accuracy_among_usable": 0.7481371087928465, "mean_response_ms": 163.87187118225037, "median_ms": 139.16606250131736, "p95_ms": 271.94698334860743, "attempts": 1342, "recoveries": 0, "first_statuses": { "ok": 1342 }, "provider_failures": 0, "model_output_failures": 0, "probability_warnings": 0, "eventual": { "n": 1342, "correct": 1004, "answered": 1342, "failed": 0, "wrong_valid": 338, "accuracy": 0.7481371087928465 }, "cost": { "known_usd": null, "known_decimal": null, "known_usd_per_1000": null, "unknown_attempts": 1342, "exact_usd": null, "public_free_access": true, "accounted_access_cost_usd": 0.0, "scope": "Captured promotional free access; no monetary receipts, not a permanent production price" }, "families": { "F01": { "n": 43, "correct": 40, "answered": 43, "failed": 0, "wrong_valid": 3, "accuracy": 0.9302325581395349 }, "F02": { "n": 46, "correct": 44, "answered": 46, "failed": 0, "wrong_valid": 2, "accuracy": 0.9565217391304348 }, "F03": { "n": 52, "correct": 38, "answered": 52, "failed": 0, "wrong_valid": 14, "accuracy": 0.7307692307692307 }, "F04": { "n": 54, "correct": 48, "answered": 54, "failed": 0, "wrong_valid": 6, "accuracy": 0.8888888888888888 }, "F05": { "n": 49, "correct": 44, "answered": 49, "failed": 0, "wrong_valid": 5, "accuracy": 0.8979591836734694 }, "F06": { "n": 40, "correct": 40, "answered": 40, "failed": 0, "wrong_valid": 0, "accuracy": 1.0 }, "F07": { "n": 76, "correct": 49, "answered": 76, "failed": 0, "wrong_valid": 27, "accuracy": 0.6447368421052632 }, "F08": { "n": 67, "correct": 60, "answered": 67, "failed": 0, "wrong_valid": 7, "accuracy": 0.8955223880597015 }, "F09": { "n": 71, "correct": 33, "answered": 71, "failed": 0, "wrong_valid": 38, "accuracy": 0.4647887323943662 }, "F10": { "n": 74, "correct": 43, "answered": 74, "failed": 0, "wrong_valid": 31, "accuracy": 0.581081081081081 }, "F11": { "n": 41, "correct": 30, "answered": 41, "failed": 0, "wrong_valid": 11, "accuracy": 0.7317073170731707 }, "F12": { "n": 52, "correct": 28, "answered": 52, "failed": 0, "wrong_valid": 24, "accuracy": 0.5384615384615384 }, "F13": { "n": 48, "correct": 12, "answered": 48, "failed": 0, "wrong_valid": 36, "accuracy": 0.25 }, "F14": { "n": 42, "correct": 25, "answered": 42, "failed": 0, "wrong_valid": 17, "accuracy": 0.5952380952380952 }, "F15": { "n": 49, "correct": 29, "answered": 49, "failed": 0, "wrong_valid": 20, "accuracy": 0.5918367346938775 }, "F16": { "n": 50, "correct": 25, "answered": 50, "failed": 0, "wrong_valid": 25, "accuracy": 0.5 }, "F17": { "n": 50, "correct": 46, "answered": 50, "failed": 0, "wrong_valid": 4, "accuracy": 0.92 }, "F18": { "n": 50, "correct": 34, "answered": 50, "failed": 0, "wrong_valid": 16, "accuracy": 0.68 }, "F19": { "n": 58, "correct": 54, "answered": 58, "failed": 0, "wrong_valid": 4, "accuracy": 0.9310344827586207 }, "F20": { "n": 43, "correct": 38, "answered": 43, "failed": 0, "wrong_valid": 5, "accuracy": 0.8837209302325582 }, "F21": { "n": 43, "correct": 41, "answered": 43, "failed": 0, "wrong_valid": 2, "accuracy": 0.9534883720930233 }, "F22": { "n": 51, "correct": 46, "answered": 51, "failed": 0, "wrong_valid": 5, "accuracy": 0.9019607843137255 }, "F23": { "n": 52, "correct": 50, "answered": 52, "failed": 0, "wrong_valid": 2, "accuracy": 0.9615384615384616 }, "F24": { "n": 43, "correct": 40, "answered": 43, "failed": 0, "wrong_valid": 3, "accuracy": 0.9302325581395349 }, "F25": { "n": 50, "correct": 45, "answered": 50, "failed": 0, "wrong_valid": 5, "accuracy": 0.9 }, "F26": { "n": 8, "correct": 3, "answered": 8, "failed": 0, "wrong_valid": 5, "accuracy": 0.375 }, "F27": { "n": 8, "correct": 4, "answered": 8, "failed": 0, "wrong_valid": 4, "accuracy": 0.5 }, "F28": { "n": 8, "correct": 3, "answered": 8, "failed": 0, "wrong_valid": 5, "accuracy": 0.375 }, "F29": { "n": 8, "correct": 3, "answered": 8, "failed": 0, "wrong_valid": 5, "accuracy": 0.375 }, "F30": { "n": 8, "correct": 4, "answered": 8, "failed": 0, "wrong_valid": 4, "accuracy": 0.5 }, "F31": { "n": 8, "correct": 5, "answered": 8, "failed": 0, "wrong_valid": 3, "accuracy": 0.625 } }, "dates_toronto": { "2026-10-21": 1342 }, "returned_identities": { "decision-models/baseten-elephant-9b-latest": 1342 }, "providers": { "Baseten direct": 1342 }, "route": "Direct \u2192 Baseten \u2192 decision-models/baseten-elephant-9b-latest", "output_token_values": { "1": 1342 }, "input_tokens": 484453, "macro_family_accuracy": 0.7099909651220829, "new_domains": { "n": 48, "correct": 22, "answered": 48, "failed": 0, "wrong_valid": 26, "accuracy": 0.4583333333333333 }, "release_exclusion_diagnostic": { "n": 1305, "correct": 990, "answered": 1305, "failed": 0, "wrong_valid": 315, "accuracy": 0.7586206896551724 }, "original_1136": { "n": 1136, "correct": 902, "answered": 1136, "failed": 0, "wrong_valid": 234, "accuracy": 0.7940140845070423 }, "added_206": { "n": 206, "correct": 102, "answered": 206, "failed": 0, "wrong_valid": 104, "accuracy": 0.49514563106796117 }, "compatibility": { "status": "ok", "reported_cost_usd": null, "accounted_cost_usd": 0.0, "cost_source": "captured_promotional_zero_tariff_not_receipt" } }, "drex_max": { "model_id": "drex_max", "label": "Drex Max", "n": 1342, "correct": 1127, "answered": 1342, "wrong_valid": 215, "failed": 0, "accuracy": 0.8397913561847988, "accuracy_among_usable": 0.8397913561847988, "mean_response_ms": 289.75990623420995, "median_ms": 285.8057909979834, "p95_ms": 444.8038062444539, "attempts": 1342, "recoveries": 0, "first_statuses": { "ok": 1342 }, "provider_failures": 0, "model_output_failures": 0, "probability_warnings": 0, "eventual": { "n": 1342, "correct": 1127, "answered": 1342, "failed": 0, "wrong_valid": 215, "accuracy": 0.8397913561847988 }, "cost": { "known_usd": 0.02842302, "known_decimal": "0.02842302", "known_usd_per_1000": 0.021179597615499254, "unknown_attempts": 0, "exact_usd": 0.02842302, "public_free_access": false, "accounted_access_cost_usd": 0.02842302, "scope": "API-reported OpenRouter charges, all attempts" }, "families": { "F01": { "n": 43, "correct": 41, "answered": 43, "failed": 0, "wrong_valid": 2, "accuracy": 0.9534883720930233 }, "F02": { "n": 46, "correct": 45, "answered": 46, "failed": 0, "wrong_valid": 1, "accuracy": 0.9782608695652174 }, "F03": { "n": 52, "correct": 50, "answered": 52, "failed": 0, "wrong_valid": 2, "accuracy": 0.9615384615384616 }, "F04": { "n": 54, "correct": 51, "answered": 54, "failed": 0, "wrong_valid": 3, "accuracy": 0.9444444444444444 }, "F05": { "n": 49, "correct": 46, "answered": 49, "failed": 0, "wrong_valid": 3, "accuracy": 0.9387755102040817 }, "F06": { "n": 40, "correct": 39, "answered": 40, "failed": 0, "wrong_valid": 1, "accuracy": 0.975 }, "F07": { "n": 76, "correct": 50, "answered": 76, "failed": 0, "wrong_valid": 26, "accuracy": 0.6578947368421053 }, "F08": { "n": 67, "correct": 65, "answered": 67, "failed": 0, "wrong_valid": 2, "accuracy": 0.9701492537313433 }, "F09": { "n": 71, "correct": 42, "answered": 71, "failed": 0, "wrong_valid": 29, "accuracy": 0.5915492957746479 }, "F10": { "n": 74, "correct": 53, "answered": 74, "failed": 0, "wrong_valid": 21, "accuracy": 0.7162162162162162 }, "F11": { "n": 41, "correct": 35, "answered": 41, "failed": 0, "wrong_valid": 6, "accuracy": 0.8536585365853658 }, "F12": { "n": 52, "correct": 29, "answered": 52, "failed": 0, "wrong_valid": 23, "accuracy": 0.5576923076923077 }, "F13": { "n": 48, "correct": 18, "answered": 48, "failed": 0, "wrong_valid": 30, "accuracy": 0.375 }, "F14": { "n": 42, "correct": 40, "answered": 42, "failed": 0, "wrong_valid": 2, "accuracy": 0.9523809523809523 }, "F15": { "n": 49, "correct": 40, "answered": 49, "failed": 0, "wrong_valid": 9, "accuracy": 0.8163265306122449 }, "F16": { "n": 50, "correct": 39, "answered": 50, "failed": 0, "wrong_valid": 11, "accuracy": 0.78 }, "F17": { "n": 50, "correct": 46, "answered": 50, "failed": 0, "wrong_valid": 4, "accuracy": 0.92 }, "F18": { "n": 50, "correct": 46, "answered": 50, "failed": 0, "wrong_valid": 4, "accuracy": 0.92 }, "F19": { "n": 58, "correct": 54, "answered": 58, "failed": 0, "wrong_valid": 4, "accuracy": 0.9310344827586207 }, "F20": { "n": 43, "correct": 39, "answered": 43, "failed": 0, "wrong_valid": 4, "accuracy": 0.9069767441860465 }, "F21": { "n": 43, "correct": 42, "answered": 43, "failed": 0, "wrong_valid": 1, "accuracy": 0.9767441860465116 }, "F22": { "n": 51, "correct": 48, "answered": 51, "failed": 0, "wrong_valid": 3, "accuracy": 0.9411764705882353 }, "F23": { "n": 52, "correct": 52, "answered": 52, "failed": 0, "wrong_valid": 0, "accuracy": 1.0 }, "F24": { "n": 43, "correct": 42, "answered": 43, "failed": 0, "wrong_valid": 1, "accuracy": 0.9767441860465116 }, "F25": { "n": 50, "correct": 46, "answered": 50, "failed": 0, "wrong_valid": 4, "accuracy": 0.92 }, "F26": { "n": 8, "correct": 5, "answered": 8, "failed": 0, "wrong_valid": 3, "accuracy": 0.625 }, "F27": { "n": 8, "correct": 4, "answered": 8, "failed": 0, "wrong_valid": 4, "accuracy": 0.5 }, "F28": { "n": 8, "correct": 5, "answered": 8, "failed": 0, "wrong_valid": 3, "accuracy": 0.625 }, "F29": { "n": 8, "correct": 5, "answered": 8, "failed": 0, "wrong_valid": 3, "accuracy": 0.625 }, "F30": { "n": 8, "correct": 3, "answered": 8, "failed": 0, "wrong_valid": 5, "accuracy": 0.375 }, "F31": { "n": 8, "correct": 7, "answered": 8, "failed": 0, "wrong_valid": 1, "accuracy": 0.875 } }, "dates_toronto": { "2026-10-21": 1342 }, "returned_identities": { "nace-ai/drex-max-20261016": 1342 }, "providers": { "DeepInfra": 1342 }, "route": "OpenRouter \u2192 DeepInfra \u2192 nace-ai/drex-max", "output_token_values": { "169": 177, "170": 263, "171": 212, "160": 34, "164": 44, "173": 32, "162": 68, "168": 94, "159": 11, "163": 70, "165": 50, "167": 49, "172": 117, "157": 2, "161": 65, "166": 45, "158": 8, "156": 1 }, "input_tokens": 473717, "macro_family_accuracy": 0.8109694050743982, "new_domains": { "n": 48, "correct": 29, "answered": 48, "failed": 0, "wrong_valid": 19, "accuracy": 0.6041666666666666 }, "release_exclusion_diagnostic": { "n": 1305, "correct": 1112, "answered": 1305, "failed": 0, "wrong_valid": 193, "accuracy": 0.8521072796934865 }, "original_1136": { "n": 1136, "correct": 1004, "answered": 1136, "failed": 0, "wrong_valid": 132, "accuracy": 0.8838028169014085 }, "added_206": { "n": 206, "correct": 123, "answered": 206, "failed": 0, "wrong_valid": 83, "accuracy": 0.5970873786407767 }, "compatibility": { "status": "ok", "reported_cost_usd": 7.2e-06, "accounted_cost_usd": 7.2e-06, "cost_source": "api_reported" } } }, "uniform_guess_baseline": 0.2, "limits": [ "One consolidated development overview, not a held-out scored release;37 author-development cases retain permanent release exclusions.", "Questions include dependent variants, correlated model routes, and unbalanced category counts. No independent-trial confidence claims.", "Six domains have8 questions each; their apparent perfect scores are not reliability guarantees.", "Timing pools retain original test dates, providers and request loads; this was not a synchronized rerun.", "Costs retain captured API charges/tariffs; unknown costs are lower bounds, not zero.", "First-attempt provider failures are unsuccessful delivered decisions, not wrong reasoning answers.", "Ordinary LLMs without complete1342 coverage are references only, never ranked against the full-bank cohort.", "No new model calls, unchanged question reruns or model removals. This is a public developmental overview, not a held-out release." ], "latency_definition": "Median and linearly interpolated p95 from all1342 raw first-attempt durations; includes failures, excludes retry durations and wait/queue times.", "reference_model_ids": [ "reference_sol", "reference_sonnet", "reference_qwen" ], "tested_llm_reference_count": 3, "all_tested_primary_cases": 38918, "decision_model_ids": [ "mammoth1", "drex_max", "respan02", "pplx_decider12", "nimble_v4_max", "d3_turbo", "liquid_ultra", "amos_large", "kettei_fat", "jev114", "jev113", "mercury_decide", "elephant1", "liquid_d1", "clef", "pplx_flash12", "microsoft_decision", "drex20", "clef_flash", "clef_omni", "drex15", "kev4b", "luna_decisions", "tev1_4b_native", "classify1", "solar_flash" ], "model_kinds": { "respan02": "Decision model", "pplx_decider12": "Decision model", "nimble_v4_max": "Decision model", "jev114": "Decision model", "liquid_ultra": "Decision model", "mercury_decide": "Decision model", "jev113": "Decision model", "luna_decisions": "Decision model", "microsoft_decision": "Decision model", "liquid_d1": "Decision model", "pplx_flash12": "Decision model", "clef": "Decision model", "clef_flash": "Decision model", "clef_omni": "Decision model", "solar_flash": "Decision model", "drex20": "Decision model", "drex15": "Decision model", "classify1": "Decision model", "kev4b": "Decision model", "tev1_4b_native": "Decision model", "reference_sol": "LLM reference", "reference_sonnet": "LLM reference", "reference_qwen": "LLM reference", "d3_turbo": "Decision model", "amos_large": "Decision model", "kettei_fat": "Decision model", "mammoth1": "Decision model", "elephant1": "Decision model", "drex_max": "Decision model" }, "source_native_statistics_sha256": "e1348fb78c3b5bdc88874ba4af23681f37aaa972414d8910b7ce401c3a730950", "reference_evidence_bindings": { "reference_sol": { "source_summary_sha256": "fb440bc91730ab5e0c6974d283cf5f73602105f9eb0017a0ffb771a49f637e92", "completion_sha256": "a4906472c00c336c8b0ef312f215abc0ee6610295d3cd36fcea04298eb5db821", "private_source_scoring_bindings": [ "33a04e1b7ac8b3a3020423a37781e1ca5a3939081c6d350fd6a532a00bad77c2", "3b41037fba1cb02f42d0044a0dd758f1cc588de86fe0d2a5f90b142a41f353bb", "471b808d20d9be2608ef60fddbf8526bf9015af3d6a4d976b7a2b20843ad46e1", "4ca81b991c4002e70aead0d56b9f53c21f1e70d9b1fef1a4a1f2e32f32d5b56c", "4cd7c58fe4be67dadb3644fc54d46d8e4a40aaa8ed73c2005b6dbe52a4341a36", "6cb31d4192e256c5b6fa56d480025634bfbdca9ce8b2b68558ed44fe6bfa6c69", "8376ddce67cd9bb5f62d685d0309b4de5706f091084fbe72aed7dc91d1bf4dec", "aac4b2c2ece0f6cf37b81794031d93a8c3f53d032da1eb127d14f62be34781a6", "b5dba10a8c9fbcbd0fc79f62e11a9a08594fdc187cf3ead7b5bb872a43bf1b5f", "bb7ff1f81ac1b35bb93c25a5ac7dce48a0d86ff48cda711304b9932864c1bc3a", "c99db093ae3517dcacc03319f00a461acbfd19e0eb22cb0c5a5dd636f1ee9271", "ca5aa924ee10efe60bca09e02bb7fbb68740d12b243ca7bd619ceb5fc952c7bb", "dbac830fe3cad5f469f950fe2c3b1c2e575cb6d00c2bdaec4b82c46f41b0b8ab", "f6f685ea96ebd626804931278e03256bd502f0dc811a8694d9417a4ad5b64c01", "fbe6f2b49ad7c99416fb45dc05b8b9db37a3850017d7bfef81644d09afe9bcd4" ], "operational_evidence": {} }, "reference_sonnet": { "source_summary_sha256": "a35287115c5f86a381b8ab420407f52051c2cba6689717406b033e5093bc7f0c", "completion_sha256": "96a498a27d8cb92845a5e867ce89c303e2501081307b7003e7ef65ce088244f2", "private_source_scoring_bindings": [ "33a04e1b7ac8b3a3020423a37781e1ca5a3939081c6d350fd6a532a00bad77c2", "3b41037fba1cb02f42d0044a0dd758f1cc588de86fe0d2a5f90b142a41f353bb", "471b808d20d9be2608ef60fddbf8526bf9015af3d6a4d976b7a2b20843ad46e1", "4ca81b991c4002e70aead0d56b9f53c21f1e70d9b1fef1a4a1f2e32f32d5b56c", "4cd7c58fe4be67dadb3644fc54d46d8e4a40aaa8ed73c2005b6dbe52a4341a36", "6cb31d4192e256c5b6fa56d480025634bfbdca9ce8b2b68558ed44fe6bfa6c69", "8376ddce67cd9bb5f62d685d0309b4de5706f091084fbe72aed7dc91d1bf4dec", "aac4b2c2ece0f6cf37b81794031d93a8c3f53d032da1eb127d14f62be34781a6", "b5dba10a8c9fbcbd0fc79f62e11a9a08594fdc187cf3ead7b5bb872a43bf1b5f", "bb7ff1f81ac1b35bb93c25a5ac7dce48a0d86ff48cda711304b9932864c1bc3a", "c99db093ae3517dcacc03319f00a461acbfd19e0eb22cb0c5a5dd636f1ee9271", "ca5aa924ee10efe60bca09e02bb7fbb68740d12b243ca7bd619ceb5fc952c7bb", "dbac830fe3cad5f469f950fe2c3b1c2e575cb6d00c2bdaec4b82c46f41b0b8ab", "f6f685ea96ebd626804931278e03256bd502f0dc811a8694d9417a4ad5b64c01", "fbe6f2b49ad7c99416fb45dc05b8b9db37a3850017d7bfef81644d09afe9bcd4" ], "operational_evidence": {} }, "reference_qwen": { "source_summary_sha256": "ce52e97aaec9dff8137852ce917e065cde038f5c370622589b5f0c235d638331", "completion_sha256": "bc5bda846e49c4545ce4e3d00f4b19c6f7a4315f3f46299a5b95435b3019ac38", "private_source_scoring_bindings": [ "33a04e1b7ac8b3a3020423a37781e1ca5a3939081c6d350fd6a532a00bad77c2", "3b41037fba1cb02f42d0044a0dd758f1cc588de86fe0d2a5f90b142a41f353bb", "471b808d20d9be2608ef60fddbf8526bf9015af3d6a4d976b7a2b20843ad46e1", "4ca81b991c4002e70aead0d56b9f53c21f1e70d9b1fef1a4a1f2e32f32d5b56c", "4cd7c58fe4be67dadb3644fc54d46d8e4a40aaa8ed73c2005b6dbe52a4341a36", "6cb31d4192e256c5b6fa56d480025634bfbdca9ce8b2b68558ed44fe6bfa6c69", "8376ddce67cd9bb5f62d685d0309b4de5706f091084fbe72aed7dc91d1bf4dec", "aac4b2c2ece0f6cf37b81794031d93a8c3f53d032da1eb127d14f62be34781a6", "b5dba10a8c9fbcbd0fc79f62e11a9a08594fdc187cf3ead7b5bb872a43bf1b5f", "bb7ff1f81ac1b35bb93c25a5ac7dce48a0d86ff48cda711304b9932864c1bc3a", "c99db093ae3517dcacc03319f00a461acbfd19e0eb22cb0c5a5dd636f1ee9271", "ca5aa924ee10efe60bca09e02bb7fbb68740d12b243ca7bd619ceb5fc952c7bb", "dbac830fe3cad5f469f950fe2c3b1c2e575cb6d00c2bdaec4b82c46f41b0b8ab", "f6f685ea96ebd626804931278e03256bd502f0dc811a8694d9417a4ad5b64c01", "fbe6f2b49ad7c99416fb45dc05b8b9db37a3850017d7bfef81644d09afe9bcd4" ], "operational_evidence": { "operator_events.jsonl": "ea476ad58192f6a79368dfd4ea53adb0d48cec347379b70db4e39f45dd365abc", "provider_cooldown.json": "4a659427b0e9833596ddc39b9a18b7b82c109dd63d0d9ba11c93ee44056423b0", "provider_cooldown_extension.json": "333b79894b3662389f76873467cd1c3ed32ceca5bf6b5bc1ab3fc4142b419045", "pacing_receipt.json": "12111cf8279413f890c138902870abeed5f969c3989bf35200a4191a03a416e4", "pacing_waits.jsonl": "d6abe3197024998697a681a14cd2133ccf1a5cc37886adeec5072d97048d12c3" } } }, "question_bank_sha256": "2db8cf8353f12b609d4df5330d94f53390c9c2919677dc793df4bb2d3cff7076", "decision_model_count": 26, "added_model_ids": [ "mammoth1", "elephant1", "drex_max" ], "updated_date_toronto": "2026-10-21", "prior_statistics_sha256": "c31683c5e211e301a6389f770b2441e12b66e2f80f9c3a124fdbfc166165a4b0", "direct_evidence_bindings": { "d3_turbo": { "manifest.json": "e2dddd70e2ff6e887751bdc05f50cf01d72068bb9278e77ae923736fdf5163fe", "software.json": "a6a99bb579834587f2dcb5a34452b2ab4632ba4f8fe4ce2bfc0062c18ca27efe", "model_metadata.json": "a32ae0e379f4925c9bf07894b2853ac789ff92f5dc274250ba732c88bad0f7c5", "probe_dispatch.json": "2b4f8c34e467b55807817eeb2c160fdfcdc4dba08b54e9cf95bc06e208c6413b", "compatibility_probe.json": "80eaf0b091526becdfb21ef312f14709231387c37d4343004d685cba59e38e4d", "dispatches.jsonl": "3a23e59f60d2fa643f02c014e56080a452529fcdbcf8f4c91d6429a0564b0a94", "attempts.jsonl": "aabe666145d6a24bbd504c2cf0f2a181d585ff2f8ef8353d2ddfa5e41bc03021", "results.jsonl": "e3ef638cfb6b094fe624a86f3e378d0e5c93fdc9f360744ed7a66052f4b562c6", "completion.json": "26c9687c4031db7d957937cebd5bf5391f2955b4b3574600f324fb86e9c8fc2e" }, "amos_large": { "manifest.json": "e0647fdf0743969bf08e668bbdf2896630e51bfe910ae40cc7409350807c2fae", "software.json": "a6a99bb579834587f2dcb5a34452b2ab4632ba4f8fe4ce2bfc0062c18ca27efe", "model_metadata.json": "e2a4e3a59cf87dc708ab25103db2d82bc46bceeeed201f9cac8e409849a88503", "probe_dispatch.json": "ae5045ac0325920cc46d9ccd9d6b91b9e321c8e7846434fd99dc45155bd15d7c", "compatibility_probe.json": "923ec73d8ad96a8367088603ef6d67e7e1e400c468c25dd67ab751c73dc19bcb", "dispatches.jsonl": "a09a139f6f685c488c8fbd6d4eb56700fde8635ec9827360888e5ee50437e29e", "attempts.jsonl": "2ab2ffbb01ec7ab609839b5b3cdcd7d92496a771d76009ae8d9df438bf6fbe37", "results.jsonl": "0189941dd1502ad63ecf55e44879fb63f768c67e1d1204f213d006cc2ece7e68", "completion.json": "8789c3d8969e4e2c111800e16edd5dc74bdf9fda8e81c063c2e9fcec90d6c831" }, "kettei_fat": { "manifest.json": "c0fb09c04bba4eb6fcaeaceb643b729f85552291b3805dac7e754ed286d55943", "software.json": "a6a99bb579834587f2dcb5a34452b2ab4632ba4f8fe4ce2bfc0062c18ca27efe", "model_metadata.json": "3842b8979235fea42e52fd867414ec12d89181a777ac7f00646af4cff0f65980", "probe_dispatch.json": "8f632423adcee7d2b1b6022b4caa82c03f9259eac6c82af392e7c1284889f8f4", "compatibility_probe.json": "f115a687b4ef4850105e4eb0beebf419da424e5493cd7c6d40cdf8c4f735190b", "dispatches.jsonl": "32f2d6f80726a98096a5b38832a187c6f1cb9cc6aeae60035e86c6e5d67c47ef", "attempts.jsonl": "7c20266cdad35eeefd2eeeff19e3309f0b328fb4b3980eb7d00091201f946c38", "results.jsonl": "eb4d4e73cabf177a82031d33fe77ce22a6f51748430920d1164a6f4e3ef5b849", "completion.json": "dd1a5b0444b9a841bb142857c5444f0f41c94ec4d4b04aec4d7bce5227cb24f7", "account_rate_amendment.json": "87c84f03393d1072ea99784d59280fdd139e865a897038a658207515375e68ac" } }, "direct_source_summary_sha256": "6bd97c172fb1ba44ed8c9ee731aeaab6b6fd876184f3c45c066468c452f7b573", "direct_scoring_authority_sha256": [ "33a04e1b7ac8b3a3020423a37781e1ca5a3939081c6d350fd6a532a00bad77c2", "3b41037fba1cb02f42d0044a0dd758f1cc588de86fe0d2a5f90b142a41f353bb", "471b808d20d9be2608ef60fddbf8526bf9015af3d6a4d976b7a2b20843ad46e1", "4ca81b991c4002e70aead0d56b9f53c21f1e70d9b1fef1a4a1f2e32f32d5b56c", "4cd7c58fe4be67dadb3644fc54d46d8e4a40aaa8ed73c2005b6dbe52a4341a36", "8376ddce67cd9bb5f62d685d0309b4de5706f091084fbe72aed7dc91d1bf4dec", "aac4b2c2ece0f6cf37b81794031d93a8c3f53d032da1eb127d14f62be34781a6", "b5dba10a8c9fbcbd0fc79f62e11a9a08594fdc187cf3ead7b5bb872a43bf1b5f", "bb7ff1f81ac1b35bb93c25a5ac7dce48a0d86ff48cda711304b9932864c1bc3a", "c99db093ae3517dcacc03319f00a461acbfd19e0eb22cb0c5a5dd636f1ee9271", "ca5aa924ee10efe60bca09e02bb7fbb68740d12b243ca7bd619ceb5fc952c7bb", "dbac830fe3cad5f469f950fe2c3b1c2e575cb6d00c2bdaec4b82c46f41b0b8ab", "f6f685ea96ebd626804931278e03256bd502f0dc811a8694d9417a4ad5b64c01", "fbe6f2b49ad7c99416fb45dc05b8b9db37a3850017d7bfef81644d09afe9bcd4" ], "direct_analysis_source_sha256": { "analyze_direct_decisions_2026_10_20.py": "2517c3e6f8d32d2a9e0d403d8fc7a02f883883c42a3e008f7453addd43384916", "test_analyze_direct_decisions_2026_10_20.py": "526ad74fedc0f49e184aa62a246309667ef908bd7d002d7af72788ab792f0497", "llm_references_2026-10-19/statistics.json": "cff708651203a54f3ddb7e17782f8c2abdbc8900a92f81d7492afa6e8a90b435", "llm_references_2026-10-19/category_guide.json": "183f7fa8ace26704616423f307e0d8f0acb923a6b29c78e837ebf1ddaa7be823" }, "total_http_attempts": 39084, "total_recoveries": 166 }