{ "status": "COMPLETE_200_EACH_FIVE_CONFIGURATIONS", "questions": 200, "image_files": 227, "models": { "typesafe/jev-1.14": { "scheduled": 200, "completed_first": 200, "correct_first": 141, "accuracy_scheduled": 0.705, "usable_first": 195, "coverage": 0.975, "accuracy_usable": 0.7230769230769231, "wilson95": [ 0.6384128547538122, 0.7638605614315084 ], "statuses_first": { "ok": 195, "http_failure": 5 }, "eventual_correct": 141, "eventual_usable": 195, "retries": 0, "first_cost": { "reported_usd": 0.023976876, "exact_decimal_usd": "0.023976876", "known_records": 195, "missing_records": 5 }, "core_cost": { "reported_usd": 0.023976876, "exact_decimal_usd": "0.023976876", "known_records": 195, "missing_records": 5 }, "core_cost_per_scheduled_usd": 0.00011988438, "first_cost_per_first_correct_usd": 0.0001700487659574468, "core_cost_per_first_correct_usd": 0.0001700487659574468, "latency_first_all_ms": { "median": 307.1314584813081, "p90": 556.7650381126441, "p95": 723.9778851129809 }, "latency_first_usable_ms": { "median": 307.59666697122157, "p90": 558.100433612708, "p95": 730.1787706150203 }, "tokens_first": { "input_tokens": { "total": 570878, "covered_requests": 195 }, "output_tokens": { "total": 9216, "covered_requests": 195 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.023976876, "exact_decimal_usd": "0.023976876", "known_records": 195, "missing_records": 5 } }, "test_window": { "first": "2026-10-15T10:07:11.395723-04:00", "last": "2026-10-15T10:18:02.646747-04:00" }, "identities": [ [ "typesafe/jev-1.14-20261014", "TypeSafe" ] ], "groups": { "family_id": { "V01": { "scheduled": 10, "completed_first": 10, "correct_first": 10, "accuracy_scheduled": 1.0, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 1.0, "wilson95": [ 0.7224672001371107, 0.9999999999999999 ], "statuses_first": { "ok": 10 }, "eventual_correct": 10, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.000949158, "exact_decimal_usd": "0.000949158", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.000949158, "exact_decimal_usd": "0.000949158", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 9.49158e-05, "first_cost_per_first_correct_usd": 9.49158e-05, "core_cost_per_first_correct_usd": 9.49158e-05, "latency_first_all_ms": { "median": 311.47908300044946, "p90": 382.44847499299794, "p95": 418.17498751333903 }, "latency_first_usable_ms": { "median": 311.47908300044946, "p90": 382.44847499299794, "p95": 418.17498751333903 }, "tokens_first": { "input_tokens": { "total": 22599, "covered_requests": 10 }, "output_tokens": { "total": 450, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.000949158, "exact_decimal_usd": "0.000949158", "known_records": 10, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:07:13.705458-04:00", "last": "2026-10-15T10:17:59.009998-04:00" }, "identities": [ [ "typesafe/jev-1.14-20261014", "TypeSafe" ] ] }, "V02": { "scheduled": 10, "completed_first": 10, "correct_first": 9, "accuracy_scheduled": 0.9, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.9, "wilson95": [ 0.5958499732047615, 0.9821237869049271 ], "statuses_first": { "ok": 10 }, "eventual_correct": 9, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.000852768, "exact_decimal_usd": "0.000852768", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.000852768, "exact_decimal_usd": "0.000852768", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 8.527679999999999e-05, "first_cost_per_first_correct_usd": 9.475199999999999e-05, "core_cost_per_first_correct_usd": 9.475199999999999e-05, "latency_first_all_ms": { "median": 324.1271454899106, "p90": 418.4982244390994, "p95": 488.8334872259291 }, "latency_first_usable_ms": { "median": 324.1271454899106, "p90": 418.4982244390994, "p95": 488.8334872259291 }, "tokens_first": { "input_tokens": { "total": 20304, "covered_requests": 10 }, "output_tokens": { "total": 450, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.000852768, "exact_decimal_usd": "0.000852768", "known_records": 10, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:07:15.783254-04:00", "last": "2026-10-15T10:17:51.001980-04:00" }, "identities": [ [ "typesafe/jev-1.14-20261014", "TypeSafe" ] ] }, "V03": { "scheduled": 10, "completed_first": 10, "correct_first": 9, "accuracy_scheduled": 0.9, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.9, "wilson95": [ 0.5958499732047615, 0.9821237869049271 ], "statuses_first": { "ok": 10 }, "eventual_correct": 9, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.001148616, "exact_decimal_usd": "0.001148616", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.001148616, "exact_decimal_usd": "0.001148616", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00011486160000000001, "first_cost_per_first_correct_usd": 0.000127624, "core_cost_per_first_correct_usd": 0.000127624, "latency_first_all_ms": { "median": 292.70347897545435, "p90": 362.83472140203224, "p95": 424.0629021980566 }, "latency_first_usable_ms": { "median": 292.70347897545435, "p90": 362.83472140203224, "p95": 424.0629021980566 }, "tokens_first": { "input_tokens": { "total": 27348, "covered_requests": 10 }, "output_tokens": { "total": 450, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.001148616, "exact_decimal_usd": "0.001148616", "known_records": 10, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:07:14.348804-04:00", "last": "2026-10-15T10:17:54.424243-04:00" }, "identities": [ [ "typesafe/jev-1.14-20261014", "TypeSafe" ] ] }, "V04": { "scheduled": 10, "completed_first": 10, "correct_first": 4, "accuracy_scheduled": 0.4, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.4, "wilson95": [ 0.16818032970623614, 0.6873262302663417 ], "statuses_first": { "ok": 10 }, "eventual_correct": 4, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.001156848, "exact_decimal_usd": "0.001156848", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.001156848, "exact_decimal_usd": "0.001156848", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.0001156848, "first_cost_per_first_correct_usd": 0.000289212, "core_cost_per_first_correct_usd": 0.000289212, "latency_first_all_ms": { "median": 318.8891044992488, "p90": 523.9550541969948, "p95": 556.1949855880811 }, "latency_first_usable_ms": { "median": 318.8891044992488, "p90": 523.9550541969948, "p95": 556.1949855880811 }, "tokens_first": { "input_tokens": { "total": 27544, "covered_requests": 10 }, "output_tokens": { "total": 520, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.001156848, "exact_decimal_usd": "0.001156848", "known_records": 10, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:07:13.030379-04:00", "last": "2026-10-15T10:16:59.946500-04:00" }, "identities": [ [ "typesafe/jev-1.14-20261014", "TypeSafe" ] ] }, "V05": { "scheduled": 10, "completed_first": 10, "correct_first": 7, "accuracy_scheduled": 0.7, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.7, "wilson95": [ 0.39677814746114537, 0.8922087325936989 ], "statuses_first": { "ok": 10 }, "eventual_correct": 7, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.001756566, "exact_decimal_usd": "0.001756566", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.001756566, "exact_decimal_usd": "0.001756566", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.0001756566, "first_cost_per_first_correct_usd": 0.000250938, "core_cost_per_first_correct_usd": 0.000250938, "latency_first_all_ms": { "median": 346.1596669803839, "p90": 407.55444167298265, "p95": 416.82811681821477 }, "latency_first_usable_ms": { "median": 346.1596669803839, "p90": 407.55444167298265, "p95": 416.82811681821477 }, "tokens_first": { "input_tokens": { "total": 41823, "covered_requests": 10 }, "output_tokens": { "total": 492, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.001756566, "exact_decimal_usd": "0.001756566", "known_records": 10, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:07:36.892775-04:00", "last": "2026-10-15T10:17:50.655906-04:00" }, "identities": [ [ "typesafe/jev-1.14-20261014", "TypeSafe" ] ] }, "V06": { "scheduled": 10, "completed_first": 10, "correct_first": 3, "accuracy_scheduled": 0.3, "usable_first": 9, "coverage": 0.9, "accuracy_usable": 0.3333333333333333, "wilson95": [ 0.10779126740630099, 0.6032218525388546 ], "statuses_first": { "http_failure": 1, "ok": 9 }, "eventual_correct": 3, "eventual_usable": 9, "retries": 0, "first_cost": { "reported_usd": 0.000868602, "exact_decimal_usd": "0.000868602", "known_records": 9, "missing_records": 1 }, "core_cost": { "reported_usd": 0.000868602, "exact_decimal_usd": "0.000868602", "known_records": 9, "missing_records": 1 }, "core_cost_per_scheduled_usd": 8.68602e-05, "first_cost_per_first_correct_usd": 0.000289534, "core_cost_per_first_correct_usd": 0.000289534, "latency_first_all_ms": { "median": 339.09693799796514, "p90": 488.9910374942701, "p95": 568.1591437401946 }, "latency_first_usable_ms": { "median": 340.7041669706814, "p90": 506.5839499933645, "p95": 576.955599989742 }, "tokens_first": { "input_tokens": { "total": 20681, "covered_requests": 9 }, "output_tokens": { "total": 405, "covered_requests": 9 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.000868602, "exact_decimal_usd": "0.000868602", "known_records": 9, "missing_records": 1 } }, "test_window": { "first": "2026-10-15T10:07:41.864889-04:00", "last": "2026-10-15T10:18:00.269954-04:00" }, "identities": [ [ "typesafe/jev-1.14-20261014", "TypeSafe" ] ] }, "V07": { "scheduled": 10, "completed_first": 10, "correct_first": 4, "accuracy_scheduled": 0.4, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.4, "wilson95": [ 0.16818032970623614, 0.6873262302663417 ], "statuses_first": { "ok": 10 }, "eventual_correct": 4, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.000922362, "exact_decimal_usd": "0.000922362", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.000922362, "exact_decimal_usd": "0.000922362", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 9.22362e-05, "first_cost_per_first_correct_usd": 0.0002305905, "core_cost_per_first_correct_usd": 0.0002305905, "latency_first_all_ms": { "median": 243.13114548567683, "p90": 469.6074372332075, "p95": 799.3127811205333 }, "latency_first_usable_ms": { "median": 243.13114548567683, "p90": 469.6074372332075, "p95": 799.3127811205333 }, "tokens_first": { "input_tokens": { "total": 21961, "covered_requests": 10 }, "output_tokens": { "total": 443, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.000922362, "exact_decimal_usd": "0.000922362", "known_records": 10, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:12:48.548251-04:00", "last": "2026-10-15T10:17:56.940993-04:00" }, "identities": [ [ "typesafe/jev-1.14-20261014", "TypeSafe" ] ] }, "V08": { "scheduled": 10, "completed_first": 10, "correct_first": 3, "accuracy_scheduled": 0.3, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.3, "wilson95": [ 0.10779126740630099, 0.6032218525388546 ], "statuses_first": { "ok": 10 }, "eventual_correct": 3, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.001307964, "exact_decimal_usd": "0.001307964", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.001307964, "exact_decimal_usd": "0.001307964", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.0001307964, "first_cost_per_first_correct_usd": 0.000435988, "core_cost_per_first_correct_usd": 0.000435988, "latency_first_all_ms": { "median": 312.1634165290743, "p90": 542.0387963706157, "p95": 1054.2246897035498 }, "latency_first_usable_ms": { "median": 312.1634165290743, "p90": 542.0387963706157, "p95": 1054.2246897035498 }, "tokens_first": { "input_tokens": { "total": 31142, "covered_requests": 10 }, "output_tokens": { "total": 520, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.001307964, "exact_decimal_usd": "0.001307964", "known_records": 10, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:07:15.325860-04:00", "last": "2026-10-15T10:18:02.646747-04:00" }, "identities": [ [ "typesafe/jev-1.14-20261014", "TypeSafe" ] ] }, "V09": { "scheduled": 10, "completed_first": 10, "correct_first": 6, "accuracy_scheduled": 0.6, "usable_first": 8, "coverage": 0.8, "accuracy_usable": 0.75, "wilson95": [ 0.31267376973365824, 0.8318196702937638 ], "statuses_first": { "ok": 8, "http_failure": 2 }, "eventual_correct": 6, "eventual_usable": 8, "retries": 0, "first_cost": { "reported_usd": 0.00082488, "exact_decimal_usd": "0.000824880", "known_records": 8, "missing_records": 2 }, "core_cost": { "reported_usd": 0.00082488, "exact_decimal_usd": "0.000824880", "known_records": 8, "missing_records": 2 }, "core_cost_per_scheduled_usd": 8.2488e-05, "first_cost_per_first_correct_usd": 0.00013748, "core_cost_per_first_correct_usd": 0.00013748, "latency_first_all_ms": { "median": 330.7088125147857, "p90": 530.9136954892892, "p95": 537.1000142622506 }, "latency_first_usable_ms": { "median": 394.894542027032, "p90": 533.6631704994943, "p95": 538.4747517673532 }, "tokens_first": { "input_tokens": { "total": 19640, "covered_requests": 8 }, "output_tokens": { "total": 360, "covered_requests": 8 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.00082488, "exact_decimal_usd": "0.000824880", "known_records": 8, "missing_records": 2 } }, "test_window": { "first": "2026-10-15T10:07:11.395723-04:00", "last": "2026-10-15T10:18:00.680324-04:00" }, "identities": [ [ "typesafe/jev-1.14-20261014", "TypeSafe" ] ] }, "V10": { "scheduled": 10, "completed_first": 10, "correct_first": 6, "accuracy_scheduled": 0.6, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.6, "wilson95": [ 0.31267376973365824, 0.8318196702937638 ], "statuses_first": { "ok": 10 }, "eventual_correct": 6, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.00175056, "exact_decimal_usd": "0.001750560", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.00175056, "exact_decimal_usd": "0.001750560", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00017505599999999999, "first_cost_per_first_correct_usd": 0.00029176, "core_cost_per_first_correct_usd": 0.00029176, "latency_first_all_ms": { "median": 271.81402101996355, "p90": 492.3379337298684, "p95": 524.4179463712497 }, "latency_first_usable_ms": { "median": 271.81402101996355, "p90": 492.3379337298684, "p95": 524.4179463712497 }, "tokens_first": { "input_tokens": { "total": 41680, "covered_requests": 10 }, "output_tokens": { "total": 513, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.00175056, "exact_decimal_usd": "0.001750560", "known_records": 10, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:07:35.228320-04:00", "last": "2026-10-15T10:16:56.202490-04:00" }, "identities": [ [ "typesafe/jev-1.14-20261014", "TypeSafe" ] ] }, "V11": { "scheduled": 10, "completed_first": 10, "correct_first": 9, "accuracy_scheduled": 0.9, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.9, "wilson95": [ 0.5958499732047615, 0.9821237869049271 ], "statuses_first": { "ok": 10 }, "eventual_correct": 9, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.001562652, "exact_decimal_usd": "0.001562652", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.001562652, "exact_decimal_usd": "0.001562652", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.0001562652, "first_cost_per_first_correct_usd": 0.000173628, "core_cost_per_first_correct_usd": 0.000173628, "latency_first_all_ms": { "median": 276.3017500110436, "p90": 475.5063045478891, "p95": 530.6814232986652 }, "latency_first_usable_ms": { "median": 276.3017500110436, "p90": 475.5063045478891, "p95": 530.6814232986652 }, "tokens_first": { "input_tokens": { "total": 37206, "covered_requests": 10 }, "output_tokens": { "total": 506, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.001562652, "exact_decimal_usd": "0.001562652", "known_records": 10, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:07:30.710022-04:00", "last": "2026-10-15T10:17:58.163703-04:00" }, "identities": [ [ "typesafe/jev-1.14-20261014", "TypeSafe" ] ] }, "V12": { "scheduled": 10, "completed_first": 10, "correct_first": 7, "accuracy_scheduled": 0.7, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.7, "wilson95": [ 0.39677814746114537, 0.8922087325936989 ], "statuses_first": { "ok": 10 }, "eventual_correct": 7, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.001003716, "exact_decimal_usd": "0.001003716", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.001003716, "exact_decimal_usd": "0.001003716", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00010037159999999999, "first_cost_per_first_correct_usd": 0.000143388, "core_cost_per_first_correct_usd": 0.000143388, "latency_first_all_ms": { "median": 253.46404148149304, "p90": 352.74991672486044, "p95": 651.3509793730911 }, "latency_first_usable_ms": { "median": 253.46404148149304, "p90": 352.74991672486044, "p95": 651.3509793730911 }, "tokens_first": { "input_tokens": { "total": 23898, "covered_requests": 10 }, "output_tokens": { "total": 450, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.001003716, "exact_decimal_usd": "0.001003716", "known_records": 10, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:07:12.457146-04:00", "last": "2026-10-15T10:17:52.362188-04:00" }, "identities": [ [ "typesafe/jev-1.14-20261014", "TypeSafe" ] ] }, "V13": { "scheduled": 10, "completed_first": 10, "correct_first": 7, "accuracy_scheduled": 0.7, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.7, "wilson95": [ 0.39677814746114537, 0.8922087325936989 ], "statuses_first": { "ok": 10 }, "eventual_correct": 7, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.00087192, "exact_decimal_usd": "0.000871920", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.00087192, "exact_decimal_usd": "0.000871920", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 8.7192e-05, "first_cost_per_first_correct_usd": 0.00012456, "core_cost_per_first_correct_usd": 0.00012456, "latency_first_all_ms": { "median": 257.7387089841068, "p90": 312.1735711756628, "p95": 332.7696400956483 }, "latency_first_usable_ms": { "median": 257.7387089841068, "p90": 312.1735711756628, "p95": 332.7696400956483 }, "tokens_first": { "input_tokens": { "total": 20760, "covered_requests": 10 }, "output_tokens": { "total": 478, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.00087192, "exact_decimal_usd": "0.000871920", "known_records": 10, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:12:46.717518-04:00", "last": "2026-10-15T10:17:48.869080-04:00" }, "identities": [ [ "typesafe/jev-1.14-20261014", "TypeSafe" ] ] }, "V14": { "scheduled": 10, "completed_first": 10, "correct_first": 6, "accuracy_scheduled": 0.6, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.6, "wilson95": [ 0.31267376973365824, 0.8318196702937638 ], "statuses_first": { "ok": 10 }, "eventual_correct": 6, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.001396836, "exact_decimal_usd": "0.001396836", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.001396836, "exact_decimal_usd": "0.001396836", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.0001396836, "first_cost_per_first_correct_usd": 0.000232806, "core_cost_per_first_correct_usd": 0.000232806, "latency_first_all_ms": { "median": 439.378042035969, "p90": 644.2251167027279, "p95": 746.181016846094 }, "latency_first_usable_ms": { "median": 439.378042035969, "p90": 644.2251167027279, "p95": 746.181016846094 }, "tokens_first": { "input_tokens": { "total": 33258, "covered_requests": 10 }, "output_tokens": { "total": 464, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.001396836, "exact_decimal_usd": "0.001396836", "known_records": 10, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:07:17.572723-04:00", "last": "2026-10-15T10:17:55.526223-04:00" }, "identities": [ [ "typesafe/jev-1.14-20261014", "TypeSafe" ] ] }, "V15": { "scheduled": 10, "completed_first": 10, "correct_first": 10, "accuracy_scheduled": 1.0, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 1.0, "wilson95": [ 0.7224672001371107, 0.9999999999999999 ], "statuses_first": { "ok": 10 }, "eventual_correct": 10, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.000759906, "exact_decimal_usd": "0.000759906", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.000759906, "exact_decimal_usd": "0.000759906", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 7.59906e-05, "first_cost_per_first_correct_usd": 7.59906e-05, "core_cost_per_first_correct_usd": 7.59906e-05, "latency_first_all_ms": { "median": 276.33725001942366, "p90": 401.69398782309145, "p95": 421.23161890194746 }, "latency_first_usable_ms": { "median": 276.33725001942366, "p90": 401.69398782309145, "p95": 421.23161890194746 }, "tokens_first": { "input_tokens": { "total": 18093, "covered_requests": 10 }, "output_tokens": { "total": 485, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.000759906, "exact_decimal_usd": "0.000759906", "known_records": 10, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:07:18.346195-04:00", "last": "2026-10-15T10:17:55.812653-04:00" }, "identities": [ [ "typesafe/jev-1.14-20261014", "TypeSafe" ] ] }, "V16": { "scheduled": 10, "completed_first": 10, "correct_first": 10, "accuracy_scheduled": 1.0, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 1.0, "wilson95": [ 0.7224672001371107, 0.9999999999999999 ], "statuses_first": { "ok": 10 }, "eventual_correct": 10, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.001933764, "exact_decimal_usd": "0.001933764", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.001933764, "exact_decimal_usd": "0.001933764", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.0001933764, "first_cost_per_first_correct_usd": 0.0001933764, "core_cost_per_first_correct_usd": 0.0001933764, "latency_first_all_ms": { "median": 724.9781044956762, "p90": 863.9710791991092, "p95": 872.7509355929215 }, "latency_first_usable_ms": { "median": 724.9781044956762, "p90": 863.9710791991092, "p95": 872.7509355929215 }, "tokens_first": { "input_tokens": { "total": 46042, "covered_requests": 10 }, "output_tokens": { "total": 450, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.001933764, "exact_decimal_usd": "0.001933764", "known_records": 10, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:07:18.849911-04:00", "last": "2026-10-15T10:16:14.600605-04:00" }, "identities": [ [ "typesafe/jev-1.14-20261014", "TypeSafe" ] ] }, "V17": { "scheduled": 10, "completed_first": 10, "correct_first": 7, "accuracy_scheduled": 0.7, "usable_first": 9, "coverage": 0.9, "accuracy_usable": 0.7777777777777778, "wilson95": [ 0.39677814746114537, 0.8922087325936989 ], "statuses_first": { "http_failure": 1, "ok": 9 }, "eventual_correct": 7, "eventual_usable": 9, "retries": 0, "first_cost": { "reported_usd": 0.001618008, "exact_decimal_usd": "0.001618008", "known_records": 9, "missing_records": 1 }, "core_cost": { "reported_usd": 0.001618008, "exact_decimal_usd": "0.001618008", "known_records": 9, "missing_records": 1 }, "core_cost_per_scheduled_usd": 0.0001618008, "first_cost_per_first_correct_usd": 0.000231144, "core_cost_per_first_correct_usd": 0.000231144, "latency_first_all_ms": { "median": 348.70856199995615, "p90": 499.8364711354951, "p95": 611.2870895740341 }, "latency_first_usable_ms": { "median": 335.5609160498716, "p90": 524.6032752329484, "p95": 623.6704916227608 }, "tokens_first": { "input_tokens": { "total": 38524, "covered_requests": 9 }, "output_tokens": { "total": 405, "covered_requests": 9 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.001618008, "exact_decimal_usd": "0.001618008", "known_records": 9, "missing_records": 1 } }, "test_window": { "first": "2026-10-15T10:12:55.626734-04:00", "last": "2026-10-15T10:17:47.734696-04:00" }, "identities": [ [ "typesafe/jev-1.14-20261014", "TypeSafe" ] ] }, "V18": { "scheduled": 10, "completed_first": 10, "correct_first": 10, "accuracy_scheduled": 1.0, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 1.0, "wilson95": [ 0.7224672001371107, 0.9999999999999999 ], "statuses_first": { "ok": 10 }, "eventual_correct": 10, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.001150674, "exact_decimal_usd": "0.001150674", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.001150674, "exact_decimal_usd": "0.001150674", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.0001150674, "first_cost_per_first_correct_usd": 0.0001150674, "core_cost_per_first_correct_usd": 0.0001150674, "latency_first_all_ms": { "median": 304.2913539975416, "p90": 472.1901208162307, "p95": 527.7864769042934 }, "latency_first_usable_ms": { "median": 304.2913539975416, "p90": 472.1901208162307, "p95": 527.7864769042934 }, "tokens_first": { "input_tokens": { "total": 27397, "covered_requests": 10 }, "output_tokens": { "total": 513, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.001150674, "exact_decimal_usd": "0.001150674", "known_records": 10, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:07:31.679879-04:00", "last": "2026-10-15T10:16:49.553740-04:00" }, "identities": [ [ "typesafe/jev-1.14-20261014", "TypeSafe" ] ] }, "V19": { "scheduled": 10, "completed_first": 10, "correct_first": 4, "accuracy_scheduled": 0.4, "usable_first": 9, "coverage": 0.9, "accuracy_usable": 0.4444444444444444, "wilson95": [ 0.16818032970623614, 0.6873262302663417 ], "statuses_first": { "ok": 9, "http_failure": 1 }, "eventual_correct": 4, "eventual_usable": 9, "retries": 0, "first_cost": { "reported_usd": 0.001053192, "exact_decimal_usd": "0.001053192", "known_records": 9, "missing_records": 1 }, "core_cost": { "reported_usd": 0.001053192, "exact_decimal_usd": "0.001053192", "known_records": 9, "missing_records": 1 }, "core_cost_per_scheduled_usd": 0.00010531920000000001, "first_cost_per_first_correct_usd": 0.000263298, "core_cost_per_first_correct_usd": 0.000263298, "latency_first_all_ms": { "median": 290.68006252055056, "p90": 449.7430168732534, "p95": 598.8504039502001 }, "latency_first_usable_ms": { "median": 311.2247500102967, "p90": 482.87799177924177, "p95": 615.4178914031944 }, "tokens_first": { "input_tokens": { "total": 25076, "covered_requests": 9 }, "output_tokens": { "total": 405, "covered_requests": 9 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.001053192, "exact_decimal_usd": "0.001053192", "known_records": 9, "missing_records": 1 } }, "test_window": { "first": "2026-10-15T10:07:34.686185-04:00", "last": "2026-10-15T10:17:00.614733-04:00" }, "identities": [ [ "typesafe/jev-1.14-20261014", "TypeSafe" ] ] }, "V20": { "scheduled": 10, "completed_first": 10, "correct_first": 10, "accuracy_scheduled": 1.0, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 1.0, "wilson95": [ 0.7224672001371107, 0.9999999999999999 ], "statuses_first": { "ok": 10 }, "eventual_correct": 10, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.001087884, "exact_decimal_usd": "0.001087884", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.001087884, "exact_decimal_usd": "0.001087884", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00010878839999999999, "first_cost_per_first_correct_usd": 0.00010878839999999999, "core_cost_per_first_correct_usd": 0.00010878839999999999, "latency_first_all_ms": { "median": 287.16600002371706, "p90": 388.6692583677359, "p95": 422.2544836986344 }, "latency_first_usable_ms": { "median": 287.16600002371706, "p90": 388.6692583677359, "p95": 422.2544836986344 }, "tokens_first": { "input_tokens": { "total": 25902, "covered_requests": 10 }, "output_tokens": { "total": 457, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.001087884, "exact_decimal_usd": "0.001087884", "known_records": 10, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:07:38.124023-04:00", "last": "2026-10-15T10:17:57.214876-04:00" }, "identities": [ [ "typesafe/jev-1.14-20261014", "TypeSafe" ] ] } }, "image_count": { "1": { "scheduled": 177, "completed_first": 177, "correct_first": 125, "accuracy_scheduled": 0.7062146892655368, "usable_first": 175, "coverage": 0.9887005649717514, "accuracy_usable": 0.7142857142857143, "wilson95": [ 0.6353029701887516, 0.7683655274889858 ], "statuses_first": { "ok": 175, "http_failure": 2 }, "eventual_correct": 125, "eventual_usable": 175, "retries": 0, "first_cost": { "reported_usd": 0.020183856, "exact_decimal_usd": "0.020183856", "known_records": 175, "missing_records": 2 }, "core_cost": { "reported_usd": 0.020183856, "exact_decimal_usd": "0.020183856", "known_records": 175, "missing_records": 2 }, "core_cost_per_scheduled_usd": 0.00011403308474576272, "first_cost_per_first_correct_usd": 0.000161470848, "core_cost_per_first_correct_usd": 0.000161470848, "latency_first_all_ms": { "median": 297.20383300445974, "p90": 467.79242499032995, "p95": 583.8775748037733 }, "latency_first_usable_ms": { "median": 297.9260829742998, "p90": 468.9943249919451, "p95": 584.1249457094818 }, "tokens_first": { "input_tokens": { "total": 480568, "covered_requests": 175 }, "output_tokens": { "total": 8316, "covered_requests": 175 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.020183856, "exact_decimal_usd": "0.020183856", "known_records": 175, "missing_records": 2 } }, "test_window": { "first": "2026-10-15T10:07:12.457146-04:00", "last": "2026-10-15T10:18:02.646747-04:00" }, "identities": [ [ "typesafe/jev-1.14-20261014", "TypeSafe" ] ] }, "2": { "scheduled": 19, "completed_first": 19, "correct_first": 16, "accuracy_scheduled": 0.8421052631578947, "usable_first": 18, "coverage": 0.9473684210526315, "accuracy_usable": 0.8888888888888888, "wilson95": [ 0.6243452472967093, 0.9447952836146585 ], "statuses_first": { "ok": 18, "http_failure": 1 }, "eventual_correct": 16, "eventual_usable": 18, "retries": 0, "first_cost": { "reported_usd": 0.003443118, "exact_decimal_usd": "0.003443118", "known_records": 18, "missing_records": 1 }, "core_cost": { "reported_usd": 0.003443118, "exact_decimal_usd": "0.003443118", "known_records": 18, "missing_records": 1 }, "core_cost_per_scheduled_usd": 0.00018121673684210526, "first_cost_per_first_correct_usd": 0.000215194875, "core_cost_per_first_correct_usd": 0.000215194875, "latency_first_all_ms": { "median": 539.647415978834, "p90": 858.3370000356808, "p95": 863.9710791991092 }, "latency_first_usable_ms": { "median": 618.3388534700498, "p90": 858.7973750312813, "p95": 864.9466187984217 }, "tokens_first": { "input_tokens": { "total": 81979, "covered_requests": 18 }, "output_tokens": { "total": 810, "covered_requests": 18 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.003443118, "exact_decimal_usd": "0.003443118", "known_records": 18, "missing_records": 1 } }, "test_window": { "first": "2026-10-15T10:07:18.849911-04:00", "last": "2026-10-15T10:17:47.734696-04:00" }, "identities": [ [ "typesafe/jev-1.14-20261014", "TypeSafe" ] ] }, "3": { "scheduled": 4, "completed_first": 4, "correct_first": 0, "accuracy_scheduled": 0.0, "usable_first": 2, "coverage": 0.5, "accuracy_usable": 0.0, "wilson95": [ 0.0, 0.4898908364545973 ], "statuses_first": { "ok": 2, "http_failure": 2 }, "eventual_correct": 0, "eventual_usable": 2, "retries": 0, "first_cost": { "reported_usd": 0.000349902, "exact_decimal_usd": "0.000349902", "known_records": 2, "missing_records": 2 }, "core_cost": { "reported_usd": 0.000349902, "exact_decimal_usd": "0.000349902", "known_records": 2, "missing_records": 2 }, "core_cost_per_scheduled_usd": 8.74755e-05, "first_cost_per_first_correct_usd": null, "core_cost_per_first_correct_usd": null, "latency_first_all_ms": { "median": 380.06260397378355, "p90": 539.1621205199044, "p95": 541.2242267775582 }, "latency_first_usable_ms": { "median": 536.4126455096994, "p90": 541.9115955301095, "p95": 542.5989642826607 }, "tokens_first": { "input_tokens": { "total": 8331, "covered_requests": 2 }, "output_tokens": { "total": 90, "covered_requests": 2 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.000349902, "exact_decimal_usd": "0.000349902", "known_records": 2, "missing_records": 2 } }, "test_window": { "first": "2026-10-15T10:07:11.395723-04:00", "last": "2026-10-15T10:17:02.406983-04:00" }, "identities": [ [ "typesafe/jev-1.14-20261014", "TypeSafe" ] ] } }, "analysis_subset": { "perception_control": { "scheduled": 13, "completed_first": 13, "correct_first": 11, "accuracy_scheduled": 0.8461538461538461, "usable_first": 11, "coverage": 0.8461538461538461, "accuracy_usable": 1.0, "wilson95": [ 0.5776536898051745, 0.9567418216419186 ], "statuses_first": { "ok": 11, "http_failure": 2 }, "eventual_correct": 11, "eventual_usable": 11, "retries": 0, "first_cost": { "reported_usd": 0.00121212, "exact_decimal_usd": "0.001212120", "known_records": 11, "missing_records": 2 }, "core_cost": { "reported_usd": 0.00121212, "exact_decimal_usd": "0.001212120", "known_records": 11, "missing_records": 2 }, "core_cost_per_scheduled_usd": 9.324e-05, "first_cost_per_first_correct_usd": 0.00011019272727272726, "core_cost_per_first_correct_usd": 0.00011019272727272726, "latency_first_all_ms": { "median": 251.32141704671085, "p90": 776.0582500020976, "p95": 869.824316794984 }, "latency_first_usable_ms": { "median": 297.19395795837045, "p90": 862.0200000004843, "p95": 871.775395993609 }, "tokens_first": { "input_tokens": { "total": 28860, "covered_requests": 11 }, "output_tokens": { "total": 509, "covered_requests": 11 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.00121212, "exact_decimal_usd": "0.001212120", "known_records": 11, "missing_records": 2 } }, "test_window": { "first": "2026-10-15T10:07:13.030379-04:00", "last": "2026-10-15T10:17:02.406983-04:00" }, "identities": [ [ "typesafe/jev-1.14-20261014", "TypeSafe" ] ] }, "reasoning_and_grounded_decision": { "scheduled": 187, "completed_first": 187, "correct_first": 130, "accuracy_scheduled": 0.6951871657754011, "usable_first": 184, "coverage": 0.983957219251337, "accuracy_usable": 0.7065217391304348, "wilson95": [ 0.6258302287724811, 0.7566862351610055 ], "statuses_first": { "ok": 184, "http_failure": 3 }, "eventual_correct": 130, "eventual_usable": 184, "retries": 0, "first_cost": { "reported_usd": 0.022764756, "exact_decimal_usd": "0.022764756", "known_records": 184, "missing_records": 3 }, "core_cost": { "reported_usd": 0.022764756, "exact_decimal_usd": "0.022764756", "known_records": 184, "missing_records": 3 }, "core_cost_per_scheduled_usd": 0.00012173666310160428, "first_cost_per_first_correct_usd": 0.0001751135076923077, "core_cost_per_first_correct_usd": 0.0001751135076923077, "latency_first_all_ms": { "median": 307.59666697122157, "p90": 548.5709834261799, "p95": 702.134571474744 }, "latency_first_usable_ms": { "median": 308.2788125029765, "p90": 552.5344712194058, "p95": 702.2747652226826 }, "tokens_first": { "input_tokens": { "total": 542018, "covered_requests": 184 }, "output_tokens": { "total": 8707, "covered_requests": 184 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.022764756, "exact_decimal_usd": "0.022764756", "known_records": 184, "missing_records": 3 } }, "test_window": { "first": "2026-10-15T10:07:11.395723-04:00", "last": "2026-10-15T10:18:02.646747-04:00" }, "identities": [ [ "typesafe/jev-1.14-20261014", "TypeSafe" ] ] } }, "difficulty_proposed": { "demanding": { "scheduled": 55, "completed_first": 55, "correct_first": 31, "accuracy_scheduled": 0.5636363636363636, "usable_first": 52, "coverage": 0.9454545454545454, "accuracy_usable": 0.5961538461538461, "wilson95": [ 0.43269816346414447, 0.6862655763936809 ], "statuses_first": { "ok": 52, "http_failure": 3 }, "eventual_correct": 31, "eventual_usable": 52, "retries": 0, "first_cost": { "reported_usd": 0.006770778, "exact_decimal_usd": "0.006770778", "known_records": 52, "missing_records": 3 }, "core_cost": { "reported_usd": 0.006770778, "exact_decimal_usd": "0.006770778", "known_records": 52, "missing_records": 3 }, "core_cost_per_scheduled_usd": 0.00012310505454545453, "first_cost_per_first_correct_usd": 0.00021841219354838708, "core_cost_per_first_correct_usd": 0.00021841219354838708, "latency_first_all_ms": { "median": 301.19566700886935, "p90": 655.2308172103951, "p95": 850.9207169059664 }, "latency_first_usable_ms": { "median": 304.137520986842, "p90": 689.9179547908714, "p95": 852.3126168642193 }, "tokens_first": { "input_tokens": { "total": 161209, "covered_requests": 52 }, "output_tokens": { "total": 2459, "covered_requests": 52 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.006770778, "exact_decimal_usd": "0.006770778", "known_records": 52, "missing_records": 3 } }, "test_window": { "first": "2026-10-15T10:07:14.348804-04:00", "last": "2026-10-15T10:17:58.163703-04:00" }, "identities": [ [ "typesafe/jev-1.14-20261014", "TypeSafe" ] ] }, "moderate": { "scheduled": 107, "completed_first": 107, "correct_first": 78, "accuracy_scheduled": 0.7289719626168224, "usable_first": 106, "coverage": 0.9906542056074766, "accuracy_usable": 0.7358490566037735, "wilson95": [ 0.6379084109244534, 0.804164442693129 ], "statuses_first": { "ok": 106, "http_failure": 1 }, "eventual_correct": 78, "eventual_usable": 106, "retries": 0, "first_cost": { "reported_usd": 0.013022226, "exact_decimal_usd": "0.013022226", "known_records": 106, "missing_records": 1 }, "core_cost": { "reported_usd": 0.013022226, "exact_decimal_usd": "0.013022226", "known_records": 106, "missing_records": 1 }, "core_cost_per_scheduled_usd": 0.00012170304672897195, "first_cost_per_first_correct_usd": 0.00016695161538461537, "core_cost_per_first_correct_usd": 0.00016695161538461537, "latency_first_all_ms": { "median": 312.95891699846834, "p90": 548.5709834261799, "p95": 674.3916786741456 }, "latency_first_usable_ms": { "median": 313.9607709890697, "p90": 549.8921460239217, "p95": 678.1647807219997 }, "tokens_first": { "input_tokens": { "total": 310053, "covered_requests": 106 }, "output_tokens": { "total": 4973, "covered_requests": 106 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.013022226, "exact_decimal_usd": "0.013022226", "known_records": 106, "missing_records": 1 } }, "test_window": { "first": "2026-10-15T10:07:11.395723-04:00", "last": "2026-10-15T10:18:02.646747-04:00" }, "identities": [ [ "typesafe/jev-1.14-20261014", "TypeSafe" ] ] }, "straightforward": { "scheduled": 38, "completed_first": 38, "correct_first": 32, "accuracy_scheduled": 0.8421052631578947, "usable_first": 37, "coverage": 0.9736842105263158, "accuracy_usable": 0.8648648648648649, "wilson95": [ 0.6958318822077686, 0.9255613658484106 ], "statuses_first": { "ok": 37, "http_failure": 1 }, "eventual_correct": 32, "eventual_usable": 37, "retries": 0, "first_cost": { "reported_usd": 0.004183872, "exact_decimal_usd": "0.004183872", "known_records": 37, "missing_records": 1 }, "core_cost": { "reported_usd": 0.004183872, "exact_decimal_usd": "0.004183872", "known_records": 37, "missing_records": 1 }, "core_cost_per_scheduled_usd": 0.0001101018947368421, "first_cost_per_first_correct_usd": 0.000130746, "core_cost_per_first_correct_usd": 0.000130746, "latency_first_all_ms": { "median": 299.87377050565556, "p90": 467.4720874114429, "p95": 662.3593499913112 }, "latency_first_usable_ms": { "median": 301.8214580370113, "p90": 470.0176582089625, "p95": 667.3700499930414 }, "tokens_first": { "input_tokens": { "total": 99616, "covered_requests": 37 }, "output_tokens": { "total": 1784, "covered_requests": 37 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.004183872, "exact_decimal_usd": "0.004183872", "known_records": 37, "missing_records": 1 } }, "test_window": { "first": "2026-10-15T10:07:13.030379-04:00", "last": "2026-10-15T10:18:01.553861-04:00" }, "identities": [ [ "typesafe/jev-1.14-20261014", "TypeSafe" ] ] } }, "visual_medium": { "chart": { "scheduled": 13, "completed_first": 13, "correct_first": 10, "accuracy_scheduled": 0.7692307692307693, "usable_first": 13, "coverage": 1.0, "accuracy_usable": 0.7692307692307693, "wilson95": [ 0.49743624053272495, 0.9182047128150143 ], "statuses_first": { "ok": 13 }, "eventual_correct": 10, "eventual_usable": 13, "retries": 0, "first_cost": { "reported_usd": 0.001164492, "exact_decimal_usd": "0.001164492", "known_records": 13, "missing_records": 0 }, "core_cost": { "reported_usd": 0.001164492, "exact_decimal_usd": "0.001164492", "known_records": 13, "missing_records": 0 }, "core_cost_per_scheduled_usd": 8.957630769230769e-05, "first_cost_per_first_correct_usd": 0.00011644920000000001, "core_cost_per_first_correct_usd": 0.00011644920000000001, "latency_first_all_ms": { "median": 248.77520801965147, "p90": 310.4991334024817, "p95": 328.08113361243153 }, "latency_first_usable_ms": { "median": 248.77520801965147, "p90": 310.4991334024817, "p95": 328.08113361243153 }, "tokens_first": { "input_tokens": { "total": 27726, "covered_requests": 13 }, "output_tokens": { "total": 627, "covered_requests": 13 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.001164492, "exact_decimal_usd": "0.001164492", "known_records": 13, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:07:31.679879-04:00", "last": "2026-10-15T10:17:48.869080-04:00" }, "identities": [ [ "typesafe/jev-1.14-20261014", "TypeSafe" ] ] }, "diagram": { "scheduled": 92, "completed_first": 92, "correct_first": 54, "accuracy_scheduled": 0.5869565217391305, "usable_first": 90, "coverage": 0.9782608695652174, "accuracy_usable": 0.6, "wilson95": [ 0.48483335095613705, 0.6821090161246245 ], "statuses_first": { "ok": 90, "http_failure": 2 }, "eventual_correct": 54, "eventual_usable": 90, "retries": 0, "first_cost": { "reported_usd": 0.01175664, "exact_decimal_usd": "0.011756640", "known_records": 90, "missing_records": 2 }, "core_cost": { "reported_usd": 0.01175664, "exact_decimal_usd": "0.011756640", "known_records": 90, "missing_records": 2 }, "core_cost_per_scheduled_usd": 0.00012778956521739132, "first_cost_per_first_correct_usd": 0.00021771555555555558, "core_cost_per_first_correct_usd": 0.00021771555555555558, "latency_first_all_ms": { "median": 279.45129200816154, "p90": 453.67086722981196, "p95": 547.8702876716854 }, "latency_first_usable_ms": { "median": 280.26297950418666, "p90": 456.2392756284681, "p95": 554.7768793767316 }, "tokens_first": { "input_tokens": { "total": 279920, "covered_requests": 90 }, "output_tokens": { "total": 4337, "covered_requests": 90 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.01175664, "exact_decimal_usd": "0.011756640", "known_records": 90, "missing_records": 2 } }, "test_window": { "first": "2026-10-15T10:07:12.457146-04:00", "last": "2026-10-15T10:18:02.646747-04:00" }, "identities": [ [ "typesafe/jev-1.14-20261014", "TypeSafe" ] ] }, "diagram_or_published_illustration": { "scheduled": 8, "completed_first": 8, "correct_first": 7, "accuracy_scheduled": 0.875, "usable_first": 7, "coverage": 0.875, "accuracy_usable": 1.0, "wilson95": [ 0.5291118177871464, 0.9775825085499433 ], "statuses_first": { "ok": 7, "http_failure": 1 }, "eventual_correct": 7, "eventual_usable": 7, "retries": 0, "first_cost": { "reported_usd": 0.000584304, "exact_decimal_usd": "0.000584304", "known_records": 7, "missing_records": 1 }, "core_cost": { "reported_usd": 0.000584304, "exact_decimal_usd": "0.000584304", "known_records": 7, "missing_records": 1 }, "core_cost_per_scheduled_usd": 7.3038e-05, "first_cost_per_first_correct_usd": 8.3472e-05, "core_cost_per_first_correct_usd": 8.3472e-05, "latency_first_all_ms": { "median": 277.9233750188723, "p90": 313.8923250255175, "p95": 341.86047501279967 }, "latency_first_usable_ms": { "median": 274.52245802851394, "p90": 289.1502000275068, "p95": 289.5349125319626 }, "tokens_first": { "input_tokens": { "total": 13912, "covered_requests": 7 }, "output_tokens": { "total": 315, "covered_requests": 7 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.000584304, "exact_decimal_usd": "0.000584304", "known_records": 7, "missing_records": 1 } }, "test_window": { "first": "2026-10-15T10:07:13.705458-04:00", "last": "2026-10-15T10:16:01.597968-04:00" }, "identities": [ [ "typesafe/jev-1.14-20261014", "TypeSafe" ] ] }, "document": { "scheduled": 12, "completed_first": 12, "correct_first": 8, "accuracy_scheduled": 0.6666666666666666, "usable_first": 12, "coverage": 1.0, "accuracy_usable": 0.6666666666666666, "wilson95": [ 0.39062208887279953, 0.8618799089087867 ], "statuses_first": { "ok": 12 }, "eventual_correct": 8, "eventual_usable": 12, "retries": 0, "first_cost": { "reported_usd": 0.001625274, "exact_decimal_usd": "0.001625274", "known_records": 12, "missing_records": 0 }, "core_cost": { "reported_usd": 0.001625274, "exact_decimal_usd": "0.001625274", "known_records": 12, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00013543949999999998, "first_cost_per_first_correct_usd": 0.00020315925, "core_cost_per_first_correct_usd": 0.00020315925, "latency_first_all_ms": { "median": 439.378042035969, "p90": 615.061220905045, "p95": 723.5241501475683 }, "latency_first_usable_ms": { "median": 439.378042035969, "p90": 615.061220905045, "p95": 723.5241501475683 }, "tokens_first": { "input_tokens": { "total": 38697, "covered_requests": 12 }, "output_tokens": { "total": 568, "covered_requests": 12 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.001625274, "exact_decimal_usd": "0.001625274", "known_records": 12, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:07:17.572723-04:00", "last": "2026-10-15T10:17:55.526223-04:00" }, "identities": [ [ "typesafe/jev-1.14-20261014", "TypeSafe" ] ] }, "photograph": { "scheduled": 64, "completed_first": 64, "correct_first": 51, "accuracy_scheduled": 0.796875, "usable_first": 62, "coverage": 0.96875, "accuracy_usable": 0.8225806451612904, "wilson95": [ 0.6828636426898583, 0.8772658218081634 ], "statuses_first": { "ok": 62, "http_failure": 2 }, "eventual_correct": 51, "eventual_usable": 62, "retries": 0, "first_cost": { "reported_usd": 0.007967568, "exact_decimal_usd": "0.007967568", "known_records": 62, "missing_records": 2 }, "core_cost": { "reported_usd": 0.007967568, "exact_decimal_usd": "0.007967568", "known_records": 62, "missing_records": 2 }, "core_cost_per_scheduled_usd": 0.00012449325, "first_cost_per_first_correct_usd": 0.00015622682352941174, "core_cost_per_first_correct_usd": 0.00015622682352941174, "latency_first_all_ms": { "median": 359.7170209977776, "p90": 716.6408832999879, "p95": 769.1977686394238 }, "latency_first_usable_ms": { "median": 362.439957971219, "p90": 720.7054331083782, "p95": 771.6965895349857 }, "tokens_first": { "input_tokens": { "total": 189704, "covered_requests": 62 }, "output_tokens": { "total": 2839, "covered_requests": 62 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.007967568, "exact_decimal_usd": "0.007967568", "known_records": 62, "missing_records": 2 } }, "test_window": { "first": "2026-10-15T10:07:11.395723-04:00", "last": "2026-10-15T10:18:00.680324-04:00" }, "identities": [ [ "typesafe/jev-1.14-20261014", "TypeSafe" ] ] }, "screenshot": { "scheduled": 11, "completed_first": 11, "correct_first": 11, "accuracy_scheduled": 1.0, "usable_first": 11, "coverage": 1.0, "accuracy_usable": 1.0, "wilson95": [ 0.7411670330319684, 1.0 ], "statuses_first": { "ok": 11 }, "eventual_correct": 11, "eventual_usable": 11, "retries": 0, "first_cost": { "reported_usd": 0.000878598, "exact_decimal_usd": "0.000878598", "known_records": 11, "missing_records": 0 }, "core_cost": { "reported_usd": 0.000878598, "exact_decimal_usd": "0.000878598", "known_records": 11, "missing_records": 0 }, "core_cost_per_scheduled_usd": 7.987254545454545e-05, "first_cost_per_first_correct_usd": 7.987254545454545e-05, "core_cost_per_first_correct_usd": 7.987254545454545e-05, "latency_first_all_ms": { "median": 257.36616703215986, "p90": 397.3522920277901, "p95": 419.06077100429684 }, "latency_first_usable_ms": { "median": 257.36616703215986, "p90": 397.3522920277901, "p95": 419.06077100429684 }, "tokens_first": { "input_tokens": { "total": 20919, "covered_requests": 11 }, "output_tokens": { "total": 530, "covered_requests": 11 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.000878598, "exact_decimal_usd": "0.000878598", "known_records": 11, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:07:18.346195-04:00", "last": "2026-10-15T10:17:57.214876-04:00" }, "identities": [ [ "typesafe/jev-1.14-20261014", "TypeSafe" ] ] } }, "stage": { "1": { "scheduled": 100, "completed_first": 100, "correct_first": 73, "accuracy_scheduled": 0.73, "usable_first": 97, "coverage": 0.97, "accuracy_usable": 0.7525773195876289, "wilson95": [ 0.6356788323205499, 0.8073041585042365 ], "statuses_first": { "ok": 97, "http_failure": 3 }, "eventual_correct": 73, "eventual_usable": 97, "retries": 0, "first_cost": { "reported_usd": 0.011731146, "exact_decimal_usd": "0.011731146", "known_records": 97, "missing_records": 3 }, "core_cost": { "reported_usd": 0.011731146, "exact_decimal_usd": "0.011731146", "known_records": 97, "missing_records": 3 }, "core_cost_per_scheduled_usd": 0.00011731146, "first_cost_per_first_correct_usd": 0.0001607006301369863, "core_cost_per_first_correct_usd": 0.0001607006301369863, "latency_first_all_ms": { "median": 310.0928540225141, "p90": 561.5901583107191, "p95": 752.5710333691674 }, "latency_first_usable_ms": { "median": 311.2247500102967, "p90": 568.8543832045981, "p95": 767.660383414476 }, "tokens_first": { "input_tokens": { "total": 279313, "covered_requests": 97 }, "output_tokens": { "total": 4589, "covered_requests": 97 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.011731146, "exact_decimal_usd": "0.011731146", "known_records": 97, "missing_records": 3 } }, "test_window": { "first": "2026-10-15T10:07:11.395723-04:00", "last": "2026-10-15T10:14:29.837384-04:00" }, "identities": [ [ "typesafe/jev-1.14-20261014", "TypeSafe" ] ] }, "2": { "scheduled": 100, "completed_first": 100, "correct_first": 68, "accuracy_scheduled": 0.68, "usable_first": 98, "coverage": 0.98, "accuracy_usable": 0.6938775510204082, "wilson95": [ 0.5833738138954762, 0.7633085267500092 ], "statuses_first": { "ok": 98, "http_failure": 2 }, "eventual_correct": 68, "eventual_usable": 98, "retries": 0, "first_cost": { "reported_usd": 0.01224573, "exact_decimal_usd": "0.012245730", "known_records": 98, "missing_records": 2 }, "core_cost": { "reported_usd": 0.01224573, "exact_decimal_usd": "0.012245730", "known_records": 98, "missing_records": 2 }, "core_cost_per_scheduled_usd": 0.0001224573, "first_cost_per_first_correct_usd": 0.00018008426470588235, "core_cost_per_first_correct_usd": 0.00018008426470588235, "latency_first_all_ms": { "median": 296.3485204963945, "p90": 540.0113076844718, "p95": 702.5432026857742 }, "latency_first_usable_ms": { "median": 298.62768700695597, "p90": 540.7390910957474, "p95": 704.668940088595 }, "tokens_first": { "input_tokens": { "total": 291565, "covered_requests": 98 }, "output_tokens": { "total": 4627, "covered_requests": 98 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.01224573, "exact_decimal_usd": "0.012245730", "known_records": 98, "missing_records": 2 } }, "test_window": { "first": "2026-10-15T10:15:28.470618-04:00", "last": "2026-10-15T10:18:02.646747-04:00" }, "identities": [ [ "typesafe/jev-1.14-20261014", "TypeSafe" ] ] } } }, "families_at_least_8_of_10": [ "V01", "V02", "V03", "V11", "V15", "V16", "V18", "V20" ], "families_at_most_4_of_10": [ "V04", "V06", "V07", "V08", "V19" ] }, "openai/gpt-6-luna-decisions": { "scheduled": 200, "completed_first": 200, "correct_first": 121, "accuracy_scheduled": 0.605, "usable_first": 198, "coverage": 0.99, "accuracy_usable": 0.6111111111111112, "wilson95": [ 0.5358834915928536, 0.6701589898679203 ], "statuses_first": { "ok": 198, "http_failure": 2 }, "eventual_correct": 121, "eventual_usable": 199, "retries": 2, "first_cost": { "reported_usd": 0.0224498, "exact_decimal_usd": "0.0224498", "known_records": 198, "missing_records": 2 }, "core_cost": { "reported_usd": 0.0225462, "exact_decimal_usd": "0.0225462", "known_records": 199, "missing_records": 3 }, "core_cost_per_scheduled_usd": 0.00011273099999999999, "first_cost_per_first_correct_usd": 0.00018553553719008264, "core_cost_per_first_correct_usd": 0.00018633223140495866, "latency_first_all_ms": { "median": 383.0100414925255, "p90": 658.4688036120496, "p95": 799.3361333152274 }, "latency_first_usable_ms": { "median": 383.0100414925255, "p90": 654.8458034929354, "p95": 791.6830687376204 }, "tokens_first": { "input_tokens": { "total": 224498, "covered_requests": 198 }, "output_tokens": { "total": 0, "covered_requests": 198 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.0225462, "exact_decimal_usd": "0.0225462", "known_records": 199, "missing_records": 3 } }, "test_window": { "first": "2026-10-15T10:07:11.994059-04:00", "last": "2026-10-15T10:18:02.345676-04:00" }, "identities": [ [ "openai/gpt-6-luna-decisions-20261006", "OpenAI" ] ], "groups": { "family_id": { "V01": { "scheduled": 10, "completed_first": 10, "correct_first": 9, "accuracy_scheduled": 0.9, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.9, "wilson95": [ 0.5958499732047615, 0.9821237869049271 ], "statuses_first": { "ok": 10 }, "eventual_correct": 9, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.0008925, "exact_decimal_usd": "0.0008925", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0008925, "exact_decimal_usd": "0.0008925", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 8.925e-05, "first_cost_per_first_correct_usd": 9.916666666666666e-05, "core_cost_per_first_correct_usd": 9.916666666666666e-05, "latency_first_all_ms": { "median": 421.61418750765733, "p90": 498.45727075589815, "p95": 562.7162393764592 }, "latency_first_usable_ms": { "median": 421.61418750765733, "p90": 498.45727075589815, "p95": 562.7162393764592 }, "tokens_first": { "input_tokens": { "total": 8925, "covered_requests": 10 }, "output_tokens": { "total": 0, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.0008925, "exact_decimal_usd": "0.0008925", "known_records": 10, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:07:13.982947-04:00", "last": "2026-10-15T10:17:59.493281-04:00" }, "identities": [ [ "openai/gpt-6-luna-decisions-20261006", "OpenAI" ] ] }, "V02": { "scheduled": 10, "completed_first": 10, "correct_first": 5, "accuracy_scheduled": 0.5, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.5, "wilson95": [ 0.236593090512564, 0.7634069094874361 ], "statuses_first": { "ok": 10 }, "eventual_correct": 5, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.0008034, "exact_decimal_usd": "0.0008034", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0008034, "exact_decimal_usd": "0.0008034", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 8.034e-05, "first_cost_per_first_correct_usd": 0.00016068, "core_cost_per_first_correct_usd": 0.00016068, "latency_first_all_ms": { "median": 425.944916496519, "p90": 645.9133253491018, "p95": 667.218725188286 }, "latency_first_usable_ms": { "median": 425.944916496519, "p90": 645.9133253491018, "p95": 667.218725188286 }, "tokens_first": { "input_tokens": { "total": 8034, "covered_requests": 10 }, "output_tokens": { "total": 0, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.0008034, "exact_decimal_usd": "0.0008034", "known_records": 10, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:07:16.083658-04:00", "last": "2026-10-15T10:17:51.446420-04:00" }, "identities": [ [ "openai/gpt-6-luna-decisions-20261006", "OpenAI" ] ] }, "V03": { "scheduled": 10, "completed_first": 10, "correct_first": 6, "accuracy_scheduled": 0.6, "usable_first": 9, "coverage": 0.9, "accuracy_usable": 0.6666666666666666, "wilson95": [ 0.31267376973365824, 0.8318196702937638 ], "statuses_first": { "ok": 9, "http_failure": 1 }, "eventual_correct": 6, "eventual_usable": 10, "retries": 1, "first_cost": { "reported_usd": 0.0009707, "exact_decimal_usd": "0.0009707", "known_records": 9, "missing_records": 1 }, "core_cost": { "reported_usd": 0.0010671, "exact_decimal_usd": "0.0010671", "known_records": 10, "missing_records": 1 }, "core_cost_per_scheduled_usd": 0.00010671000000000001, "first_cost_per_first_correct_usd": 0.00016178333333333333, "core_cost_per_first_correct_usd": 0.00017785, "latency_first_all_ms": { "median": 354.3738125008531, "p90": 914.2782832728682, "p95": 3063.9076206250998 }, "latency_first_usable_ms": { "median": 350.1283749938011, "p90": 423.80317500792444, "p95": 430.19302499014884 }, "tokens_first": { "input_tokens": { "total": 9707, "covered_requests": 9 }, "output_tokens": { "total": 0, "covered_requests": 9 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.0010671, "exact_decimal_usd": "0.0010671", "known_records": 10, "missing_records": 1 } }, "test_window": { "first": "2026-10-15T10:07:14.629770-04:00", "last": "2026-10-15T10:17:53.904170-04:00" }, "identities": [ [ "openai/gpt-6-luna-decisions-20261006", "OpenAI" ] ] }, "V04": { "scheduled": 10, "completed_first": 10, "correct_first": 3, "accuracy_scheduled": 0.3, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.3, "wilson95": [ 0.10779126740630099, 0.6032218525388546 ], "statuses_first": { "ok": 10 }, "eventual_correct": 3, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.001079, "exact_decimal_usd": "0.0010790", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.001079, "exact_decimal_usd": "0.0010790", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00010790000000000001, "first_cost_per_first_correct_usd": 0.0003596666666666667, "core_cost_per_first_correct_usd": 0.0003596666666666667, "latency_first_all_ms": { "median": 337.86208351375535, "p90": 422.2380834107753, "p95": 458.057333700708 }, "latency_first_usable_ms": { "median": 337.86208351375535, "p90": 422.2380834107753, "p95": 458.057333700708 }, "tokens_first": { "input_tokens": { "total": 10790, "covered_requests": 10 }, "output_tokens": { "total": 0, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.001079, "exact_decimal_usd": "0.0010790", "known_records": 10, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:07:13.379427-04:00", "last": "2026-10-15T10:17:00.315232-04:00" }, "identities": [ [ "openai/gpt-6-luna-decisions-20261006", "OpenAI" ] ] }, "V05": { "scheduled": 10, "completed_first": 10, "correct_first": 3, "accuracy_scheduled": 0.3, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.3, "wilson95": [ 0.10779126740630099, 0.6032218525388546 ], "statuses_first": { "ok": 10 }, "eventual_correct": 3, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.0016484, "exact_decimal_usd": "0.0016484", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0016484, "exact_decimal_usd": "0.0016484", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00016484, "first_cost_per_first_correct_usd": 0.0005494666666666666, "core_cost_per_first_correct_usd": 0.0005494666666666666, "latency_first_all_ms": { "median": 427.14949999935925, "p90": 508.0684333166573, "p95": 535.886570662842 }, "latency_first_usable_ms": { "median": 427.14949999935925, "p90": 508.0684333166573, "p95": 535.886570662842 }, "tokens_first": { "input_tokens": { "total": 16484, "covered_requests": 10 }, "output_tokens": { "total": 0, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.0016484, "exact_decimal_usd": "0.0016484", "known_records": 10, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:07:37.230171-04:00", "last": "2026-10-15T10:17:50.332403-04:00" }, "identities": [ [ "openai/gpt-6-luna-decisions-20261006", "OpenAI" ] ] }, "V06": { "scheduled": 10, "completed_first": 10, "correct_first": 8, "accuracy_scheduled": 0.8, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.8, "wilson95": [ 0.49016247153664183, 0.9433178485456247 ], "statuses_first": { "ok": 10 }, "eventual_correct": 8, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.0008332, "exact_decimal_usd": "0.0008332", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0008332, "exact_decimal_usd": "0.0008332", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 8.332e-05, "first_cost_per_first_correct_usd": 0.00010415, "core_cost_per_first_correct_usd": 0.00010415, "latency_first_all_ms": { "median": 428.78175000078045, "p90": 735.3764420258813, "p95": 782.4684920196886 }, "latency_first_usable_ms": { "median": 428.78175000078045, "p90": 735.3764420258813, "p95": 782.4684920196886 }, "tokens_first": { "input_tokens": { "total": 8332, "covered_requests": 10 }, "output_tokens": { "total": 0, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.0008332, "exact_decimal_usd": "0.0008332", "known_records": 10, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:11:47.555887-04:00", "last": "2026-10-15T10:17:59.854598-04:00" }, "identities": [ [ "openai/gpt-6-luna-decisions-20261006", "OpenAI" ] ] }, "V07": { "scheduled": 10, "completed_first": 10, "correct_first": 4, "accuracy_scheduled": 0.4, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.4, "wilson95": [ 0.16818032970623614, 0.6873262302663417 ], "statuses_first": { "ok": 10 }, "eventual_correct": 4, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.000859, "exact_decimal_usd": "0.0008590", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.000859, "exact_decimal_usd": "0.0008590", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 8.59e-05, "first_cost_per_first_correct_usd": 0.00021475, "core_cost_per_first_correct_usd": 0.00021475, "latency_first_all_ms": { "median": 348.72843750054017, "p90": 390.20364999305457, "p95": 417.48107498278836 }, "latency_first_usable_ms": { "median": 348.72843750054017, "p90": 390.20364999305457, "p95": 417.48107498278836 }, "tokens_first": { "input_tokens": { "total": 8590, "covered_requests": 10 }, "output_tokens": { "total": 0, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.000859, "exact_decimal_usd": "0.0008590", "known_records": 10, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:12:48.228239-04:00", "last": "2026-10-15T10:17:56.514460-04:00" }, "identities": [ [ "openai/gpt-6-luna-decisions-20261006", "OpenAI" ] ] }, "V08": { "scheduled": 10, "completed_first": 10, "correct_first": 4, "accuracy_scheduled": 0.4, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.4, "wilson95": [ 0.16818032970623614, 0.6873262302663417 ], "statuses_first": { "ok": 10 }, "eventual_correct": 4, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.0012433, "exact_decimal_usd": "0.0012433", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0012433, "exact_decimal_usd": "0.0012433", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00012433, "first_cost_per_first_correct_usd": 0.000310825, "core_cost_per_first_correct_usd": 0.000310825, "latency_first_all_ms": { "median": 319.9297085229773, "p90": 482.92779171606526, "p95": 518.6816043569705 }, "latency_first_usable_ms": { "median": 319.9297085229773, "p90": 482.92779171606526, "p95": 518.6816043569705 }, "tokens_first": { "input_tokens": { "total": 12433, "covered_requests": 10 }, "output_tokens": { "total": 0, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.0012433, "exact_decimal_usd": "0.0012433", "known_records": 10, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:07:14.988723-04:00", "last": "2026-10-15T10:18:02.345676-04:00" }, "identities": [ [ "openai/gpt-6-luna-decisions-20261006", "OpenAI" ] ] }, "V09": { "scheduled": 10, "completed_first": 10, "correct_first": 7, "accuracy_scheduled": 0.7, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.7, "wilson95": [ 0.39677814746114537, 0.8922087325936989 ], "statuses_first": { "ok": 10 }, "eventual_correct": 7, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.0008952, "exact_decimal_usd": "0.0008952", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0008952, "exact_decimal_usd": "0.0008952", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 8.952e-05, "first_cost_per_first_correct_usd": 0.0001278857142857143, "core_cost_per_first_correct_usd": 0.0001278857142857143, "latency_first_all_ms": { "median": 420.47631248715334, "p90": 492.41625391296105, "p95": 587.7978979639008 }, "latency_first_usable_ms": { "median": 420.47631248715334, "p90": 492.41625391296105, "p95": 587.7978979639008 }, "tokens_first": { "input_tokens": { "total": 8952, "covered_requests": 10 }, "output_tokens": { "total": 0, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.0008952, "exact_decimal_usd": "0.0008952", "known_records": 10, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:07:11.994059-04:00", "last": "2026-10-15T10:18:01.197828-04:00" }, "identities": [ [ "openai/gpt-6-luna-decisions-20261006", "OpenAI" ] ] }, "V10": { "scheduled": 10, "completed_first": 10, "correct_first": 1, "accuracy_scheduled": 0.1, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.1, "wilson95": [ 0.017876213095072896, 0.4041500267952385 ], "statuses_first": { "ok": 10 }, "eventual_correct": 1, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.0016513, "exact_decimal_usd": "0.0016513", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0016513, "exact_decimal_usd": "0.0016513", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00016513, "first_cost_per_first_correct_usd": 0.0016513, "core_cost_per_first_correct_usd": 0.0016513, "latency_first_all_ms": { "median": 351.22393700294197, "p90": 432.7413547376636, "p95": 508.75432288448775 }, "latency_first_usable_ms": { "median": 351.22393700294197, "p90": 432.7413547376636, "p95": 508.75432288448775 }, "tokens_first": { "input_tokens": { "total": 16513, "covered_requests": 10 }, "output_tokens": { "total": 0, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.0016513, "exact_decimal_usd": "0.0016513", "known_records": 10, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:07:34.965212-04:00", "last": "2026-10-15T10:16:55.859397-04:00" }, "identities": [ [ "openai/gpt-6-luna-decisions-20261006", "OpenAI" ] ] }, "V11": { "scheduled": 10, "completed_first": 10, "correct_first": 3, "accuracy_scheduled": 0.3, "usable_first": 9, "coverage": 0.9, "accuracy_usable": 0.3333333333333333, "wilson95": [ 0.10779126740630099, 0.6032218525388546 ], "statuses_first": { "ok": 9, "http_failure": 1 }, "eventual_correct": 3, "eventual_usable": 9, "retries": 1, "first_cost": { "reported_usd": 0.0012898, "exact_decimal_usd": "0.0012898", "known_records": 9, "missing_records": 1 }, "core_cost": { "reported_usd": 0.0012898, "exact_decimal_usd": "0.0012898", "known_records": 9, "missing_records": 2 }, "core_cost_per_scheduled_usd": 0.00012898, "first_cost_per_first_correct_usd": 0.00042993333333333334, "core_cost_per_first_correct_usd": 0.00042993333333333334, "latency_first_all_ms": { "median": 405.1826665236149, "p90": 838.3968127600375, "p95": 1821.6182188916696 }, "latency_first_usable_ms": { "median": 415.01454199897125, "p90": 1056.8904585670684, "p95": 1930.865041795186 }, "tokens_first": { "input_tokens": { "total": 12898, "covered_requests": 9 }, "output_tokens": { "total": 0, "covered_requests": 9 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.0012898, "exact_decimal_usd": "0.0012898", "known_records": 9, "missing_records": 2 } }, "test_window": { "first": "2026-10-15T10:07:31.153417-04:00", "last": "2026-10-15T10:17:58.598283-04:00" }, "identities": [ [ "openai/gpt-6-luna-decisions-20261006", "OpenAI" ] ] }, "V12": { "scheduled": 10, "completed_first": 10, "correct_first": 6, "accuracy_scheduled": 0.6, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.6, "wilson95": [ 0.31267376973365824, 0.8318196702937638 ], "statuses_first": { "ok": 10 }, "eventual_correct": 6, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.0009626, "exact_decimal_usd": "0.0009626", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0009626, "exact_decimal_usd": "0.0009626", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 9.626e-05, "first_cost_per_first_correct_usd": 0.00016043333333333332, "core_cost_per_first_correct_usd": 0.00016043333333333332, "latency_first_all_ms": { "median": 310.3932914964389, "p90": 362.026374990819, "p95": 374.2202499852283 }, "latency_first_usable_ms": { "median": 310.3932914964389, "p90": 362.026374990819, "p95": 374.2202499852283 }, "tokens_first": { "input_tokens": { "total": 9626, "covered_requests": 10 }, "output_tokens": { "total": 0, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.0009626, "exact_decimal_usd": "0.0009626", "known_records": 10, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:07:12.712202-04:00", "last": "2026-10-15T10:17:52.727374-04:00" }, "identities": [ [ "openai/gpt-6-luna-decisions-20261006", "OpenAI" ] ] }, "V13": { "scheduled": 10, "completed_first": 10, "correct_first": 8, "accuracy_scheduled": 0.8, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.8, "wilson95": [ 0.49016247153664183, 0.9433178485456247 ], "statuses_first": { "ok": 10 }, "eventual_correct": 8, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.0008153, "exact_decimal_usd": "0.0008153", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0008153, "exact_decimal_usd": "0.0008153", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 8.153e-05, "first_cost_per_first_correct_usd": 0.0001019125, "core_cost_per_first_correct_usd": 0.0001019125, "latency_first_all_ms": { "median": 341.15008349181153, "p90": 545.0714084145142, "p95": 1275.8451211993797 }, "latency_first_usable_ms": { "median": 341.15008349181153, "p90": 545.0714084145142, "p95": 1275.8451211993797 }, "tokens_first": { "input_tokens": { "total": 8153, "covered_requests": 10 }, "output_tokens": { "total": 0, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.0008153, "exact_decimal_usd": "0.0008153", "known_records": 10, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:12:47.007646-04:00", "last": "2026-10-15T10:17:49.252501-04:00" }, "identities": [ [ "openai/gpt-6-luna-decisions-20261006", "OpenAI" ] ] }, "V14": { "scheduled": 10, "completed_first": 10, "correct_first": 6, "accuracy_scheduled": 0.6, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.6, "wilson95": [ 0.31267376973365824, 0.8318196702937638 ], "statuses_first": { "ok": 10 }, "eventual_correct": 6, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.0013049, "exact_decimal_usd": "0.0013049", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0013049, "exact_decimal_usd": "0.0013049", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00013048999999999998, "first_cost_per_first_correct_usd": 0.00021748333333333333, "core_cost_per_first_correct_usd": 0.00021748333333333333, "latency_first_all_ms": { "median": 583.3093539986294, "p90": 865.9100749995558, "p95": 947.315975022502 }, "latency_first_usable_ms": { "median": 583.3093539986294, "p90": 865.9100749995558, "p95": 947.315975022502 }, "tokens_first": { "input_tokens": { "total": 13049, "covered_requests": 10 }, "output_tokens": { "total": 0, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.0013049, "exact_decimal_usd": "0.0013049", "known_records": 10, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:07:16.726783-04:00", "last": "2026-10-15T10:17:55.024715-04:00" }, "identities": [ [ "openai/gpt-6-luna-decisions-20261006", "OpenAI" ] ] }, "V15": { "scheduled": 10, "completed_first": 10, "correct_first": 9, "accuracy_scheduled": 0.9, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.9, "wilson95": [ 0.5958499732047615, 0.9821237869049271 ], "statuses_first": { "ok": 10 }, "eventual_correct": 9, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.0007308, "exact_decimal_usd": "0.0007308", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0007308, "exact_decimal_usd": "0.0007308", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 7.308e-05, "first_cost_per_first_correct_usd": 8.12e-05, "core_cost_per_first_correct_usd": 8.12e-05, "latency_first_all_ms": { "median": 303.7292920053005, "p90": 531.8107169819991, "p95": 1087.1587544796043 }, "latency_first_usable_ms": { "median": 303.7292920053005, "p90": 531.8107169819991, "p95": 1087.1587544796043 }, "tokens_first": { "input_tokens": { "total": 7308, "covered_requests": 10 }, "output_tokens": { "total": 0, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.0007308, "exact_decimal_usd": "0.0007308", "known_records": 10, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:07:18.060685-04:00", "last": "2026-10-15T10:17:56.223779-04:00" }, "identities": [ [ "openai/gpt-6-luna-decisions-20261006", "OpenAI" ] ] }, "V16": { "scheduled": 10, "completed_first": 10, "correct_first": 9, "accuracy_scheduled": 0.9, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.9, "wilson95": [ 0.5958499732047615, 0.9821237869049271 ], "statuses_first": { "ok": 10 }, "eventual_correct": 9, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.0017647, "exact_decimal_usd": "0.0017647", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0017647, "exact_decimal_usd": "0.0017647", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00017647, "first_cost_per_first_correct_usd": 0.00019607777777777777, "core_cost_per_first_correct_usd": 0.00019607777777777777, "latency_first_all_ms": { "median": 776.9975414848886, "p90": 942.9941416077778, "p95": 1351.352341301389 }, "latency_first_usable_ms": { "median": 776.9975414848886, "p90": 942.9941416077778, "p95": 1351.352341301389 }, "tokens_first": { "input_tokens": { "total": 17647, "covered_requests": 10 }, "output_tokens": { "total": 0, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.0017647, "exact_decimal_usd": "0.0017647", "known_records": 10, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:07:19.659081-04:00", "last": "2026-10-15T10:16:13.832218-04:00" }, "identities": [ [ "openai/gpt-6-luna-decisions-20261006", "OpenAI" ] ] }, "V17": { "scheduled": 10, "completed_first": 10, "correct_first": 6, "accuracy_scheduled": 0.6, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.6, "wilson95": [ 0.31267376973365824, 0.8318196702937638 ], "statuses_first": { "ok": 10 }, "eventual_correct": 6, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.00158, "exact_decimal_usd": "0.0015800", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.00158, "exact_decimal_usd": "0.0015800", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.000158, "first_cost_per_first_correct_usd": 0.00026333333333333336, "core_cost_per_first_correct_usd": 0.00026333333333333336, "latency_first_all_ms": { "median": 487.15229146182537, "p90": 579.959787480766, "p95": 651.4222062512997 }, "latency_first_usable_ms": { "median": 487.15229146182537, "p90": 579.959787480766, "p95": 651.4222062512997 }, "tokens_first": { "input_tokens": { "total": 15800, "covered_requests": 10 }, "output_tokens": { "total": 0, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.00158, "exact_decimal_usd": "0.0015800", "known_records": 10, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:13:44.274847-04:00", "last": "2026-10-15T10:17:47.356589-04:00" }, "identities": [ [ "openai/gpt-6-luna-decisions-20261006", "OpenAI" ] ] }, "V18": { "scheduled": 10, "completed_first": 10, "correct_first": 8, "accuracy_scheduled": 0.8, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.8, "wilson95": [ 0.49016247153664183, 0.9433178485456247 ], "statuses_first": { "ok": 10 }, "eventual_correct": 8, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.0010653, "exact_decimal_usd": "0.0010653", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0010653, "exact_decimal_usd": "0.0010653", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00010653, "first_cost_per_first_correct_usd": 0.0001331625, "core_cost_per_first_correct_usd": 0.0001331625, "latency_first_all_ms": { "median": 388.1018545071129, "p90": 502.17169101815665, "p95": 578.9474285149479 }, "latency_first_usable_ms": { "median": 388.1018545071129, "p90": 502.17169101815665, "p95": 578.9474285149479 }, "tokens_first": { "input_tokens": { "total": 10653, "covered_requests": 10 }, "output_tokens": { "total": 0, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.0010653, "exact_decimal_usd": "0.0010653", "known_records": 10, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:07:31.927755-04:00", "last": "2026-10-15T10:16:50.070730-04:00" }, "identities": [ [ "openai/gpt-6-luna-decisions-20261006", "OpenAI" ] ] }, "V19": { "scheduled": 10, "completed_first": 10, "correct_first": 6, "accuracy_scheduled": 0.6, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.6, "wilson95": [ 0.31267376973365824, 0.8318196702937638 ], "statuses_first": { "ok": 10 }, "eventual_correct": 6, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.0010483, "exact_decimal_usd": "0.0010483", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0010483, "exact_decimal_usd": "0.0010483", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00010483000000000001, "first_cost_per_first_correct_usd": 0.00017471666666666667, "core_cost_per_first_correct_usd": 0.00017471666666666667, "latency_first_all_ms": { "median": 394.88264598185197, "p90": 515.0464500009548, "p95": 540.0290999881689 }, "latency_first_usable_ms": { "median": 394.88264598185197, "p90": 515.0464500009548, "p95": 540.0290999881689 }, "tokens_first": { "input_tokens": { "total": 10483, "covered_requests": 10 }, "output_tokens": { "total": 0, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.0010483, "exact_decimal_usd": "0.0010483", "known_records": 10, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:07:34.170082-04:00", "last": "2026-10-15T10:17:01.049430-04:00" }, "identities": [ [ "openai/gpt-6-luna-decisions-20261006", "OpenAI" ] ] }, "V20": { "scheduled": 10, "completed_first": 10, "correct_first": 10, "accuracy_scheduled": 1.0, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 1.0, "wilson95": [ 0.7224672001371107, 0.9999999999999999 ], "statuses_first": { "ok": 10 }, "eventual_correct": 10, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.0010121, "exact_decimal_usd": "0.0010121", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0010121, "exact_decimal_usd": "0.0010121", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00010120999999999999, "first_cost_per_first_correct_usd": 0.00010120999999999999, "core_cost_per_first_correct_usd": 0.00010120999999999999, "latency_first_all_ms": { "median": 344.08339552464895, "p90": 458.1350294814911, "p95": 489.4994107628008 }, "latency_first_usable_ms": { "median": 344.08339552464895, "p90": 458.1350294814911, "p95": 489.4994107628008 }, "tokens_first": { "input_tokens": { "total": 10121, "covered_requests": 10 }, "output_tokens": { "total": 0, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.0010121, "exact_decimal_usd": "0.0010121", "known_records": 10, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:07:37.669644-04:00", "last": "2026-10-15T10:17:57.695063-04:00" }, "identities": [ [ "openai/gpt-6-luna-decisions-20261006", "OpenAI" ] ] } }, "image_count": { "1": { "scheduled": 177, "completed_first": 177, "correct_first": 105, "accuracy_scheduled": 0.5932203389830508, "usable_first": 175, "coverage": 0.9887005649717514, "accuracy_usable": 0.6, "wilson95": [ 0.5196170968111699, 0.662863182960958 ], "statuses_first": { "ok": 175, "http_failure": 2 }, "eventual_correct": 105, "eventual_usable": 176, "retries": 2, "first_cost": { "reported_usd": 0.0187689, "exact_decimal_usd": "0.0187689", "known_records": 175, "missing_records": 2 }, "core_cost": { "reported_usd": 0.0188653, "exact_decimal_usd": "0.0188653", "known_records": 176, "missing_records": 3 }, "core_cost_per_scheduled_usd": 0.00010658361581920904, "first_cost_per_first_correct_usd": 0.0001787514285714286, "core_cost_per_first_correct_usd": 0.00017966952380952384, "latency_first_all_ms": { "median": 369.9526669806801, "p90": 600.7380335824564, "p95": 699.4876084150743 }, "latency_first_usable_ms": { "median": 369.9526669806801, "p90": 592.8753664018585, "p95": 689.9063750228378 }, "tokens_first": { "input_tokens": { "total": 187689, "covered_requests": 175 }, "output_tokens": { "total": 0, "covered_requests": 175 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.0188653, "exact_decimal_usd": "0.0188653", "known_records": 176, "missing_records": 3 } }, "test_window": { "first": "2026-10-15T10:07:12.712202-04:00", "last": "2026-10-15T10:18:02.345676-04:00" }, "identities": [ [ "openai/gpt-6-luna-decisions-20261006", "OpenAI" ] ] }, "2": { "scheduled": 19, "completed_first": 19, "correct_first": 14, "accuracy_scheduled": 0.7368421052631579, "usable_first": 19, "coverage": 1.0, "accuracy_usable": 0.7368421052631579, "wilson95": [ 0.5120844910218489, 0.8819358765321748 ], "statuses_first": { "ok": 19 }, "eventual_correct": 14, "eventual_usable": 19, "retries": 0, "first_cost": { "reported_usd": 0.0032449, "exact_decimal_usd": "0.0032449", "known_records": 19, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0032449, "exact_decimal_usd": "0.0032449", "known_records": 19, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.0001707842105263158, "first_cost_per_first_correct_usd": 0.00023177857142857146, "core_cost_per_first_correct_usd": 0.00023177857142857146, "latency_first_all_ms": { "median": 594.2235420225188, "p90": 848.1419750140049, "p95": 942.9941416077762 }, "latency_first_usable_ms": { "median": 594.2235420225188, "p90": 848.1419750140049, "p95": 942.9941416077762 }, "tokens_first": { "input_tokens": { "total": 32449, "covered_requests": 19 }, "output_tokens": { "total": 0, "covered_requests": 19 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.0032449, "exact_decimal_usd": "0.0032449", "known_records": 19, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:07:19.659081-04:00", "last": "2026-10-15T10:17:47.356589-04:00" }, "identities": [ [ "openai/gpt-6-luna-decisions-20261006", "OpenAI" ] ] }, "3": { "scheduled": 4, "completed_first": 4, "correct_first": 2, "accuracy_scheduled": 0.5, "usable_first": 4, "coverage": 1.0, "accuracy_usable": 0.5, "wilson95": [ 0.15003898915214947, 0.8499610108478506 ], "statuses_first": { "ok": 4 }, "eventual_correct": 2, "eventual_usable": 4, "retries": 0, "first_cost": { "reported_usd": 0.000436, "exact_decimal_usd": "0.0004360", "known_records": 4, "missing_records": 0 }, "core_cost": { "reported_usd": 0.000436, "exact_decimal_usd": "0.0004360", "known_records": 4, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.000109, "first_cost_per_first_correct_usd": 0.000218, "core_cost_per_first_correct_usd": 0.000218, "latency_first_all_ms": { "median": 421.27493748557754, "p90": 611.1701918998733, "p95": 647.174866957357 }, "latency_first_usable_ms": { "median": 421.27493748557754, "p90": 611.1701918998733, "p95": 647.174866957357 }, "tokens_first": { "input_tokens": { "total": 4360, "covered_requests": 4 }, "output_tokens": { "total": 0, "covered_requests": 4 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.000436, "exact_decimal_usd": "0.0004360", "known_records": 4, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:07:11.994059-04:00", "last": "2026-10-15T10:17:02.170923-04:00" }, "identities": [ [ "openai/gpt-6-luna-decisions-20261006", "OpenAI" ] ] } }, "analysis_subset": { "perception_control": { "scheduled": 13, "completed_first": 13, "correct_first": 10, "accuracy_scheduled": 0.7692307692307693, "usable_first": 13, "coverage": 1.0, "accuracy_usable": 0.7692307692307693, "wilson95": [ 0.49743624053272495, 0.9182047128150143 ], "statuses_first": { "ok": 13 }, "eventual_correct": 10, "eventual_usable": 13, "retries": 0, "first_cost": { "reported_usd": 0.0012355, "exact_decimal_usd": "0.0012355", "known_records": 13, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0012355, "exact_decimal_usd": "0.0012355", "known_records": 13, "missing_records": 0 }, "core_cost_per_scheduled_usd": 9.503846153846154e-05, "first_cost_per_first_correct_usd": 0.00012355, "core_cost_per_first_correct_usd": 0.00012355, "latency_first_all_ms": { "median": 382.53795803757384, "p90": 818.4342084103265, "p95": 1215.232941403518 }, "latency_first_usable_ms": { "median": 382.53795803757384, "p90": 818.4342084103265, "p95": 1215.232941403518 }, "tokens_first": { "input_tokens": { "total": 12355, "covered_requests": 13 }, "output_tokens": { "total": 0, "covered_requests": 13 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.0012355, "exact_decimal_usd": "0.0012355", "known_records": 13, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:07:13.379427-04:00", "last": "2026-10-15T10:17:02.170923-04:00" }, "identities": [ [ "openai/gpt-6-luna-decisions-20261006", "OpenAI" ] ] }, "reasoning_and_grounded_decision": { "scheduled": 187, "completed_first": 187, "correct_first": 111, "accuracy_scheduled": 0.5935828877005348, "usable_first": 185, "coverage": 0.9893048128342246, "accuracy_usable": 0.6, "wilson95": [ 0.5219888403244819, 0.6614094643011623 ], "statuses_first": { "ok": 185, "http_failure": 2 }, "eventual_correct": 111, "eventual_usable": 186, "retries": 2, "first_cost": { "reported_usd": 0.0212143, "exact_decimal_usd": "0.0212143", "known_records": 185, "missing_records": 2 }, "core_cost": { "reported_usd": 0.0213107, "exact_decimal_usd": "0.0213107", "known_records": 186, "missing_records": 3 }, "core_cost_per_scheduled_usd": 0.00011396096256684492, "first_cost_per_first_correct_usd": 0.00019111981981981982, "core_cost_per_first_correct_usd": 0.00019198828828828827, "latency_first_all_ms": { "median": 383.3428329671733, "p90": 646.4951916248539, "p95": 787.2464747924822 }, "latency_first_usable_ms": { "median": 383.3428329671733, "p90": 635.4973584297112, "p95": 776.1888993903992 }, "tokens_first": { "input_tokens": { "total": 212143, "covered_requests": 185 }, "output_tokens": { "total": 0, "covered_requests": 185 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.0213107, "exact_decimal_usd": "0.0213107", "known_records": 186, "missing_records": 3 } }, "test_window": { "first": "2026-10-15T10:07:11.994059-04:00", "last": "2026-10-15T10:18:02.345676-04:00" }, "identities": [ [ "openai/gpt-6-luna-decisions-20261006", "OpenAI" ] ] } }, "difficulty_proposed": { "demanding": { "scheduled": 55, "completed_first": 55, "correct_first": 33, "accuracy_scheduled": 0.6, "usable_first": 55, "coverage": 1.0, "accuracy_usable": 0.6, "wilson95": [ 0.46812799210121386, 0.7188150276753689 ], "statuses_first": { "ok": 55 }, "eventual_correct": 33, "eventual_usable": 55, "retries": 0, "first_cost": { "reported_usd": 0.0065344, "exact_decimal_usd": "0.0065344", "known_records": 55, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0065344, "exact_decimal_usd": "0.0065344", "known_records": 55, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00011880727272727273, "first_cost_per_first_correct_usd": 0.00019801212121212122, "core_cost_per_first_correct_usd": 0.00019801212121212122, "latency_first_all_ms": { "median": 360.371041984763, "p90": 591.211658983957, "p95": 714.2066874017471 }, "latency_first_usable_ms": { "median": 360.371041984763, "p90": 591.211658983957, "p95": 714.2066874017471 }, "tokens_first": { "input_tokens": { "total": 65344, "covered_requests": 55 }, "output_tokens": { "total": 0, "covered_requests": 55 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.0065344, "exact_decimal_usd": "0.0065344", "known_records": 55, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:07:14.629770-04:00", "last": "2026-10-15T10:17:58.598283-04:00" }, "identities": [ [ "openai/gpt-6-luna-decisions-20261006", "OpenAI" ] ] }, "moderate": { "scheduled": 107, "completed_first": 107, "correct_first": 61, "accuracy_scheduled": 0.5700934579439252, "usable_first": 105, "coverage": 0.9813084112149533, "accuracy_usable": 0.580952380952381, "wilson95": [ 0.4754691120338291, 0.6598593125429819 ], "statuses_first": { "ok": 105, "http_failure": 2 }, "eventual_correct": 61, "eventual_usable": 106, "retries": 2, "first_cost": { "reported_usd": 0.0119582, "exact_decimal_usd": "0.0119582", "known_records": 105, "missing_records": 2 }, "core_cost": { "reported_usd": 0.0120546, "exact_decimal_usd": "0.0120546", "known_records": 106, "missing_records": 3 }, "core_cost_per_scheduled_usd": 0.00011265981308411216, "first_cost_per_first_correct_usd": 0.0001960360655737705, "core_cost_per_first_correct_usd": 0.00019761639344262295, "latency_first_all_ms": { "median": 398.48841703496873, "p90": 702.268325025216, "p95": 841.8490126146933 }, "latency_first_usable_ms": { "median": 398.48841703496873, "p90": 675.4037414211782, "p95": 821.7710836092009 }, "tokens_first": { "input_tokens": { "total": 119582, "covered_requests": 105 }, "output_tokens": { "total": 0, "covered_requests": 105 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.0120546, "exact_decimal_usd": "0.0120546", "known_records": 106, "missing_records": 3 } }, "test_window": { "first": "2026-10-15T10:07:11.994059-04:00", "last": "2026-10-15T10:18:02.345676-04:00" }, "identities": [ [ "openai/gpt-6-luna-decisions-20261006", "OpenAI" ] ] }, "straightforward": { "scheduled": 38, "completed_first": 38, "correct_first": 27, "accuracy_scheduled": 0.7105263157894737, "usable_first": 38, "coverage": 1.0, "accuracy_usable": 0.7105263157894737, "wilson95": [ 0.5524286232424794, 0.8299672217151693 ], "statuses_first": { "ok": 38 }, "eventual_correct": 27, "eventual_usable": 38, "retries": 0, "first_cost": { "reported_usd": 0.0039572, "exact_decimal_usd": "0.0039572", "known_records": 38, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0039572, "exact_decimal_usd": "0.0039572", "known_records": 38, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00010413684210526316, "first_cost_per_first_correct_usd": 0.00014656296296296296, "core_cost_per_first_correct_usd": 0.00014656296296296296, "latency_first_all_ms": { "median": 391.2228334811516, "p90": 638.8860794715586, "p95": 862.5508295197492 }, "latency_first_usable_ms": { "median": 391.2228334811516, "p90": 638.8860794715586, "p95": 862.5508295197492 }, "tokens_first": { "input_tokens": { "total": 39572, "covered_requests": 38 }, "output_tokens": { "total": 0, "covered_requests": 38 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.0039572, "exact_decimal_usd": "0.0039572", "known_records": 38, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:07:13.379427-04:00", "last": "2026-10-15T10:18:02.003947-04:00" }, "identities": [ [ "openai/gpt-6-luna-decisions-20261006", "OpenAI" ] ] } }, "visual_medium": { "chart": { "scheduled": 13, "completed_first": 13, "correct_first": 11, "accuracy_scheduled": 0.8461538461538461, "usable_first": 13, "coverage": 1.0, "accuracy_usable": 0.8461538461538461, "wilson95": [ 0.5776536898051745, 0.9567418216419186 ], "statuses_first": { "ok": 13 }, "eventual_correct": 11, "eventual_usable": 13, "retries": 0, "first_cost": { "reported_usd": 0.0010936, "exact_decimal_usd": "0.0010936", "known_records": 13, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0010936, "exact_decimal_usd": "0.0010936", "known_records": 13, "missing_records": 0 }, "core_cost_per_scheduled_usd": 8.412307692307691e-05, "first_cost_per_first_correct_usd": 9.941818181818181e-05, "core_cost_per_first_correct_usd": 9.941818181818181e-05, "latency_first_all_ms": { "median": 342.36524999141693, "p90": 430.11041720164945, "p95": 1067.8287589922525 }, "latency_first_usable_ms": { "median": 342.36524999141693, "p90": 430.11041720164945, "p95": 1067.8287589922525 }, "tokens_first": { "input_tokens": { "total": 10936, "covered_requests": 13 }, "output_tokens": { "total": 0, "covered_requests": 13 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.0010936, "exact_decimal_usd": "0.0010936", "known_records": 13, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:07:31.927755-04:00", "last": "2026-10-15T10:17:49.252501-04:00" }, "identities": [ [ "openai/gpt-6-luna-decisions-20261006", "OpenAI" ] ] }, "diagram": { "scheduled": 92, "completed_first": 92, "correct_first": 41, "accuracy_scheduled": 0.44565217391304346, "usable_first": 91, "coverage": 0.9891304347826086, "accuracy_usable": 0.45054945054945056, "wilson95": [ 0.34829808027402587, 0.5473629403004981 ], "statuses_first": { "ok": 91, "http_failure": 1 }, "eventual_correct": 41, "eventual_usable": 91, "retries": 1, "first_cost": { "reported_usd": 0.0109563, "exact_decimal_usd": "0.0109563", "known_records": 91, "missing_records": 1 }, "core_cost": { "reported_usd": 0.0109563, "exact_decimal_usd": "0.0109563", "known_records": 91, "missing_records": 2 }, "core_cost_per_scheduled_usd": 0.00011909021739130435, "first_cost_per_first_correct_usd": 0.00026722682926829267, "core_cost_per_first_correct_usd": 0.00026722682926829267, "latency_first_all_ms": { "median": 352.6751249737572, "p90": 491.9871755933856, "p95": 528.5162313899492 }, "latency_first_usable_ms": { "median": 350.1283749938011, "p90": 493.87658399064094, "p95": 530.8725209906697 }, "tokens_first": { "input_tokens": { "total": 109563, "covered_requests": 91 }, "output_tokens": { "total": 0, "covered_requests": 91 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.0109563, "exact_decimal_usd": "0.0109563", "known_records": 91, "missing_records": 2 } }, "test_window": { "first": "2026-10-15T10:07:12.712202-04:00", "last": "2026-10-15T10:18:02.345676-04:00" }, "identities": [ [ "openai/gpt-6-luna-decisions-20261006", "OpenAI" ] ] }, "diagram_or_published_illustration": { "scheduled": 8, "completed_first": 8, "correct_first": 7, "accuracy_scheduled": 0.875, "usable_first": 8, "coverage": 1.0, "accuracy_usable": 0.875, "wilson95": [ 0.5291118177871464, 0.9775825085499433 ], "statuses_first": { "ok": 8 }, "eventual_correct": 7, "eventual_usable": 8, "retries": 0, "first_cost": { "reported_usd": 0.0006443, "exact_decimal_usd": "0.0006443", "known_records": 8, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0006443, "exact_decimal_usd": "0.0006443", "known_records": 8, "missing_records": 0 }, "core_cost_per_scheduled_usd": 8.05375e-05, "first_cost_per_first_correct_usd": 9.204285714285714e-05, "core_cost_per_first_correct_usd": 9.204285714285714e-05, "latency_first_all_ms": { "median": 450.57345801615156, "p90": 588.0678086890839, "p95": 607.5215083430521 }, "latency_first_usable_ms": { "median": 450.57345801615156, "p90": 588.0678086890839, "p95": 607.5215083430521 }, "tokens_first": { "input_tokens": { "total": 6443, "covered_requests": 8 }, "output_tokens": { "total": 0, "covered_requests": 8 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.0006443, "exact_decimal_usd": "0.0006443", "known_records": 8, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:07:13.982947-04:00", "last": "2026-10-15T10:16:02.083271-04:00" }, "identities": [ [ "openai/gpt-6-luna-decisions-20261006", "OpenAI" ] ] }, "document": { "scheduled": 12, "completed_first": 12, "correct_first": 6, "accuracy_scheduled": 0.5, "usable_first": 12, "coverage": 1.0, "accuracy_usable": 0.5, "wilson95": [ 0.2537815976337061, 0.7462184023662939 ], "statuses_first": { "ok": 12 }, "eventual_correct": 6, "eventual_usable": 12, "retries": 0, "first_cost": { "reported_usd": 0.0015278, "exact_decimal_usd": "0.0015278", "known_records": 12, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0015278, "exact_decimal_usd": "0.0015278", "known_records": 12, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00012731666666666665, "first_cost_per_first_correct_usd": 0.0002546333333333333, "core_cost_per_first_correct_usd": 0.0002546333333333333, "latency_first_all_ms": { "median": 576.552624523174, "p90": 840.9769540943671, "p95": 929.225775017403 }, "latency_first_usable_ms": { "median": 576.552624523174, "p90": 840.9769540943671, "p95": 929.225775017403 }, "tokens_first": { "input_tokens": { "total": 15278, "covered_requests": 12 }, "output_tokens": { "total": 0, "covered_requests": 12 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.0015278, "exact_decimal_usd": "0.0015278", "known_records": 12, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:07:16.726783-04:00", "last": "2026-10-15T10:17:55.024715-04:00" }, "identities": [ [ "openai/gpt-6-luna-decisions-20261006", "OpenAI" ] ] }, "photograph": { "scheduled": 64, "completed_first": 64, "correct_first": 46, "accuracy_scheduled": 0.71875, "usable_first": 63, "coverage": 0.984375, "accuracy_usable": 0.7301587301587301, "wilson95": [ 0.5986606971779259, 0.8140662766627217 ], "statuses_first": { "ok": 63, "http_failure": 1 }, "eventual_correct": 46, "eventual_usable": 64, "retries": 1, "first_cost": { "reported_usd": 0.0073861, "exact_decimal_usd": "0.0073861", "known_records": 63, "missing_records": 1 }, "core_cost": { "reported_usd": 0.0074825, "exact_decimal_usd": "0.0074825", "known_records": 64, "missing_records": 1 }, "core_cost_per_scheduled_usd": 0.0001169140625, "first_cost_per_first_correct_usd": 0.0001605673913043478, "core_cost_per_first_correct_usd": 0.00016266304347826085, "latency_first_all_ms": { "median": 454.07843799330294, "p90": 782.4438248877415, "p95": 844.48225631495 }, "latency_first_usable_ms": { "median": 446.0422089905478, "p90": 761.1281829886138, "p95": 826.3790253084153 }, "tokens_first": { "input_tokens": { "total": 73861, "covered_requests": 63 }, "output_tokens": { "total": 0, "covered_requests": 63 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.0074825, "exact_decimal_usd": "0.0074825", "known_records": 64, "missing_records": 1 } }, "test_window": { "first": "2026-10-15T10:07:11.994059-04:00", "last": "2026-10-15T10:18:01.197828-04:00" }, "identities": [ [ "openai/gpt-6-luna-decisions-20261006", "OpenAI" ] ] }, "screenshot": { "scheduled": 11, "completed_first": 11, "correct_first": 10, "accuracy_scheduled": 0.9090909090909091, "usable_first": 11, "coverage": 1.0, "accuracy_usable": 0.9090909090909091, "wilson95": [ 0.6226415635484043, 0.9837678271141151 ], "statuses_first": { "ok": 11 }, "eventual_correct": 10, "eventual_usable": 11, "retries": 0, "first_cost": { "reported_usd": 0.0008417, "exact_decimal_usd": "0.0008417", "known_records": 11, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0008417, "exact_decimal_usd": "0.0008417", "known_records": 11, "missing_records": 0 }, "core_cost_per_scheduled_usd": 7.651818181818182e-05, "first_cost_per_first_correct_usd": 8.417e-05, "core_cost_per_first_correct_usd": 8.417e-05, "latency_first_all_ms": { "median": 318.34133400116116, "p90": 451.16516697453335, "p95": 1046.8359794758726 }, "latency_first_usable_ms": { "median": 318.34133400116116, "p90": 451.16516697453335, "p95": 1046.8359794758726 }, "tokens_first": { "input_tokens": { "total": 8417, "covered_requests": 11 }, "output_tokens": { "total": 0, "covered_requests": 11 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.0008417, "exact_decimal_usd": "0.0008417", "known_records": 11, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:07:18.060685-04:00", "last": "2026-10-15T10:17:57.695063-04:00" }, "identities": [ [ "openai/gpt-6-luna-decisions-20261006", "OpenAI" ] ] } }, "stage": { "1": { "scheduled": 100, "completed_first": 100, "correct_first": 59, "accuracy_scheduled": 0.59, "usable_first": 98, "coverage": 0.98, "accuracy_usable": 0.6020408163265306, "wilson95": [ 0.4920143005341078, 0.6813268697886348 ], "statuses_first": { "ok": 98, "http_failure": 2 }, "eventual_correct": 59, "eventual_usable": 99, "retries": 2, "first_cost": { "reported_usd": 0.0109175, "exact_decimal_usd": "0.0109175", "known_records": 98, "missing_records": 2 }, "core_cost": { "reported_usd": 0.0110139, "exact_decimal_usd": "0.0110139", "known_records": 99, "missing_records": 3 }, "core_cost_per_scheduled_usd": 0.000110139, "first_cost_per_first_correct_usd": 0.00018504237288135593, "core_cost_per_first_correct_usd": 0.00018667627118644067, "latency_first_all_ms": { "median": 399.1647709917743, "p90": 688.9848750259262, "p95": 781.8991597945568 }, "latency_first_usable_ms": { "median": 399.1647709917743, "p90": 684.7829169186298, "p95": 765.7831579301272 }, "tokens_first": { "input_tokens": { "total": 109175, "covered_requests": 98 }, "output_tokens": { "total": 0, "covered_requests": 98 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.0110139, "exact_decimal_usd": "0.0110139", "known_records": 99, "missing_records": 3 } }, "test_window": { "first": "2026-10-15T10:07:11.994059-04:00", "last": "2026-10-15T10:14:29.568423-04:00" }, "identities": [ [ "openai/gpt-6-luna-decisions-20261006", "OpenAI" ] ] }, "2": { "scheduled": 100, "completed_first": 100, "correct_first": 62, "accuracy_scheduled": 0.62, "usable_first": 100, "coverage": 1.0, "accuracy_usable": 0.62, "wilson95": [ 0.5220975529551107, 0.7090240074752127 ], "statuses_first": { "ok": 100 }, "eventual_correct": 62, "eventual_usable": 100, "retries": 0, "first_cost": { "reported_usd": 0.0115323, "exact_decimal_usd": "0.0115323", "known_records": 100, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0115323, "exact_decimal_usd": "0.0115323", "known_records": 100, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00011532300000000001, "first_cost_per_first_correct_usd": 0.00018600483870967742, "core_cost_per_first_correct_usd": 0.00018600483870967742, "latency_first_all_ms": { "median": 379.4493129826151, "p90": 605.971879779827, "p95": 800.2138812153133 }, "latency_first_usable_ms": { "median": 379.4493129826151, "p90": 605.971879779827, "p95": 800.2138812153133 }, "tokens_first": { "input_tokens": { "total": 115323, "covered_requests": 100 }, "output_tokens": { "total": 0, "covered_requests": 100 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 0, "reasoning_text_present_attempts": 0, "phase_cost": { "decisions": { "reported_usd": 0.0115323, "exact_decimal_usd": "0.0115323", "known_records": 100, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:15:27.898899-04:00", "last": "2026-10-15T10:18:02.345676-04:00" }, "identities": [ [ "openai/gpt-6-luna-decisions-20261006", "OpenAI" ] ] } } }, "families_at_least_8_of_10": [ "V01", "V06", "V13", "V15", "V16", "V18", "V20" ], "families_at_most_4_of_10": [ "V04", "V05", "V07", "V08", "V10", "V11" ] }, "openai/gpt-6-luna": { "scheduled": 200, "completed_first": 200, "correct_first": 117, "accuracy_scheduled": 0.585, "usable_first": 200, "coverage": 1.0, "accuracy_usable": 0.585, "wilson95": [ 0.5157391490134403, 0.6510571455024242 ], "statuses_first": { "ok": 200 }, "eventual_correct": 117, "eventual_usable": 200, "retries": 0, "first_cost": { "reported_usd": 0.0295817, "exact_decimal_usd": "0.029581700", "known_records": 200, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0295817, "exact_decimal_usd": "0.029581700", "known_records": 200, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.0001479085, "first_cost_per_first_correct_usd": 0.00025283504273504275, "core_cost_per_first_correct_usd": 0.00025283504273504275, "latency_first_all_ms": { "median": 2354.4554370164406, "p90": 4474.265654064947, "p95": 5057.388358071326 }, "latency_first_usable_ms": { "median": 2354.4554370164406, "p90": 4474.265654064947, "p95": 5057.388358071326 }, "tokens_first": { "prompt_tokens": { "total": 246335, "covered_requests": 200 }, "completion_tokens": { "total": 1000, "covered_requests": 200 } }, "cached_tokens": 0, "cache_write_tokens": 177928, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 200, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.026887025, "exact_decimal_usd": "0.026887025", "known_records": 180, "missing_records": 0 }, "pilot": { "reported_usd": 0.002694675, "exact_decimal_usd": "0.002694675", "known_records": 20, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:34:21.852048-04:00", "last": "2026-10-15T11:03:31.672835-04:00" }, "identities": [ [ "openai/gpt-6-luna", "OpenAI" ] ], "groups": { "family_id": { "V01": { "scheduled": 10, "completed_first": 10, "correct_first": 8, "accuracy_scheduled": 0.8, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.8, "wilson95": [ 0.49016247153664183, 0.9433178485456247 ], "statuses_first": { "ok": 10 }, "eventual_correct": 8, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.00109725, "exact_decimal_usd": "0.001097250", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.00109725, "exact_decimal_usd": "0.001097250", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.000109725, "first_cost_per_first_correct_usd": 0.00013715625, "core_cost_per_first_correct_usd": 0.00013715625, "latency_first_all_ms": { "median": 1925.0850210082717, "p90": 2978.497470822185, "p95": 3406.9136518985024 }, "latency_first_usable_ms": { "median": 1925.0850210082717, "p90": 2978.497470822185, "p95": 3406.9136518985024 }, "tokens_first": { "prompt_tokens": { "total": 9423, "covered_requests": 10 }, "completion_tokens": { "total": 50, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 5198, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.00100135, "exact_decimal_usd": "0.001001350", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 9.59e-05, "exact_decimal_usd": "0.0000959", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:09.974574-04:00", "last": "2026-10-15T11:03:11.716725-04:00" }, "identities": [ [ "openai/gpt-6-luna", "OpenAI" ] ] }, "V02": { "scheduled": 10, "completed_first": 10, "correct_first": 5, "accuracy_scheduled": 0.5, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.5, "wilson95": [ 0.236593090512564, 0.7634069094874361 ], "statuses_first": { "ok": 10 }, "eventual_correct": 5, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.000943675, "exact_decimal_usd": "0.000943675", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.000943675, "exact_decimal_usd": "0.000943675", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 9.436750000000001e-05, "first_cost_per_first_correct_usd": 0.00018873500000000001, "core_cost_per_first_correct_usd": 0.00018873500000000001, "latency_first_all_ms": { "median": 2766.3301464926917, "p90": 3779.1607494233176, "p95": 3814.292624691734 }, "latency_first_usable_ms": { "median": 2766.3301464926917, "p90": 3779.1607494233176, "p95": 3814.292624691734 }, "tokens_first": { "prompt_tokens": { "total": 8367, "covered_requests": 10 }, "completion_tokens": { "total": 50, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 3279, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.000892275, "exact_decimal_usd": "0.000892275", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 5.14e-05, "exact_decimal_usd": "0.0000514", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:22.442592-04:00", "last": "2026-10-15T11:02:00.097117-04:00" }, "identities": [ [ "openai/gpt-6-luna", "OpenAI" ] ] }, "V03": { "scheduled": 10, "completed_first": 10, "correct_first": 5, "accuracy_scheduled": 0.5, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.5, "wilson95": [ 0.236593090512564, 0.7634069094874361 ], "statuses_first": { "ok": 10 }, "eventual_correct": 5, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.001415475, "exact_decimal_usd": "0.001415475", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.001415475, "exact_decimal_usd": "0.001415475", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.0001415475, "first_cost_per_first_correct_usd": 0.000283095, "core_cost_per_first_correct_usd": 0.000283095, "latency_first_all_ms": { "median": 2444.9318959959783, "p90": 3588.723695918452, "p95": 3658.2844524644315 }, "latency_first_usable_ms": { "median": 2444.9318959959783, "p90": 3588.723695918452, "p95": 3658.2844524644315 }, "tokens_first": { "prompt_tokens": { "total": 11507, "covered_requests": 10 }, "completion_tokens": { "total": 50, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 9591, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.001243925, "exact_decimal_usd": "0.001243925", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 0.00017155, "exact_decimal_usd": "0.00017155", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:13.600130-04:00", "last": "2026-10-15T11:02:27.540854-04:00" }, "identities": [ [ "openai/gpt-6-luna", "OpenAI" ] ] }, "V04": { "scheduled": 10, "completed_first": 10, "correct_first": 3, "accuracy_scheduled": 0.3, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.3, "wilson95": [ 0.10779126740630099, 0.6032218525388546 ], "statuses_first": { "ok": 10 }, "eventual_correct": 3, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.001389425, "exact_decimal_usd": "0.001389425", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.001389425, "exact_decimal_usd": "0.001389425", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00013894249999999998, "first_cost_per_first_correct_usd": 0.0004631416666666666, "core_cost_per_first_correct_usd": 0.0004631416666666666, "latency_first_all_ms": { "median": 2285.904999997001, "p90": 6947.154550324193, "p95": 7591.181087674338 }, "latency_first_usable_ms": { "median": 2285.904999997001, "p90": 6947.154550324193, "p95": 7591.181087674338 }, "tokens_first": { "prompt_tokens": { "total": 11627, "covered_requests": 10 }, "completion_tokens": { "total": 50, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 8069, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.001305625, "exact_decimal_usd": "0.001305625", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 8.38e-05, "exact_decimal_usd": "0.0000838", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:03.989524-04:00", "last": "2026-10-15T11:00:45.935408-04:00" }, "identities": [ [ "openai/gpt-6-luna", "OpenAI" ] ] }, "V05": { "scheduled": 10, "completed_first": 10, "correct_first": 5, "accuracy_scheduled": 0.5, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.5, "wilson95": [ 0.236593090512564, 0.7634069094874361 ], "statuses_first": { "ok": 10 }, "eventual_correct": 5, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.002288425, "exact_decimal_usd": "0.002288425", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.002288425, "exact_decimal_usd": "0.002288425", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00022884250000000002, "first_cost_per_first_correct_usd": 0.00045768500000000005, "core_cost_per_first_correct_usd": 0.00045768500000000005, "latency_first_all_ms": { "median": 3056.8002704530954, "p90": 4134.119145775912, "p95": 4692.021926879532 }, "latency_first_usable_ms": { "median": 3056.8002704530954, "p90": 4134.119145775912, "p95": 4692.021926879532 }, "tokens_first": { "prompt_tokens": { "total": 18445, "covered_requests": 10 }, "completion_tokens": { "total": 50, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 16757, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.001983375, "exact_decimal_usd": "0.001983375", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 0.00030505, "exact_decimal_usd": "0.00030505", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:36:04.340677-04:00", "last": "2026-10-15T11:01:49.828725-04:00" }, "identities": [ [ "openai/gpt-6-luna", "OpenAI" ] ] }, "V06": { "scheduled": 10, "completed_first": 10, "correct_first": 4, "accuracy_scheduled": 0.4, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.4, "wilson95": [ 0.16818032970623614, 0.6873262302663417 ], "statuses_first": { "ok": 10 }, "eventual_correct": 4, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.000986225, "exact_decimal_usd": "0.000986225", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.000986225, "exact_decimal_usd": "0.000986225", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 9.862250000000001e-05, "first_cost_per_first_correct_usd": 0.00024655625, "core_cost_per_first_correct_usd": 0.00024655625, "latency_first_all_ms": { "median": 2964.130395994289, "p90": 4099.06079940265, "p95": 4282.331774680642 }, "latency_first_usable_ms": { "median": 2964.130395994289, "p90": 4099.06079940265, "p95": 4282.331774680642 }, "tokens_first": { "prompt_tokens": { "total": 8713, "covered_requests": 10 }, "completion_tokens": { "total": 50, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 3597, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.000961625, "exact_decimal_usd": "0.000961625", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 2.46e-05, "exact_decimal_usd": "0.0000246", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:36:11.381415-04:00", "last": "2026-10-15T11:03:16.682372-04:00" }, "identities": [ [ "openai/gpt-6-luna", "OpenAI" ] ] }, "V07": { "scheduled": 10, "completed_first": 10, "correct_first": 2, "accuracy_scheduled": 0.2, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.2, "wilson95": [ 0.05668215145437519, 0.5098375284633583 ], "statuses_first": { "ok": 10 }, "eventual_correct": 2, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.000992525, "exact_decimal_usd": "0.000992525", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.000992525, "exact_decimal_usd": "0.000992525", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 9.92525e-05, "first_cost_per_first_correct_usd": 0.0004962625, "core_cost_per_first_correct_usd": 0.0004962625, "latency_first_all_ms": { "median": 2294.188583502546, "p90": 2810.041291674133, "p95": 3255.7473543158258 }, "latency_first_usable_ms": { "median": 2294.188583502546, "p90": 2810.041291674133, "p95": 3255.7473543158258 }, "tokens_first": { "prompt_tokens": { "total": 9050, "covered_requests": 10 }, "completion_tokens": { "total": 50, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 2501, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.000894525, "exact_decimal_usd": "0.000894525", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 9.8e-05, "exact_decimal_usd": "0.000098", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:36:23.310633-04:00", "last": "2026-10-15T11:02:45.760023-04:00" }, "identities": [ [ "openai/gpt-6-luna", "OpenAI" ] ] }, "V08": { "scheduled": 10, "completed_first": 10, "correct_first": 6, "accuracy_scheduled": 0.6, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.6, "wilson95": [ 0.31267376973365824, 0.8318196702937638 ], "statuses_first": { "ok": 10 }, "eventual_correct": 6, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.00166045, "exact_decimal_usd": "0.001660450", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.00166045, "exact_decimal_usd": "0.001660450", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.000166045, "first_cost_per_first_correct_usd": 0.00027674166666666664, "core_cost_per_first_correct_usd": 0.00027674166666666664, "latency_first_all_ms": { "median": 1985.3351460187696, "p90": 4352.259774674894, "p95": 4610.3873873566035 }, "latency_first_usable_ms": { "median": 1985.3351460187696, "p90": 4352.259774674894, "p95": 4610.3873873566035 }, "tokens_first": { "prompt_tokens": { "total": 13550, "covered_requests": 10 }, "completion_tokens": { "total": 50, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 11218, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.00157415, "exact_decimal_usd": "0.001574150", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 8.63e-05, "exact_decimal_usd": "0.0000863", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:15.707556-04:00", "last": "2026-10-15T11:03:31.672835-04:00" }, "identities": [ [ "openai/gpt-6-luna", "OpenAI" ] ] }, "V09": { "scheduled": 10, "completed_first": 10, "correct_first": 7, "accuracy_scheduled": 0.7, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.7, "wilson95": [ 0.39677814746114537, 0.8922087325936989 ], "statuses_first": { "ok": 10 }, "eventual_correct": 7, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.00105635, "exact_decimal_usd": "0.00105635", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.00105635, "exact_decimal_usd": "0.00105635", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.000105635, "first_cost_per_first_correct_usd": 0.00015090714285714285, "core_cost_per_first_correct_usd": 0.00015090714285714285, "latency_first_all_ms": { "median": 2362.74008298642, "p90": 4950.814025342697, "p95": 5129.011700153933 }, "latency_first_usable_ms": { "median": 2362.74008298642, "p90": 4950.814025342697, "p95": 5129.011700153933 }, "tokens_first": { "prompt_tokens": { "total": 9411, "covered_requests": 10 }, "completion_tokens": { "total": 50, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 3610, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.00079255, "exact_decimal_usd": "0.00079255", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 0.0002638, "exact_decimal_usd": "0.0002638", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:34:21.852048-04:00", "last": "2026-10-15T11:03:22.645694-04:00" }, "identities": [ [ "openai/gpt-6-luna", "OpenAI" ] ] }, "V10": { "scheduled": 10, "completed_first": 10, "correct_first": 4, "accuracy_scheduled": 0.4, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.4, "wilson95": [ 0.16818032970623614, 0.6873262302663417 ], "statuses_first": { "ok": 10 }, "eventual_correct": 4, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.00230715, "exact_decimal_usd": "0.002307150", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.00230715, "exact_decimal_usd": "0.002307150", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.000230715, "first_cost_per_first_correct_usd": 0.0005767875, "core_cost_per_first_correct_usd": 0.0005767875, "latency_first_all_ms": { "median": 1652.4944790289737, "p90": 2885.665829206118, "p95": 3378.993310601799 }, "latency_first_usable_ms": { "median": 1652.4944790289737, "p90": 2885.665829206118, "p95": 3378.993310601799 }, "tokens_first": { "prompt_tokens": { "total": 18374, "covered_requests": 10 }, "completion_tokens": { "total": 50, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 17790, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.0021466, "exact_decimal_usd": "0.002146600", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 0.00016055, "exact_decimal_usd": "0.00016055", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:36:00.241393-04:00", "last": "2026-10-15T11:00:02.619161-04:00" }, "identities": [ [ "openai/gpt-6-luna", "OpenAI" ] ] }, "V11": { "scheduled": 10, "completed_first": 10, "correct_first": 5, "accuracy_scheduled": 0.5, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.5, "wilson95": [ 0.236593090512564, 0.7634069094874361 ], "statuses_first": { "ok": 10 }, "eventual_correct": 5, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.0020052, "exact_decimal_usd": "0.002005200", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0020052, "exact_decimal_usd": "0.002005200", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00020051999999999998, "first_cost_per_first_correct_usd": 0.00040103999999999997, "core_cost_per_first_correct_usd": 0.00040103999999999997, "latency_first_all_ms": { "median": 2209.0729374904186, "p90": 3453.8864538131743, "p95": 3770.357934900675 }, "latency_first_usable_ms": { "median": 2209.0729374904186, "p90": 3453.8864538131743, "p95": 3770.357934900675 }, "tokens_first": { "prompt_tokens": { "total": 16289, "covered_requests": 10 }, "completion_tokens": { "total": 50, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 14052, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.0017664, "exact_decimal_usd": "0.001766400", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 0.0002388, "exact_decimal_usd": "0.0002388", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:41.967145-04:00", "last": "2026-10-15T11:02:57.701079-04:00" }, "identities": [ [ "openai/gpt-6-luna", "OpenAI" ] ] }, "V12": { "scheduled": 10, "completed_first": 10, "correct_first": 3, "accuracy_scheduled": 0.3, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.3, "wilson95": [ 0.10779126740630099, 0.6032218525388546 ], "statuses_first": { "ok": 10 }, "eventual_correct": 3, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.00113855, "exact_decimal_usd": "0.001138550", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.00113855, "exact_decimal_usd": "0.001138550", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00011385499999999999, "first_cost_per_first_correct_usd": 0.00037951666666666663, "core_cost_per_first_correct_usd": 0.00037951666666666663, "latency_first_all_ms": { "median": 1922.0965210115537, "p90": 3800.1504752726755, "p95": 4291.3893001299575 }, "latency_first_usable_ms": { "median": 1922.0965210115537, "p90": 3800.1504752726755, "p95": 4291.3893001299575 }, "tokens_first": { "prompt_tokens": { "total": 10197, "covered_requests": 10 }, "completion_tokens": { "total": 50, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 3754, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.000957375, "exact_decimal_usd": "0.000957375", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 0.000181175, "exact_decimal_usd": "0.000181175", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:34:59.797439-04:00", "last": "2026-10-15T11:02:12.076688-04:00" }, "identities": [ [ "openai/gpt-6-luna", "OpenAI" ] ] }, "V13": { "scheduled": 10, "completed_first": 10, "correct_first": 7, "accuracy_scheduled": 0.7, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.7, "wilson95": [ 0.39677814746114537, 0.8922087325936989 ], "statuses_first": { "ok": 10 }, "eventual_correct": 7, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.00093275, "exact_decimal_usd": "0.000932750", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.00093275, "exact_decimal_usd": "0.000932750", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 9.3275e-05, "first_cost_per_first_correct_usd": 0.00013324999999999999, "core_cost_per_first_correct_usd": 0.00013324999999999999, "latency_first_all_ms": { "median": 2067.5852914864663, "p90": 5743.153037474257, "p95": 6980.978331252115 }, "latency_first_usable_ms": { "median": 2067.5852914864663, "p90": 5743.153037474257, "p95": 6980.978331252115 }, "tokens_first": { "prompt_tokens": { "total": 8479, "covered_requests": 10 }, "completion_tokens": { "total": 50, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 2394, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.00085875, "exact_decimal_usd": "0.000858750", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 7.4e-05, "exact_decimal_usd": "0.000074", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:36:16.718309-04:00", "last": "2026-10-15T11:01:40.407429-04:00" }, "identities": [ [ "openai/gpt-6-luna", "OpenAI" ] ] }, "V14": { "scheduled": 10, "completed_first": 10, "correct_first": 6, "accuracy_scheduled": 0.6, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.6, "wilson95": [ 0.31267376973365824, 0.8318196702937638 ], "statuses_first": { "ok": 10 }, "eventual_correct": 6, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.001782625, "exact_decimal_usd": "0.001782625", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.001782625, "exact_decimal_usd": "0.001782625", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.0001782625, "first_cost_per_first_correct_usd": 0.00029710416666666665, "core_cost_per_first_correct_usd": 0.00029710416666666665, "latency_first_all_ms": { "median": 2634.9312085076235, "p90": 7660.469445865601, "p95": 7671.3940774207 }, "latency_first_usable_ms": { "median": 2634.9312085076235, "p90": 7660.469445865601, "p95": 7671.3940774207 }, "tokens_first": { "prompt_tokens": { "total": 14369, "covered_requests": 10 }, "completion_tokens": { "total": 50, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 12829, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.0015967, "exact_decimal_usd": "0.001596700", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 0.000185925, "exact_decimal_usd": "0.000185925", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:29.144879-04:00", "last": "2026-10-15T11:02:36.268612-04:00" }, "identities": [ [ "openai/gpt-6-luna", "OpenAI" ] ] }, "V15": { "scheduled": 10, "completed_first": 10, "correct_first": 10, "accuracy_scheduled": 1.0, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 1.0, "wilson95": [ 0.7224672001371107, 0.9999999999999999 ], "statuses_first": { "ok": 10 }, "eventual_correct": 10, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.0008511, "exact_decimal_usd": "0.000851100", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0008511, "exact_decimal_usd": "0.000851100", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 8.511e-05, "first_cost_per_first_correct_usd": 8.511e-05, "core_cost_per_first_correct_usd": 8.511e-05, "latency_first_all_ms": { "median": 2835.0201880093664, "p90": 3664.7228460700717, "p95": 3768.0875270190877 }, "latency_first_usable_ms": { "median": 2835.0201880093664, "p90": 3664.7228460700717, "p95": 3768.0875270190877 }, "tokens_first": { "prompt_tokens": { "total": 7431, "covered_requests": 10 }, "completion_tokens": { "total": 50, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 3320, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.0007491, "exact_decimal_usd": "0.000749100", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 0.000102, "exact_decimal_usd": "0.000102", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:31.928314-04:00", "last": "2026-10-15T11:02:42.908987-04:00" }, "identities": [ [ "openai/gpt-6-luna", "OpenAI" ] ] }, "V16": { "scheduled": 10, "completed_first": 10, "correct_first": 10, "accuracy_scheduled": 1.0, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 1.0, "wilson95": [ 0.7224672001371107, 0.9999999999999999 ], "statuses_first": { "ok": 10 }, "eventual_correct": 10, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.002512125, "exact_decimal_usd": "0.002512125", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.002512125, "exact_decimal_usd": "0.002512125", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.0002512125, "first_cost_per_first_correct_usd": 0.0002512125, "core_cost_per_first_correct_usd": 0.0002512125, "latency_first_all_ms": { "median": 3254.859146021772, "p90": 5031.9971832679585, "p95": 5040.026008145651 }, "latency_first_usable_ms": { "median": 3254.859146021772, "p90": 5031.9971832679585, "p95": 5040.026008145651 }, "tokens_first": { "prompt_tokens": { "total": 19903, "covered_requests": 10 }, "completion_tokens": { "total": 50, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 19873, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.0022857, "exact_decimal_usd": "0.002285700", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 0.000226425, "exact_decimal_usd": "0.000226425", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:35.144108-04:00", "last": "2026-10-15T10:57:55.616247-04:00" }, "identities": [ [ "openai/gpt-6-luna", "OpenAI" ] ] }, "V17": { "scheduled": 10, "completed_first": 10, "correct_first": 7, "accuracy_scheduled": 0.7, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.7, "wilson95": [ 0.39677814746114537, 0.8922087325936989 ], "statuses_first": { "ok": 10 }, "eventual_correct": 7, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.002207875, "exact_decimal_usd": "0.002207875", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.002207875, "exact_decimal_usd": "0.002207875", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00022078750000000002, "first_cost_per_first_correct_usd": 0.0003154107142857143, "core_cost_per_first_correct_usd": 0.0003154107142857143, "latency_first_all_ms": { "median": 2584.812916495139, "p90": 4255.751908989623, "p95": 4745.23862148635 }, "latency_first_usable_ms": { "median": 2584.812916495139, "p90": 4255.751908989623, "p95": 4745.23862148635 }, "tokens_first": { "prompt_tokens": { "total": 17661, "covered_requests": 10 }, "completion_tokens": { "total": 50, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 16671, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.002109075, "exact_decimal_usd": "0.002109075", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 9.88e-05, "exact_decimal_usd": "0.0000988", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:36:25.521607-04:00", "last": "2026-10-15T11:01:26.496094-04:00" }, "identities": [ [ "openai/gpt-6-luna", "OpenAI" ] ] }, "V18": { "scheduled": 10, "completed_first": 10, "correct_first": 6, "accuracy_scheduled": 0.6, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.6, "wilson95": [ 0.31267376973365824, 0.8318196702937638 ], "statuses_first": { "ok": 10 }, "eventual_correct": 6, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.001390475, "exact_decimal_usd": "0.001390475", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.001390475, "exact_decimal_usd": "0.001390475", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.0001390475, "first_cost_per_first_correct_usd": 0.00023174583333333333, "core_cost_per_first_correct_usd": 0.00023174583333333333, "latency_first_all_ms": { "median": 1716.4988750009798, "p90": 3548.86961596203, "p95": 4074.965515968505 }, "latency_first_usable_ms": { "median": 1716.4988750009798, "p90": 3548.86961596203, "p95": 4074.965515968505 }, "tokens_first": { "prompt_tokens": { "total": 11439, "covered_requests": 10 }, "completion_tokens": { "total": 50, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 8863, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.001311775, "exact_decimal_usd": "0.001311775", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 7.87e-05, "exact_decimal_usd": "0.0000787", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:51.324697-04:00", "last": "2026-10-15T10:59:09.887897-04:00" }, "identities": [ [ "openai/gpt-6-luna", "OpenAI" ] ] }, "V19": { "scheduled": 10, "completed_first": 10, "correct_first": 5, "accuracy_scheduled": 0.5, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.5, "wilson95": [ 0.236593090512564, 0.7634069094874361 ], "statuses_first": { "ok": 10 }, "eventual_correct": 5, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.001360475, "exact_decimal_usd": "0.001360475", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.001360475, "exact_decimal_usd": "0.001360475", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00013604750000000001, "first_cost_per_first_correct_usd": 0.00027209500000000003, "core_cost_per_first_correct_usd": 0.00027209500000000003, "latency_first_all_ms": { "median": 1604.9335415300447, "p90": 3266.4516076853033, "p95": 3375.4818453424373 }, "latency_first_usable_ms": { "median": 1604.9335415300447, "p90": 3266.4516076853033, "p95": 3375.4818453424373 }, "tokens_first": { "prompt_tokens": { "total": 11227, "covered_requests": 10 }, "completion_tokens": { "total": 50, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 8511, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.001290875, "exact_decimal_usd": "0.001290875", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 6.96e-05, "exact_decimal_usd": "0.0000696", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:54.396102-04:00", "last": "2026-10-15T11:00:52.359280-04:00" }, "identities": [ [ "openai/gpt-6-luna", "OpenAI" ] ] }, "V20": { "scheduled": 10, "completed_first": 10, "correct_first": 9, "accuracy_scheduled": 0.9, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.9, "wilson95": [ 0.5958499732047615, 0.9821237869049271 ], "statuses_first": { "ok": 10 }, "eventual_correct": 9, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.001263575, "exact_decimal_usd": "0.001263575", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.001263575, "exact_decimal_usd": "0.001263575", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00012635750000000002, "first_cost_per_first_correct_usd": 0.00014039722222222224, "core_cost_per_first_correct_usd": 0.00014039722222222224, "latency_first_all_ms": { "median": 2233.8353534869384, "p90": 5165.980283019598, "p95": 6137.902995495822 }, "latency_first_usable_ms": { "median": 2233.8353534869384, "p90": 5165.980283019598, "p95": 6137.902995495822 }, "tokens_first": { "prompt_tokens": { "total": 10873, "covered_requests": 10 }, "completion_tokens": { "total": 50, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 6051, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.001165275, "exact_decimal_usd": "0.001165275", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 9.83e-05, "exact_decimal_usd": "0.0000983", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:36:08.314966-04:00", "last": "2026-10-15T11:02:53.819186-04:00" }, "identities": [ [ "openai/gpt-6-luna", "OpenAI" ] ] } }, "image_count": { "1": { "scheduled": 177, "completed_first": 177, "correct_first": 99, "accuracy_scheduled": 0.559322033898305, "usable_first": 177, "coverage": 1.0, "accuracy_usable": 0.559322033898305, "wilson95": [ 0.48569224102360603, 0.6304315733768389 ], "statuses_first": { "ok": 177 }, "eventual_correct": 99, "eventual_usable": 177, "retries": 0, "first_cost": { "reported_usd": 0.024427225, "exact_decimal_usd": "0.024427225", "known_records": 177, "missing_records": 0 }, "core_cost": { "reported_usd": 0.024427225, "exact_decimal_usd": "0.024427225", "known_records": 177, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.0001380069209039548, "first_cost_per_first_correct_usd": 0.00024673964646464646, "core_cost_per_first_correct_usd": 0.00024673964646464646, "latency_first_all_ms": { "median": 2250.8505419827998, "p90": 4069.7321159997955, "p95": 4918.971125246026 }, "latency_first_usable_ms": { "median": 2250.8505419827998, "p90": 4069.7321159997955, "p95": 4918.971125246026 }, "tokens_first": { "prompt_tokens": { "total": 205137, "covered_requests": 177 }, "completion_tokens": { "total": 885, "covered_requests": 177 } }, "cached_tokens": 0, "cache_write_tokens": 138841, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 177, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.022321575, "exact_decimal_usd": "0.022321575", "known_records": 160, "missing_records": 0 }, "pilot": { "reported_usd": 0.00210565, "exact_decimal_usd": "0.002105650", "known_records": 17, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:34:59.797439-04:00", "last": "2026-10-15T11:03:31.672835-04:00" }, "identities": [ [ "openai/gpt-6-luna", "OpenAI" ] ] }, "2": { "scheduled": 19, "completed_first": 19, "correct_first": 16, "accuracy_scheduled": 0.8421052631578947, "usable_first": 19, "coverage": 1.0, "accuracy_usable": 0.8421052631578947, "wilson95": [ 0.6243452472967093, 0.9447952836146585 ], "statuses_first": { "ok": 19 }, "eventual_correct": 16, "eventual_usable": 19, "retries": 0, "first_cost": { "reported_usd": 0.004583825, "exact_decimal_usd": "0.004583825", "known_records": 19, "missing_records": 0 }, "core_cost": { "reported_usd": 0.004583825, "exact_decimal_usd": "0.004583825", "known_records": 19, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00024125394736842105, "first_cost_per_first_correct_usd": 0.0002864890625, "core_cost_per_first_correct_usd": 0.0002864890625, "latency_first_all_ms": { "median": 2782.968625018839, "p90": 5033.781366574112, "p95": 5066.721883119317 }, "latency_first_usable_ms": { "median": 2782.968625018839, "p90": 5033.781366574112, "p95": 5066.721883119317 }, "tokens_first": { "prompt_tokens": { "total": 36494, "covered_requests": 19 }, "completion_tokens": { "total": 95, "covered_requests": 19 } }, "cached_tokens": 0, "cache_write_tokens": 35477, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 19, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.0042586, "exact_decimal_usd": "0.004258600", "known_records": 17, "missing_records": 0 }, "pilot": { "reported_usd": 0.000325225, "exact_decimal_usd": "0.000325225", "known_records": 2, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:35.144108-04:00", "last": "2026-10-15T11:01:26.496094-04:00" }, "identities": [ [ "openai/gpt-6-luna", "OpenAI" ] ] }, "3": { "scheduled": 4, "completed_first": 4, "correct_first": 2, "accuracy_scheduled": 0.5, "usable_first": 4, "coverage": 1.0, "accuracy_usable": 0.5, "wilson95": [ 0.15003898915214947, 0.8499610108478506 ], "statuses_first": { "ok": 4 }, "eventual_correct": 2, "eventual_usable": 4, "retries": 0, "first_cost": { "reported_usd": 0.00057065, "exact_decimal_usd": "0.00057065", "known_records": 4, "missing_records": 0 }, "core_cost": { "reported_usd": 0.00057065, "exact_decimal_usd": "0.00057065", "known_records": 4, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.0001426625, "first_cost_per_first_correct_usd": 0.000285325, "core_cost_per_first_correct_usd": 0.000285325, "latency_first_all_ms": { "median": 3746.532916498836, "p90": 4855.203387478832, "p95": 5081.206381222 }, "latency_first_usable_ms": { "median": 3746.532916498836, "p90": 4855.203387478832, "p95": 5081.206381222 }, "tokens_first": { "prompt_tokens": { "total": 4704, "covered_requests": 4 }, "completion_tokens": { "total": 20, "covered_requests": 4 } }, "cached_tokens": 0, "cache_write_tokens": 3610, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 4, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.00030685, "exact_decimal_usd": "0.00030685", "known_records": 3, "missing_records": 0 }, "pilot": { "reported_usd": 0.0002638, "exact_decimal_usd": "0.0002638", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:34:21.852048-04:00", "last": "2026-10-15T11:01:00.812542-04:00" }, "identities": [ [ "openai/gpt-6-luna", "OpenAI" ] ] } }, "analysis_subset": { "perception_control": { "scheduled": 13, "completed_first": 13, "correct_first": 13, "accuracy_scheduled": 1.0, "usable_first": 13, "coverage": 1.0, "accuracy_usable": 1.0, "wilson95": [ 0.7719046276458016, 0.9999999999999999 ], "statuses_first": { "ok": 13 }, "eventual_correct": 13, "eventual_usable": 13, "retries": 0, "first_cost": { "reported_usd": 0.001540275, "exact_decimal_usd": "0.001540275", "known_records": 13, "missing_records": 0 }, "core_cost": { "reported_usd": 0.001540275, "exact_decimal_usd": "0.001540275", "known_records": 13, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00011848269230769232, "first_cost_per_first_correct_usd": 0.00011848269230769232, "core_cost_per_first_correct_usd": 0.00011848269230769232, "latency_first_all_ms": { "median": 2504.6353340148926, "p90": 3668.6876163934357, "p95": 3735.7349497964606 }, "latency_first_usable_ms": { "median": 2504.6353340148926, "p90": 3668.6876163934357, "p95": 3735.7349497964606 }, "tokens_first": { "prompt_tokens": { "total": 13144, "covered_requests": 13 }, "completion_tokens": { "total": 65, "covered_requests": 13 } }, "cached_tokens": 0, "cache_write_tokens": 7735, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 13, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.001456475, "exact_decimal_usd": "0.001456475", "known_records": 12, "missing_records": 0 }, "pilot": { "reported_usd": 8.38e-05, "exact_decimal_usd": "0.0000838", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:03.989524-04:00", "last": "2026-10-15T11:01:00.812542-04:00" }, "identities": [ [ "openai/gpt-6-luna", "OpenAI" ] ] }, "reasoning_and_grounded_decision": { "scheduled": 187, "completed_first": 187, "correct_first": 104, "accuracy_scheduled": 0.5561497326203209, "usable_first": 187, "coverage": 1.0, "accuracy_usable": 0.5561497326203209, "wilson95": [ 0.48452070247623696, 0.6255182802991495 ], "statuses_first": { "ok": 187 }, "eventual_correct": 104, "eventual_usable": 187, "retries": 0, "first_cost": { "reported_usd": 0.028041425, "exact_decimal_usd": "0.028041425", "known_records": 187, "missing_records": 0 }, "core_cost": { "reported_usd": 0.028041425, "exact_decimal_usd": "0.028041425", "known_records": 187, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00014995414438502672, "first_cost_per_first_correct_usd": 0.0002696290865384615, "core_cost_per_first_correct_usd": 0.0002696290865384615, "latency_first_all_ms": { "median": 2320.9594580112025, "p90": 4571.763641003054, "p95": 5178.724183695156 }, "latency_first_usable_ms": { "median": 2320.9594580112025, "p90": 4571.763641003054, "p95": 5178.724183695156 }, "tokens_first": { "prompt_tokens": { "total": 233191, "covered_requests": 187 }, "completion_tokens": { "total": 935, "covered_requests": 187 } }, "cached_tokens": 0, "cache_write_tokens": 170193, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 187, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.02543055, "exact_decimal_usd": "0.025430550", "known_records": 168, "missing_records": 0 }, "pilot": { "reported_usd": 0.002610875, "exact_decimal_usd": "0.002610875", "known_records": 19, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:34:21.852048-04:00", "last": "2026-10-15T11:03:31.672835-04:00" }, "identities": [ [ "openai/gpt-6-luna", "OpenAI" ] ] } }, "difficulty_proposed": { "demanding": { "scheduled": 55, "completed_first": 55, "correct_first": 30, "accuracy_scheduled": 0.5454545454545454, "usable_first": 55, "coverage": 1.0, "accuracy_usable": 0.5454545454545454, "wilson95": [ 0.41522690241861226, 0.6697471974798342 ], "statuses_first": { "ok": 55 }, "eventual_correct": 30, "eventual_usable": 55, "retries": 0, "first_cost": { "reported_usd": 0.0086125, "exact_decimal_usd": "0.008612500", "known_records": 55, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0086125, "exact_decimal_usd": "0.008612500", "known_records": 55, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.0001565909090909091, "first_cost_per_first_correct_usd": 0.00028708333333333336, "core_cost_per_first_correct_usd": 0.00028708333333333336, "latency_first_all_ms": { "median": 2096.104208030738, "p90": 3803.973841981497, "p95": 4647.116753726731 }, "latency_first_usable_ms": { "median": 2096.104208030738, "p90": 3803.973841981497, "p95": 4647.116753726731 }, "tokens_first": { "prompt_tokens": { "total": 71044, "covered_requests": 55 }, "completion_tokens": { "total": 275, "covered_requests": 55 } }, "cached_tokens": 0, "cache_write_tokens": 54824, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 55, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.00801985, "exact_decimal_usd": "0.008019850", "known_records": 48, "missing_records": 0 }, "pilot": { "reported_usd": 0.00059265, "exact_decimal_usd": "0.00059265", "known_records": 7, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:13.600130-04:00", "last": "2026-10-15T11:02:57.701079-04:00" }, "identities": [ [ "openai/gpt-6-luna", "OpenAI" ] ] }, "moderate": { "scheduled": 107, "completed_first": 107, "correct_first": 60, "accuracy_scheduled": 0.5607476635514018, "usable_first": 107, "coverage": 1.0, "accuracy_usable": 0.5607476635514018, "wilson95": [ 0.46622561361687753, 0.6510590210163586 ], "statuses_first": { "ok": 107 }, "eventual_correct": 60, "eventual_usable": 107, "retries": 0, "first_cost": { "reported_usd": 0.015927575, "exact_decimal_usd": "0.015927575", "known_records": 107, "missing_records": 0 }, "core_cost": { "reported_usd": 0.015927575, "exact_decimal_usd": "0.015927575", "known_records": 107, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00014885584112149533, "first_cost_per_first_correct_usd": 0.0002654595833333333, "core_cost_per_first_correct_usd": 0.0002654595833333333, "latency_first_all_ms": { "median": 2421.355250000488, "p90": 4926.72770844074, "p95": 5290.0239748705635 }, "latency_first_usable_ms": { "median": 2421.355250000488, "p90": 4926.72770844074, "p95": 5290.0239748705635 }, "tokens_first": { "prompt_tokens": { "total": 132742, "covered_requests": 107 }, "completion_tokens": { "total": 535, "covered_requests": 107 } }, "cached_tokens": 0, "cache_write_tokens": 95435, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 107, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.014476575, "exact_decimal_usd": "0.014476575", "known_records": 98, "missing_records": 0 }, "pilot": { "reported_usd": 0.001451, "exact_decimal_usd": "0.001451000", "known_records": 9, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:34:21.852048-04:00", "last": "2026-10-15T11:03:31.672835-04:00" }, "identities": [ [ "openai/gpt-6-luna", "OpenAI" ] ] }, "straightforward": { "scheduled": 38, "completed_first": 38, "correct_first": 27, "accuracy_scheduled": 0.7105263157894737, "usable_first": 38, "coverage": 1.0, "accuracy_usable": 0.7105263157894737, "wilson95": [ 0.5524286232424794, 0.8299672217151693 ], "statuses_first": { "ok": 38 }, "eventual_correct": 27, "eventual_usable": 38, "retries": 0, "first_cost": { "reported_usd": 0.005041625, "exact_decimal_usd": "0.005041625", "known_records": 38, "missing_records": 0 }, "core_cost": { "reported_usd": 0.005041625, "exact_decimal_usd": "0.005041625", "known_records": 38, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00013267434210526317, "first_cost_per_first_correct_usd": 0.00018672685185185185, "core_cost_per_first_correct_usd": 0.00018672685185185185, "latency_first_all_ms": { "median": 2728.885333490325, "p90": 3971.463670383674, "p95": 4320.503783016466 }, "latency_first_usable_ms": { "median": 2728.885333490325, "p90": 3971.463670383674, "p95": 4320.503783016466 }, "tokens_first": { "prompt_tokens": { "total": 42549, "covered_requests": 38 }, "completion_tokens": { "total": 190, "covered_requests": 38 } }, "cached_tokens": 0, "cache_write_tokens": 27669, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 38, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.0043906, "exact_decimal_usd": "0.004390600", "known_records": 34, "missing_records": 0 }, "pilot": { "reported_usd": 0.000651025, "exact_decimal_usd": "0.000651025", "known_records": 4, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:03.989524-04:00", "last": "2026-10-15T11:03:29.891194-04:00" }, "identities": [ [ "openai/gpt-6-luna", "OpenAI" ] ] } }, "visual_medium": { "chart": { "scheduled": 13, "completed_first": 13, "correct_first": 9, "accuracy_scheduled": 0.6923076923076923, "usable_first": 13, "coverage": 1.0, "accuracy_usable": 0.6923076923076923, "wilson95": [ 0.42369343180549035, 0.8731929634428948 ], "statuses_first": { "ok": 13 }, "eventual_correct": 9, "eventual_usable": 13, "retries": 0, "first_cost": { "reported_usd": 0.00126785, "exact_decimal_usd": "0.001267850", "known_records": 13, "missing_records": 0 }, "core_cost": { "reported_usd": 0.00126785, "exact_decimal_usd": "0.001267850", "known_records": 13, "missing_records": 0 }, "core_cost_per_scheduled_usd": 9.752692307692309e-05, "first_cost_per_first_correct_usd": 0.00014087222222222223, "core_cost_per_first_correct_usd": 0.00014087222222222223, "latency_first_all_ms": { "median": 2143.991124990862, "p90": 5071.36701657437, "p95": 6568.369899992827 }, "latency_first_usable_ms": { "median": 2143.991124990862, "p90": 5071.36701657437, "p95": 6568.369899992827 }, "tokens_first": { "prompt_tokens": { "total": 11400, "covered_requests": 13 }, "completion_tokens": { "total": 65, "covered_requests": 13 } }, "cached_tokens": 0, "cache_write_tokens": 3814, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 13, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.00111515, "exact_decimal_usd": "0.001115150", "known_records": 11, "missing_records": 0 }, "pilot": { "reported_usd": 0.0001527, "exact_decimal_usd": "0.0001527", "known_records": 2, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:51.324697-04:00", "last": "2026-10-15T11:01:40.407429-04:00" }, "identities": [ [ "openai/gpt-6-luna", "OpenAI" ] ] }, "diagram": { "scheduled": 92, "completed_first": 92, "correct_first": 39, "accuracy_scheduled": 0.42391304347826086, "usable_first": 92, "coverage": 1.0, "accuracy_usable": 0.42391304347826086, "wilson95": [ 0.32797984904074834, 0.5259455797635852 ], "statuses_first": { "ok": 92 }, "eventual_correct": 39, "eventual_usable": 92, "retries": 0, "first_cost": { "reported_usd": 0.014658175, "exact_decimal_usd": "0.014658175", "known_records": 92, "missing_records": 0 }, "core_cost": { "reported_usd": 0.014658175, "exact_decimal_usd": "0.014658175", "known_records": 92, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.0001593279891304348, "first_cost_per_first_correct_usd": 0.00037585064102564104, "core_cost_per_first_correct_usd": 0.00037585064102564104, "latency_first_all_ms": { "median": 2186.135020980146, "p90": 3998.7474207824566, "p95": 4514.376601876576 }, "latency_first_usable_ms": { "median": 2186.135020980146, "p90": 3998.7474207824566, "p95": 4514.376601876576 }, "tokens_first": { "prompt_tokens": { "total": 121351, "covered_requests": 92 }, "completion_tokens": { "total": 460, "covered_requests": 92 } }, "cached_tokens": 0, "cache_write_tokens": 91723, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 92, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.01323875, "exact_decimal_usd": "0.013238750", "known_records": 82, "missing_records": 0 }, "pilot": { "reported_usd": 0.001419425, "exact_decimal_usd": "0.001419425", "known_records": 10, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:34:59.797439-04:00", "last": "2026-10-15T11:03:31.672835-04:00" }, "identities": [ [ "openai/gpt-6-luna", "OpenAI" ] ] }, "diagram_or_published_illustration": { "scheduled": 8, "completed_first": 8, "correct_first": 7, "accuracy_scheduled": 0.875, "usable_first": 8, "coverage": 1.0, "accuracy_usable": 0.875, "wilson95": [ 0.5291118177871464, 0.9775825085499433 ], "statuses_first": { "ok": 8 }, "eventual_correct": 7, "eventual_usable": 8, "retries": 0, "first_cost": { "reported_usd": 0.000748975, "exact_decimal_usd": "0.000748975", "known_records": 8, "missing_records": 0 }, "core_cost": { "reported_usd": 0.000748975, "exact_decimal_usd": "0.000748975", "known_records": 8, "missing_records": 0 }, "core_cost_per_scheduled_usd": 9.3621875e-05, "first_cost_per_first_correct_usd": 0.00010699642857142858, "core_cost_per_first_correct_usd": 0.00010699642857142858, "latency_first_all_ms": { "median": 1765.2746874955483, "p90": 3017.098083702149, "p95": 3426.2139583384846 }, "latency_first_usable_ms": { "median": 1765.2746874955483, "p90": 3017.098083702149, "p95": 3426.2139583384846 }, "tokens_first": { "prompt_tokens": { "total": 6691, "covered_requests": 8 }, "completion_tokens": { "total": 40, "covered_requests": 8 } }, "cached_tokens": 0, "cache_write_tokens": 2395, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 8, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.000502875, "exact_decimal_usd": "0.000502875", "known_records": 5, "missing_records": 0 }, "pilot": { "reported_usd": 0.0002461, "exact_decimal_usd": "0.0002461", "known_records": 3, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:09.974574-04:00", "last": "2026-10-15T10:56:48.843027-04:00" }, "identities": [ [ "openai/gpt-6-luna", "OpenAI" ] ] }, "document": { "scheduled": 12, "completed_first": 12, "correct_first": 7, "accuracy_scheduled": 0.5833333333333334, "usable_first": 12, "coverage": 1.0, "accuracy_usable": 0.5833333333333334, "wilson95": [ 0.31951131254954973, 0.8067396863412435 ], "statuses_first": { "ok": 12 }, "eventual_correct": 7, "eventual_usable": 12, "retries": 0, "first_cost": { "reported_usd": 0.0020705, "exact_decimal_usd": "0.002070500", "known_records": 12, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0020705, "exact_decimal_usd": "0.002070500", "known_records": 12, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00017254166666666665, "first_cost_per_first_correct_usd": 0.00029578571428571425, "core_cost_per_first_correct_usd": 0.00029578571428571425, "latency_first_all_ms": { "median": 2505.096645996673, "p90": 7380.745724972804, "p95": 7668.966381519567 }, "latency_first_usable_ms": { "median": 2505.096645996673, "p90": 7380.745724972804, "p95": 7668.966381519567 }, "tokens_first": { "prompt_tokens": { "total": 16744, "covered_requests": 12 }, "completion_tokens": { "total": 60, "covered_requests": 12 } }, "cached_tokens": 0, "cache_write_tokens": 14644, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 12, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.001884575, "exact_decimal_usd": "0.001884575", "known_records": 11, "missing_records": 0 }, "pilot": { "reported_usd": 0.000185925, "exact_decimal_usd": "0.000185925", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:29.144879-04:00", "last": "2026-10-15T11:02:36.268612-04:00" }, "identities": [ [ "openai/gpt-6-luna", "OpenAI" ] ] }, "photograph": { "scheduled": 64, "completed_first": 64, "correct_first": 44, "accuracy_scheduled": 0.6875, "usable_first": 64, "coverage": 1.0, "accuracy_usable": 0.6875, "wilson95": [ 0.5660766429725185, 0.7876893346051795 ], "statuses_first": { "ok": 64 }, "eventual_correct": 44, "eventual_usable": 64, "retries": 0, "first_cost": { "reported_usd": 0.00983205, "exact_decimal_usd": "0.009832050", "known_records": 64, "missing_records": 0 }, "core_cost": { "reported_usd": 0.00983205, "exact_decimal_usd": "0.009832050", "known_records": 64, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00015362578125, "first_cost_per_first_correct_usd": 0.0002234556818181818, "core_cost_per_first_correct_usd": 0.0002234556818181818, "latency_first_all_ms": { "median": 2698.0487289838493, "p90": 4772.4854953819895, "p95": 5018.180668671266 }, "latency_first_usable_ms": { "median": 2698.0487289838493, "p90": 4772.4854953819895, "p95": 5018.180668671266 }, "tokens_first": { "prompt_tokens": { "total": 81513, "covered_requests": 64 }, "completion_tokens": { "total": 320, "covered_requests": 64 } }, "cached_tokens": 0, "cache_write_tokens": 60830, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 64, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.009243525, "exact_decimal_usd": "0.009243525", "known_records": 61, "missing_records": 0 }, "pilot": { "reported_usd": 0.000588525, "exact_decimal_usd": "0.000588525", "known_records": 3, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:34:21.852048-04:00", "last": "2026-10-15T11:03:22.645694-04:00" }, "identities": [ [ "openai/gpt-6-luna", "OpenAI" ] ] }, "screenshot": { "scheduled": 11, "completed_first": 11, "correct_first": 11, "accuracy_scheduled": 1.0, "usable_first": 11, "coverage": 1.0, "accuracy_usable": 1.0, "wilson95": [ 0.7411670330319684, 1.0 ], "statuses_first": { "ok": 11 }, "eventual_correct": 11, "eventual_usable": 11, "retries": 0, "first_cost": { "reported_usd": 0.00100415, "exact_decimal_usd": "0.001004150", "known_records": 11, "missing_records": 0 }, "core_cost": { "reported_usd": 0.00100415, "exact_decimal_usd": "0.001004150", "known_records": 11, "missing_records": 0 }, "core_cost_per_scheduled_usd": 9.128636363636365e-05, "first_cost_per_first_correct_usd": 9.128636363636365e-05, "core_cost_per_first_correct_usd": 9.128636363636365e-05, "latency_first_all_ms": { "median": 2923.265209014062, "p90": 3871.4522079681046, "p95": 5490.638957970077 }, "latency_first_usable_ms": { "median": 2923.265209014062, "p90": 3871.4522079681046, "p95": 5490.638957970077 }, "tokens_first": { "prompt_tokens": { "total": 8636, "covered_requests": 11 }, "completion_tokens": { "total": 55, "covered_requests": 11 } }, "cached_tokens": 0, "cache_write_tokens": 4522, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 11, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.00090215, "exact_decimal_usd": "0.000902150", "known_records": 10, "missing_records": 0 }, "pilot": { "reported_usd": 0.000102, "exact_decimal_usd": "0.000102", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:31.928314-04:00", "last": "2026-10-15T11:02:53.819186-04:00" }, "identities": [ [ "openai/gpt-6-luna", "OpenAI" ] ] } }, "stage": { "1": { "scheduled": 100, "completed_first": 100, "correct_first": 53, "accuracy_scheduled": 0.53, "usable_first": 100, "coverage": 1.0, "accuracy_usable": 0.53, "wilson95": [ 0.4328885697009936, 0.6248918204065872 ], "statuses_first": { "ok": 100 }, "eventual_correct": 53, "eventual_usable": 100, "retries": 0, "first_cost": { "reported_usd": 0.01445495, "exact_decimal_usd": "0.014454950", "known_records": 100, "missing_records": 0 }, "core_cost": { "reported_usd": 0.01445495, "exact_decimal_usd": "0.014454950", "known_records": 100, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00014454949999999998, "first_cost_per_first_correct_usd": 0.0002727349056603773, "core_cost_per_first_correct_usd": 0.0002727349056603773, "latency_first_all_ms": { "median": 2216.9610415003262, "p90": 3898.788833041908, "p95": 4568.045951472594 }, "latency_first_usable_ms": { "median": 2216.9610415003262, "p90": 3898.788833041908, "p95": 4568.045951472594 }, "tokens_first": { "prompt_tokens": { "total": 121050, "covered_requests": 100 }, "completion_tokens": { "total": 500, "covered_requests": 100 } }, "cached_tokens": 0, "cache_write_tokens": 83998, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 100, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.011760275, "exact_decimal_usd": "0.011760275", "known_records": 80, "missing_records": 0 }, "pilot": { "reported_usd": 0.002694675, "exact_decimal_usd": "0.002694675", "known_records": 20, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:34:21.852048-04:00", "last": "2026-10-15T10:53:31.425935-04:00" }, "identities": [ [ "openai/gpt-6-luna", "OpenAI" ] ] }, "2": { "scheduled": 100, "completed_first": 100, "correct_first": 64, "accuracy_scheduled": 0.64, "usable_first": 100, "coverage": 1.0, "accuracy_usable": 0.64, "wilson95": [ 0.5423540245160874, 0.7272877959859567 ], "statuses_first": { "ok": 100 }, "eventual_correct": 64, "eventual_usable": 100, "retries": 0, "first_cost": { "reported_usd": 0.01512675, "exact_decimal_usd": "0.015126750", "known_records": 100, "missing_records": 0 }, "core_cost": { "reported_usd": 0.01512675, "exact_decimal_usd": "0.015126750", "known_records": 100, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.0001512675, "first_cost_per_first_correct_usd": 0.00023635546875, "core_cost_per_first_correct_usd": 0.00023635546875, "latency_first_all_ms": { "median": 2604.6565834840294, "p90": 4887.694804248167, "p95": 5252.7889413322555 }, "latency_first_usable_ms": { "median": 2604.6565834840294, "p90": 4887.694804248167, "p95": 5252.7889413322555 }, "tokens_first": { "prompt_tokens": { "total": 125285, "covered_requests": 100 }, "completion_tokens": { "total": 500, "covered_requests": 100 } }, "cached_tokens": 0, "cache_write_tokens": 93930, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 100, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.01512675, "exact_decimal_usd": "0.015126750", "known_records": 100, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:53:33.528239-04:00", "last": "2026-10-15T11:03:31.672835-04:00" }, "identities": [ [ "openai/gpt-6-luna", "OpenAI" ] ] } } }, "families_at_least_8_of_10": [ "V01", "V15", "V16", "V20" ], "families_at_most_4_of_10": [ "V04", "V06", "V07", "V10", "V12" ] }, "openai/gpt-5.4-nano": { "scheduled": 200, "completed_first": 200, "correct_first": 70, "accuracy_scheduled": 0.35, "usable_first": 196, "coverage": 0.98, "accuracy_usable": 0.35714285714285715, "wilson95": [ 0.28728824127358077, 0.4183653566395992 ], "statuses_first": { "ok": 196, "invalid_output": 4 }, "eventual_correct": 70, "eventual_usable": 196, "retries": 0, "first_cost": { "reported_usd": 0.0505245, "exact_decimal_usd": "0.05052450", "known_records": 200, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0505245, "exact_decimal_usd": "0.05052450", "known_records": 200, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.0002526225, "first_cost_per_first_correct_usd": 0.0007217785714285714, "core_cost_per_first_correct_usd": 0.0007217785714285714, "latency_first_all_ms": { "median": 1604.5854165276978, "p90": 2655.022999359062, "p95": 2977.7548517973614 }, "latency_first_usable_ms": { "median": 1604.5854165276978, "p90": 2659.8973329819273, "p95": 2993.473926995648 }, "tokens_first": { "prompt_tokens": { "total": 246335, "covered_requests": 200 }, "completion_tokens": { "total": 1006, "covered_requests": 200 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 200, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.0457945, "exact_decimal_usd": "0.04579450", "known_records": 180, "missing_records": 0 }, "pilot": { "reported_usd": 0.00473, "exact_decimal_usd": "0.00473000", "known_records": 20, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:34:19.268947-04:00", "last": "2026-10-15T11:03:33.731149-04:00" }, "identities": [ [ "openai/gpt-5.4-nano", "OpenAI" ] ], "groups": { "family_id": { "V01": { "scheduled": 10, "completed_first": 10, "correct_first": 6, "accuracy_scheduled": 0.6, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.6, "wilson95": [ 0.31267376973365824, 0.8318196702937638 ], "statuses_first": { "ok": 10 }, "eventual_correct": 6, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.0019471, "exact_decimal_usd": "0.00194710", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0019471, "exact_decimal_usd": "0.00194710", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00019471, "first_cost_per_first_correct_usd": 0.00032451666666666665, "core_cost_per_first_correct_usd": 0.00032451666666666665, "latency_first_all_ms": { "median": 1851.8058125046082, "p90": 2915.079937508562, "p95": 3742.4040937708905 }, "latency_first_usable_ms": { "median": 1851.8058125046082, "p90": 2915.079937508562, "p95": 3742.4040937708905 }, "tokens_first": { "prompt_tokens": { "total": 9423, "covered_requests": 10 }, "completion_tokens": { "total": 50, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.00175405, "exact_decimal_usd": "0.00175405", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 0.00019305, "exact_decimal_usd": "0.00019305", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:07.642291-04:00", "last": "2026-10-15T11:03:08.130176-04:00" }, "identities": [ [ "openai/gpt-5.4-nano", "OpenAI" ] ] }, "V02": { "scheduled": 10, "completed_first": 10, "correct_first": 4, "accuracy_scheduled": 0.4, "usable_first": 8, "coverage": 0.8, "accuracy_usable": 0.5, "wilson95": [ 0.16818032970623614, 0.6873262302663417 ], "statuses_first": { "ok": 8, "invalid_output": 2 }, "eventual_correct": 4, "eventual_usable": 8, "retries": 0, "first_cost": { "reported_usd": 0.0017359, "exact_decimal_usd": "0.00173590", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0017359, "exact_decimal_usd": "0.00173590", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00017359, "first_cost_per_first_correct_usd": 0.000433975, "core_cost_per_first_correct_usd": 0.000433975, "latency_first_all_ms": { "median": 1553.697646013461, "p90": 2440.817196521675, "p95": 2580.6630777457026 }, "latency_first_usable_ms": { "median": 1553.697646013461, "p90": 2502.970921510132, "p95": 2611.7399402399315 }, "tokens_first": { "prompt_tokens": { "total": 8367, "covered_requests": 10 }, "completion_tokens": { "total": 50, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.00163185, "exact_decimal_usd": "0.00163185", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 0.00010405, "exact_decimal_usd": "0.00010405", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:20.783239-04:00", "last": "2026-10-15T11:01:56.226495-04:00" }, "identities": [ [ "openai/gpt-5.4-nano", "OpenAI" ] ] }, "V03": { "scheduled": 10, "completed_first": 10, "correct_first": 3, "accuracy_scheduled": 0.3, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.3, "wilson95": [ 0.10779126740630099, 0.6032218525388546 ], "statuses_first": { "ok": 10 }, "eventual_correct": 3, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.0023639, "exact_decimal_usd": "0.00236390", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0023639, "exact_decimal_usd": "0.00236390", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00023639, "first_cost_per_first_correct_usd": 0.0007879666666666666, "core_cost_per_first_correct_usd": 0.0007879666666666666, "latency_first_all_ms": { "median": 1368.0678959935904, "p90": 2200.2253252896476, "p95": 2499.9592251493586 }, "latency_first_usable_ms": { "median": 1368.0678959935904, "p90": 2200.2253252896476, "p95": 2499.9592251493586 }, "tokens_first": { "prompt_tokens": { "total": 11507, "covered_requests": 10 }, "completion_tokens": { "total": 50, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.00208705, "exact_decimal_usd": "0.00208705", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 0.00027685, "exact_decimal_usd": "0.00027685", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:12.348498-04:00", "last": "2026-10-15T11:02:29.061261-04:00" }, "identities": [ [ "openai/gpt-5.4-nano", "OpenAI" ] ] }, "V04": { "scheduled": 10, "completed_first": 10, "correct_first": 3, "accuracy_scheduled": 0.3, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.3, "wilson95": [ 0.10779126740630099, 0.6032218525388546 ], "statuses_first": { "ok": 10 }, "eventual_correct": 3, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.0023879, "exact_decimal_usd": "0.00238790", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0023879, "exact_decimal_usd": "0.00238790", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00023879, "first_cost_per_first_correct_usd": 0.0007959666666666667, "core_cost_per_first_correct_usd": 0.0007959666666666667, "latency_first_all_ms": { "median": 1392.9223125160206, "p90": 2049.8541916022077, "p95": 2079.531428811606 }, "latency_first_usable_ms": { "median": 1392.9223125160206, "p90": 2049.8541916022077, "p95": 2079.531428811606 }, "tokens_first": { "prompt_tokens": { "total": 11627, "covered_requests": 10 }, "completion_tokens": { "total": 50, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.00221905, "exact_decimal_usd": "0.00221905", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 0.00016885, "exact_decimal_usd": "0.00016885", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:06.808309-04:00", "last": "2026-10-15T11:00:48.421728-04:00" }, "identities": [ [ "openai/gpt-5.4-nano", "OpenAI" ] ] }, "V05": { "scheduled": 10, "completed_first": 10, "correct_first": 3, "accuracy_scheduled": 0.3, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.3, "wilson95": [ 0.10779126740630099, 0.6032218525388546 ], "statuses_first": { "ok": 10 }, "eventual_correct": 3, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.0037515, "exact_decimal_usd": "0.00375150", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0037515, "exact_decimal_usd": "0.00375150", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00037515, "first_cost_per_first_correct_usd": 0.0012505, "core_cost_per_first_correct_usd": 0.0012505, "latency_first_all_ms": { "median": 1526.0820834955666, "p90": 2195.6440840032883, "p95": 2252.6337715040427 }, "latency_first_usable_ms": { "median": 1526.0820834955666, "p90": 2195.6440840032883, "p95": 2252.6337715040427 }, "tokens_first": { "prompt_tokens": { "total": 18445, "covered_requests": 10 }, "completion_tokens": { "total": 50, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.00326105, "exact_decimal_usd": "0.00326105", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 0.00049045, "exact_decimal_usd": "0.00049045", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:36:02.774518-04:00", "last": "2026-10-15T11:01:51.273809-04:00" }, "identities": [ [ "openai/gpt-5.4-nano", "OpenAI" ] ] }, "V06": { "scheduled": 10, "completed_first": 10, "correct_first": 4, "accuracy_scheduled": 0.4, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.4, "wilson95": [ 0.16818032970623614, 0.6873262302663417 ], "statuses_first": { "ok": 10 }, "eventual_correct": 4, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.0018051, "exact_decimal_usd": "0.00180510", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0018051, "exact_decimal_usd": "0.00180510", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00018051, "first_cost_per_first_correct_usd": 0.000451275, "core_cost_per_first_correct_usd": 0.000451275, "latency_first_all_ms": { "median": 1428.611416980857, "p90": 2399.617932800902, "p95": 2463.7913834041683 }, "latency_first_usable_ms": { "median": 1428.611416980857, "p90": 2399.617932800902, "p95": 2463.7913834041683 }, "tokens_first": { "prompt_tokens": { "total": 8713, "covered_requests": 10 }, "completion_tokens": { "total": 50, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.00175465, "exact_decimal_usd": "0.00175465", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 5.045e-05, "exact_decimal_usd": "0.00005045", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:36:15.132673-04:00", "last": "2026-10-15T11:03:18.625909-04:00" }, "identities": [ [ "openai/gpt-5.4-nano", "OpenAI" ] ] }, "V07": { "scheduled": 10, "completed_first": 10, "correct_first": 2, "accuracy_scheduled": 0.2, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.2, "wilson95": [ 0.05668215145437519, 0.5098375284633583 ], "statuses_first": { "ok": 10 }, "eventual_correct": 2, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.0018725, "exact_decimal_usd": "0.00187250", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0018725, "exact_decimal_usd": "0.00187250", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00018725, "first_cost_per_first_correct_usd": 0.00093625, "core_cost_per_first_correct_usd": 0.00093625, "latency_first_all_ms": { "median": 1241.0856869828422, "p90": 1884.53443312319, "p95": 1928.237196081318 }, "latency_first_usable_ms": { "median": 1241.0856869828422, "p90": 1884.53443312319, "p95": 1928.237196081318 }, "tokens_first": { "prompt_tokens": { "total": 9050, "covered_requests": 10 }, "completion_tokens": { "total": 50, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.00167525, "exact_decimal_usd": "0.00167525", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 0.00019725, "exact_decimal_usd": "0.00019725", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:36:22.403475-04:00", "last": "2026-10-15T11:02:44.073465-04:00" }, "identities": [ [ "openai/gpt-5.4-nano", "OpenAI" ] ] }, "V08": { "scheduled": 10, "completed_first": 10, "correct_first": 4, "accuracy_scheduled": 0.4, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.4, "wilson95": [ 0.16818032970623614, 0.6873262302663417 ], "statuses_first": { "ok": 10 }, "eventual_correct": 4, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.0027725, "exact_decimal_usd": "0.00277250", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0027725, "exact_decimal_usd": "0.00277250", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00027725, "first_cost_per_first_correct_usd": 0.000693125, "core_cost_per_first_correct_usd": 0.000693125, "latency_first_all_ms": { "median": 1627.850895485608, "p90": 1833.2066086004488, "p95": 1871.5671582904179 }, "latency_first_usable_ms": { "median": 1627.850895485608, "p90": 1833.2066086004488, "p95": 1871.5671582904179 }, "tokens_first": { "prompt_tokens": { "total": 13550, "covered_requests": 10 }, "completion_tokens": { "total": 50, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.00259865, "exact_decimal_usd": "0.00259865", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 0.00017385, "exact_decimal_usd": "0.00017385", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:17.988218-04:00", "last": "2026-10-15T11:03:33.731149-04:00" }, "identities": [ [ "openai/gpt-5.4-nano", "OpenAI" ] ] }, "V09": { "scheduled": 10, "completed_first": 10, "correct_first": 3, "accuracy_scheduled": 0.3, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.3, "wilson95": [ 0.10779126740630099, 0.6032218525388546 ], "statuses_first": { "ok": 10 }, "eventual_correct": 3, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.0019447, "exact_decimal_usd": "0.00194470", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0019447, "exact_decimal_usd": "0.00194470", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00019447, "first_cost_per_first_correct_usd": 0.0006482333333333333, "core_cost_per_first_correct_usd": 0.0006482333333333333, "latency_first_all_ms": { "median": 1781.036228989251, "p90": 2424.1510124877095, "p95": 2493.7053187401034 }, "latency_first_usable_ms": { "median": 1781.036228989251, "p90": 2424.1510124877095, "p95": 2493.7053187401034 }, "tokens_first": { "prompt_tokens": { "total": 9411, "covered_requests": 10 }, "completion_tokens": { "total": 50, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.00152025, "exact_decimal_usd": "0.00152025", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 0.00042445, "exact_decimal_usd": "0.00042445", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:34:19.268947-04:00", "last": "2026-10-15T11:03:20.145361-04:00" }, "identities": [ [ "openai/gpt-5.4-nano", "OpenAI" ] ] }, "V10": { "scheduled": 10, "completed_first": 10, "correct_first": 1, "accuracy_scheduled": 0.1, "usable_first": 9, "coverage": 0.9, "accuracy_usable": 0.1111111111111111, "wilson95": [ 0.017876213095072896, 0.4041500267952385 ], "statuses_first": { "ok": 9, "invalid_output": 1 }, "eventual_correct": 1, "eventual_usable": 9, "retries": 0, "first_cost": { "reported_usd": 0.0037373, "exact_decimal_usd": "0.00373730", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0037373, "exact_decimal_usd": "0.00373730", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00037372999999999996, "first_cost_per_first_correct_usd": 0.0037373, "core_cost_per_first_correct_usd": 0.0037373, "latency_first_all_ms": { "median": 1497.710312018171, "p90": 1734.0370750112922, "p95": 1890.1604124956061 }, "latency_first_usable_ms": { "median": 1536.412957997527, "p90": 1768.7311500078067, "p95": 1907.5074499938637 }, "tokens_first": { "prompt_tokens": { "total": 18374, "covered_requests": 10 }, "completion_tokens": { "total": 50, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.00347805, "exact_decimal_usd": "0.00347805", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 0.00025925, "exact_decimal_usd": "0.00025925", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:57.512762-04:00", "last": "2026-10-15T11:00:00.949711-04:00" }, "identities": [ [ "openai/gpt-5.4-nano", "OpenAI" ] ] }, "V11": { "scheduled": 10, "completed_first": 10, "correct_first": 3, "accuracy_scheduled": 0.3, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.3, "wilson95": [ 0.10779126740630099, 0.6032218525388546 ], "statuses_first": { "ok": 10 }, "eventual_correct": 3, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.0033203, "exact_decimal_usd": "0.00332030", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0033203, "exact_decimal_usd": "0.00332030", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00033203, "first_cost_per_first_correct_usd": 0.0011067666666666667, "core_cost_per_first_correct_usd": 0.0011067666666666667, "latency_first_all_ms": { "median": 2789.458270970499, "p90": 3871.3047744240607, "p95": 4266.996449732686 }, "latency_first_usable_ms": { "median": 2789.458270970499, "p90": 3871.3047744240607, "p95": 4266.996449732686 }, "tokens_first": { "prompt_tokens": { "total": 16289, "covered_requests": 10 }, "completion_tokens": { "total": 50, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.00293585, "exact_decimal_usd": "0.00293585", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 0.00038445, "exact_decimal_usd": "0.00038445", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:45.206853-04:00", "last": "2026-10-15T11:03:02.383844-04:00" }, "identities": [ [ "openai/gpt-5.4-nano", "OpenAI" ] ] }, "V12": { "scheduled": 10, "completed_first": 10, "correct_first": 4, "accuracy_scheduled": 0.4, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.4, "wilson95": [ 0.16818032970623614, 0.6873262302663417 ], "statuses_first": { "ok": 10 }, "eventual_correct": 4, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.0021019, "exact_decimal_usd": "0.00210190", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0021019, "exact_decimal_usd": "0.00210190", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00021019, "first_cost_per_first_correct_usd": 0.000525475, "core_cost_per_first_correct_usd": 0.000525475, "latency_first_all_ms": { "median": 1330.7068335125223, "p90": 2738.7199038115787, "p95": 3969.1957224014886 }, "latency_first_usable_ms": { "median": 1330.7068335125223, "p90": 2738.7199038115787, "p95": 3969.1957224014886 }, "tokens_first": { "prompt_tokens": { "total": 10197, "covered_requests": 10 }, "completion_tokens": { "total": 50, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.00180965, "exact_decimal_usd": "0.00180965", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 0.00029225, "exact_decimal_usd": "0.00029225", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:01.358273-04:00", "last": "2026-10-15T11:02:07.286847-04:00" }, "identities": [ [ "openai/gpt-5.4-nano", "OpenAI" ] ] }, "V13": { "scheduled": 10, "completed_first": 10, "correct_first": 1, "accuracy_scheduled": 0.1, "usable_first": 9, "coverage": 0.9, "accuracy_usable": 0.1111111111111111, "wilson95": [ 0.017876213095072896, 0.4041500267952385 ], "statuses_first": { "ok": 9, "invalid_output": 1 }, "eventual_correct": 1, "eventual_usable": 9, "retries": 0, "first_cost": { "reported_usd": 0.0017583, "exact_decimal_usd": "0.00175830", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0017583, "exact_decimal_usd": "0.00175830", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00017583, "first_cost_per_first_correct_usd": 0.0017583, "core_cost_per_first_correct_usd": 0.0017583, "latency_first_all_ms": { "median": 1693.450708000455, "p90": 2206.930258282227, "p95": 2522.876983144669 }, "latency_first_usable_ms": { "median": 1668.5943329939619, "p90": 2277.1406415849924, "p95": 2557.9821747960523 }, "tokens_first": { "prompt_tokens": { "total": 8479, "covered_requests": 10 }, "completion_tokens": { "total": 50, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.00160905, "exact_decimal_usd": "0.00160905", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 0.00014925, "exact_decimal_usd": "0.00014925", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:36:19.654681-04:00", "last": "2026-10-15T11:01:38.627081-04:00" }, "identities": [ [ "openai/gpt-5.4-nano", "OpenAI" ] ] }, "V14": { "scheduled": 10, "completed_first": 10, "correct_first": 3, "accuracy_scheduled": 0.3, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.3, "wilson95": [ 0.10779126740630099, 0.6032218525388546 ], "statuses_first": { "ok": 10 }, "eventual_correct": 3, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.0029363, "exact_decimal_usd": "0.00293630", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0029363, "exact_decimal_usd": "0.00293630", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00029363, "first_cost_per_first_correct_usd": 0.0009787666666666668, "core_cost_per_first_correct_usd": 0.0009787666666666668, "latency_first_all_ms": { "median": 2090.433895966271, "p90": 3105.3232204925735, "p95": 3299.082026732503 }, "latency_first_usable_ms": { "median": 2090.433895966271, "p90": 3105.3232204925735, "p95": 3299.082026732503 }, "tokens_first": { "prompt_tokens": { "total": 14369, "covered_requests": 10 }, "completion_tokens": { "total": 50, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.00263645, "exact_decimal_usd": "0.00263645", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 0.00029985, "exact_decimal_usd": "0.00029985", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:25.143392-04:00", "last": "2026-10-15T11:02:33.929912-04:00" }, "identities": [ [ "openai/gpt-5.4-nano", "OpenAI" ] ] }, "V15": { "scheduled": 10, "completed_first": 10, "correct_first": 7, "accuracy_scheduled": 0.7, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.7, "wilson95": [ 0.39677814746114537, 0.8922087325936989 ], "statuses_first": { "ok": 10 }, "eventual_correct": 7, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.0015487, "exact_decimal_usd": "0.00154870", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0015487, "exact_decimal_usd": "0.00154870", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00015487, "first_cost_per_first_correct_usd": 0.00022124285714285715, "core_cost_per_first_correct_usd": 0.00022124285714285715, "latency_first_all_ms": { "median": 1456.3130000024103, "p90": 2110.162562795449, "p95": 2402.716656413394 }, "latency_first_usable_ms": { "median": 1456.3130000024103, "p90": 2110.162562795449, "p95": 2402.716656413394 }, "tokens_first": { "prompt_tokens": { "total": 7431, "covered_requests": 10 }, "completion_tokens": { "total": 50, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.00134345, "exact_decimal_usd": "0.00134345", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 0.00020525, "exact_decimal_usd": "0.00020525", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:33.610890-04:00", "last": "2026-10-15T11:02:38.326085-04:00" }, "identities": [ [ "openai/gpt-5.4-nano", "OpenAI" ] ] }, "V16": { "scheduled": 10, "completed_first": 10, "correct_first": 5, "accuracy_scheduled": 0.5, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.5, "wilson95": [ 0.236593090512564, 0.7634069094874361 ], "statuses_first": { "ok": 10 }, "eventual_correct": 5, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.0040431, "exact_decimal_usd": "0.00404310", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0040431, "exact_decimal_usd": "0.00404310", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00040431, "first_cost_per_first_correct_usd": 0.00080862, "core_cost_per_first_correct_usd": 0.00080862, "latency_first_all_ms": { "median": 1849.047936993884, "p90": 2651.311154762516, "p95": 2896.886035855277 }, "latency_first_usable_ms": { "median": 1849.047936993884, "p90": 2651.311154762516, "p95": 2896.886035855277 }, "tokens_first": { "prompt_tokens": { "total": 19903, "covered_requests": 10 }, "completion_tokens": { "total": 50, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.00367845, "exact_decimal_usd": "0.00367845", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 0.00036465, "exact_decimal_usd": "0.00036465", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:37.973853-04:00", "last": "2026-10-15T10:57:50.742735-04:00" }, "identities": [ [ "openai/gpt-5.4-nano", "OpenAI" ] ] }, "V17": { "scheduled": 10, "completed_first": 10, "correct_first": 5, "accuracy_scheduled": 0.5, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.5, "wilson95": [ 0.236593090512564, 0.7634069094874361 ], "statuses_first": { "ok": 10 }, "eventual_correct": 5, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.0035947, "exact_decimal_usd": "0.00359470", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0035947, "exact_decimal_usd": "0.00359470", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00035947, "first_cost_per_first_correct_usd": 0.00071894, "core_cost_per_first_correct_usd": 0.00071894, "latency_first_all_ms": { "median": 1973.4128124837298, "p90": 2386.3744331116322, "p95": 2526.5453835570947 }, "latency_first_usable_ms": { "median": 1973.4128124837298, "p90": 2386.3744331116322, "p95": 2526.5453835570947 }, "tokens_first": { "prompt_tokens": { "total": 17661, "covered_requests": 10 }, "completion_tokens": { "total": 50, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.00339585, "exact_decimal_usd": "0.00339585", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 0.00019885, "exact_decimal_usd": "0.00019885", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:36:29.189817-04:00", "last": "2026-10-15T11:01:23.782436-04:00" }, "identities": [ [ "openai/gpt-5.4-nano", "OpenAI" ] ] }, "V18": { "scheduled": 10, "completed_first": 10, "correct_first": 2, "accuracy_scheduled": 0.2, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.2, "wilson95": [ 0.05668215145437519, 0.5098375284633583 ], "statuses_first": { "ok": 10 }, "eventual_correct": 2, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.0023503, "exact_decimal_usd": "0.00235030", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0023503, "exact_decimal_usd": "0.00235030", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00023503, "first_cost_per_first_correct_usd": 0.00117515, "core_cost_per_first_correct_usd": 0.00117515, "latency_first_all_ms": { "median": 1729.7380625095684, "p90": 3020.19128719694, "p95": 3228.839206093107 }, "latency_first_usable_ms": { "median": 1729.7380625095684, "p90": 3020.19128719694, "p95": 3228.839206093107 }, "tokens_first": { "prompt_tokens": { "total": 11439, "covered_requests": 10 }, "completion_tokens": { "total": 50, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.00219165, "exact_decimal_usd": "0.00219165", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 0.00015865, "exact_decimal_usd": "0.00015865", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:48.335622-04:00", "last": "2026-10-15T10:59:06.144751-04:00" }, "identities": [ [ "openai/gpt-5.4-nano", "OpenAI" ] ] }, "V19": { "scheduled": 10, "completed_first": 10, "correct_first": 3, "accuracy_scheduled": 0.3, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.3, "wilson95": [ 0.10779126740630099, 0.6032218525388546 ], "statuses_first": { "ok": 10 }, "eventual_correct": 3, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.0023079, "exact_decimal_usd": "0.00230790", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0023079, "exact_decimal_usd": "0.00230790", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00023078999999999998, "first_cost_per_first_correct_usd": 0.0007693, "core_cost_per_first_correct_usd": 0.0007693, "latency_first_all_ms": { "median": 1677.883271011524, "p90": 1858.644437487237, "p95": 2096.9180312415115 }, "latency_first_usable_ms": { "median": 1677.883271011524, "p90": 1858.644437487237, "p95": 2096.9180312415115 }, "tokens_first": { "prompt_tokens": { "total": 11227, "covered_requests": 10 }, "completion_tokens": { "total": 50, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.00216745, "exact_decimal_usd": "0.00216745", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 0.00014045, "exact_decimal_usd": "0.00014045", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:55.925475-04:00", "last": "2026-10-15T11:00:49.539123-04:00" }, "identities": [ [ "openai/gpt-5.4-nano", "OpenAI" ] ] }, "V20": { "scheduled": 10, "completed_first": 10, "correct_first": 4, "accuracy_scheduled": 0.4, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.4, "wilson95": [ 0.16818032970623614, 0.6873262302663417 ], "statuses_first": { "ok": 10 }, "eventual_correct": 4, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.0022446, "exact_decimal_usd": "0.00224460", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0022446, "exact_decimal_usd": "0.00224460", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00022445999999999998, "first_cost_per_first_correct_usd": 0.00056115, "core_cost_per_first_correct_usd": 0.00056115, "latency_first_all_ms": { "median": 1592.4897294898983, "p90": 2207.024033612106, "p95": 2274.627495801542 }, "latency_first_usable_ms": { "median": 1592.4897294898983, "p90": 2207.024033612106, "p95": 2274.627495801542 }, "tokens_first": { "prompt_tokens": { "total": 10873, "covered_requests": 10 }, "completion_tokens": { "total": 56, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.00204675, "exact_decimal_usd": "0.00204675", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 0.00019785, "exact_decimal_usd": "0.00019785", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:36:06.533509-04:00", "last": "2026-10-15T11:02:56.158525-04:00" }, "identities": [ [ "openai/gpt-5.4-nano", "OpenAI" ] ] } }, "image_count": { "1": { "scheduled": 177, "completed_first": 177, "correct_first": 58, "accuracy_scheduled": 0.327683615819209, "usable_first": 173, "coverage": 0.9774011299435028, "accuracy_usable": 0.3352601156069364, "wilson95": [ 0.2628371166532165, 0.3998508510407289 ], "statuses_first": { "ok": 173, "invalid_output": 4 }, "eventual_correct": 58, "eventual_usable": 173, "retries": 0, "first_cost": { "reported_usd": 0.04214115, "exact_decimal_usd": "0.04214115", "known_records": 177, "missing_records": 0 }, "core_cost": { "reported_usd": 0.04214115, "exact_decimal_usd": "0.04214115", "known_records": 177, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.000238085593220339, "first_cost_per_first_correct_usd": 0.000726571551724138, "core_cost_per_first_correct_usd": 0.000726571551724138, "latency_first_all_ms": { "median": 1546.6453339904547, "p90": 2658.678749576211, "p95": 2989.544158196076 }, "latency_first_usable_ms": { "median": 1546.6453339904547, "p90": 2663.5530831990764, "p95": 3005.2632333943625 }, "tokens_first": { "prompt_tokens": { "total": 205137, "covered_requests": 177 }, "completion_tokens": { "total": 891, "covered_requests": 177 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 177, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.0383991, "exact_decimal_usd": "0.03839910", "known_records": 160, "missing_records": 0 }, "pilot": { "reported_usd": 0.00374205, "exact_decimal_usd": "0.00374205", "known_records": 17, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:01.358273-04:00", "last": "2026-10-15T11:03:33.731149-04:00" }, "identities": [ [ "openai/gpt-5.4-nano", "OpenAI" ] ] }, "2": { "scheduled": 19, "completed_first": 19, "correct_first": 9, "accuracy_scheduled": 0.47368421052631576, "usable_first": 19, "coverage": 1.0, "accuracy_usable": 0.47368421052631576, "wilson95": [ 0.2732980516117956, 0.6829219075488682 ], "statuses_first": { "ok": 19 }, "eventual_correct": 9, "eventual_usable": 19, "retries": 0, "first_cost": { "reported_usd": 0.00741755, "exact_decimal_usd": "0.00741755", "known_records": 19, "missing_records": 0 }, "core_cost": { "reported_usd": 0.00741755, "exact_decimal_usd": "0.00741755", "known_records": 19, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.0003903973684210526, "first_cost_per_first_correct_usd": 0.0008241722222222223, "core_cost_per_first_correct_usd": 0.0008241722222222223, "latency_first_all_ms": { "median": 1850.6625829613768, "p90": 2610.734433971811, "p95": 2714.2907922971053 }, "latency_first_usable_ms": { "median": 1850.6625829613768, "p90": 2610.734433971811, "p95": 2714.2907922971053 }, "tokens_first": { "prompt_tokens": { "total": 36494, "covered_requests": 19 }, "completion_tokens": { "total": 95, "covered_requests": 19 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 19, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.00685405, "exact_decimal_usd": "0.00685405", "known_records": 17, "missing_records": 0 }, "pilot": { "reported_usd": 0.0005635, "exact_decimal_usd": "0.00056350", "known_records": 2, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:37.973853-04:00", "last": "2026-10-15T11:01:23.782436-04:00" }, "identities": [ [ "openai/gpt-5.4-nano", "OpenAI" ] ] }, "3": { "scheduled": 4, "completed_first": 4, "correct_first": 3, "accuracy_scheduled": 0.75, "usable_first": 4, "coverage": 1.0, "accuracy_usable": 0.75, "wilson95": [ 0.30064184258240184, 0.9544127391902995 ], "statuses_first": { "ok": 4 }, "eventual_correct": 3, "eventual_usable": 4, "retries": 0, "first_cost": { "reported_usd": 0.0009658, "exact_decimal_usd": "0.00096580", "known_records": 4, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0009658, "exact_decimal_usd": "0.00096580", "known_records": 4, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00024145, "first_cost_per_first_correct_usd": 0.0003219333333333333, "core_cost_per_first_correct_usd": 0.0003219333333333333, "latency_first_all_ms": { "median": 2403.4853334887885, "p90": 2516.8900874909014, "p95": 2540.0748562416993 }, "latency_first_usable_ms": { "median": 2403.4853334887885, "p90": 2516.8900874909014, "p95": 2540.0748562416993 }, "tokens_first": { "prompt_tokens": { "total": 4704, "covered_requests": 4 }, "completion_tokens": { "total": 20, "covered_requests": 4 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 4, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.00054135, "exact_decimal_usd": "0.00054135", "known_records": 3, "missing_records": 0 }, "pilot": { "reported_usd": 0.00042445, "exact_decimal_usd": "0.00042445", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:34:19.268947-04:00", "last": "2026-10-15T11:01:03.612376-04:00" }, "identities": [ [ "openai/gpt-5.4-nano", "OpenAI" ] ] } }, "analysis_subset": { "perception_control": { "scheduled": 13, "completed_first": 13, "correct_first": 9, "accuracy_scheduled": 0.6923076923076923, "usable_first": 13, "coverage": 1.0, "accuracy_usable": 0.6923076923076923, "wilson95": [ 0.42369343180549035, 0.8731929634428948 ], "statuses_first": { "ok": 13 }, "eventual_correct": 9, "eventual_usable": 13, "retries": 0, "first_cost": { "reported_usd": 0.00271005, "exact_decimal_usd": "0.00271005", "known_records": 13, "missing_records": 0 }, "core_cost": { "reported_usd": 0.00271005, "exact_decimal_usd": "0.00271005", "known_records": 13, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00020846538461538462, "first_cost_per_first_correct_usd": 0.0003011166666666667, "core_cost_per_first_correct_usd": 0.0003011166666666667, "latency_first_all_ms": { "median": 1569.3130000145175, "p90": 2331.7937169922516, "p95": 2558.8429501978676 }, "latency_first_usable_ms": { "median": 1569.3130000145175, "p90": 2331.7937169922516, "p95": 2558.8429501978676 }, "tokens_first": { "prompt_tokens": { "total": 13144, "covered_requests": 13 }, "completion_tokens": { "total": 65, "covered_requests": 13 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 13, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.0025412, "exact_decimal_usd": "0.00254120", "known_records": 12, "missing_records": 0 }, "pilot": { "reported_usd": 0.00016885, "exact_decimal_usd": "0.00016885", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:06.808309-04:00", "last": "2026-10-15T11:01:03.612376-04:00" }, "identities": [ [ "openai/gpt-5.4-nano", "OpenAI" ] ] }, "reasoning_and_grounded_decision": { "scheduled": 187, "completed_first": 187, "correct_first": 61, "accuracy_scheduled": 0.32620320855614976, "usable_first": 183, "coverage": 0.9786096256684492, "accuracy_usable": 0.3333333333333333, "wilson95": [ 0.2630945298963469, 0.3963086186560281 ], "statuses_first": { "ok": 183, "invalid_output": 4 }, "eventual_correct": 61, "eventual_usable": 183, "retries": 0, "first_cost": { "reported_usd": 0.04781445, "exact_decimal_usd": "0.04781445", "known_records": 187, "missing_records": 0 }, "core_cost": { "reported_usd": 0.04781445, "exact_decimal_usd": "0.04781445", "known_records": 187, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00025569224598930484, "first_cost_per_first_correct_usd": 0.0007838434426229509, "core_cost_per_first_correct_usd": 0.0007838434426229509, "latency_first_all_ms": { "median": 1605.0389170413837, "p90": 2658.678749576211, "p95": 3028.841846191789 }, "latency_first_usable_ms": { "median": 1605.0389170413837, "p90": 2663.5530831990764, "p95": 3044.5609213900757 }, "tokens_first": { "prompt_tokens": { "total": 233191, "covered_requests": 187 }, "completion_tokens": { "total": 941, "covered_requests": 187 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 187, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.0432533, "exact_decimal_usd": "0.04325330", "known_records": 168, "missing_records": 0 }, "pilot": { "reported_usd": 0.00456115, "exact_decimal_usd": "0.00456115", "known_records": 19, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:34:19.268947-04:00", "last": "2026-10-15T11:03:33.731149-04:00" }, "identities": [ [ "openai/gpt-5.4-nano", "OpenAI" ] ] } }, "difficulty_proposed": { "demanding": { "scheduled": 55, "completed_first": 55, "correct_first": 15, "accuracy_scheduled": 0.2727272727272727, "usable_first": 54, "coverage": 0.9818181818181818, "accuracy_usable": 0.2777777777777778, "wilson95": [ 0.17280751939105832, 0.40232198111670814 ], "statuses_first": { "ok": 54, "invalid_output": 1 }, "eventual_correct": 15, "eventual_usable": 54, "retries": 0, "first_cost": { "reported_usd": 0.01456005, "exact_decimal_usd": "0.01456005", "known_records": 55, "missing_records": 0 }, "core_cost": { "reported_usd": 0.01456005, "exact_decimal_usd": "0.01456005", "known_records": 55, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.0002647281818181818, "first_cost_per_first_correct_usd": 0.00097067, "core_cost_per_first_correct_usd": 0.00097067, "latency_first_all_ms": { "median": 1568.755166954361, "p90": 2387.9343336215243, "p95": 2758.340699906693 }, "latency_first_usable_ms": { "median": 1586.4435414841864, "p90": 2393.385833723005, "p95": 2773.732441556056 }, "tokens_first": { "prompt_tokens": { "total": 71044, "covered_requests": 55 }, "completion_tokens": { "total": 281, "covered_requests": 55 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 55, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.0134335, "exact_decimal_usd": "0.01343350", "known_records": 48, "missing_records": 0 }, "pilot": { "reported_usd": 0.00112655, "exact_decimal_usd": "0.00112655", "known_records": 7, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:12.348498-04:00", "last": "2026-10-15T11:03:02.383844-04:00" }, "identities": [ [ "openai/gpt-5.4-nano", "OpenAI" ] ] }, "moderate": { "scheduled": 107, "completed_first": 107, "correct_first": 36, "accuracy_scheduled": 0.3364485981308411, "usable_first": 104, "coverage": 0.9719626168224299, "accuracy_usable": 0.34615384615384615, "wilson95": [ 0.2539726954082616, 0.4302609805791796 ], "statuses_first": { "ok": 104, "invalid_output": 3 }, "eventual_correct": 36, "eventual_usable": 104, "retries": 0, "first_cost": { "reported_usd": 0.02721715, "exact_decimal_usd": "0.02721715", "known_records": 107, "missing_records": 0 }, "core_cost": { "reported_usd": 0.02721715, "exact_decimal_usd": "0.02721715", "known_records": 107, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.0002543658878504673, "first_cost_per_first_correct_usd": 0.0007560319444444444, "core_cost_per_first_correct_usd": 0.0007560319444444444, "latency_first_all_ms": { "median": 1614.757958042901, "p90": 2678.1381004140712, "p95": 3006.4893087837827 }, "latency_first_usable_ms": { "median": 1584.3942294886801, "p90": 2686.704425222706, "p95": 3029.4548838865003 }, "tokens_first": { "prompt_tokens": { "total": 132742, "covered_requests": 107 }, "completion_tokens": { "total": 535, "covered_requests": 107 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 107, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.0247369, "exact_decimal_usd": "0.02473690", "known_records": 98, "missing_records": 0 }, "pilot": { "reported_usd": 0.00248025, "exact_decimal_usd": "0.00248025", "known_records": 9, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:34:19.268947-04:00", "last": "2026-10-15T11:03:33.731149-04:00" }, "identities": [ [ "openai/gpt-5.4-nano", "OpenAI" ] ] }, "straightforward": { "scheduled": 38, "completed_first": 38, "correct_first": 19, "accuracy_scheduled": 0.5, "usable_first": 38, "coverage": 1.0, "accuracy_usable": 0.5, "wilson95": [ 0.34849928364395105, 0.6515007163560489 ], "statuses_first": { "ok": 38 }, "eventual_correct": 19, "eventual_usable": 38, "retries": 0, "first_cost": { "reported_usd": 0.0087473, "exact_decimal_usd": "0.00874730", "known_records": 38, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0087473, "exact_decimal_usd": "0.00874730", "known_records": 38, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00023019210526315788, "first_cost_per_first_correct_usd": 0.00046038421052631577, "core_cost_per_first_correct_usd": 0.00046038421052631577, "latency_first_all_ms": { "median": 1652.169312525075, "p90": 2661.5430662583094, "p95": 2788.6758875043593 }, "latency_first_usable_ms": { "median": 1652.169312525075, "p90": 2661.5430662583094, "p95": 2788.6758875043593 }, "tokens_first": { "prompt_tokens": { "total": 42549, "covered_requests": 38 }, "completion_tokens": { "total": 190, "covered_requests": 38 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 38, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.0076241, "exact_decimal_usd": "0.00762410", "known_records": 34, "missing_records": 0 }, "pilot": { "reported_usd": 0.0011232, "exact_decimal_usd": "0.00112320", "known_records": 4, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:06.808309-04:00", "last": "2026-10-15T11:03:26.392370-04:00" }, "identities": [ [ "openai/gpt-5.4-nano", "OpenAI" ] ] } }, "visual_medium": { "chart": { "scheduled": 13, "completed_first": 13, "correct_first": 1, "accuracy_scheduled": 0.07692307692307693, "usable_first": 12, "coverage": 0.9230769230769231, "accuracy_usable": 0.08333333333333333, "wilson95": [ 0.013710421242556586, 0.3331395092109959 ], "statuses_first": { "ok": 12, "invalid_output": 1 }, "eventual_correct": 1, "eventual_usable": 12, "retries": 0, "first_cost": { "reported_usd": 0.00236125, "exact_decimal_usd": "0.00236125", "known_records": 13, "missing_records": 0 }, "core_cost": { "reported_usd": 0.00236125, "exact_decimal_usd": "0.00236125", "known_records": 13, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00018163461538461538, "first_cost_per_first_correct_usd": 0.00236125, "core_cost_per_first_correct_usd": 0.00236125, "latency_first_all_ms": { "median": 1690.9599589998834, "p90": 2698.4029414015827, "p95": 2892.824258003384 }, "latency_first_usable_ms": { "median": 1679.7771459969226, "p90": 2768.613324704347, "p95": 2899.574326752918 }, "tokens_first": { "prompt_tokens": { "total": 11400, "covered_requests": 13 }, "completion_tokens": { "total": 65, "covered_requests": 13 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 13, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.00205335, "exact_decimal_usd": "0.00205335", "known_records": 11, "missing_records": 0 }, "pilot": { "reported_usd": 0.0003079, "exact_decimal_usd": "0.00030790", "known_records": 2, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:48.335622-04:00", "last": "2026-10-15T11:01:38.627081-04:00" }, "identities": [ [ "openai/gpt-5.4-nano", "OpenAI" ] ] }, "diagram": { "scheduled": 92, "completed_first": 92, "correct_first": 26, "accuracy_scheduled": 0.2826086956521739, "usable_first": 91, "coverage": 0.9891304347826086, "accuracy_usable": 0.2857142857142857, "wilson95": [ 0.2007567930217749, 0.3818872892763214 ], "statuses_first": { "ok": 91, "invalid_output": 1 }, "eventual_correct": 26, "eventual_usable": 91, "retries": 0, "first_cost": { "reported_usd": 0.0248527, "exact_decimal_usd": "0.02485270", "known_records": 92, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0248527, "exact_decimal_usd": "0.02485270", "known_records": 92, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.0002701380434782609, "first_cost_per_first_correct_usd": 0.0009558730769230768, "core_cost_per_first_correct_usd": 0.0009558730769230768, "latency_first_all_ms": { "median": 1495.3306245151907, "p90": 2341.527024691459, "p95": 2968.213350279257 }, "latency_first_usable_ms": { "median": 1505.5188330006786, "p90": 2342.230957990978, "p95": 2975.8685419801623 }, "tokens_first": { "prompt_tokens": { "total": 121351, "covered_requests": 92 }, "completion_tokens": { "total": 466, "covered_requests": 92 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 92, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.0224186, "exact_decimal_usd": "0.02241860", "known_records": 82, "missing_records": 0 }, "pilot": { "reported_usd": 0.0024341, "exact_decimal_usd": "0.00243410", "known_records": 10, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:01.358273-04:00", "last": "2026-10-15T11:03:33.731149-04:00" }, "identities": [ [ "openai/gpt-5.4-nano", "OpenAI" ] ] }, "diagram_or_published_illustration": { "scheduled": 8, "completed_first": 8, "correct_first": 3, "accuracy_scheduled": 0.375, "usable_first": 8, "coverage": 1.0, "accuracy_usable": 0.375, "wilson95": [ 0.13684428582359742, 0.6942576053973727 ], "statuses_first": { "ok": 8 }, "eventual_correct": 3, "eventual_usable": 8, "retries": 0, "first_cost": { "reported_usd": 0.0013882, "exact_decimal_usd": "0.00138820", "known_records": 8, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0013882, "exact_decimal_usd": "0.00138820", "known_records": 8, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.000173525, "first_cost_per_first_correct_usd": 0.0004627333333333333, "core_cost_per_first_correct_usd": 0.0004627333333333333, "latency_first_all_ms": { "median": 1587.2239585150965, "p90": 2152.6723418035544, "p95": 2253.9488374080975 }, "latency_first_usable_ms": { "median": 1587.2239585150965, "p90": 2152.6723418035544, "p95": 2253.9488374080975 }, "tokens_first": { "prompt_tokens": { "total": 6691, "covered_requests": 8 }, "completion_tokens": { "total": 40, "covered_requests": 8 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 8, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.00089225, "exact_decimal_usd": "0.00089225", "known_records": 5, "missing_records": 0 }, "pilot": { "reported_usd": 0.00049595, "exact_decimal_usd": "0.00049595", "known_records": 3, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:07.642291-04:00", "last": "2026-10-15T10:56:51.227982-04:00" }, "identities": [ [ "openai/gpt-5.4-nano", "OpenAI" ] ] }, "document": { "scheduled": 12, "completed_first": 12, "correct_first": 3, "accuracy_scheduled": 0.25, "usable_first": 12, "coverage": 1.0, "accuracy_usable": 0.25, "wilson95": [ 0.08894166839405471, 0.5323053349335657 ], "statuses_first": { "ok": 12 }, "eventual_correct": 3, "eventual_usable": 12, "retries": 0, "first_cost": { "reported_usd": 0.0034238, "exact_decimal_usd": "0.00342380", "known_records": 12, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0034238, "exact_decimal_usd": "0.00342380", "known_records": 12, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00028531666666666667, "first_cost_per_first_correct_usd": 0.0011412666666666667, "core_cost_per_first_correct_usd": 0.0011412666666666667, "latency_first_all_ms": { "median": 1769.18524998473, "p90": 3022.638162196381, "p95": 3256.0245142347408 }, "latency_first_usable_ms": { "median": 1769.18524998473, "p90": 3022.638162196381, "p95": 3256.0245142347408 }, "tokens_first": { "prompt_tokens": { "total": 16744, "covered_requests": 12 }, "completion_tokens": { "total": 60, "covered_requests": 12 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 12, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.00312395, "exact_decimal_usd": "0.00312395", "known_records": 11, "missing_records": 0 }, "pilot": { "reported_usd": 0.00029985, "exact_decimal_usd": "0.00029985", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:25.143392-04:00", "last": "2026-10-15T11:02:33.929912-04:00" }, "identities": [ [ "openai/gpt-5.4-nano", "OpenAI" ] ] }, "photograph": { "scheduled": 64, "completed_first": 64, "correct_first": 30, "accuracy_scheduled": 0.46875, "usable_first": 62, "coverage": 0.96875, "accuracy_usable": 0.4838709677419355, "wilson95": [ 0.35175987425176036, 0.5892791294852899 ], "statuses_first": { "ok": 62, "invalid_output": 2 }, "eventual_correct": 30, "eventual_usable": 62, "retries": 0, "first_cost": { "reported_usd": 0.0167026, "exact_decimal_usd": "0.01670260", "known_records": 64, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0167026, "exact_decimal_usd": "0.01670260", "known_records": 64, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.000260978125, "first_cost_per_first_correct_usd": 0.0005567533333333334, "core_cost_per_first_correct_usd": 0.0005567533333333334, "latency_first_all_ms": { "median": 1782.5791249924805, "p90": 2636.684778856579, "p95": 2729.621950100409 }, "latency_first_usable_ms": { "median": 1812.2091249679215, "p90": 2648.0978702544235, "p95": 2730.694066704018 }, "tokens_first": { "prompt_tokens": { "total": 81513, "covered_requests": 64 }, "completion_tokens": { "total": 320, "covered_requests": 64 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 64, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.01571565, "exact_decimal_usd": "0.01571565", "known_records": 61, "missing_records": 0 }, "pilot": { "reported_usd": 0.00098695, "exact_decimal_usd": "0.00098695", "known_records": 3, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:34:19.268947-04:00", "last": "2026-10-15T11:03:20.145361-04:00" }, "identities": [ [ "openai/gpt-5.4-nano", "OpenAI" ] ] }, "screenshot": { "scheduled": 11, "completed_first": 11, "correct_first": 7, "accuracy_scheduled": 0.6363636363636364, "usable_first": 11, "coverage": 1.0, "accuracy_usable": 0.6363636363636364, "wilson95": [ 0.35380117450784887, 0.8483352890463243 ], "statuses_first": { "ok": 11 }, "eventual_correct": 7, "eventual_usable": 11, "retries": 0, "first_cost": { "reported_usd": 0.00179595, "exact_decimal_usd": "0.00179595", "known_records": 11, "missing_records": 0 }, "core_cost": { "reported_usd": 0.00179595, "exact_decimal_usd": "0.00179595", "known_records": 11, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00016326818181818182, "first_cost_per_first_correct_usd": 0.0002565642857142857, "core_cost_per_first_correct_usd": 0.0002565642857142857, "latency_first_all_ms": { "median": 1486.0830419929698, "p90": 2045.150541991461, "p95": 2370.210646011401 }, "latency_first_usable_ms": { "median": 1486.0830419929698, "p90": 2045.150541991461, "p95": 2370.210646011401 }, "tokens_first": { "prompt_tokens": { "total": 8636, "covered_requests": 11 }, "completion_tokens": { "total": 55, "covered_requests": 11 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 11, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.0015907, "exact_decimal_usd": "0.00159070", "known_records": 10, "missing_records": 0 }, "pilot": { "reported_usd": 0.00020525, "exact_decimal_usd": "0.00020525", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:33.610890-04:00", "last": "2026-10-15T11:02:56.158525-04:00" }, "identities": [ [ "openai/gpt-5.4-nano", "OpenAI" ] ] } }, "stage": { "1": { "scheduled": 100, "completed_first": 100, "correct_first": 31, "accuracy_scheduled": 0.31, "usable_first": 98, "coverage": 0.98, "accuracy_usable": 0.3163265306122449, "wilson95": [ 0.22779697212376038, 0.40626055719489407 ], "statuses_first": { "ok": 98, "invalid_output": 2 }, "eventual_correct": 31, "eventual_usable": 98, "retries": 0, "first_cost": { "reported_usd": 0.024835, "exact_decimal_usd": "0.02483500", "known_records": 100, "missing_records": 0 }, "core_cost": { "reported_usd": 0.024835, "exact_decimal_usd": "0.02483500", "known_records": 100, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.00024835, "first_cost_per_first_correct_usd": 0.0008011290322580645, "core_cost_per_first_correct_usd": 0.0008011290322580645, "latency_first_all_ms": { "median": 1559.1453124943655, "p90": 2569.2041708854963, "p95": 2903.042047872441 }, "latency_first_usable_ms": { "median": 1559.1453124943655, "p90": 2575.3230126807466, "p95": 2910.492893675109 }, "tokens_first": { "prompt_tokens": { "total": 121050, "covered_requests": 100 }, "completion_tokens": { "total": 500, "covered_requests": 100 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 100, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.020105, "exact_decimal_usd": "0.02010500", "known_records": 80, "missing_records": 0 }, "pilot": { "reported_usd": 0.00473, "exact_decimal_usd": "0.00473000", "known_records": 20, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:34:19.268947-04:00", "last": "2026-10-15T10:53:33.510301-04:00" }, "identities": [ [ "openai/gpt-5.4-nano", "OpenAI" ] ] }, "2": { "scheduled": 100, "completed_first": 100, "correct_first": 39, "accuracy_scheduled": 0.39, "usable_first": 98, "coverage": 0.98, "accuracy_usable": 0.3979591836734694, "wilson95": [ 0.3001687400639441, 0.4879698295415926 ], "statuses_first": { "ok": 98, "invalid_output": 2 }, "eventual_correct": 39, "eventual_usable": 98, "retries": 0, "first_cost": { "reported_usd": 0.0256895, "exact_decimal_usd": "0.02568950", "known_records": 100, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0256895, "exact_decimal_usd": "0.02568950", "known_records": 100, "missing_records": 0 }, "core_cost_per_scheduled_usd": 0.000256895, "first_cost_per_first_correct_usd": 0.0006587051282051283, "core_cost_per_first_correct_usd": 0.0006587051282051283, "latency_first_all_ms": { "median": 1642.0549165341072, "p90": 2681.1670002760366, "p95": 3071.67379228922 }, "latency_first_usable_ms": { "median": 1642.0549165341072, "p90": 2684.3011668883264, "p95": 3110.1804588892232 }, "tokens_first": { "prompt_tokens": { "total": 125285, "covered_requests": 100 }, "completion_tokens": { "total": 506, "covered_requests": 100 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 100, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.0256895, "exact_decimal_usd": "0.02568950", "known_records": 100, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:53:39.284903-04:00", "last": "2026-10-15T11:03:33.731149-04:00" }, "identities": [ [ "openai/gpt-5.4-nano", "OpenAI" ] ] } } }, "families_at_least_8_of_10": [], "families_at_most_4_of_10": [ "V02", "V03", "V04", "V05", "V06", "V07", "V08", "V09", "V10", "V11", "V12", "V13", "V14", "V18", "V19", "V20" ] }, "google/gemini-2.5-flash-lite": { "scheduled": 200, "completed_first": 200, "correct_first": 75, "accuracy_scheduled": 0.375, "usable_first": 180, "coverage": 0.9, "accuracy_usable": 0.4166666666666667, "wilson95": [ 0.31085436858409826, 0.44385696301021843 ], "statuses_first": { "ok": 180, "invalid_output": 19, "http_failure": 1 }, "eventual_correct": 75, "eventual_usable": 181, "retries": 1, "first_cost": { "reported_usd": 0.0086061, "exact_decimal_usd": "0.0086061", "known_records": 199, "missing_records": 1 }, "core_cost": { "reported_usd": 0.0086404, "exact_decimal_usd": "0.0086404", "known_records": 200, "missing_records": 1 }, "core_cost_per_scheduled_usd": 4.3202e-05, "first_cost_per_first_correct_usd": 0.000114748, "core_cost_per_first_correct_usd": 0.00011520533333333333, "latency_first_all_ms": { "median": 872.8391879994888, "p90": 1313.775479112519, "p95": 1742.8930548165222 }, "latency_first_usable_ms": { "median": 869.6297289861832, "p90": 1321.611895301612, "p95": 1772.883231277228 }, "tokens_first": { "prompt_tokens": { "total": 83161, "covered_requests": 199 }, "completion_tokens": { "total": 725, "covered_requests": 199 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 200, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.0077658, "exact_decimal_usd": "0.0077658", "known_records": 180, "missing_records": 1 }, "pilot": { "reported_usd": 0.0008746, "exact_decimal_usd": "0.0008746", "known_records": 20, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:34:24.116351-04:00", "last": "2026-10-15T11:03:32.479724-04:00" }, "identities": [ [ "google/gemini-2.5-flash-lite", "Google AI Studio" ] ], "groups": { "family_id": { "V01": { "scheduled": 10, "completed_first": 10, "correct_first": 5, "accuracy_scheduled": 0.5, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.5, "wilson95": [ 0.236593090512564, 0.7634069094874361 ], "statuses_first": { "ok": 10 }, "eventual_correct": 5, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.0003727, "exact_decimal_usd": "0.0003727", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0003727, "exact_decimal_usd": "0.0003727", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 3.727e-05, "first_cost_per_first_correct_usd": 7.454e-05, "core_cost_per_first_correct_usd": 7.454e-05, "latency_first_all_ms": { "median": 895.6783754983917, "p90": 1051.6756746917963, "p95": 1182.316149852704 }, "latency_first_usable_ms": { "median": 895.6783754983917, "p90": 1051.6756746917963, "p95": 1182.316149852704 }, "tokens_first": { "prompt_tokens": { "total": 3687, "covered_requests": 10 }, "completion_tokens": { "total": 10, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.0003378, "exact_decimal_usd": "0.0003378", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 3.49e-05, "exact_decimal_usd": "0.0000349", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:09.191257-04:00", "last": "2026-10-15T11:03:09.087848-04:00" }, "identities": [ [ "google/gemini-2.5-flash-lite", "Google AI Studio" ] ] }, "V02": { "scheduled": 10, "completed_first": 10, "correct_first": 3, "accuracy_scheduled": 0.3, "usable_first": 9, "coverage": 0.9, "accuracy_usable": 0.3333333333333333, "wilson95": [ 0.10779126740630099, 0.6032218525388546 ], "statuses_first": { "ok": 9, "http_failure": 1 }, "eventual_correct": 3, "eventual_usable": 10, "retries": 1, "first_cost": { "reported_usd": 0.0003348, "exact_decimal_usd": "0.0003348", "known_records": 9, "missing_records": 1 }, "core_cost": { "reported_usd": 0.0003691, "exact_decimal_usd": "0.0003691", "known_records": 10, "missing_records": 1 }, "core_cost_per_scheduled_usd": 3.6910000000000004e-05, "first_cost_per_first_correct_usd": 0.0001116, "core_cost_per_first_correct_usd": 0.00012303333333333333, "latency_first_all_ms": { "median": 832.3354169842787, "p90": 1094.6192338713445, "p95": 1191.2866584374567 }, "latency_first_usable_ms": { "median": 854.2008749791421, "p90": 1116.100883774925, "p95": 1202.0274833892472 }, "tokens_first": { "prompt_tokens": { "total": 3312, "covered_requests": 9 }, "completion_tokens": { "total": 9, "covered_requests": 9 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.0003328, "exact_decimal_usd": "0.0003328", "known_records": 9, "missing_records": 1 }, "pilot": { "reported_usd": 3.63e-05, "exact_decimal_usd": "0.0000363", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:19.760733-04:00", "last": "2026-10-15T11:01:54.639268-04:00" }, "identities": [ [ "google/gemini-2.5-flash-lite", "Google AI Studio" ] ] }, "V03": { "scheduled": 10, "completed_first": 10, "correct_first": 4, "accuracy_scheduled": 0.4, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.4, "wilson95": [ 0.16818032970623614, 0.6873262302663417 ], "statuses_first": { "ok": 10 }, "eventual_correct": 4, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.0003818, "exact_decimal_usd": "0.0003818", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0003818, "exact_decimal_usd": "0.0003818", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 3.8180000000000004e-05, "first_cost_per_first_correct_usd": 9.545e-05, "core_cost_per_first_correct_usd": 9.545e-05, "latency_first_all_ms": { "median": 980.7420415163506, "p90": 1228.4179503156336, "p95": 1500.4633501725032 }, "latency_first_usable_ms": { "median": 980.7420415163506, "p90": 1228.4179503156336, "p95": 1500.4633501725032 }, "tokens_first": { "prompt_tokens": { "total": 3778, "covered_requests": 10 }, "completion_tokens": { "total": 10, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.0003434, "exact_decimal_usd": "0.0003434", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 3.84e-05, "exact_decimal_usd": "0.0000384", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:11.630201-04:00", "last": "2026-10-15T11:02:30.861706-04:00" }, "identities": [ [ "google/gemini-2.5-flash-lite", "Google AI Studio" ] ] }, "V04": { "scheduled": 10, "completed_first": 10, "correct_first": 4, "accuracy_scheduled": 0.4, "usable_first": 8, "coverage": 0.8, "accuracy_usable": 0.5, "wilson95": [ 0.16818032970623614, 0.6873262302663417 ], "statuses_first": { "ok": 8, "invalid_output": 2 }, "eventual_correct": 4, "eventual_usable": 8, "retries": 0, "first_cost": { "reported_usd": 0.000398, "exact_decimal_usd": "0.0003980", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.000398, "exact_decimal_usd": "0.0003980", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 3.9800000000000005e-05, "first_cost_per_first_correct_usd": 9.95e-05, "core_cost_per_first_correct_usd": 9.95e-05, "latency_first_all_ms": { "median": 813.7315420026425, "p90": 825.4739493946545, "p95": 830.956224683905 }, "latency_first_usable_ms": { "median": 806.9233960122801, "p90": 824.0266210748814, "p95": 824.1411435359623 }, "tokens_first": { "prompt_tokens": { "total": 3692, "covered_requests": 10 }, "completion_tokens": { "total": 72, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.0003628, "exact_decimal_usd": "0.0003628", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 3.52e-05, "exact_decimal_usd": "0.0000352", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:03.149012-04:00", "last": "2026-10-15T11:00:46.742798-04:00" }, "identities": [ [ "google/gemini-2.5-flash-lite", "Google AI Studio" ] ] }, "V05": { "scheduled": 10, "completed_first": 10, "correct_first": 1, "accuracy_scheduled": 0.1, "usable_first": 9, "coverage": 0.9, "accuracy_usable": 0.1111111111111111, "wilson95": [ 0.017876213095072896, 0.4041500267952385 ], "statuses_first": { "ok": 9, "invalid_output": 1 }, "eventual_correct": 1, "eventual_usable": 9, "retries": 0, "first_cost": { "reported_usd": 0.0004044, "exact_decimal_usd": "0.0004044", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0004044, "exact_decimal_usd": "0.0004044", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 4.044e-05, "first_cost_per_first_correct_usd": 0.0004044, "core_cost_per_first_correct_usd": 0.0004044, "latency_first_all_ms": { "median": 923.0467079905793, "p90": 1592.9607872094493, "p95": 1686.4795810979556 }, "latency_first_usable_ms": { "median": 956.0376250301488, "p90": 1613.7427414068952, "p95": 1696.8705581966788 }, "tokens_first": { "prompt_tokens": { "total": 3880, "covered_requests": 10 }, "completion_tokens": { "total": 41, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.0003653, "exact_decimal_usd": "0.0003653", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 3.91e-05, "exact_decimal_usd": "0.0000391", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:36:01.904656-04:00", "last": "2026-10-15T11:01:46.535530-04:00" }, "identities": [ [ "google/gemini-2.5-flash-lite", "Google AI Studio" ] ] }, "V06": { "scheduled": 10, "completed_first": 10, "correct_first": 3, "accuracy_scheduled": 0.3, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.3, "wilson95": [ 0.10779126740630099, 0.6032218525388546 ], "statuses_first": { "ok": 10 }, "eventual_correct": 3, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.000374, "exact_decimal_usd": "0.0003740", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.000374, "exact_decimal_usd": "0.0003740", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 3.74e-05, "first_cost_per_first_correct_usd": 0.00012466666666666667, "core_cost_per_first_correct_usd": 0.00012466666666666667, "latency_first_all_ms": { "median": 1012.751771020703, "p90": 1865.8701203006763, "p95": 4859.921289639889 }, "latency_first_usable_ms": { "median": 1012.751771020703, "p90": 1865.8701203006763, "p95": 4859.921289639889 }, "tokens_first": { "prompt_tokens": { "total": 3700, "covered_requests": 10 }, "completion_tokens": { "total": 10, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.0003354, "exact_decimal_usd": "0.0003354", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 3.86e-05, "exact_decimal_usd": "0.0000386", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:36:14.508348-04:00", "last": "2026-10-15T11:03:12.615988-04:00" }, "identities": [ [ "google/gemini-2.5-flash-lite", "Google AI Studio" ] ] }, "V07": { "scheduled": 10, "completed_first": 10, "correct_first": 4, "accuracy_scheduled": 0.4, "usable_first": 9, "coverage": 0.9, "accuracy_usable": 0.4444444444444444, "wilson95": [ 0.16818032970623614, 0.6873262302663417 ], "statuses_first": { "ok": 9, "invalid_output": 1 }, "eventual_correct": 4, "eventual_usable": 9, "retries": 0, "first_cost": { "reported_usd": 0.0003597, "exact_decimal_usd": "0.0003597", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0003597, "exact_decimal_usd": "0.0003597", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 3.597e-05, "first_cost_per_first_correct_usd": 8.9925e-05, "core_cost_per_first_correct_usd": 8.9925e-05, "latency_first_all_ms": { "median": 875.3029165090993, "p90": 978.1810750253499, "p95": 996.5071000100579 }, "latency_first_usable_ms": { "median": 844.7897910373285, "p90": 982.2535250219516, "p95": 998.5433250083588 }, "tokens_first": { "prompt_tokens": { "total": 3553, "covered_requests": 10 }, "completion_tokens": { "total": 11, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.0003228, "exact_decimal_usd": "0.0003228", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 3.69e-05, "exact_decimal_usd": "0.0000369", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:36:24.828806-04:00", "last": "2026-10-15T11:02:46.689641-04:00" }, "identities": [ [ "google/gemini-2.5-flash-lite", "Google AI Studio" ] ] }, "V08": { "scheduled": 10, "completed_first": 10, "correct_first": 1, "accuracy_scheduled": 0.1, "usable_first": 6, "coverage": 0.6, "accuracy_usable": 0.16666666666666666, "wilson95": [ 0.017876213095072896, 0.4041500267952385 ], "statuses_first": { "ok": 6, "invalid_output": 4 }, "eventual_correct": 1, "eventual_usable": 6, "retries": 0, "first_cost": { "reported_usd": 0.0004498, "exact_decimal_usd": "0.0004498", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0004498, "exact_decimal_usd": "0.0004498", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 4.498e-05, "first_cost_per_first_correct_usd": 0.0004498, "core_cost_per_first_correct_usd": 0.0004498, "latency_first_all_ms": { "median": 875.2141045115422, "p90": 910.4118662769906, "p95": 910.5481411446817 }, "latency_first_usable_ms": { "median": 846.8770415056497, "p90": 903.9711455116048, "p95": 907.3277807619888 }, "tokens_first": { "prompt_tokens": { "total": 3966, "covered_requests": 10 }, "completion_tokens": { "total": 133, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.000411, "exact_decimal_usd": "0.0004110", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 3.88e-05, "exact_decimal_usd": "0.0000388", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:17.303121-04:00", "last": "2026-10-15T11:03:32.479724-04:00" }, "identities": [ [ "google/gemini-2.5-flash-lite", "Google AI Studio" ] ] }, "V09": { "scheduled": 10, "completed_first": 10, "correct_first": 5, "accuracy_scheduled": 0.5, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.5, "wilson95": [ 0.236593090512564, 0.7634069094874361 ], "statuses_first": { "ok": 10 }, "eventual_correct": 5, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.0006018, "exact_decimal_usd": "0.0006018", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0006018, "exact_decimal_usd": "0.0006018", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 6.0179999999999996e-05, "first_cost_per_first_correct_usd": 0.00012035999999999999, "core_cost_per_first_correct_usd": 0.00012035999999999999, "latency_first_all_ms": { "median": 960.5399789870717, "p90": 1935.796074708921, "p95": 5578.009412344537 }, "latency_first_usable_ms": { "median": 960.5399789870717, "p90": 1935.796074708921, "p95": 5578.009412344537 }, "tokens_first": { "prompt_tokens": { "total": 5978, "covered_requests": 10 }, "completion_tokens": { "total": 10, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.0005112, "exact_decimal_usd": "0.0005112", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 9.06e-05, "exact_decimal_usd": "0.0000906", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:34:24.116351-04:00", "last": "2026-10-15T11:03:23.778964-04:00" }, "identities": [ [ "google/gemini-2.5-flash-lite", "Google AI Studio" ] ] }, "V10": { "scheduled": 10, "completed_first": 10, "correct_first": 4, "accuracy_scheduled": 0.4, "usable_first": 9, "coverage": 0.9, "accuracy_usable": 0.4444444444444444, "wilson95": [ 0.16818032970623614, 0.6873262302663417 ], "statuses_first": { "ok": 9, "invalid_output": 1 }, "eventual_correct": 4, "eventual_usable": 9, "retries": 0, "first_cost": { "reported_usd": 0.0004394, "exact_decimal_usd": "0.0004394", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0004394, "exact_decimal_usd": "0.0004394", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 4.394e-05, "first_cost_per_first_correct_usd": 0.00010985, "core_cost_per_first_correct_usd": 0.00010985, "latency_first_all_ms": { "median": 775.0462290132418, "p90": 2946.0936249932215, "p95": 10551.827687513996 }, "latency_first_usable_ms": { "median": 647.6314170286059, "p90": 4636.25674999785, "p95": 11396.909250016319 }, "tokens_first": { "prompt_tokens": { "total": 4350, "covered_requests": 10 }, "completion_tokens": { "total": 11, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.000395, "exact_decimal_usd": "0.0003950", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 4.44e-05, "exact_decimal_usd": "0.0000444", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:59.576370-04:00", "last": "2026-10-15T10:59:59.236983-04:00" }, "identities": [ [ "google/gemini-2.5-flash-lite", "Google AI Studio" ] ] }, "V11": { "scheduled": 10, "completed_first": 10, "correct_first": 2, "accuracy_scheduled": 0.2, "usable_first": 7, "coverage": 0.7, "accuracy_usable": 0.2857142857142857, "wilson95": [ 0.05668215145437519, 0.5098375284633583 ], "statuses_first": { "ok": 7, "invalid_output": 3 }, "eventual_correct": 2, "eventual_usable": 7, "retries": 0, "first_cost": { "reported_usd": 0.0004313, "exact_decimal_usd": "0.0004313", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0004313, "exact_decimal_usd": "0.0004313", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 4.313e-05, "first_cost_per_first_correct_usd": 0.00021565, "core_cost_per_first_correct_usd": 0.00021565, "latency_first_all_ms": { "median": 878.8659584824927, "p90": 1887.3918714118186, "p95": 2544.6507896936946 }, "latency_first_usable_ms": { "median": 871.1060839705169, "p90": 1851.5450831735516, "p95": 2526.7273955745604 }, "tokens_first": { "prompt_tokens": { "total": 3905, "covered_requests": 10 }, "completion_tokens": { "total": 102, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.0003939, "exact_decimal_usd": "0.0003939", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 3.74e-05, "exact_decimal_usd": "0.0000374", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:44.232188-04:00", "last": "2026-10-15T11:03:03.535915-04:00" }, "identities": [ [ "google/gemini-2.5-flash-lite", "Google AI Studio" ] ] }, "V12": { "scheduled": 10, "completed_first": 10, "correct_first": 2, "accuracy_scheduled": 0.2, "usable_first": 7, "coverage": 0.7, "accuracy_usable": 0.2857142857142857, "wilson95": [ 0.05668215145437519, 0.5098375284633583 ], "statuses_first": { "ok": 7, "invalid_output": 3 }, "eventual_correct": 2, "eventual_usable": 7, "retries": 0, "first_cost": { "reported_usd": 0.0004466, "exact_decimal_usd": "0.0004466", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0004466, "exact_decimal_usd": "0.0004466", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 4.466e-05, "first_cost_per_first_correct_usd": 0.0002233, "core_cost_per_first_correct_usd": 0.0002233, "latency_first_all_ms": { "median": 831.1490835039876, "p90": 1386.0226255608718, "p95": 2050.916937796863 }, "latency_first_usable_ms": { "median": 748.9465000107884, "p90": 1591.43192479387, "p95": 2153.621587413362 }, "tokens_first": { "prompt_tokens": { "total": 4054, "covered_requests": 10 }, "completion_tokens": { "total": 103, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.0004019, "exact_decimal_usd": "0.0004019", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 4.47e-05, "exact_decimal_usd": "0.0000447", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:02.290579-04:00", "last": "2026-10-15T11:02:14.807098-04:00" }, "identities": [ [ "google/gemini-2.5-flash-lite", "Google AI Studio" ] ] }, "V13": { "scheduled": 10, "completed_first": 10, "correct_first": 1, "accuracy_scheduled": 0.1, "usable_first": 9, "coverage": 0.9, "accuracy_usable": 0.1111111111111111, "wilson95": [ 0.017876213095072896, 0.4041500267952385 ], "statuses_first": { "ok": 9, "invalid_output": 1 }, "eventual_correct": 1, "eventual_usable": 9, "retries": 0, "first_cost": { "reported_usd": 0.0003918, "exact_decimal_usd": "0.0003918", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0003918, "exact_decimal_usd": "0.0003918", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 3.918e-05, "first_cost_per_first_correct_usd": 0.0003918, "core_cost_per_first_correct_usd": 0.0003918, "latency_first_all_ms": { "median": 711.4671664894558, "p90": 786.4795330562629, "p95": 839.9021835299208 }, "latency_first_usable_ms": { "median": 708.0300829838961, "p90": 757.8641330241226, "p95": 766.2359829875641 }, "tokens_first": { "prompt_tokens": { "total": 3754, "covered_requests": 10 }, "completion_tokens": { "total": 41, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.0003547, "exact_decimal_usd": "0.0003547", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 3.71e-05, "exact_decimal_usd": "0.0000371", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:36:21.690024-04:00", "last": "2026-10-15T11:01:39.248535-04:00" }, "identities": [ [ "google/gemini-2.5-flash-lite", "Google AI Studio" ] ] }, "V14": { "scheduled": 10, "completed_first": 10, "correct_first": 4, "accuracy_scheduled": 0.4, "usable_first": 9, "coverage": 0.9, "accuracy_usable": 0.4444444444444444, "wilson95": [ 0.16818032970623614, 0.6873262302663417 ], "statuses_first": { "ok": 9, "invalid_output": 1 }, "eventual_correct": 4, "eventual_usable": 9, "retries": 0, "first_cost": { "reported_usd": 0.0003907, "exact_decimal_usd": "0.0003907", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0003907, "exact_decimal_usd": "0.0003907", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 3.9070000000000004e-05, "first_cost_per_first_correct_usd": 9.7675e-05, "core_cost_per_first_correct_usd": 9.7675e-05, "latency_first_all_ms": { "median": 1189.0777920198161, "p90": 1659.589771175524, "p95": 1914.4483025855145 }, "latency_first_usable_ms": { "median": 1088.7465000269003, "p90": 1716.2250003777444, "p95": 1942.765917186625 }, "tokens_first": { "prompt_tokens": { "total": 3743, "covered_requests": 10 }, "completion_tokens": { "total": 41, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.0003536, "exact_decimal_usd": "0.0003536", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 3.71e-05, "exact_decimal_usd": "0.0000371", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:27.748161-04:00", "last": "2026-10-15T11:02:31.978026-04:00" }, "identities": [ [ "google/gemini-2.5-flash-lite", "Google AI Studio" ] ] }, "V15": { "scheduled": 10, "completed_first": 10, "correct_first": 8, "accuracy_scheduled": 0.8, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.8, "wilson95": [ 0.49016247153664183, 0.9433178485456247 ], "statuses_first": { "ok": 10 }, "eventual_correct": 8, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.0003951, "exact_decimal_usd": "0.0003951", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0003951, "exact_decimal_usd": "0.0003951", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 3.951e-05, "first_cost_per_first_correct_usd": 4.93875e-05, "core_cost_per_first_correct_usd": 4.93875e-05, "latency_first_all_ms": { "median": 869.3090625165496, "p90": 1008.8045497133862, "p95": 1046.112774850917 }, "latency_first_usable_ms": { "median": 869.3090625165496, "p90": 1008.8045497133862, "p95": 1046.112774850917 }, "tokens_first": { "prompt_tokens": { "total": 3911, "covered_requests": 10 }, "completion_tokens": { "total": 10, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.0003539, "exact_decimal_usd": "0.0003539", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 4.12e-05, "exact_decimal_usd": "0.0000412", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:31.091292-04:00", "last": "2026-10-15T11:02:39.293358-04:00" }, "identities": [ [ "google/gemini-2.5-flash-lite", "Google AI Studio" ] ] }, "V16": { "scheduled": 10, "completed_first": 10, "correct_first": 6, "accuracy_scheduled": 0.6, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.6, "wilson95": [ 0.31267376973365824, 0.8318196702937638 ], "statuses_first": { "ok": 10 }, "eventual_correct": 6, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.0006283, "exact_decimal_usd": "0.0006283", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0006283, "exact_decimal_usd": "0.0006283", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 6.283e-05, "first_cost_per_first_correct_usd": 0.00010471666666666667, "core_cost_per_first_correct_usd": 0.00010471666666666667, "latency_first_all_ms": { "median": 1335.6530210003257, "p90": 1971.8305912974756, "p95": 2011.2354411248816 }, "latency_first_usable_ms": { "median": 1335.6530210003257, "p90": 1971.8305912974756, "p95": 2011.2354411248816 }, "tokens_first": { "prompt_tokens": { "total": 6243, "covered_requests": 10 }, "completion_tokens": { "total": 10, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.0005664, "exact_decimal_usd": "0.0005664", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 6.19e-05, "exact_decimal_usd": "0.0000619", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:40.619897-04:00", "last": "2026-10-15T10:57:52.116082-04:00" }, "identities": [ [ "google/gemini-2.5-flash-lite", "Google AI Studio" ] ] }, "V17": { "scheduled": 10, "completed_first": 10, "correct_first": 4, "accuracy_scheduled": 0.4, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.4, "wilson95": [ 0.16818032970623614, 0.6873262302663417 ], "statuses_first": { "ok": 10 }, "eventual_correct": 4, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.000614, "exact_decimal_usd": "0.0006140", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.000614, "exact_decimal_usd": "0.0006140", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 6.14e-05, "first_cost_per_first_correct_usd": 0.0001535, "core_cost_per_first_correct_usd": 0.0001535, "latency_first_all_ms": { "median": 796.2948329804931, "p90": 1076.8806663982105, "p95": 1102.3313542100368 }, "latency_first_usable_ms": { "median": 796.2948329804931, "p90": 1076.8806663982105, "p95": 1102.3313542100368 }, "tokens_first": { "prompt_tokens": { "total": 6100, "covered_requests": 10 }, "completion_tokens": { "total": 10, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.0005511, "exact_decimal_usd": "0.0005511", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 6.29e-05, "exact_decimal_usd": "0.0000629", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:36:28.032210-04:00", "last": "2026-10-15T11:01:21.509494-04:00" }, "identities": [ [ "google/gemini-2.5-flash-lite", "Google AI Studio" ] ] }, "V18": { "scheduled": 10, "completed_first": 10, "correct_first": 5, "accuracy_scheduled": 0.5, "usable_first": 10, "coverage": 1.0, "accuracy_usable": 0.5, "wilson95": [ 0.236593090512564, 0.7634069094874361 ], "statuses_first": { "ok": 10 }, "eventual_correct": 5, "eventual_usable": 10, "retries": 0, "first_cost": { "reported_usd": 0.0003916, "exact_decimal_usd": "0.0003916", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0003916, "exact_decimal_usd": "0.0003916", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 3.916e-05, "first_cost_per_first_correct_usd": 7.832e-05, "core_cost_per_first_correct_usd": 7.832e-05, "latency_first_all_ms": { "median": 820.2183959947433, "p90": 1078.1526874983683, "p95": 1147.9884687461888 }, "latency_first_usable_ms": { "median": 820.2183959947433, "p90": 1078.1526874983683, "p95": 1147.9884687461888 }, "tokens_first": { "prompt_tokens": { "total": 3876, "covered_requests": 10 }, "completion_tokens": { "total": 10, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.000349, "exact_decimal_usd": "0.0003490", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 4.26e-05, "exact_decimal_usd": "0.0000426", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:53.034948-04:00", "last": "2026-10-15T10:59:07.104169-04:00" }, "identities": [ [ "google/gemini-2.5-flash-lite", "Google AI Studio" ] ] }, "V19": { "scheduled": 10, "completed_first": 10, "correct_first": 2, "accuracy_scheduled": 0.2, "usable_first": 9, "coverage": 0.9, "accuracy_usable": 0.2222222222222222, "wilson95": [ 0.05668215145437519, 0.5098375284633583 ], "statuses_first": { "ok": 9, "invalid_output": 1 }, "eventual_correct": 2, "eventual_usable": 9, "retries": 0, "first_cost": { "reported_usd": 0.0004227, "exact_decimal_usd": "0.0004227", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0004227, "exact_decimal_usd": "0.0004227", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 4.227e-05, "first_cost_per_first_correct_usd": 0.00021135, "core_cost_per_first_correct_usd": 0.00021135, "latency_first_all_ms": { "median": 861.3024374644738, "p90": 1098.6532208276913, "p95": 1116.0742769250646 }, "latency_first_usable_ms": { "median": 858.2065829541534, "p90": 1080.6179002160206, "p95": 1107.0566166192293 }, "tokens_first": { "prompt_tokens": { "total": 4067, "covered_requests": 10 }, "completion_tokens": { "total": 40, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.0003833, "exact_decimal_usd": "0.0003833", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 3.94e-05, "exact_decimal_usd": "0.0000394", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:53.718106-04:00", "last": "2026-10-15T11:00:50.261116-04:00" }, "identities": [ [ "google/gemini-2.5-flash-lite", "Google AI Studio" ] ] }, "V20": { "scheduled": 10, "completed_first": 10, "correct_first": 7, "accuracy_scheduled": 0.7, "usable_first": 9, "coverage": 0.9, "accuracy_usable": 0.7777777777777778, "wilson95": [ 0.39677814746114537, 0.8922087325936989 ], "statuses_first": { "ok": 9, "invalid_output": 1 }, "eventual_correct": 7, "eventual_usable": 9, "retries": 0, "first_cost": { "reported_usd": 0.0003776, "exact_decimal_usd": "0.0003776", "known_records": 10, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0003776, "exact_decimal_usd": "0.0003776", "known_records": 10, "missing_records": 0 }, "core_cost_per_scheduled_usd": 3.7760000000000004e-05, "first_cost_per_first_correct_usd": 5.3942857142857145e-05, "core_cost_per_first_correct_usd": 5.3942857142857145e-05, "latency_first_all_ms": { "median": 822.4068539857399, "p90": 934.2706210853066, "p95": 945.4602270474425 }, "latency_first_usable_ms": { "median": 820.396333001554, "p90": 936.7572001880035, "p95": 946.7035165987909 }, "tokens_first": { "prompt_tokens": { "total": 3612, "covered_requests": 10 }, "completion_tokens": { "total": 41, "covered_requests": 10 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 10, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.0003405, "exact_decimal_usd": "0.0003405", "known_records": 9, "missing_records": 0 }, "pilot": { "reported_usd": 3.71e-05, "exact_decimal_usd": "0.0000371", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:36:10.614732-04:00", "last": "2026-10-15T11:02:54.653807-04:00" }, "identities": [ [ "google/gemini-2.5-flash-lite", "Google AI Studio" ] ] } }, "image_count": { "1": { "scheduled": 177, "completed_first": 177, "correct_first": 64, "accuracy_scheduled": 0.3615819209039548, "usable_first": 157, "coverage": 0.8870056497175142, "accuracy_usable": 0.40764331210191085, "wilson95": [ 0.2944352048446036, 0.43460922822102477 ], "statuses_first": { "ok": 157, "invalid_output": 19, "http_failure": 1 }, "eventual_correct": 64, "eventual_usable": 158, "retries": 1, "first_cost": { "reported_usd": 0.0070386, "exact_decimal_usd": "0.0070386", "known_records": 176, "missing_records": 1 }, "core_cost": { "reported_usd": 0.0070729, "exact_decimal_usd": "0.0070729", "known_records": 177, "missing_records": 1 }, "core_cost_per_scheduled_usd": 3.995988700564972e-05, "first_cost_per_first_correct_usd": 0.000109978125, "core_cost_per_first_correct_usd": 0.0001105140625, "latency_first_all_ms": { "median": 863.1905840011314, "p90": 1213.4246497880667, "p95": 1578.333974804263 }, "latency_first_usable_ms": { "median": 854.2008749791421, "p90": 1204.5118827954866, "p95": 1578.333974804263 }, "tokens_first": { "prompt_tokens": { "total": 67578, "covered_requests": 176 }, "completion_tokens": { "total": 702, "covered_requests": 176 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 177, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.0064137, "exact_decimal_usd": "0.0064137", "known_records": 160, "missing_records": 1 }, "pilot": { "reported_usd": 0.0006592, "exact_decimal_usd": "0.0006592", "known_records": 17, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:02.290579-04:00", "last": "2026-10-15T11:03:32.479724-04:00" }, "identities": [ [ "google/gemini-2.5-flash-lite", "Google AI Studio" ] ] }, "2": { "scheduled": 19, "completed_first": 19, "correct_first": 9, "accuracy_scheduled": 0.47368421052631576, "usable_first": 19, "coverage": 1.0, "accuracy_usable": 0.47368421052631576, "wilson95": [ 0.2732980516117956, 0.6829219075488682 ], "statuses_first": { "ok": 19 }, "eventual_correct": 9, "eventual_usable": 19, "retries": 0, "first_cost": { "reported_usd": 0.0012054, "exact_decimal_usd": "0.0012054", "known_records": 19, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0012054, "exact_decimal_usd": "0.0012054", "known_records": 19, "missing_records": 0 }, "core_cost_per_scheduled_usd": 6.344210526315788e-05, "first_cost_per_first_correct_usd": 0.00013393333333333333, "core_cost_per_first_correct_usd": 0.00013393333333333333, "latency_first_all_ms": { "median": 1127.7820420218632, "p90": 1558.9916580007407, "p95": 1971.8305912974754 }, "latency_first_usable_ms": { "median": 1127.7820420218632, "p90": 1558.9916580007407, "p95": 1971.8305912974754 }, "tokens_first": { "prompt_tokens": { "total": 11978, "covered_requests": 19 }, "completion_tokens": { "total": 19, "covered_requests": 19 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 19, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.0010806, "exact_decimal_usd": "0.0010806", "known_records": 17, "missing_records": 0 }, "pilot": { "reported_usd": 0.0001248, "exact_decimal_usd": "0.0001248", "known_records": 2, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:40.619897-04:00", "last": "2026-10-15T11:01:21.509494-04:00" }, "identities": [ [ "google/gemini-2.5-flash-lite", "Google AI Studio" ] ] }, "3": { "scheduled": 4, "completed_first": 4, "correct_first": 2, "accuracy_scheduled": 0.5, "usable_first": 4, "coverage": 1.0, "accuracy_usable": 0.5, "wilson95": [ 0.15003898915214947, 0.8499610108478506 ], "statuses_first": { "ok": 4 }, "eventual_correct": 2, "eventual_usable": 4, "retries": 0, "first_cost": { "reported_usd": 0.0003621, "exact_decimal_usd": "0.0003621", "known_records": 4, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0003621, "exact_decimal_usd": "0.0003621", "known_records": 4, "missing_records": 0 }, "core_cost_per_scheduled_usd": 9.0525e-05, "first_cost_per_first_correct_usd": 0.00018105, "core_cost_per_first_correct_usd": 0.00018105, "latency_first_all_ms": { "median": 952.6684579614084, "p90": 6745.684387278745, "p95": 7982.953568629452 }, "latency_first_usable_ms": { "median": 952.6684579614084, "p90": 6745.684387278745, "p95": 7982.953568629452 }, "tokens_first": { "prompt_tokens": { "total": 3605, "covered_requests": 4 }, "completion_tokens": { "total": 4, "covered_requests": 4 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 4, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.0002715, "exact_decimal_usd": "0.0002715", "known_records": 3, "missing_records": 0 }, "pilot": { "reported_usd": 9.06e-05, "exact_decimal_usd": "0.0000906", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:34:24.116351-04:00", "last": "2026-10-15T11:01:01.584346-04:00" }, "identities": [ [ "google/gemini-2.5-flash-lite", "Google AI Studio" ] ] } }, "analysis_subset": { "perception_control": { "scheduled": 13, "completed_first": 13, "correct_first": 9, "accuracy_scheduled": 0.6923076923076923, "usable_first": 13, "coverage": 1.0, "accuracy_usable": 0.6923076923076923, "wilson95": [ 0.42369343180549035, 0.8731929634428948 ], "statuses_first": { "ok": 13 }, "eventual_correct": 9, "eventual_usable": 13, "retries": 0, "first_cost": { "reported_usd": 0.0006689, "exact_decimal_usd": "0.0006689", "known_records": 13, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0006689, "exact_decimal_usd": "0.0006689", "known_records": 13, "missing_records": 0 }, "core_cost_per_scheduled_usd": 5.145384615384616e-05, "first_cost_per_first_correct_usd": 7.432222222222223e-05, "core_cost_per_first_correct_usd": 7.432222222222223e-05, "latency_first_all_ms": { "median": 821.5159589890391, "p90": 1171.8859001994133, "p95": 1248.1495584012007 }, "latency_first_usable_ms": { "median": 821.5159589890391, "p90": 1171.8859001994133, "p95": 1248.1495584012007 }, "tokens_first": { "prompt_tokens": { "total": 6637, "covered_requests": 13 }, "completion_tokens": { "total": 13, "covered_requests": 13 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 13, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.0006337, "exact_decimal_usd": "0.0006337", "known_records": 12, "missing_records": 0 }, "pilot": { "reported_usd": 3.52e-05, "exact_decimal_usd": "0.0000352", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:03.149012-04:00", "last": "2026-10-15T11:01:01.584346-04:00" }, "identities": [ [ "google/gemini-2.5-flash-lite", "Google AI Studio" ] ] }, "reasoning_and_grounded_decision": { "scheduled": 187, "completed_first": 187, "correct_first": 66, "accuracy_scheduled": 0.35294117647058826, "usable_first": 167, "coverage": 0.893048128342246, "accuracy_usable": 0.39520958083832336, "wilson95": [ 0.2880359502153428, 0.42376671394435916 ], "statuses_first": { "ok": 167, "invalid_output": 19, "http_failure": 1 }, "eventual_correct": 66, "eventual_usable": 168, "retries": 1, "first_cost": { "reported_usd": 0.0079372, "exact_decimal_usd": "0.0079372", "known_records": 186, "missing_records": 1 }, "core_cost": { "reported_usd": 0.0079715, "exact_decimal_usd": "0.0079715", "known_records": 187, "missing_records": 1 }, "core_cost_per_scheduled_usd": 4.26283422459893e-05, "first_cost_per_first_correct_usd": 0.00012026060606060607, "core_cost_per_first_correct_usd": 0.00012078030303030302, "latency_first_all_ms": { "median": 874.5722920284607, "p90": 1316.2320414092392, "p95": 1763.1564252253152 }, "latency_first_usable_ms": { "median": 870.1500419992954, "p90": 1323.0120831984095, "p95": 1777.7514874993358 }, "tokens_first": { "prompt_tokens": { "total": 76524, "covered_requests": 186 }, "completion_tokens": { "total": 712, "covered_requests": 186 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 187, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.0071321, "exact_decimal_usd": "0.0071321", "known_records": 168, "missing_records": 1 }, "pilot": { "reported_usd": 0.0008394, "exact_decimal_usd": "0.0008394", "known_records": 19, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:34:24.116351-04:00", "last": "2026-10-15T11:03:32.479724-04:00" }, "identities": [ [ "google/gemini-2.5-flash-lite", "Google AI Studio" ] ] } }, "difficulty_proposed": { "demanding": { "scheduled": 55, "completed_first": 55, "correct_first": 12, "accuracy_scheduled": 0.21818181818181817, "usable_first": 44, "coverage": 0.8, "accuracy_usable": 0.2727272727272727, "wilson95": [ 0.12946024041123025, 0.34370034021840024 ], "statuses_first": { "ok": 44, "invalid_output": 11 }, "eventual_correct": 12, "eventual_usable": 44, "retries": 0, "first_cost": { "reported_usd": 0.0024295, "exact_decimal_usd": "0.0024295", "known_records": 55, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0024295, "exact_decimal_usd": "0.0024295", "known_records": 55, "missing_records": 0 }, "core_cost_per_scheduled_usd": 4.417272727272727e-05, "first_cost_per_first_correct_usd": 0.00020245833333333333, "core_cost_per_first_correct_usd": 0.00020245833333333333, "latency_first_all_ms": { "median": 858.2065829541534, "p90": 1206.5051161916927, "p95": 1834.1261210967778 }, "latency_first_usable_ms": { "median": 842.4616459815297, "p90": 1207.5017328897957, "p95": 1941.2160222098473 }, "tokens_first": { "prompt_tokens": { "total": 22959, "covered_requests": 55 }, "completion_tokens": { "total": 334, "covered_requests": 55 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 55, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.0021344, "exact_decimal_usd": "0.0021344", "known_records": 48, "missing_records": 0 }, "pilot": { "reported_usd": 0.0002951, "exact_decimal_usd": "0.0002951", "known_records": 7, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:11.630201-04:00", "last": "2026-10-15T11:03:03.535915-04:00" }, "identities": [ [ "google/gemini-2.5-flash-lite", "Google AI Studio" ] ] }, "moderate": { "scheduled": 107, "completed_first": 107, "correct_first": 38, "accuracy_scheduled": 0.35514018691588783, "usable_first": 100, "coverage": 0.9345794392523364, "accuracy_usable": 0.38, "wilson95": [ 0.27092916384095944, 0.4493920920336314 ], "statuses_first": { "ok": 100, "invalid_output": 7 }, "eventual_correct": 38, "eventual_usable": 100, "retries": 0, "first_cost": { "reported_usd": 0.0046231, "exact_decimal_usd": "0.0046231", "known_records": 107, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0046231, "exact_decimal_usd": "0.0046231", "known_records": 107, "missing_records": 0 }, "core_cost_per_scheduled_usd": 4.320654205607476e-05, "first_cost_per_first_correct_usd": 0.00012166052631578946, "core_cost_per_first_correct_usd": 0.00012166052631578946, "latency_first_all_ms": { "median": 882.350999978371, "p90": 1321.6865836177026, "p95": 1593.7218292849134 }, "latency_first_usable_ms": { "median": 878.4616460034158, "p90": 1372.5316997210032, "p95": 1611.8067336239615 }, "tokens_first": { "prompt_tokens": { "total": 44935, "covered_requests": 107 }, "completion_tokens": { "total": 324, "covered_requests": 107 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 107, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.0042193, "exact_decimal_usd": "0.0042193", "known_records": 98, "missing_records": 0 }, "pilot": { "reported_usd": 0.0004038, "exact_decimal_usd": "0.0004038", "known_records": 9, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:34:24.116351-04:00", "last": "2026-10-15T11:03:32.479724-04:00" }, "identities": [ [ "google/gemini-2.5-flash-lite", "Google AI Studio" ] ] }, "straightforward": { "scheduled": 38, "completed_first": 38, "correct_first": 25, "accuracy_scheduled": 0.6578947368421053, "usable_first": 36, "coverage": 0.9473684210526315, "accuracy_usable": 0.6944444444444444, "wilson95": [ 0.4989210355178308, 0.7878758482004058 ], "statuses_first": { "ok": 36, "http_failure": 1, "invalid_output": 1 }, "eventual_correct": 25, "eventual_usable": 37, "retries": 1, "first_cost": { "reported_usd": 0.0015535, "exact_decimal_usd": "0.0015535", "known_records": 37, "missing_records": 1 }, "core_cost": { "reported_usd": 0.0015878, "exact_decimal_usd": "0.0015878", "known_records": 38, "missing_records": 1 }, "core_cost_per_scheduled_usd": 4.178421052631579e-05, "first_cost_per_first_correct_usd": 6.214e-05, "core_cost_per_first_correct_usd": 6.3512e-05, "latency_first_all_ms": { "median": 888.768021017313, "p90": 1316.8133752071299, "p95": 1409.5458580617553 }, "latency_first_usable_ms": { "median": 900.3880830132402, "p90": 1319.3845420028083, "p95": 1452.2473747638287 }, "tokens_first": { "prompt_tokens": { "total": 15267, "covered_requests": 37 }, "completion_tokens": { "total": 67, "covered_requests": 37 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 38, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.0014121, "exact_decimal_usd": "0.0014121", "known_records": 34, "missing_records": 1 }, "pilot": { "reported_usd": 0.0001757, "exact_decimal_usd": "0.0001757", "known_records": 4, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:03.149012-04:00", "last": "2026-10-15T11:03:24.676133-04:00" }, "identities": [ [ "google/gemini-2.5-flash-lite", "Google AI Studio" ] ] } }, "visual_medium": { "chart": { "scheduled": 13, "completed_first": 13, "correct_first": 1, "accuracy_scheduled": 0.07692307692307693, "usable_first": 12, "coverage": 0.9230769230769231, "accuracy_usable": 0.08333333333333333, "wilson95": [ 0.013710421242556586, 0.3331395092109959 ], "statuses_first": { "ok": 12, "invalid_output": 1 }, "eventual_correct": 1, "eventual_usable": 12, "retries": 0, "first_cost": { "reported_usd": 0.0005217, "exact_decimal_usd": "0.0005217", "known_records": 13, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0005217, "exact_decimal_usd": "0.0005217", "known_records": 13, "missing_records": 0 }, "core_cost_per_scheduled_usd": 4.0130769230769235e-05, "first_cost_per_first_correct_usd": 0.0005217, "core_cost_per_first_correct_usd": 0.0005217, "latency_first_all_ms": { "median": 714.9042499950156, "p90": 850.8924497640692, "p95": 872.2538833739236 }, "latency_first_usable_ms": { "median": 711.4671664894558, "p90": 816.9331085984595, "p95": 838.0927166814217 }, "tokens_first": { "prompt_tokens": { "total": 5041, "covered_requests": 13 }, "completion_tokens": { "total": 44, "covered_requests": 13 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 13, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.000442, "exact_decimal_usd": "0.0004420", "known_records": 11, "missing_records": 0 }, "pilot": { "reported_usd": 7.97e-05, "exact_decimal_usd": "0.0000797", "known_records": 2, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:53.034948-04:00", "last": "2026-10-15T11:01:39.248535-04:00" }, "identities": [ [ "google/gemini-2.5-flash-lite", "Google AI Studio" ] ] }, "diagram": { "scheduled": 92, "completed_first": 92, "correct_first": 23, "accuracy_scheduled": 0.25, "usable_first": 75, "coverage": 0.8152173913043478, "accuracy_usable": 0.30666666666666664, "wilson95": [ 0.17275255342528545, 0.3472881412175253 ], "statuses_first": { "ok": 75, "invalid_output": 17 }, "eventual_correct": 23, "eventual_usable": 75, "retries": 0, "first_cost": { "reported_usd": 0.0037938, "exact_decimal_usd": "0.0037938", "known_records": 92, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0037938, "exact_decimal_usd": "0.0037938", "known_records": 92, "missing_records": 0 }, "core_cost_per_scheduled_usd": 4.123695652173913e-05, "first_cost_per_first_correct_usd": 0.00016494782608695652, "core_cost_per_first_correct_usd": 0.00016494782608695652, "latency_first_all_ms": { "median": 852.1228329627775, "p90": 1132.4396957759745, "p95": 1648.2988084637332 }, "latency_first_usable_ms": { "median": 841.8457079678774, "p90": 1080.033466825262, "p95": 1634.5246956043404 }, "tokens_first": { "prompt_tokens": { "total": 35714, "covered_requests": 92 }, "completion_tokens": { "total": 556, "covered_requests": 92 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 92, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.0034009, "exact_decimal_usd": "0.0034009", "known_records": 82, "missing_records": 0 }, "pilot": { "reported_usd": 0.0003929, "exact_decimal_usd": "0.0003929", "known_records": 10, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:02.290579-04:00", "last": "2026-10-15T11:03:32.479724-04:00" }, "identities": [ [ "google/gemini-2.5-flash-lite", "Google AI Studio" ] ] }, "diagram_or_published_illustration": { "scheduled": 8, "completed_first": 8, "correct_first": 6, "accuracy_scheduled": 0.75, "usable_first": 8, "coverage": 1.0, "accuracy_usable": 0.75, "wilson95": [ 0.40927543031016883, 0.9285207872478909 ], "statuses_first": { "ok": 8 }, "eventual_correct": 6, "eventual_usable": 8, "retries": 0, "first_cost": { "reported_usd": 0.0003318, "exact_decimal_usd": "0.0003318", "known_records": 8, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0003318, "exact_decimal_usd": "0.0003318", "known_records": 8, "missing_records": 0 }, "core_cost_per_scheduled_usd": 4.1475e-05, "first_cost_per_first_correct_usd": 5.5299999999999996e-05, "core_cost_per_first_correct_usd": 5.5299999999999996e-05, "latency_first_all_ms": { "median": 812.1454374922905, "p90": 1044.9742875993252, "p95": 1086.3781648105942 }, "latency_first_usable_ms": { "median": 812.1454374922905, "p90": 1044.9742875993252, "p95": 1086.3781648105942 }, "tokens_first": { "prompt_tokens": { "total": 3286, "covered_requests": 8 }, "completion_tokens": { "total": 8, "covered_requests": 8 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 8, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.0001977, "exact_decimal_usd": "0.0001977", "known_records": 5, "missing_records": 0 }, "pilot": { "reported_usd": 0.0001341, "exact_decimal_usd": "0.0001341", "known_records": 3, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:09.191257-04:00", "last": "2026-10-15T10:56:49.670464-04:00" }, "identities": [ [ "google/gemini-2.5-flash-lite", "Google AI Studio" ] ] }, "document": { "scheduled": 12, "completed_first": 12, "correct_first": 6, "accuracy_scheduled": 0.5, "usable_first": 11, "coverage": 0.9166666666666666, "accuracy_usable": 0.5454545454545454, "wilson95": [ 0.2537815976337061, 0.7462184023662939 ], "statuses_first": { "ok": 11, "invalid_output": 1 }, "eventual_correct": 6, "eventual_usable": 11, "retries": 0, "first_cost": { "reported_usd": 0.0004853, "exact_decimal_usd": "0.0004853", "known_records": 12, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0004853, "exact_decimal_usd": "0.0004853", "known_records": 12, "missing_records": 0 }, "core_cost_per_scheduled_usd": 4.044166666666666e-05, "first_cost_per_first_correct_usd": 8.088333333333333e-05, "core_cost_per_first_correct_usd": 8.088333333333333e-05, "latency_first_all_ms": { "median": 1172.3385000077542, "p90": 1582.0982377743348, "p95": 1857.8130733832945 }, "latency_first_usable_ms": { "median": 1088.7465000269003, "p90": 1602.954541973304, "p95": 1886.130687984405 }, "tokens_first": { "prompt_tokens": { "total": 4681, "covered_requests": 12 }, "completion_tokens": { "total": 43, "covered_requests": 12 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 12, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.0004482, "exact_decimal_usd": "0.0004482", "known_records": 11, "missing_records": 0 }, "pilot": { "reported_usd": 3.71e-05, "exact_decimal_usd": "0.0000371", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:27.748161-04:00", "last": "2026-10-15T11:02:31.978026-04:00" }, "identities": [ [ "google/gemini-2.5-flash-lite", "Google AI Studio" ] ] }, "photograph": { "scheduled": 64, "completed_first": 64, "correct_first": 31, "accuracy_scheduled": 0.484375, "usable_first": 63, "coverage": 0.984375, "accuracy_usable": 0.49206349206349204, "wilson95": [ 0.3663355166543264, 0.6041839852141988 ], "statuses_first": { "ok": 63, "http_failure": 1 }, "eventual_correct": 31, "eventual_usable": 64, "retries": 1, "first_cost": { "reported_usd": 0.0030424, "exact_decimal_usd": "0.0030424", "known_records": 63, "missing_records": 1 }, "core_cost": { "reported_usd": 0.0030767, "exact_decimal_usd": "0.0030767", "known_records": 64, "missing_records": 1 }, "core_cost_per_scheduled_usd": 4.80734375e-05, "first_cost_per_first_correct_usd": 9.814193548387096e-05, "core_cost_per_first_correct_usd": 9.924838709677419e-05, "latency_first_all_ms": { "median": 952.0659380068537, "p90": 1384.3036536243746, "p95": 1934.489176806527 }, "latency_first_usable_ms": { "median": 954.8134590149857, "p90": 1389.8479494266214, "p95": 1944.017437205184 }, "tokens_first": { "prompt_tokens": { "total": 30172, "covered_requests": 63 }, "completion_tokens": { "total": 63, "covered_requests": 63 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 64, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.0028871, "exact_decimal_usd": "0.0028871", "known_records": 61, "missing_records": 1 }, "pilot": { "reported_usd": 0.0001896, "exact_decimal_usd": "0.0001896", "known_records": 3, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:34:24.116351-04:00", "last": "2026-10-15T11:03:23.778964-04:00" }, "identities": [ [ "google/gemini-2.5-flash-lite", "Google AI Studio" ] ] }, "screenshot": { "scheduled": 11, "completed_first": 11, "correct_first": 8, "accuracy_scheduled": 0.7272727272727273, "usable_first": 11, "coverage": 1.0, "accuracy_usable": 0.7272727272727273, "wilson95": [ 0.43435469882387084, 0.902539407099751 ], "statuses_first": { "ok": 11 }, "eventual_correct": 8, "eventual_usable": 11, "retries": 0, "first_cost": { "reported_usd": 0.0004311, "exact_decimal_usd": "0.0004311", "known_records": 11, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0004311, "exact_decimal_usd": "0.0004311", "known_records": 11, "missing_records": 0 }, "core_cost_per_scheduled_usd": 3.9190909090909094e-05, "first_cost_per_first_correct_usd": 5.38875e-05, "core_cost_per_first_correct_usd": 5.38875e-05, "latency_first_all_ms": { "median": 822.6377500104718, "p90": 1000.5138330161572, "p95": 1041.9674165023025 }, "latency_first_usable_ms": { "median": 822.6377500104718, "p90": 1000.5138330161572, "p95": 1041.9674165023025 }, "tokens_first": { "prompt_tokens": { "total": 4267, "covered_requests": 11 }, "completion_tokens": { "total": 11, "covered_requests": 11 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 11, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.0003899, "exact_decimal_usd": "0.0003899", "known_records": 10, "missing_records": 0 }, "pilot": { "reported_usd": 4.12e-05, "exact_decimal_usd": "0.0000412", "known_records": 1, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:35:31.091292-04:00", "last": "2026-10-15T11:02:54.653807-04:00" }, "identities": [ [ "google/gemini-2.5-flash-lite", "Google AI Studio" ] ] } }, "stage": { "1": { "scheduled": 100, "completed_first": 100, "correct_first": 40, "accuracy_scheduled": 0.4, "usable_first": 94, "coverage": 0.94, "accuracy_usable": 0.425531914893617, "wilson95": [ 0.3094012864324589, 0.4979974132089382 ], "statuses_first": { "ok": 94, "invalid_output": 6 }, "eventual_correct": 40, "eventual_usable": 94, "retries": 0, "first_cost": { "reported_usd": 0.0043126, "exact_decimal_usd": "0.0043126", "known_records": 100, "missing_records": 0 }, "core_cost": { "reported_usd": 0.0043126, "exact_decimal_usd": "0.0043126", "known_records": 100, "missing_records": 0 }, "core_cost_per_scheduled_usd": 4.3126e-05, "first_cost_per_first_correct_usd": 0.000107815, "core_cost_per_first_correct_usd": 0.000107815, "latency_first_all_ms": { "median": 847.1294789633248, "p90": 1219.8686583898964, "p95": 1394.7187520359876 }, "latency_first_usable_ms": { "median": 847.1294789633248, "p90": 1207.4530375946779, "p95": 1396.6822643502383 }, "tokens_first": { "prompt_tokens": { "total": 41994, "covered_requests": 100 }, "completion_tokens": { "total": 283, "covered_requests": 100 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 100, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.003438, "exact_decimal_usd": "0.0034380", "known_records": 80, "missing_records": 0 }, "pilot": { "reported_usd": 0.0008746, "exact_decimal_usd": "0.0008746", "known_records": 20, "missing_records": 0 } }, "test_window": { "first": "2026-10-15T10:34:24.116351-04:00", "last": "2026-10-15T10:53:29.586112-04:00" }, "identities": [ [ "google/gemini-2.5-flash-lite", "Google AI Studio" ] ] }, "2": { "scheduled": 100, "completed_first": 100, "correct_first": 35, "accuracy_scheduled": 0.35, "usable_first": 86, "coverage": 0.86, "accuracy_usable": 0.4069767441860465, "wilson95": [ 0.26364248243097527, 0.44745556703112027 ], "statuses_first": { "ok": 86, "invalid_output": 13, "http_failure": 1 }, "eventual_correct": 35, "eventual_usable": 87, "retries": 1, "first_cost": { "reported_usd": 0.0042935, "exact_decimal_usd": "0.0042935", "known_records": 99, "missing_records": 1 }, "core_cost": { "reported_usd": 0.0043278, "exact_decimal_usd": "0.0043278", "known_records": 100, "missing_records": 1 }, "core_cost_per_scheduled_usd": 4.3277999999999995e-05, "first_cost_per_first_correct_usd": 0.00012267142857142857, "core_cost_per_first_correct_usd": 0.00012365142857142858, "latency_first_all_ms": { "median": 901.8202085280791, "p90": 1356.7413330078134, "p95": 1789.1521541372636 }, "latency_first_usable_ms": { "median": 920.7560000068042, "p90": 1401.7323330044746, "p95": 1917.305062248488 }, "tokens_first": { "prompt_tokens": { "total": 41167, "covered_requests": 99 }, "completion_tokens": { "total": 442, "covered_requests": 99 } }, "cached_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens_reported": 0, "reasoning_covered_attempts": 100, "reasoning_text_present_attempts": 0, "phase_cost": { "extension": { "reported_usd": 0.0043278, "exact_decimal_usd": "0.0043278", "known_records": 100, "missing_records": 1 } }, "test_window": { "first": "2026-10-15T10:53:38.435236-04:00", "last": "2026-10-15T11:03:32.479724-04:00" }, "identities": [ [ "google/gemini-2.5-flash-lite", "Google AI Studio" ] ] } } }, "families_at_least_8_of_10": [ "V15" ], "families_at_most_4_of_10": [ "V02", "V03", "V04", "V05", "V06", "V07", "V08", "V10", "V11", "V12", "V13", "V14", "V17", "V19" ] } }, "families": { "V01": "Attribute binding and set logic", "V02": "Conditional counting", "V03": "Spatial relationships", "V04": "Connectivity and reachability", "V05": "Navigation and planning", "V06": "Rotation and reflection", "V07": "Geometry, folding and fit", "V08": "Visual rule application", "V09": "Sequential state tracking", "V10": "Constraint satisfaction", "V11": "Flowcharts and decision trees", "V12": "Causal and counterfactual reasoning", "V13": "Charts and quantitative comparisons", "V14": "Document reconciliation", "V15": "Interface state decisions", "V16": "Inspection against a reference", "V17": "Matching across images", "V18": "Image and text consistency", "V19": "Perspective and knowledge tracking", "V20": "Insufficient evidence" }, "all_model_ids": [ "typesafe/jev-1.14", "openai/gpt-6-luna-decisions", "openai/gpt-6-luna", "openai/gpt-5.4-nano", "google/gemini-2.5-flash-lite" ], "extension_first_requests": 540, "extension_attempts": 541, "extension_cost": { "reported_usd": 0.080447325, "exact_decimal_usd": "0.080447325", "known_records": 540, "missing_records": 1 }, "reused_pilot_first_requests": 60, "llm_total_cost": { "reported_usd": 0.0887466, "exact_decimal_usd": "0.088746600", "known_records": 600, "missing_records": 1 }, "decision_probes_excluded_from_core_cost": { "reported_usd": 0.000573136, "exact_decimal_usd": "0.000573136", "known_records": 4, "missing_records": 0 }, "total_scored_core_cost": { "reported_usd": 0.135269676, "exact_decimal_usd": "0.135269676", "known_records": 994, "missing_records": 9 }, "common_usable_scores": { "typesafe/jev-1.14": 128, "openai/gpt-6-luna-decisions": 105, "openai/gpt-6-luna": 99, "openai/gpt-5.4-nano": 60, "google/gemini-2.5-flash-lite": 72 }, "pairwise": [ { "a": "typesafe/jev-1.14", "b": "openai/gpt-6-luna-decisions", "items": 200, "counts": { "neither_correct": 38, "a_only": 41, "both_correct": 100, "b_only": 21 }, "both_usable": 193, "choice_agreement": 115, "agreement_rate_among_both_usable": 0.5958549222797928, "both_usable_scores": { "typesafe/jev-1.14": 139, "openai/gpt-6-luna-decisions": 117 }, "mcnemar_exact_two_sided_unadjusted_p": 0.015134080997447081 }, { "a": "typesafe/jev-1.14", "b": "openai/gpt-6-luna", "items": 200, "counts": { "neither_correct": 38, "a_only": 45, "both_correct": 96, "b_only": 21 }, "both_usable": 195, "choice_agreement": 106, "agreement_rate_among_both_usable": 0.5435897435897435, "both_usable_scores": { "typesafe/jev-1.14": 141, "openai/gpt-6-luna": 112 }, "mcnemar_exact_two_sided_unadjusted_p": 0.004272405031651673 }, { "a": "typesafe/jev-1.14", "b": "openai/gpt-5.4-nano", "items": 200, "counts": { "b_only": 19, "neither_correct": 40, "a_only": 90, "both_correct": 51 }, "both_usable": 191, "choice_agreement": 65, "agreement_rate_among_both_usable": 0.3403141361256545, "both_usable_scores": { "typesafe/jev-1.14": 140, "openai/gpt-5.4-nano": 67 }, "mcnemar_exact_two_sided_unadjusted_p": 3.0984729664322627e-12 }, { "a": "typesafe/jev-1.14", "b": "google/gemini-2.5-flash-lite", "items": 200, "counts": { "b_only": 14, "a_only": 80, "neither_correct": 45, "both_correct": 61 }, "both_usable": 175, "choice_agreement": 70, "agreement_rate_among_both_usable": 0.4, "both_usable_scores": { "typesafe/jev-1.14": 131, "google/gemini-2.5-flash-lite": 72 }, "mcnemar_exact_two_sided_unadjusted_p": 2.120124047968334e-12 }, { "a": "openai/gpt-6-luna-decisions", "b": "openai/gpt-6-luna", "items": 200, "counts": { "neither_correct": 50, "both_correct": 88, "b_only": 29, "a_only": 33 }, "both_usable": 198, "choice_agreement": 104, "agreement_rate_among_both_usable": 0.5252525252525253, "both_usable_scores": { "openai/gpt-6-luna-decisions": 121, "openai/gpt-6-luna": 116 }, "mcnemar_exact_two_sided_unadjusted_p": 0.7035366713552734 }, { "a": "openai/gpt-6-luna-decisions", "b": "openai/gpt-5.4-nano", "items": 200, "counts": { "b_only": 26, "neither_correct": 53, "both_correct": 44, "a_only": 77 }, "both_usable": 194, "choice_agreement": 64, "agreement_rate_among_both_usable": 0.32989690721649484, "both_usable_scores": { "openai/gpt-6-luna-decisions": 120, "openai/gpt-5.4-nano": 70 }, "mcnemar_exact_two_sided_unadjusted_p": 4.945479817326375e-07 }, { "a": "openai/gpt-6-luna-decisions", "b": "google/gemini-2.5-flash-lite", "items": 200, "counts": { "b_only": 21, "neither_correct": 58, "a_only": 67, "both_correct": 54 }, "both_usable": 178, "choice_agreement": 80, "agreement_rate_among_both_usable": 0.449438202247191, "both_usable_scores": { "openai/gpt-6-luna-decisions": 110, "google/gemini-2.5-flash-lite": 75 }, "mcnemar_exact_two_sided_unadjusted_p": 9.203075999074401e-07 }, { "a": "openai/gpt-6-luna", "b": "openai/gpt-5.4-nano", "items": 200, "counts": { "b_only": 20, "neither_correct": 63, "both_correct": 50, "a_only": 67 }, "both_usable": 196, "choice_agreement": 73, "agreement_rate_among_both_usable": 0.37244897959183676, "both_usable_scores": { "openai/gpt-6-luna": 115, "openai/gpt-5.4-nano": 70 }, "mcnemar_exact_two_sided_unadjusted_p": 4.3053681250011924e-07 }, { "a": "openai/gpt-6-luna", "b": "google/gemini-2.5-flash-lite", "items": 200, "counts": { "b_only": 17, "neither_correct": 66, "a_only": 59, "both_correct": 58 }, "both_usable": 180, "choice_agreement": 79, "agreement_rate_among_both_usable": 0.4388888888888889, "both_usable_scores": { "openai/gpt-6-luna": 106, "google/gemini-2.5-flash-lite": 75 }, "mcnemar_exact_two_sided_unadjusted_p": 1.3968718113140024e-06 }, { "a": "openai/gpt-5.4-nano", "b": "google/gemini-2.5-flash-lite", "items": 200, "counts": { "both_correct": 34, "b_only": 41, "neither_correct": 89, "a_only": 36 }, "both_usable": 177, "choice_agreement": 63, "agreement_rate_among_both_usable": 0.3559322033898305, "both_usable_scores": { "openai/gpt-5.4-nano": 63, "google/gemini-2.5-flash-lite": 75 }, "mcnemar_exact_two_sided_unadjusted_p": 0.6487911623106449 } ], "distinct_scene_groups": 200, "uniform_random_choice_expected_accuracy": 0.2344166666666666, "option_count_distribution": { "5": 64, "4": 135, "3": 1 }, "extension_retry_wait_seconds": 2, "public_release_date": "2026-10-15", "common_usable_first_items": 170, "no_model_correct_count": 14, "all_five_usable_wrong_count": 11, "no_model_correct_with_coverage_failure_count": 3, "answer_key_policy": "External private key; no per-item gold labels or correctness flags in this release", "images_included": 221, "images_referenced_only": 6 }