{ "schema_version": 1, "models": [ { "id": "pretrained-4b", "label": "Pretrained Qwen3.5-4B", "note": "Original pretrained weights. The single-clip demonstration lets thinking finish at the model\u2019s own end token, with no output-token cutoff or sentence limit. This demo includes the Qwen-recommended thinking presence penalty of 1.5; the earlier speed and cup-checkpoint runs used 0. The prompt gives the overall tracking task and asks reasoning to use the overhead history. Wrist images remain available, so camera focus is a prompt instruction. The capped speed trials use the earlier robot-subtask prompt. Open the task/question panel to inspect each recording.", "benchmark_groups": [ { "thinking": false, "output_token_cap": 128, "requests": 9, "generated_tokens": 291, "decode_tokens": 282, "weighted_decode_ms_per_token": 24.458465414082116, "weighted_decode_tokens_per_second": 40.88563951458065, "token_p50_ms": 24.116783868521452, "token_p95_ms": 26.394780911505222, "median_request_ttft_ms": 86.98154520243406, "median_request_generation_ms": 703.1395165249705, "completed_responses": 9, "finished_thinking": 0, "peak_allocated_bytes": 9751848960, "complete_answers": 9, "ended_during_thinking": 0 }, { "thinking": true, "output_token_cap": 128, "requests": 9, "generated_tokens": 1152, "decode_tokens": 1143, "weighted_decode_ms_per_token": 24.91690364461767, "weighted_decode_tokens_per_second": 40.133397562662694, "token_p50_ms": 24.56353697925806, "token_p95_ms": 26.79282296448946, "median_request_ttft_ms": 83.15128646790981, "median_request_generation_ms": 3266.7781906202435, "completed_responses": 0, "finished_thinking": 0, "peak_allocated_bytes": 9751603200, "complete_answers": 0, "ended_during_thinking": 0 }, { "thinking": true, "output_token_cap": 256, "requests": 9, "generated_tokens": 2304, "decode_tokens": 2295, "weighted_decode_ms_per_token": 24.872802278282595, "weighted_decode_tokens_per_second": 40.20455712274683, "token_p50_ms": 24.52925778925419, "token_p95_ms": 26.718663144856688, "median_request_ttft_ms": 82.68214948475361, "median_request_generation_ms": 6431.44086189568, "completed_responses": 0, "finished_thinking": 0, "peak_allocated_bytes": 9751603200, "complete_answers": 0, "ended_during_thinking": 0 }, { "thinking": true, "output_token_cap": 512, "requests": 9, "generated_tokens": 4608, "decode_tokens": 4599, "weighted_decode_ms_per_token": 24.94245734240736, "weighted_decode_tokens_per_second": 40.09228065511381, "token_p50_ms": 24.613408371806145, "token_p95_ms": 26.969467662274837, "median_request_ttft_ms": 84.06988810747862, "median_request_generation_ms": 12888.269694522023, "completed_responses": 0, "finished_thinking": 0, "peak_allocated_bytes": 9751603200, "complete_answers": 0, "ended_during_thinking": 0 } ], "evaluation_groups": [ { "thinking": false, "output_token_cap": 128, "requests": 1, "generated_tokens": 11, "decode_tokens": 10, "weighted_decode_ms_per_token": 24.11205731332302, "weighted_decode_tokens_per_second": 41.47302683489617, "token_p50_ms": 24.04729835689068, "token_p95_ms": 24.505381286144257, "median_request_ttft_ms": 82.9090615734458, "median_request_generation_ms": 324.22545924782753, "completed_responses": 1, "finished_thinking": 0, "peak_allocated_bytes": 9636953600 }, { "thinking": true, "output_token_cap": null, "requests": 1, "generated_tokens": 9552, "decode_tokens": 9551, "weighted_decode_ms_per_token": 25.460018550230377, "weighted_decode_tokens_per_second": 39.27726910438372, "token_p50_ms": 24.850010871887207, "token_p95_ms": 27.52943802624941, "median_request_ttft_ms": 83.99572968482971, "median_request_generation_ms": 243252.97097861767, "completed_responses": 1, "finished_thinking": 1, "peak_allocated_bytes": 9694846976 } ], "requests": [ { "folder": "cup_eval_excluded", "episode_index": 242, "thinking": false, "output_token_cap": 128, "repeat": 0, "file": "data/pretrained-4b/base_measured_001_cup_eval_excluded_000242_direct_128_r0.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_excluded", "episode_index": 242, "thinking": true, "output_token_cap": 128, "repeat": 0, "file": "data/pretrained-4b/base_measured_002_cup_eval_excluded_000242_think_128_r0.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_excluded", "episode_index": 242, "thinking": true, "output_token_cap": 256, "repeat": 0, "file": "data/pretrained-4b/base_measured_003_cup_eval_excluded_000242_think_256_r0.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_excluded", "episode_index": 242, "thinking": true, "output_token_cap": 512, "repeat": 0, "file": "data/pretrained-4b/base_measured_004_cup_eval_excluded_000242_think_512_r0.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_6mixes", "episode_index": 0, "thinking": false, "output_token_cap": 128, "repeat": 0, "file": "data/pretrained-4b/base_measured_005_cup_eval_6mixes_000000_direct_128_r0.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_6mixes", "episode_index": 0, "thinking": true, "output_token_cap": 128, "repeat": 0, "file": "data/pretrained-4b/base_measured_006_cup_eval_6mixes_000000_think_128_r0.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_6mixes", "episode_index": 0, "thinking": true, "output_token_cap": 256, "repeat": 0, "file": "data/pretrained-4b/base_measured_007_cup_eval_6mixes_000000_think_256_r0.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_6mixes", "episode_index": 0, "thinking": true, "output_token_cap": 512, "repeat": 0, "file": "data/pretrained-4b/base_measured_008_cup_eval_6mixes_000000_think_512_r0.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_4cups", "episode_index": 15, "thinking": false, "output_token_cap": 128, "repeat": 0, "file": "data/pretrained-4b/base_measured_009_cup_eval_4cups_000015_direct_128_r0.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_4cups", "episode_index": 15, "thinking": true, "output_token_cap": 128, "repeat": 0, "file": "data/pretrained-4b/base_measured_010_cup_eval_4cups_000015_think_128_r0.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_4cups", "episode_index": 15, "thinking": true, "output_token_cap": 256, "repeat": 0, "file": "data/pretrained-4b/base_measured_011_cup_eval_4cups_000015_think_256_r0.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_4cups", "episode_index": 15, "thinking": true, "output_token_cap": 512, "repeat": 0, "file": "data/pretrained-4b/base_measured_012_cup_eval_4cups_000015_think_512_r0.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_excluded", "episode_index": 242, "thinking": true, "output_token_cap": 512, "repeat": 1, "file": "data/pretrained-4b/base_measured_013_cup_eval_excluded_000242_think_512_r1.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_excluded", "episode_index": 242, "thinking": true, "output_token_cap": 256, "repeat": 1, "file": "data/pretrained-4b/base_measured_014_cup_eval_excluded_000242_think_256_r1.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_excluded", "episode_index": 242, "thinking": true, "output_token_cap": 128, "repeat": 1, "file": "data/pretrained-4b/base_measured_015_cup_eval_excluded_000242_think_128_r1.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_excluded", "episode_index": 242, "thinking": false, "output_token_cap": 128, "repeat": 1, "file": "data/pretrained-4b/base_measured_016_cup_eval_excluded_000242_direct_128_r1.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_6mixes", "episode_index": 0, "thinking": true, "output_token_cap": 512, "repeat": 1, "file": "data/pretrained-4b/base_measured_017_cup_eval_6mixes_000000_think_512_r1.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_6mixes", "episode_index": 0, "thinking": true, "output_token_cap": 256, "repeat": 1, "file": "data/pretrained-4b/base_measured_018_cup_eval_6mixes_000000_think_256_r1.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_6mixes", "episode_index": 0, "thinking": true, "output_token_cap": 128, "repeat": 1, "file": "data/pretrained-4b/base_measured_019_cup_eval_6mixes_000000_think_128_r1.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_6mixes", "episode_index": 0, "thinking": false, "output_token_cap": 128, "repeat": 1, "file": "data/pretrained-4b/base_measured_020_cup_eval_6mixes_000000_direct_128_r1.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_4cups", "episode_index": 15, "thinking": true, "output_token_cap": 512, "repeat": 1, "file": "data/pretrained-4b/base_measured_021_cup_eval_4cups_000015_think_512_r1.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_4cups", "episode_index": 15, "thinking": true, "output_token_cap": 256, "repeat": 1, "file": "data/pretrained-4b/base_measured_022_cup_eval_4cups_000015_think_256_r1.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_4cups", "episode_index": 15, "thinking": true, "output_token_cap": 128, "repeat": 1, "file": "data/pretrained-4b/base_measured_023_cup_eval_4cups_000015_think_128_r1.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_4cups", "episode_index": 15, "thinking": false, "output_token_cap": 128, "repeat": 1, "file": "data/pretrained-4b/base_measured_024_cup_eval_4cups_000015_direct_128_r1.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_excluded", "episode_index": 242, "thinking": false, "output_token_cap": 128, "repeat": 2, "file": "data/pretrained-4b/base_measured_025_cup_eval_excluded_000242_direct_128_r2.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_excluded", "episode_index": 242, "thinking": true, "output_token_cap": 128, "repeat": 2, "file": "data/pretrained-4b/base_measured_026_cup_eval_excluded_000242_think_128_r2.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_excluded", "episode_index": 242, "thinking": true, "output_token_cap": 256, "repeat": 2, "file": "data/pretrained-4b/base_measured_027_cup_eval_excluded_000242_think_256_r2.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_excluded", "episode_index": 242, "thinking": true, "output_token_cap": 512, "repeat": 2, "file": "data/pretrained-4b/base_measured_028_cup_eval_excluded_000242_think_512_r2.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_6mixes", "episode_index": 0, "thinking": false, "output_token_cap": 128, "repeat": 2, "file": "data/pretrained-4b/base_measured_029_cup_eval_6mixes_000000_direct_128_r2.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_6mixes", "episode_index": 0, "thinking": true, "output_token_cap": 128, "repeat": 2, "file": "data/pretrained-4b/base_measured_030_cup_eval_6mixes_000000_think_128_r2.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_6mixes", "episode_index": 0, "thinking": true, "output_token_cap": 256, "repeat": 2, "file": "data/pretrained-4b/base_measured_031_cup_eval_6mixes_000000_think_256_r2.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_6mixes", "episode_index": 0, "thinking": true, "output_token_cap": 512, "repeat": 2, "file": "data/pretrained-4b/base_measured_032_cup_eval_6mixes_000000_think_512_r2.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_4cups", "episode_index": 15, "thinking": false, "output_token_cap": 128, "repeat": 2, "file": "data/pretrained-4b/base_measured_033_cup_eval_4cups_000015_direct_128_r2.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_4cups", "episode_index": 15, "thinking": true, "output_token_cap": 128, "repeat": 2, "file": "data/pretrained-4b/base_measured_034_cup_eval_4cups_000015_think_128_r2.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_4cups", "episode_index": 15, "thinking": true, "output_token_cap": 256, "repeat": 2, "file": "data/pretrained-4b/base_measured_035_cup_eval_4cups_000015_think_256_r2.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_4cups", "episode_index": 15, "thinking": true, "output_token_cap": 512, "repeat": 2, "file": "data/pretrained-4b/base_measured_036_cup_eval_4cups_000015_think_512_r2.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_excluded", "episode_index": 242, "thinking": false, "output_token_cap": 128, "repeat": 0, "file": "data/pretrained-4b/base_natural_demo_001_cup_eval_excluded_000242_direct_128_r0.json", "evaluation": true, "run_kind": "natural_demo" }, { "folder": "cup_eval_excluded", "episode_index": 242, "thinking": true, "output_token_cap": null, "repeat": 0, "file": "data/pretrained-4b/base_natural_demo_002_cup_eval_excluded_000242_think_None_r0.json", "evaluation": true, "run_kind": "natural_demo" } ], "accuracy": [ { "thinking": false, "folder": "cup_eval_excluded", "total": 1, "correct": 0, "location_correct": 0, "completed": 1 }, { "thinking": true, "folder": "cup_eval_excluded", "total": 1, "correct": 0, "location_correct": 0, "completed": 1 } ], "comparison_clips": 1 }, { "id": "cup-oat-oct15-1000", "label": "Cup OAT \u00b7 Oct 15 \u00b7 step 1,000", "note": "Three-arm cup checkpoint: subtask + narration + 128 latent slots. The 128 slots remain in both modes; native thinking adds generated reasoning. Uses the checkpoint\u2019s original prompt, normalized robot state and matched 120-frame history cap. This thinking mode was not used during its robot fine-tuning.", "benchmark_groups": [ { "thinking": false, "output_token_cap": 128, "requests": 9, "generated_tokens": 21, "decode_tokens": 12, "weighted_decode_ms_per_token": 24.62580963037908, "weighted_decode_tokens_per_second": 40.60780193664667, "token_p50_ms": 24.24186933785677, "token_p95_ms": 26.358068734407425, "median_request_ttft_ms": 83.26379023492336, "median_request_generation_ms": 127.25390959531069, "completed_responses": 9, "finished_thinking": 0, "peak_allocated_bytes": 9786337280, "complete_answers": 9, "ended_during_thinking": 0 }, { "thinking": true, "output_token_cap": 128, "requests": 9, "generated_tokens": 854, "decode_tokens": 845, "weighted_decode_ms_per_token": 24.715575932927386, "weighted_decode_tokens_per_second": 40.460315499577234, "token_p50_ms": 24.537323974072933, "token_p95_ms": 25.987096689641472, "median_request_ttft_ms": 83.39486829936504, "median_request_generation_ms": 3211.925271898508, "completed_responses": 3, "finished_thinking": 0, "peak_allocated_bytes": 9786059776, "complete_answers": 0, "ended_during_thinking": 3 }, { "thinking": true, "output_token_cap": 256, "requests": 9, "generated_tokens": 1502, "decode_tokens": 1493, "weighted_decode_ms_per_token": 24.745157795065573, "weighted_decode_tokens_per_second": 40.41194678497503, "token_p50_ms": 24.51496571302414, "token_p95_ms": 26.099389046430588, "median_request_ttft_ms": 83.65150075405836, "median_request_generation_ms": 6367.885256186128, "completed_responses": 4, "finished_thinking": 0, "peak_allocated_bytes": 9786059776, "complete_answers": 0, "ended_during_thinking": 4 }, { "thinking": true, "output_token_cap": 512, "requests": 9, "generated_tokens": 2582, "decode_tokens": 2573, "weighted_decode_ms_per_token": 24.788340379176695, "weighted_decode_tokens_per_second": 40.3415470621843, "token_p50_ms": 24.581137113273144, "token_p95_ms": 26.436991430819038, "median_request_ttft_ms": 83.94002821296453, "median_request_generation_ms": 7821.5283919125795, "completed_responses": 5, "finished_thinking": 1, "peak_allocated_bytes": 9786059776, "complete_answers": 0, "ended_during_thinking": 4 } ], "evaluation_groups": [ { "thinking": false, "output_token_cap": 128, "requests": 47, "generated_tokens": 109, "decode_tokens": 62, "weighted_decode_ms_per_token": 25.11101486461778, "weighted_decode_tokens_per_second": 39.823161484764675, "token_p50_ms": 24.644566234201193, "token_p95_ms": 26.461595902219415, "median_request_ttft_ms": 94.45523098111153, "median_request_generation_ms": 122.52608686685562, "completed_responses": 47, "finished_thinking": 0, "peak_allocated_bytes": 9790509056 }, { "thinking": true, "output_token_cap": 2048, "requests": 47, "generated_tokens": 28666, "decode_tokens": 28619, "weighted_decode_ms_per_token": 24.99268692359211, "weighted_decode_tokens_per_second": 40.0117043460437, "token_p50_ms": 24.57366418093443, "token_p95_ms": 27.38036140799522, "median_request_ttft_ms": 91.26534406095743, "median_request_generation_ms": 5151.83371398598, "completed_responses": 42, "finished_thinking": 13, "peak_allocated_bytes": 9790350336 } ], "requests": [ { "folder": "cup_eval_excluded", "episode_index": 242, "thinking": false, "output_token_cap": 128, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_measured_001_cup_eval_excluded_000242_direct_128_r0.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_excluded", "episode_index": 242, "thinking": true, "output_token_cap": 128, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_measured_002_cup_eval_excluded_000242_think_128_r0.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_excluded", "episode_index": 242, "thinking": true, "output_token_cap": 256, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_measured_003_cup_eval_excluded_000242_think_256_r0.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_excluded", "episode_index": 242, "thinking": true, "output_token_cap": 512, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_measured_004_cup_eval_excluded_000242_think_512_r0.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_6mixes", "episode_index": 0, "thinking": false, "output_token_cap": 128, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_measured_005_cup_eval_6mixes_000000_direct_128_r0.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_6mixes", "episode_index": 0, "thinking": true, "output_token_cap": 128, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_measured_006_cup_eval_6mixes_000000_think_128_r0.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_6mixes", "episode_index": 0, "thinking": true, "output_token_cap": 256, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_measured_007_cup_eval_6mixes_000000_think_256_r0.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_6mixes", "episode_index": 0, "thinking": true, "output_token_cap": 512, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_measured_008_cup_eval_6mixes_000000_think_512_r0.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_4cups", "episode_index": 15, "thinking": false, "output_token_cap": 128, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_measured_009_cup_eval_4cups_000015_direct_128_r0.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_4cups", "episode_index": 15, "thinking": true, "output_token_cap": 128, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_measured_010_cup_eval_4cups_000015_think_128_r0.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_4cups", "episode_index": 15, "thinking": true, "output_token_cap": 256, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_measured_011_cup_eval_4cups_000015_think_256_r0.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_4cups", "episode_index": 15, "thinking": true, "output_token_cap": 512, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_measured_012_cup_eval_4cups_000015_think_512_r0.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_excluded", "episode_index": 242, "thinking": true, "output_token_cap": 512, "repeat": 1, "file": "data/cup-oat-oct15-1000/oat_measured_013_cup_eval_excluded_000242_think_512_r1.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_excluded", "episode_index": 242, "thinking": true, "output_token_cap": 256, "repeat": 1, "file": "data/cup-oat-oct15-1000/oat_measured_014_cup_eval_excluded_000242_think_256_r1.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_excluded", "episode_index": 242, "thinking": true, "output_token_cap": 128, "repeat": 1, "file": "data/cup-oat-oct15-1000/oat_measured_015_cup_eval_excluded_000242_think_128_r1.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_excluded", "episode_index": 242, "thinking": false, "output_token_cap": 128, "repeat": 1, "file": "data/cup-oat-oct15-1000/oat_measured_016_cup_eval_excluded_000242_direct_128_r1.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_6mixes", "episode_index": 0, "thinking": true, "output_token_cap": 512, "repeat": 1, "file": "data/cup-oat-oct15-1000/oat_measured_017_cup_eval_6mixes_000000_think_512_r1.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_6mixes", "episode_index": 0, "thinking": true, "output_token_cap": 256, "repeat": 1, "file": "data/cup-oat-oct15-1000/oat_measured_018_cup_eval_6mixes_000000_think_256_r1.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_6mixes", "episode_index": 0, "thinking": true, "output_token_cap": 128, "repeat": 1, "file": "data/cup-oat-oct15-1000/oat_measured_019_cup_eval_6mixes_000000_think_128_r1.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_6mixes", "episode_index": 0, "thinking": false, "output_token_cap": 128, "repeat": 1, "file": "data/cup-oat-oct15-1000/oat_measured_020_cup_eval_6mixes_000000_direct_128_r1.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_4cups", "episode_index": 15, "thinking": true, "output_token_cap": 512, "repeat": 1, "file": "data/cup-oat-oct15-1000/oat_measured_021_cup_eval_4cups_000015_think_512_r1.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_4cups", "episode_index": 15, "thinking": true, "output_token_cap": 256, "repeat": 1, "file": "data/cup-oat-oct15-1000/oat_measured_022_cup_eval_4cups_000015_think_256_r1.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_4cups", "episode_index": 15, "thinking": true, "output_token_cap": 128, "repeat": 1, "file": "data/cup-oat-oct15-1000/oat_measured_023_cup_eval_4cups_000015_think_128_r1.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_4cups", "episode_index": 15, "thinking": false, "output_token_cap": 128, "repeat": 1, "file": "data/cup-oat-oct15-1000/oat_measured_024_cup_eval_4cups_000015_direct_128_r1.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_excluded", "episode_index": 242, "thinking": false, "output_token_cap": 128, "repeat": 2, "file": "data/cup-oat-oct15-1000/oat_measured_025_cup_eval_excluded_000242_direct_128_r2.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_excluded", "episode_index": 242, "thinking": true, "output_token_cap": 128, "repeat": 2, "file": "data/cup-oat-oct15-1000/oat_measured_026_cup_eval_excluded_000242_think_128_r2.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_excluded", "episode_index": 242, "thinking": true, "output_token_cap": 256, "repeat": 2, "file": "data/cup-oat-oct15-1000/oat_measured_027_cup_eval_excluded_000242_think_256_r2.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_excluded", "episode_index": 242, "thinking": true, "output_token_cap": 512, "repeat": 2, "file": "data/cup-oat-oct15-1000/oat_measured_028_cup_eval_excluded_000242_think_512_r2.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_6mixes", "episode_index": 0, "thinking": false, "output_token_cap": 128, "repeat": 2, "file": "data/cup-oat-oct15-1000/oat_measured_029_cup_eval_6mixes_000000_direct_128_r2.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_6mixes", "episode_index": 0, "thinking": true, "output_token_cap": 128, "repeat": 2, "file": "data/cup-oat-oct15-1000/oat_measured_030_cup_eval_6mixes_000000_think_128_r2.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_6mixes", "episode_index": 0, "thinking": true, "output_token_cap": 256, "repeat": 2, "file": "data/cup-oat-oct15-1000/oat_measured_031_cup_eval_6mixes_000000_think_256_r2.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_6mixes", "episode_index": 0, "thinking": true, "output_token_cap": 512, "repeat": 2, "file": "data/cup-oat-oct15-1000/oat_measured_032_cup_eval_6mixes_000000_think_512_r2.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_4cups", "episode_index": 15, "thinking": false, "output_token_cap": 128, "repeat": 2, "file": "data/cup-oat-oct15-1000/oat_measured_033_cup_eval_4cups_000015_direct_128_r2.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_4cups", "episode_index": 15, "thinking": true, "output_token_cap": 128, "repeat": 2, "file": "data/cup-oat-oct15-1000/oat_measured_034_cup_eval_4cups_000015_think_128_r2.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_4cups", "episode_index": 15, "thinking": true, "output_token_cap": 256, "repeat": 2, "file": "data/cup-oat-oct15-1000/oat_measured_035_cup_eval_4cups_000015_think_256_r2.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_4cups", "episode_index": 15, "thinking": true, "output_token_cap": 512, "repeat": 2, "file": "data/cup-oat-oct15-1000/oat_measured_036_cup_eval_4cups_000015_think_512_r2.json", "evaluation": false, "run_kind": "speed" }, { "folder": "cup_eval_excluded", "episode_index": 242, "thinking": false, "output_token_cap": 128, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_001_cup_eval_excluded_000242_direct_128_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 242, "thinking": true, "output_token_cap": 2048, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_002_cup_eval_excluded_000242_think_2048_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 226, "thinking": false, "output_token_cap": 128, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_003_cup_eval_excluded_000226_direct_128_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 226, "thinking": true, "output_token_cap": 2048, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_004_cup_eval_excluded_000226_think_2048_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 210, "thinking": false, "output_token_cap": 128, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_005_cup_eval_excluded_000210_direct_128_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 210, "thinking": true, "output_token_cap": 2048, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_006_cup_eval_excluded_000210_think_2048_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 194, "thinking": false, "output_token_cap": 128, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_007_cup_eval_excluded_000194_direct_128_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 194, "thinking": true, "output_token_cap": 2048, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_008_cup_eval_excluded_000194_think_2048_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 177, "thinking": false, "output_token_cap": 128, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_009_cup_eval_excluded_000177_direct_128_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 177, "thinking": true, "output_token_cap": 2048, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_010_cup_eval_excluded_000177_think_2048_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 161, "thinking": false, "output_token_cap": 128, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_011_cup_eval_excluded_000161_direct_128_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 161, "thinking": true, "output_token_cap": 2048, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_012_cup_eval_excluded_000161_think_2048_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 145, "thinking": false, "output_token_cap": 128, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_013_cup_eval_excluded_000145_direct_128_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 145, "thinking": true, "output_token_cap": 2048, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_014_cup_eval_excluded_000145_think_2048_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 129, "thinking": false, "output_token_cap": 128, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_015_cup_eval_excluded_000129_direct_128_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 129, "thinking": true, "output_token_cap": 2048, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_016_cup_eval_excluded_000129_think_2048_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 112, "thinking": false, "output_token_cap": 128, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_017_cup_eval_excluded_000112_direct_128_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 112, "thinking": true, "output_token_cap": 2048, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_018_cup_eval_excluded_000112_think_2048_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 96, "thinking": false, "output_token_cap": 128, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_019_cup_eval_excluded_000096_direct_128_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 96, "thinking": true, "output_token_cap": 2048, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_020_cup_eval_excluded_000096_think_2048_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 80, "thinking": false, "output_token_cap": 128, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_021_cup_eval_excluded_000080_direct_128_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 80, "thinking": true, "output_token_cap": 2048, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_022_cup_eval_excluded_000080_think_2048_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 64, "thinking": false, "output_token_cap": 128, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_023_cup_eval_excluded_000064_direct_128_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 64, "thinking": true, "output_token_cap": 2048, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_024_cup_eval_excluded_000064_think_2048_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 47, "thinking": false, "output_token_cap": 128, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_025_cup_eval_excluded_000047_direct_128_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 47, "thinking": true, "output_token_cap": 2048, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_026_cup_eval_excluded_000047_think_2048_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 31, "thinking": false, "output_token_cap": 128, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_027_cup_eval_excluded_000031_direct_128_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 31, "thinking": true, "output_token_cap": 2048, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_028_cup_eval_excluded_000031_think_2048_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 13, "thinking": false, "output_token_cap": 128, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_029_cup_eval_excluded_000013_direct_128_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 13, "thinking": true, "output_token_cap": 2048, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_030_cup_eval_excluded_000013_think_2048_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 235, "thinking": false, "output_token_cap": 128, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_031_cup_eval_excluded_000235_direct_128_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 235, "thinking": true, "output_token_cap": 2048, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_032_cup_eval_excluded_000235_think_2048_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 218, "thinking": false, "output_token_cap": 128, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_033_cup_eval_excluded_000218_direct_128_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 218, "thinking": true, "output_token_cap": 2048, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_034_cup_eval_excluded_000218_think_2048_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 202, "thinking": false, "output_token_cap": 128, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_035_cup_eval_excluded_000202_direct_128_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 202, "thinking": true, "output_token_cap": 2048, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_036_cup_eval_excluded_000202_think_2048_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 186, "thinking": false, "output_token_cap": 128, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_037_cup_eval_excluded_000186_direct_128_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 186, "thinking": true, "output_token_cap": 2048, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_038_cup_eval_excluded_000186_think_2048_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 169, "thinking": false, "output_token_cap": 128, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_039_cup_eval_excluded_000169_direct_128_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 169, "thinking": true, "output_token_cap": 2048, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_040_cup_eval_excluded_000169_think_2048_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 153, "thinking": false, "output_token_cap": 128, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_041_cup_eval_excluded_000153_direct_128_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 153, "thinking": true, "output_token_cap": 2048, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_042_cup_eval_excluded_000153_think_2048_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 137, "thinking": false, "output_token_cap": 128, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_043_cup_eval_excluded_000137_direct_128_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 137, "thinking": true, "output_token_cap": 2048, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_044_cup_eval_excluded_000137_think_2048_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 121, "thinking": false, "output_token_cap": 128, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_045_cup_eval_excluded_000121_direct_128_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 121, "thinking": true, "output_token_cap": 2048, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_046_cup_eval_excluded_000121_think_2048_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 103, "thinking": false, "output_token_cap": 128, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_047_cup_eval_excluded_000103_direct_128_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 103, "thinking": true, "output_token_cap": 2048, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_048_cup_eval_excluded_000103_think_2048_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 88, "thinking": false, "output_token_cap": 128, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_049_cup_eval_excluded_000088_direct_128_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 88, "thinking": true, "output_token_cap": 2048, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_050_cup_eval_excluded_000088_think_2048_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 72, "thinking": false, "output_token_cap": 128, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_051_cup_eval_excluded_000072_direct_128_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 72, "thinking": true, "output_token_cap": 2048, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_052_cup_eval_excluded_000072_think_2048_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 56, "thinking": false, "output_token_cap": 128, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_053_cup_eval_excluded_000056_direct_128_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 56, "thinking": true, "output_token_cap": 2048, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_054_cup_eval_excluded_000056_think_2048_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 39, "thinking": false, "output_token_cap": 128, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_055_cup_eval_excluded_000039_direct_128_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 39, "thinking": true, "output_token_cap": 2048, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_056_cup_eval_excluded_000039_think_2048_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 22, "thinking": false, "output_token_cap": 128, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_057_cup_eval_excluded_000022_direct_128_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 22, "thinking": true, "output_token_cap": 2048, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_058_cup_eval_excluded_000022_think_2048_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 6, "thinking": false, "output_token_cap": 128, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_059_cup_eval_excluded_000006_direct_128_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_excluded", "episode_index": 6, "thinking": true, "output_token_cap": 2048, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_060_cup_eval_excluded_000006_think_2048_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_6mixes", "episode_index": 0, "thinking": false, "output_token_cap": 128, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_061_cup_eval_6mixes_000000_direct_128_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_6mixes", "episode_index": 0, "thinking": true, "output_token_cap": 2048, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_062_cup_eval_6mixes_000000_think_2048_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_6mixes", "episode_index": 1, "thinking": false, "output_token_cap": 128, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_063_cup_eval_6mixes_000001_direct_128_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_6mixes", "episode_index": 1, "thinking": true, "output_token_cap": 2048, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_064_cup_eval_6mixes_000001_think_2048_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_6mixes", "episode_index": 2, "thinking": false, "output_token_cap": 128, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_065_cup_eval_6mixes_000002_direct_128_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_6mixes", "episode_index": 2, "thinking": true, "output_token_cap": 2048, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_066_cup_eval_6mixes_000002_think_2048_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_6mixes", "episode_index": 3, "thinking": false, "output_token_cap": 128, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_067_cup_eval_6mixes_000003_direct_128_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_6mixes", "episode_index": 3, "thinking": true, "output_token_cap": 2048, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_068_cup_eval_6mixes_000003_think_2048_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_6mixes", "episode_index": 4, "thinking": false, "output_token_cap": 128, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_069_cup_eval_6mixes_000004_direct_128_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_6mixes", "episode_index": 4, "thinking": true, "output_token_cap": 2048, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_070_cup_eval_6mixes_000004_think_2048_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_6mixes", "episode_index": 5, "thinking": false, "output_token_cap": 128, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_071_cup_eval_6mixes_000005_direct_128_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_6mixes", "episode_index": 5, "thinking": true, "output_token_cap": 2048, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_072_cup_eval_6mixes_000005_think_2048_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_6mixes", "episode_index": 6, "thinking": false, "output_token_cap": 128, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_073_cup_eval_6mixes_000006_direct_128_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_6mixes", "episode_index": 6, "thinking": true, "output_token_cap": 2048, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_074_cup_eval_6mixes_000006_think_2048_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_6mixes", "episode_index": 7, "thinking": false, "output_token_cap": 128, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_075_cup_eval_6mixes_000007_direct_128_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_6mixes", "episode_index": 7, "thinking": true, "output_token_cap": 2048, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_076_cup_eval_6mixes_000007_think_2048_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_6mixes", "episode_index": 8, "thinking": false, "output_token_cap": 128, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_077_cup_eval_6mixes_000008_direct_128_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_6mixes", "episode_index": 8, "thinking": true, "output_token_cap": 2048, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_078_cup_eval_6mixes_000008_think_2048_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_6mixes", "episode_index": 9, "thinking": false, "output_token_cap": 128, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_079_cup_eval_6mixes_000009_direct_128_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_6mixes", "episode_index": 9, "thinking": true, "output_token_cap": 2048, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_080_cup_eval_6mixes_000009_think_2048_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_6mixes", "episode_index": 10, "thinking": false, "output_token_cap": 128, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_081_cup_eval_6mixes_000010_direct_128_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_6mixes", "episode_index": 10, "thinking": true, "output_token_cap": 2048, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_082_cup_eval_6mixes_000010_think_2048_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_6mixes", "episode_index": 11, "thinking": false, "output_token_cap": 128, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_083_cup_eval_6mixes_000011_direct_128_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_6mixes", "episode_index": 11, "thinking": true, "output_token_cap": 2048, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_084_cup_eval_6mixes_000011_think_2048_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_6mixes", "episode_index": 12, "thinking": false, "output_token_cap": 128, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_085_cup_eval_6mixes_000012_direct_128_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_6mixes", "episode_index": 12, "thinking": true, "output_token_cap": 2048, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_086_cup_eval_6mixes_000012_think_2048_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_6mixes", "episode_index": 13, "thinking": false, "output_token_cap": 128, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_087_cup_eval_6mixes_000013_direct_128_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_6mixes", "episode_index": 13, "thinking": true, "output_token_cap": 2048, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_088_cup_eval_6mixes_000013_think_2048_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_6mixes", "episode_index": 14, "thinking": false, "output_token_cap": 128, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_089_cup_eval_6mixes_000014_direct_128_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_6mixes", "episode_index": 14, "thinking": true, "output_token_cap": 2048, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_090_cup_eval_6mixes_000014_think_2048_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_4cups", "episode_index": 15, "thinking": false, "output_token_cap": 128, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_091_cup_eval_4cups_000015_direct_128_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_4cups", "episode_index": 15, "thinking": true, "output_token_cap": 2048, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_092_cup_eval_4cups_000015_think_2048_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_4cups", "episode_index": 16, "thinking": false, "output_token_cap": 128, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_093_cup_eval_4cups_000016_direct_128_r0.json", "evaluation": true, "run_kind": "evaluation" }, { "folder": "cup_eval_4cups", "episode_index": 16, "thinking": true, "output_token_cap": 2048, "repeat": 0, "file": "data/cup-oat-oct15-1000/oat_evaluation_094_cup_eval_4cups_000016_think_2048_r0.json", "evaluation": true, "run_kind": "evaluation" } ], "accuracy": [ { "thinking": false, "folder": "cup_eval_excluded", "total": 30, "correct": 23, "location_correct": 23, "completed": 30 }, { "thinking": false, "folder": "cup_eval_6mixes", "total": 15, "correct": 4, "location_correct": 4, "completed": 15 }, { "thinking": false, "folder": "cup_eval_4cups", "total": 2, "correct": 0, "location_correct": 0, "completed": 2 }, { "thinking": true, "folder": "cup_eval_excluded", "total": 30, "correct": 2, "location_correct": 2, "completed": 3 }, { "thinking": true, "folder": "cup_eval_6mixes", "total": 15, "correct": 0, "location_correct": 0, "completed": 0 }, { "thinking": true, "folder": "cup_eval_4cups", "total": 2, "correct": 0, "location_correct": 0, "completed": 0 } ], "comparison_clips": 47 } ], "default_model": "pretrained-4b", "evaluation_pending": false, "pending_models": [], "status_note": "The 47-clip cup-checkpoint comparison is complete. The original-model demonstration without a thinking cutoff is complete.", "runtime_note": "Transformers eager generation \u00b7 FlashAttention2 \u00b7 BF16 \u00b7 batch 1 \u00b7 one H100. Fresh attention cache for every request. Model and kernels warmed up. Thinking uses temperature 1, top-p 0.95, top-k 20; direct output is greedy. The natural-completion demo adds presence penalty 1.5 for thinking; earlier capped runs use 0. Timing includes sampling and token delivery. Offline video reading, input checks, resizing, and CPU prompt preparation happen before timed generation and are shown separately below. Cup OAT direct answers are very short, so their decode speed uses only one or two subsequent tokens per trial. Results do not represent an optimized serving-engine ceiling.", "provenance": "Recorded October 15, 2026 \u00b7 Frozen cup evaluation inputs \u00b7 No robot actions or new training", "built_at": 1792131762.3877463 }