{ "source": "Su et al., GPT 6 Astra as an Embodied Policy; public_results/data.json at 8f3d362b077d8efb77e2a7274d5b2c20e2243846", "scope": "Selected summary fields only; score uses native 0..1 scale. Not original model or checker traces.", "cases": [ { "id": "mix__organize_table__standard__g0__l0", "case_id": "organize_table__standard__g0__l0", "method": "mix", "task": "organize_table", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 0, "reset_seed": 0, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0.5, "native_complete": true, "steps": 1000, "limit": 1000, "corrected_steps": 425, "chunks": 125, "tokens": { "totalTokens": 20838354, "inputTokens": 20788102, "cachedInputTokens": 20368128, "cacheWriteInputTokens": 0, "outputTokens": 50252, "reasoningOutputTokens": 19900 } }, { "id": "mix__classify_objects_by_language__standard__g0__l0", "case_id": "classify_objects_by_language__standard__g0__l0", "method": "mix", "task": "classify_objects_by_language", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 0, "reset_seed": 0, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "success", "success": true, "score": 1, "native_complete": true, "steps": 794, "limit": 1100, "corrected_steps": 130, "chunks": 73, "tokens": { "totalTokens": 11263361, "inputTokens": 11234999, "cachedInputTokens": 10962816, "cacheWriteInputTokens": 0, "outputTokens": 28362, "reasoningOutputTokens": 9623 } }, { "id": "mix__imitate_sorting_sequence__standard__g0__l0", "case_id": "imitate_sorting_sequence__standard__g0__l0", "method": "mix", "task": "imitate_sorting_sequence", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 0, "reset_seed": 0, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "success", "success": true, "score": 1, "native_complete": true, "steps": 1436, "limit": 1600, "corrected_steps": 210, "chunks": 126, "tokens": { "totalTokens": 17627393, "inputTokens": 17597307, "cachedInputTokens": 17276416, "cacheWriteInputTokens": 0, "outputTokens": 30086, "reasoningOutputTokens": 8470 } }, { "id": "mix__arrange_largest_number__standard__g0__l0", "case_id": "arrange_largest_number__standard__g0__l0", "method": "mix", "task": "arrange_largest_number", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 0, "reset_seed": 0, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "success", "success": true, "score": 1, "native_complete": true, "steps": 717, "limit": 1050, "corrected_steps": 140, "chunks": 67, "tokens": { "totalTokens": 11140693, "inputTokens": 11112366, "cachedInputTokens": 10829312, "cacheWriteInputTokens": 0, "outputTokens": 28327, "reasoningOutputTokens": 5977 } }, { "id": "mix__pack_objects_into_box__standard__g0__l0", "case_id": "pack_objects_into_box__standard__g0__l0", "method": "mix", "task": "pack_objects_into_box", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 0, "reset_seed": 0, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0.5, "native_complete": true, "steps": 1300, "limit": 1300, "corrected_steps": 310, "chunks": 130, "tokens": { "totalTokens": 18329856, "inputTokens": 18280141, "cachedInputTokens": 17860096, "cacheWriteInputTokens": 0, "outputTokens": 49715, "reasoningOutputTokens": 18385 } }, { "id": "mix__classify_objects__standard__g0__l0", "case_id": "classify_objects__standard__g0__l0", "method": "mix", "task": "classify_objects", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 0, "reset_seed": 0, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "success", "success": true, "score": 1, "native_complete": true, "steps": 867, "limit": 1100, "corrected_steps": 15, "chunks": 61, "tokens": { "totalTokens": 9897377, "inputTokens": 9879799, "cachedInputTokens": 9621248, "cacheWriteInputTokens": 0, "outputTokens": 17578, "reasoningOutputTokens": 2781 } }, { "id": "mix__build_tower__standard__g0__l0", "case_id": "build_tower__standard__g0__l0", "method": "mix", "task": "build_tower", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 0, "reset_seed": 0, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0.1, "native_complete": true, "steps": 1050, "limit": 1050, "corrected_steps": 190, "chunks": 97, "tokens": { "totalTokens": 14050329, "inputTokens": 14014861, "cachedInputTokens": 13698816, "cacheWriteInputTokens": 0, "outputTokens": 35468, "reasoningOutputTokens": 12365 } }, { "id": "mix__make_kong__standard__g0__l0", "case_id": "make_kong__standard__g0__l0", "method": "mix", "task": "make_kong", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 0, "reset_seed": 0, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0, "native_complete": true, "steps": 600, "limit": 600, "corrected_steps": 180, "chunks": 67, "tokens": { "totalTokens": 13403851, "inputTokens": 13373700, "cachedInputTokens": 13123584, "cacheWriteInputTokens": 0, "outputTokens": 30151, "reasoningOutputTokens": 11655 } }, { "id": "mix__fold_clothes__standard__g0__l0", "case_id": "fold_clothes__standard__g0__l0", "method": "mix", "task": "fold_clothes", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 0, "reset_seed": 0, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "success", "success": true, "score": 1, "native_complete": true, "steps": 299, "limit": 500, "corrected_steps": 0, "chunks": 20, "tokens": { "totalTokens": 4961745, "inputTokens": 4952855, "cachedInputTokens": 4823936, "cacheWriteInputTokens": 0, "outputTokens": 8890, "reasoningOutputTokens": 572 } }, { "id": "mix__put_bottles_into_dustbin__standard__g0__l0", "case_id": "put_bottles_into_dustbin__standard__g0__l0", "method": "mix", "task": "put_bottles_into_dustbin", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 0, "reset_seed": 0, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "success", "success": true, "score": 1, "native_complete": true, "steps": 485, "limit": 700, "corrected_steps": 0, "chunks": 34, "tokens": { "totalTokens": 4911846, "inputTokens": 4902071, "cachedInputTokens": 4776064, "cacheWriteInputTokens": 0, "outputTokens": 9775, "reasoningOutputTokens": 638 } }, { "id": "mix__organize_table__standard__g0__l1", "case_id": "organize_table__standard__g0__l1", "method": "mix", "task": "organize_table", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 1, "reset_seed": 1, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0.75, "native_complete": true, "steps": 1000, "limit": 1000, "corrected_steps": 255, "chunks": 105, "tokens": { "totalTokens": 14768921, "inputTokens": 14740122, "cachedInputTokens": 14311424, "cacheWriteInputTokens": 0, "outputTokens": 28799, "reasoningOutputTokens": 9197 } }, { "id": "mix__classify_objects_by_language__standard__g0__l1", "case_id": "classify_objects_by_language__standard__g0__l1", "method": "mix", "task": "classify_objects_by_language", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 1, "reset_seed": 1, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0, "native_complete": true, "steps": 1100, "limit": 1100, "corrected_steps": 0, "chunks": 74, "tokens": { "totalTokens": 5030010, "inputTokens": 5019392, "cachedInputTokens": 4889728, "cacheWriteInputTokens": 0, "outputTokens": 10618, "reasoningOutputTokens": 1441 } }, { "id": "mix__imitate_sorting_sequence__standard__g0__l1", "case_id": "imitate_sorting_sequence__standard__g0__l1", "method": "mix", "task": "imitate_sorting_sequence", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 1, "reset_seed": 1, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "success", "success": true, "score": 1, "native_complete": true, "steps": 1554, "limit": 1600, "corrected_steps": 185, "chunks": 135, "tokens": { "totalTokens": 19149974, "inputTokens": 19098601, "cachedInputTokens": 18561280, "cacheWriteInputTokens": 0, "outputTokens": 51373, "reasoningOutputTokens": 10178 } }, { "id": "mix__arrange_largest_number__standard__g0__l1", "case_id": "arrange_largest_number__standard__g0__l1", "method": "mix", "task": "arrange_largest_number", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 1, "reset_seed": 1, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "success", "success": true, "score": 1, "native_complete": true, "steps": 589, "limit": 1050, "corrected_steps": 0, "chunks": 40, "tokens": { "totalTokens": 12443568, "inputTokens": 12431043, "cachedInputTokens": 12221952, "cacheWriteInputTokens": 0, "outputTokens": 12525, "reasoningOutputTokens": 1299 } }, { "id": "mix__pack_objects_into_box__standard__g0__l1", "case_id": "pack_objects_into_box__standard__g0__l1", "method": "mix", "task": "pack_objects_into_box", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 1, "reset_seed": 1, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0.25, "native_complete": true, "steps": 1300, "limit": 1300, "corrected_steps": 0, "chunks": 87, "tokens": { "totalTokens": 40990266, "inputTokens": 40973429, "cachedInputTokens": 40501888, "cacheWriteInputTokens": 0, "outputTokens": 16837, "reasoningOutputTokens": 1312 } }, { "id": "mix__classify_objects__standard__g0__l1", "case_id": "classify_objects__standard__g0__l1", "method": "mix", "task": "classify_objects", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 1, "reset_seed": 1, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "success", "success": true, "score": 1, "native_complete": true, "steps": 844, "limit": 1100, "corrected_steps": 135, "chunks": 77, "tokens": { "totalTokens": 10513000, "inputTokens": 10477237, "cachedInputTokens": 10143872, "cacheWriteInputTokens": 0, "outputTokens": 35763, "reasoningOutputTokens": 11053 } }, { "id": "mix__build_tower__standard__g0__l1", "case_id": "build_tower__standard__g0__l1", "method": "mix", "task": "build_tower", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 1, "reset_seed": 1, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "success", "success": true, "score": 1, "native_complete": true, "steps": 861, "limit": 1050, "corrected_steps": 95, "chunks": 71, "tokens": { "totalTokens": 12231652, "inputTokens": 12212418, "cachedInputTokens": 12011264, "cacheWriteInputTokens": 0, "outputTokens": 19234, "reasoningOutputTokens": 6308 } }, { "id": "mix__make_kong__standard__g0__l1", "case_id": "make_kong__standard__g0__l1", "method": "mix", "task": "make_kong", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 1, "reset_seed": 1, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "success", "success": true, "score": 1, "native_complete": true, "steps": 381, "limit": 600, "corrected_steps": 15, "chunks": 29, "tokens": { "totalTokens": 5101838, "inputTokens": 5090813, "cachedInputTokens": 4900992, "cacheWriteInputTokens": 0, "outputTokens": 11025, "reasoningOutputTokens": 1871 } }, { "id": "mix__fold_clothes__standard__g0__l1", "case_id": "fold_clothes__standard__g0__l1", "method": "mix", "task": "fold_clothes", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 1, "reset_seed": 1, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "success", "success": true, "score": 1, "native_complete": true, "steps": 303, "limit": 500, "corrected_steps": 0, "chunks": 21, "tokens": { "totalTokens": 2938700, "inputTokens": 2932987, "cachedInputTokens": 2825344, "cacheWriteInputTokens": 0, "outputTokens": 5713, "reasoningOutputTokens": 384 } }, { "id": "mix__put_bottles_into_dustbin__standard__g0__l1", "case_id": "put_bottles_into_dustbin__standard__g0__l1", "method": "mix", "task": "put_bottles_into_dustbin", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 1, "reset_seed": 1, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "success", "success": true, "score": 1, "native_complete": true, "steps": 397, "limit": 700, "corrected_steps": 5, "chunks": 30, "tokens": { "totalTokens": 5265586, "inputTokens": 5255493, "cachedInputTokens": 5115648, "cacheWriteInputTokens": 0, "outputTokens": 10093, "reasoningOutputTokens": 1276 } }, { "id": "mix__organize_table__standard__g0__l2", "case_id": "organize_table__standard__g0__l2", "method": "mix", "task": "organize_table", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 2, "reset_seed": 2, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0.5, "native_complete": true, "steps": 1000, "limit": 1000, "corrected_steps": 274, "chunks": 105, "tokens": { "totalTokens": 15445394, "inputTokens": 15404036, "cachedInputTokens": 15091328, "cacheWriteInputTokens": 0, "outputTokens": 41358, "reasoningOutputTokens": 14400 } }, { "id": "mix__classify_objects_by_language__standard__g0__l2", "case_id": "classify_objects_by_language__standard__g0__l2", "method": "mix", "task": "classify_objects_by_language", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 2, "reset_seed": 2, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0.1, "native_complete": true, "steps": 1100, "limit": 1100, "corrected_steps": 85, "chunks": 85, "tokens": { "totalTokens": 11968101, "inputTokens": 11945257, "cachedInputTokens": 11554048, "cacheWriteInputTokens": 0, "outputTokens": 22844, "reasoningOutputTokens": 5495 } }, { "id": "mix__imitate_sorting_sequence__standard__g0__l2", "case_id": "imitate_sorting_sequence__standard__g0__l2", "method": "mix", "task": "imitate_sorting_sequence", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 2, "reset_seed": 2, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0.15, "native_complete": true, "steps": 1600, "limit": 1600, "corrected_steps": 320, "chunks": 152, "tokens": { "totalTokens": 23005269, "inputTokens": 22956329, "cachedInputTokens": 22545536, "cacheWriteInputTokens": 0, "outputTokens": 48940, "reasoningOutputTokens": 17182 } }, { "id": "mix__arrange_largest_number__random__g0__l0", "case_id": "arrange_largest_number__random__g0__l0", "method": "mix", "task": "arrange_largest_number", "variant": "random", "seeds": { "eval_seed": 0, "layout_id": 0, "reset_seed": 0, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0.05, "native_complete": true, "steps": 1050, "limit": 1050, "corrected_steps": 328, "chunks": 117, "tokens": { "totalTokens": 18638326, "inputTokens": 18585604, "cachedInputTokens": 18019712, "cacheWriteInputTokens": 0, "outputTokens": 52722, "reasoningOutputTokens": 19306 } }, { "id": "mix__pack_objects_into_box__random__g0__l0", "case_id": "pack_objects_into_box__random__g0__l0", "method": "mix", "task": "pack_objects_into_box", "variant": "random", "seeds": { "eval_seed": 0, "layout_id": 0, "reset_seed": 0, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0.5, "native_complete": true, "steps": 1300, "limit": 1300, "corrected_steps": 0, "chunks": 87, "tokens": { "totalTokens": 11918877, "inputTokens": 11900964, "cachedInputTokens": 11649152, "cacheWriteInputTokens": 0, "outputTokens": 17913, "reasoningOutputTokens": 2077 } }, { "id": "mix__classify_objects__standard__g0__l2", "case_id": "classify_objects__standard__g0__l2", "method": "mix", "task": "classify_objects", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 2, "reset_seed": 2, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0.4, "native_complete": true, "steps": 1100, "limit": 1100, "corrected_steps": 150, "chunks": 94, "tokens": { "totalTokens": 13707363, "inputTokens": 13681114, "cachedInputTokens": 13371520, "cacheWriteInputTokens": 0, "outputTokens": 26249, "reasoningOutputTokens": 8992 } }, { "id": "mix__build_tower__standard__g0__l2", "case_id": "build_tower__standard__g0__l2", "method": "mix", "task": "build_tower", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 2, "reset_seed": 2, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "success", "success": true, "score": 1, "native_complete": true, "steps": 718, "limit": 1050, "corrected_steps": 0, "chunks": 48, "tokens": { "totalTokens": 6253310, "inputTokens": 6233229, "cachedInputTokens": 5747584, "cacheWriteInputTokens": 0, "outputTokens": 20081, "reasoningOutputTokens": 2688 } }, { "id": "mix__make_kong__standard__g0__l2", "case_id": "make_kong__standard__g0__l2", "method": "mix", "task": "make_kong", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 2, "reset_seed": 2, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0, "native_complete": true, "steps": 600, "limit": 600, "corrected_steps": 85, "chunks": 52, "tokens": { "totalTokens": 8194616, "inputTokens": 8173909, "cachedInputTokens": 7824000, "cacheWriteInputTokens": 0, "outputTokens": 20707, "reasoningOutputTokens": 6439 } }, { "id": "mix__fold_clothes__random__g0__l0", "case_id": "fold_clothes__random__g0__l0", "method": "mix", "task": "fold_clothes", "variant": "random", "seeds": { "eval_seed": 0, "layout_id": 0, "reset_seed": 0, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "success", "success": true, "score": 1, "native_complete": true, "steps": 312, "limit": 500, "corrected_steps": 15, "chunks": 24, "tokens": { "totalTokens": 3683955, "inputTokens": 3676066, "cachedInputTokens": 3504256, "cacheWriteInputTokens": 0, "outputTokens": 7889, "reasoningOutputTokens": 970 } }, { "id": "mix__put_bottles_into_dustbin__standard__g0__l2", "case_id": "put_bottles_into_dustbin__standard__g0__l2", "method": "mix", "task": "put_bottles_into_dustbin", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 2, "reset_seed": 2, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "success", "success": true, "score": 1, "native_complete": true, "steps": 436, "limit": 700, "corrected_steps": 0, "chunks": 30, "tokens": { "totalTokens": 8016190, "inputTokens": 8004618, "cachedInputTokens": 7812352, "cacheWriteInputTokens": 0, "outputTokens": 11572, "reasoningOutputTokens": 859 } }, { "id": "mix__organize_table__standard__g0__l3", "case_id": "organize_table__standard__g0__l3", "method": "mix", "task": "organize_table", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 3, "reset_seed": 3, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0.5, "native_complete": true, "steps": 1000, "limit": 1000, "corrected_steps": 60, "chunks": 81, "tokens": { "totalTokens": 11207303, "inputTokens": 11183470, "cachedInputTokens": 10832128, "cacheWriteInputTokens": 0, "outputTokens": 23833, "reasoningOutputTokens": 5697 } }, { "id": "mix__classify_objects_by_language__standard__g0__l3", "case_id": "classify_objects_by_language__standard__g0__l3", "method": "mix", "task": "classify_objects_by_language", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 3, "reset_seed": 3, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0.4, "native_complete": true, "steps": 1100, "limit": 1100, "corrected_steps": 315, "chunks": 120, "tokens": { "totalTokens": 16359852, "inputTokens": 16330809, "cachedInputTokens": 15948800, "cacheWriteInputTokens": 0, "outputTokens": 29043, "reasoningOutputTokens": 10237 } }, { "id": "mix__imitate_sorting_sequence__standard__g0__l3", "case_id": "imitate_sorting_sequence__standard__g0__l3", "method": "mix", "task": "imitate_sorting_sequence", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 3, "reset_seed": 3, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0.5, "native_complete": true, "steps": 1600, "limit": 1600, "corrected_steps": 315, "chunks": 158, "tokens": { "totalTokens": 27872887, "inputTokens": 27816419, "cachedInputTokens": 27085184, "cacheWriteInputTokens": 0, "outputTokens": 56468, "reasoningOutputTokens": 21940 } }, { "id": "mix__arrange_largest_number__random__g0__l1", "case_id": "arrange_largest_number__random__g0__l1", "method": "mix", "task": "arrange_largest_number", "variant": "random", "seeds": { "eval_seed": 0, "layout_id": 1, "reset_seed": 1, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0.4, "native_complete": true, "steps": 1050, "limit": 1050, "corrected_steps": 395, "chunks": 127, "tokens": { "totalTokens": 23096559, "inputTokens": 23040657, "cachedInputTokens": 22413056, "cacheWriteInputTokens": 0, "outputTokens": 55902, "reasoningOutputTokens": 23012 } }, { "id": "mix__pack_objects_into_box__random__g0__l1", "case_id": "pack_objects_into_box__random__g0__l1", "method": "mix", "task": "pack_objects_into_box", "variant": "random", "seeds": { "eval_seed": 0, "layout_id": 1, "reset_seed": 1, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "success", "success": true, "score": 1, "native_complete": true, "steps": 1022, "limit": 1300, "corrected_steps": 205, "chunks": 99, "tokens": { "totalTokens": 14468914, "inputTokens": 14441667, "cachedInputTokens": 13949568, "cacheWriteInputTokens": 0, "outputTokens": 27247, "reasoningOutputTokens": 6467 } }, { "id": "mix__classify_objects__standard__g0__l3", "case_id": "classify_objects__standard__g0__l3", "method": "mix", "task": "classify_objects", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 3, "reset_seed": 3, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0.15, "native_complete": true, "steps": 1100, "limit": 1100, "corrected_steps": 60, "chunks": 82, "tokens": { "totalTokens": 11624271, "inputTokens": 11591685, "cachedInputTokens": 11248640, "cacheWriteInputTokens": 0, "outputTokens": 32586, "reasoningOutputTokens": 11261 } }, { "id": "mix__build_tower__standard__g0__l3", "case_id": "build_tower__standard__g0__l3", "method": "mix", "task": "build_tower", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 3, "reset_seed": 3, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "success", "success": true, "score": 1, "native_complete": true, "steps": 721, "limit": 1050, "corrected_steps": 0, "chunks": 49, "tokens": { "totalTokens": 6167017, "inputTokens": 6153777, "cachedInputTokens": 5981952, "cacheWriteInputTokens": 0, "outputTokens": 13240, "reasoningOutputTokens": 1521 } }, { "id": "mix__make_kong__standard__g0__l3", "case_id": "make_kong__standard__g0__l3", "method": "mix", "task": "make_kong", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 3, "reset_seed": 3, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0, "native_complete": true, "steps": 600, "limit": 600, "corrected_steps": 0, "chunks": 40, "tokens": { "totalTokens": 6804042, "inputTokens": 6781216, "cachedInputTokens": 6619648, "cacheWriteInputTokens": 0, "outputTokens": 22826, "reasoningOutputTokens": 1156 } }, { "id": "mix__fold_clothes__random__g0__l1", "case_id": "fold_clothes__random__g0__l1", "method": "mix", "task": "fold_clothes", "variant": "random", "seeds": { "eval_seed": 0, "layout_id": 1, "reset_seed": 1, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "success", "success": true, "score": 1, "native_complete": true, "steps": 302, "limit": 500, "corrected_steps": 10, "chunks": 22, "tokens": { "totalTokens": 2982395, "inputTokens": 2971898, "cachedInputTokens": 2814592, "cacheWriteInputTokens": 0, "outputTokens": 10497, "reasoningOutputTokens": 2211 } }, { "id": "mix__put_bottles_into_dustbin__standard__g0__l3", "case_id": "put_bottles_into_dustbin__standard__g0__l3", "method": "mix", "task": "put_bottles_into_dustbin", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 3, "reset_seed": 3, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "success", "success": true, "score": 1, "native_complete": true, "steps": 410, "limit": 700, "corrected_steps": 0, "chunks": 29, "tokens": { "totalTokens": 7297183, "inputTokens": 7285874, "cachedInputTokens": 7120384, "cacheWriteInputTokens": 0, "outputTokens": 11309, "reasoningOutputTokens": 380 } }, { "id": "mix__organize_table__standard__g0__l4", "case_id": "organize_table__standard__g0__l4", "method": "mix", "task": "organize_table", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 4, "reset_seed": 4, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0.75, "native_complete": true, "steps": 1000, "limit": 1000, "corrected_steps": 238, "chunks": 101, "tokens": { "totalTokens": 15518336, "inputTokens": 15487699, "cachedInputTokens": 15172096, "cacheWriteInputTokens": 0, "outputTokens": 30637, "reasoningOutputTokens": 9635 } }, { "id": "mix__classify_objects_by_language__standard__g0__l4", "case_id": "classify_objects_by_language__standard__g0__l4", "method": "mix", "task": "classify_objects_by_language", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 4, "reset_seed": 4, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0.4, "native_complete": true, "steps": 1100, "limit": 1100, "corrected_steps": 181, "chunks": 103, "tokens": { "totalTokens": 27497595, "inputTokens": 27462387, "cachedInputTokens": 27062528, "cacheWriteInputTokens": 0, "outputTokens": 35208, "reasoningOutputTokens": 11489 } }, { "id": "mix__imitate_sorting_sequence__standard__g0__l4", "case_id": "imitate_sorting_sequence__standard__g0__l4", "method": "mix", "task": "imitate_sorting_sequence", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 4, "reset_seed": 4, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0, "native_complete": true, "steps": 165, "limit": 1600, "corrected_steps": 5, "chunks": 12, "tokens": { "totalTokens": 4942693, "inputTokens": 4921195, "cachedInputTokens": 4346624, "cacheWriteInputTokens": 0, "outputTokens": 21498, "reasoningOutputTokens": 4166 } }, { "id": "mix__arrange_largest_number__random__g0__l2", "case_id": "arrange_largest_number__random__g0__l2", "method": "mix", "task": "arrange_largest_number", "variant": "random", "seeds": { "eval_seed": 0, "layout_id": 2, "reset_seed": 2, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0.05, "native_complete": true, "steps": 1050, "limit": 1050, "corrected_steps": 310, "chunks": 115, "tokens": { "totalTokens": 15116214, "inputTokens": 15084284, "cachedInputTokens": 14701184, "cacheWriteInputTokens": 0, "outputTokens": 31930, "reasoningOutputTokens": 14331 } }, { "id": "mix__pack_objects_into_box__random__g0__l2", "case_id": "pack_objects_into_box__random__g0__l2", "method": "mix", "task": "pack_objects_into_box", "variant": "random", "seeds": { "eval_seed": 0, "layout_id": 2, "reset_seed": 2, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0.25, "native_complete": true, "steps": 1300, "limit": 1300, "corrected_steps": 265, "chunks": 124, "tokens": { "totalTokens": 17302211, "inputTokens": 17249484, "cachedInputTokens": 16902912, "cacheWriteInputTokens": 0, "outputTokens": 52727, "reasoningOutputTokens": 16313 } }, { "id": "mix__classify_objects__standard__g0__l4", "case_id": "classify_objects__standard__g0__l4", "method": "mix", "task": "classify_objects", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 4, "reset_seed": 4, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "success", "success": true, "score": 1, "native_complete": true, "steps": 888, "limit": 1100, "corrected_steps": 5, "chunks": 61, "tokens": { "totalTokens": 10224391, "inputTokens": 10210863, "cachedInputTokens": 9972608, "cacheWriteInputTokens": 0, "outputTokens": 13528, "reasoningOutputTokens": 1811 } }, { "id": "mix__build_tower__standard__g0__l4", "case_id": "build_tower__standard__g0__l4", "method": "mix", "task": "build_tower", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 4, "reset_seed": 4, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0.1, "native_complete": true, "steps": 1050, "limit": 1050, "corrected_steps": 263, "chunks": 108, "tokens": { "totalTokens": 17793239, "inputTokens": 17747694, "cachedInputTokens": 17259776, "cacheWriteInputTokens": 0, "outputTokens": 45545, "reasoningOutputTokens": 22284 } }, { "id": "mix__make_kong__standard__g0__l4", "case_id": "make_kong__standard__g0__l4", "method": "mix", "task": "make_kong", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 4, "reset_seed": 4, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "success", "success": true, "score": 1, "native_complete": true, "steps": 368, "limit": 600, "corrected_steps": 0, "chunks": 25, "tokens": { "totalTokens": 3898779, "inputTokens": 3890594, "cachedInputTokens": 3706624, "cacheWriteInputTokens": 0, "outputTokens": 8185, "reasoningOutputTokens": 986 } }, { "id": "mix__fold_clothes__random__g0__l2", "case_id": "fold_clothes__random__g0__l2", "method": "mix", "task": "fold_clothes", "variant": "random", "seeds": { "eval_seed": 0, "layout_id": 2, "reset_seed": 2, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "success", "success": true, "score": 1, "native_complete": true, "steps": 297, "limit": 500, "corrected_steps": 0, "chunks": 20, "tokens": { "totalTokens": 2726605, "inputTokens": 2717762, "cachedInputTokens": 2573696, "cacheWriteInputTokens": 0, "outputTokens": 8843, "reasoningOutputTokens": 1242 } }, { "id": "mix__put_bottles_into_dustbin__standard__g0__l4", "case_id": "put_bottles_into_dustbin__standard__g0__l4", "method": "mix", "task": "put_bottles_into_dustbin", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 4, "reset_seed": 4, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "success", "success": true, "score": 1, "native_complete": true, "steps": 534, "limit": 700, "corrected_steps": 0, "chunks": 37, "tokens": { "totalTokens": 6172621, "inputTokens": 6159507, "cachedInputTokens": 5900544, "cacheWriteInputTokens": 0, "outputTokens": 13114, "reasoningOutputTokens": 1309 } }, { "id": "gpt__organize_table__standard__g0__l0", "case_id": "organize_table__standard__g0__l0", "method": "gpt", "task": "organize_table", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 0, "reset_seed": 0, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0.5, "native_complete": true, "steps": 1000, "limit": 1000, "corrected_steps": 0, "chunks": 201, "tokens": { "totalTokens": 25592351, "inputTokens": 25536947, "cachedInputTokens": 25128832, "cacheWriteInputTokens": 0, "outputTokens": 55404, "reasoningOutputTokens": 30272 } }, { "id": "gpt__classify_objects_by_language__standard__g0__l0", "case_id": "classify_objects_by_language__standard__g0__l0", "method": "gpt", "task": "classify_objects_by_language", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 0, "reset_seed": 0, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0.4, "native_complete": true, "steps": 1100, "limit": 1100, "corrected_steps": 0, "chunks": 220, "tokens": { "totalTokens": 28987945, "inputTokens": 28934059, "cachedInputTokens": 28502400, "cacheWriteInputTokens": 0, "outputTokens": 53886, "reasoningOutputTokens": 30158 } }, { "id": "gpt__imitate_sorting_sequence__standard__g0__l0", "case_id": "imitate_sorting_sequence__standard__g0__l0", "method": "gpt", "task": "imitate_sorting_sequence", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 0, "reset_seed": 0, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0, "native_complete": true, "steps": 450, "limit": 1600, "corrected_steps": 0, "chunks": 90, "tokens": { "totalTokens": 9149147, "inputTokens": 9136758, "cachedInputTokens": 8997760, "cacheWriteInputTokens": 0, "outputTokens": 12389, "reasoningOutputTokens": 3980 } }, { "id": "gpt__arrange_largest_number__standard__g0__l0", "case_id": "arrange_largest_number__standard__g0__l0", "method": "gpt", "task": "arrange_largest_number", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 0, "reset_seed": 0, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "success", "success": true, "score": 1, "native_complete": true, "steps": 877, "limit": 1050, "corrected_steps": 0, "chunks": 176, "tokens": { "totalTokens": 23050663, "inputTokens": 23008367, "cachedInputTokens": 22385920, "cacheWriteInputTokens": 0, "outputTokens": 42296, "reasoningOutputTokens": 23096 } }, { "id": "gpt__pack_objects_into_box__standard__g0__l0", "case_id": "pack_objects_into_box__standard__g0__l0", "method": "gpt", "task": "pack_objects_into_box", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 0, "reset_seed": 0, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0.5, "native_complete": true, "steps": 1300, "limit": 1300, "corrected_steps": 0, "chunks": 260, "tokens": { "totalTokens": 37021230, "inputTokens": 36947920, "cachedInputTokens": 36468608, "cacheWriteInputTokens": 0, "outputTokens": 73310, "reasoningOutputTokens": 43137 } }, { "id": "gpt__classify_objects__standard__g0__l0", "case_id": "classify_objects__standard__g0__l0", "method": "gpt", "task": "classify_objects", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 0, "reset_seed": 0, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "success", "success": true, "score": 1, "native_complete": true, "steps": 737, "limit": 1100, "corrected_steps": 0, "chunks": 148, "tokens": { "totalTokens": 24918873, "inputTokens": 24888758, "cachedInputTokens": 24553984, "cacheWriteInputTokens": 0, "outputTokens": 30115, "reasoningOutputTokens": 10208 } }, { "id": "gpt__build_tower__standard__g0__l0", "case_id": "build_tower__standard__g0__l0", "method": "gpt", "task": "build_tower", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 0, "reset_seed": 0, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0.1, "native_complete": true, "steps": 1050, "limit": 1050, "corrected_steps": 0, "chunks": 210, "tokens": { "totalTokens": 28536185, "inputTokens": 28483840, "cachedInputTokens": 28046336, "cacheWriteInputTokens": 0, "outputTokens": 52345, "reasoningOutputTokens": 25871 } }, { "id": "gpt__make_kong__standard__g0__l0", "case_id": "make_kong__standard__g0__l0", "method": "gpt", "task": "make_kong", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 0, "reset_seed": 0, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0, "native_complete": true, "steps": 600, "limit": 600, "corrected_steps": 0, "chunks": 120, "tokens": { "totalTokens": 15074394, "inputTokens": 15047648, "cachedInputTokens": 14833408, "cacheWriteInputTokens": 0, "outputTokens": 26746, "reasoningOutputTokens": 13210 } }, { "id": "gpt__fold_clothes__standard__g0__l0", "case_id": "fold_clothes__standard__g0__l0", "method": "gpt", "task": "fold_clothes", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 0, "reset_seed": 0, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0, "native_complete": true, "steps": 268, "limit": 500, "corrected_steps": 0, "chunks": 54, "tokens": { "totalTokens": 6527428, "inputTokens": 6516777, "cachedInputTokens": 6370048, "cacheWriteInputTokens": 0, "outputTokens": 10651, "reasoningOutputTokens": 3526 } }, { "id": "gpt__put_bottles_into_dustbin__standard__g0__l0", "case_id": "put_bottles_into_dustbin__standard__g0__l0", "method": "gpt", "task": "put_bottles_into_dustbin", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 0, "reset_seed": 0, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0.4, "native_complete": true, "steps": 700, "limit": 700, "corrected_steps": 0, "chunks": 140, "tokens": { "totalTokens": 20863002, "inputTokens": 20835445, "cachedInputTokens": 20583424, "cacheWriteInputTokens": 0, "outputTokens": 27557, "reasoningOutputTokens": 15479 } }, { "id": "gpt__organize_table__standard__g0__l1", "case_id": "organize_table__standard__g0__l1", "method": "gpt", "task": "organize_table", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 1, "reset_seed": 1, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0, "native_complete": true, "steps": 1000, "limit": 1000, "corrected_steps": 0, "chunks": 200, "tokens": { "totalTokens": 8478389, "inputTokens": 8465048, "cachedInputTokens": 8107520, "cacheWriteInputTokens": 0, "outputTokens": 13341, "reasoningOutputTokens": 6331 } }, { "id": "gpt__imitate_sorting_sequence__standard__g0__l1", "case_id": "imitate_sorting_sequence__standard__g0__l1", "method": "gpt", "task": "imitate_sorting_sequence", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 1, "reset_seed": 1, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0, "native_complete": true, "steps": 104, "limit": 1600, "corrected_steps": 0, "chunks": 22, "tokens": { "totalTokens": 5221126, "inputTokens": 5201562, "cachedInputTokens": 4568448, "cacheWriteInputTokens": 0, "outputTokens": 19564, "reasoningOutputTokens": 6602 } }, { "id": "gpt__arrange_largest_number__standard__g0__l1", "case_id": "arrange_largest_number__standard__g0__l1", "method": "gpt", "task": "arrange_largest_number", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 1, "reset_seed": 1, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "success", "success": true, "score": 1, "native_complete": true, "steps": 735, "limit": 1050, "corrected_steps": 0, "chunks": 147, "tokens": { "totalTokens": 22556736, "inputTokens": 22525348, "cachedInputTokens": 22184576, "cacheWriteInputTokens": 0, "outputTokens": 31388, "reasoningOutputTokens": 15808 } }, { "id": "gpt__pack_objects_into_box__standard__g0__l1", "case_id": "pack_objects_into_box__standard__g0__l1", "method": "gpt", "task": "pack_objects_into_box", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 1, "reset_seed": 1, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "success", "success": true, "score": 1, "native_complete": true, "steps": 1109, "limit": 1300, "corrected_steps": 0, "chunks": 222, "tokens": { "totalTokens": 30481204, "inputTokens": 30435661, "cachedInputTokens": 30032384, "cacheWriteInputTokens": 0, "outputTokens": 45543, "reasoningOutputTokens": 21532 } }, { "id": "gpt__classify_objects__standard__g0__l1", "case_id": "classify_objects__standard__g0__l1", "method": "gpt", "task": "classify_objects", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 1, "reset_seed": 1, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "success", "success": true, "score": 1, "native_complete": true, "steps": 814, "limit": 1100, "corrected_steps": 0, "chunks": 163, "tokens": { "totalTokens": 23446028, "inputTokens": 23408987, "cachedInputTokens": 22909184, "cacheWriteInputTokens": 0, "outputTokens": 37041, "reasoningOutputTokens": 16006 } }, { "id": "gpt__build_tower__standard__g0__l1", "case_id": "build_tower__standard__g0__l1", "method": "gpt", "task": "build_tower", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 1, "reset_seed": 1, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0, "native_complete": true, "steps": 1050, "limit": 1050, "corrected_steps": 0, "chunks": 210, "tokens": { "totalTokens": 27757171, "inputTokens": 27698013, "cachedInputTokens": 27291392, "cacheWriteInputTokens": 0, "outputTokens": 59158, "reasoningOutputTokens": 31296 } }, { "id": "gpt__make_kong__standard__g0__l1", "case_id": "make_kong__standard__g0__l1", "method": "gpt", "task": "make_kong", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 1, "reset_seed": 1, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0, "native_complete": true, "steps": 600, "limit": 600, "corrected_steps": 0, "chunks": 120, "tokens": { "totalTokens": 19814593, "inputTokens": 19774777, "cachedInputTokens": 19491072, "cacheWriteInputTokens": 0, "outputTokens": 39816, "reasoningOutputTokens": 21309 } }, { "id": "gpt__fold_clothes__standard__g0__l1", "case_id": "fold_clothes__standard__g0__l1", "method": "gpt", "task": "fold_clothes", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 1, "reset_seed": 1, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "success", "success": true, "score": 1, "native_complete": true, "steps": 236, "limit": 500, "corrected_steps": 0, "chunks": 48, "tokens": { "totalTokens": 6197259, "inputTokens": 6188073, "cachedInputTokens": 6050304, "cacheWriteInputTokens": 0, "outputTokens": 9186, "reasoningOutputTokens": 3020 } }, { "id": "gpt__put_bottles_into_dustbin__standard__g0__l1", "case_id": "put_bottles_into_dustbin__standard__g0__l1", "method": "gpt", "task": "put_bottles_into_dustbin", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 1, "reset_seed": 1, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "success", "success": true, "score": 1, "native_complete": true, "steps": 602, "limit": 700, "corrected_steps": 0, "chunks": 121, "tokens": { "totalTokens": 18399564, "inputTokens": 18375991, "cachedInputTokens": 18142848, "cacheWriteInputTokens": 0, "outputTokens": 23573, "reasoningOutputTokens": 9702 } }, { "id": "gpt__organize_table__standard__g0__l2", "case_id": "organize_table__standard__g0__l2", "method": "gpt", "task": "organize_table", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 2, "reset_seed": 2, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0.25, "native_complete": true, "steps": 1000, "limit": 1000, "corrected_steps": 0, "chunks": 200, "tokens": { "totalTokens": 26732930, "inputTokens": 26686486, "cachedInputTokens": 26286208, "cacheWriteInputTokens": 0, "outputTokens": 46444, "reasoningOutputTokens": 25386 } }, { "id": "gpt__classify_objects_by_language__standard__g0__l2", "case_id": "classify_objects_by_language__standard__g0__l2", "method": "gpt", "task": "classify_objects_by_language", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 2, "reset_seed": 2, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "success", "success": true, "score": 1, "native_complete": true, "steps": 791, "limit": 1100, "corrected_steps": 0, "chunks": 159, "tokens": { "totalTokens": 25137794, "inputTokens": 25104491, "cachedInputTokens": 24804864, "cacheWriteInputTokens": 0, "outputTokens": 33303, "reasoningOutputTokens": 12111 } }, { "id": "gpt__imitate_sorting_sequence__standard__g0__l2", "case_id": "imitate_sorting_sequence__standard__g0__l2", "method": "gpt", "task": "imitate_sorting_sequence", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 2, "reset_seed": 2, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0, "native_complete": true, "steps": 378, "limit": 1600, "corrected_steps": 0, "chunks": 76, "tokens": { "totalTokens": 6301459, "inputTokens": 6292827, "cachedInputTokens": 6171392, "cacheWriteInputTokens": 0, "outputTokens": 8632, "reasoningOutputTokens": 1856 } }, { "id": "gpt__arrange_largest_number__random__g0__l0", "case_id": "arrange_largest_number__random__g0__l0", "method": "gpt", "task": "arrange_largest_number", "variant": "random", "seeds": { "eval_seed": 0, "layout_id": 0, "reset_seed": 0, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0.3, "native_complete": true, "steps": 1050, "limit": 1050, "corrected_steps": 0, "chunks": 210, "tokens": { "totalTokens": 29374714, "inputTokens": 29323994, "cachedInputTokens": 28941312, "cacheWriteInputTokens": 0, "outputTokens": 50720, "reasoningOutputTokens": 21755 } }, { "id": "gpt__pack_objects_into_box__random__g0__l0", "case_id": "pack_objects_into_box__random__g0__l0", "method": "gpt", "task": "pack_objects_into_box", "variant": "random", "seeds": { "eval_seed": 0, "layout_id": 0, "reset_seed": 0, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0.5, "native_complete": true, "steps": 1300, "limit": 1300, "corrected_steps": 0, "chunks": 260, "tokens": { "totalTokens": 35814920, "inputTokens": 35748441, "cachedInputTokens": 35296256, "cacheWriteInputTokens": 0, "outputTokens": 66479, "reasoningOutputTokens": 35271 } }, { "id": "gpt__classify_objects__standard__g0__l2", "case_id": "classify_objects__standard__g0__l2", "method": "gpt", "task": "classify_objects", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 2, "reset_seed": 2, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "success", "success": true, "score": 1, "native_complete": true, "steps": 1003, "limit": 1100, "corrected_steps": 0, "chunks": 202, "tokens": { "totalTokens": 27281537, "inputTokens": 27245621, "cachedInputTokens": 26776832, "cacheWriteInputTokens": 0, "outputTokens": 35916, "reasoningOutputTokens": 20175 } }, { "id": "gpt__build_tower__standard__g0__l2", "case_id": "build_tower__standard__g0__l2", "method": "gpt", "task": "build_tower", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 2, "reset_seed": 2, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0.3, "native_complete": true, "steps": 1050, "limit": 1050, "corrected_steps": 0, "chunks": 210, "tokens": { "totalTokens": 30325438, "inputTokens": 30275104, "cachedInputTokens": 29605888, "cacheWriteInputTokens": 0, "outputTokens": 50334, "reasoningOutputTokens": 16821 } }, { "id": "gpt__make_kong__standard__g0__l2", "case_id": "make_kong__standard__g0__l2", "method": "gpt", "task": "make_kong", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 2, "reset_seed": 2, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0, "native_complete": true, "steps": 600, "limit": 600, "corrected_steps": 0, "chunks": 120, "tokens": { "totalTokens": 21178573, "inputTokens": 21144679, "cachedInputTokens": 20809856, "cacheWriteInputTokens": 0, "outputTokens": 33894, "reasoningOutputTokens": 13296 } }, { "id": "gpt__fold_clothes__random__g0__l0", "case_id": "fold_clothes__random__g0__l0", "method": "gpt", "task": "fold_clothes", "variant": "random", "seeds": { "eval_seed": 0, "layout_id": 0, "reset_seed": 0, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0, "native_complete": true, "steps": 500, "limit": 500, "corrected_steps": 0, "chunks": 100, "tokens": { "totalTokens": 18715897, "inputTokens": 18676529, "cachedInputTokens": 17792512, "cacheWriteInputTokens": 0, "outputTokens": 39368, "reasoningOutputTokens": 10479 } }, { "id": "gpt__put_bottles_into_dustbin__standard__g0__l2", "case_id": "put_bottles_into_dustbin__standard__g0__l2", "method": "gpt", "task": "put_bottles_into_dustbin", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 2, "reset_seed": 2, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0, "native_complete": true, "steps": 700, "limit": 700, "corrected_steps": 0, "chunks": 140, "tokens": { "totalTokens": 16924289, "inputTokens": 16870955, "cachedInputTokens": 16404096, "cacheWriteInputTokens": 0, "outputTokens": 53334, "reasoningOutputTokens": 10878 } }, { "id": "gpt__organize_table__standard__g0__l3", "case_id": "organize_table__standard__g0__l3", "method": "gpt", "task": "organize_table", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 3, "reset_seed": 3, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0.5, "native_complete": true, "steps": 1000, "limit": 1000, "corrected_steps": 0, "chunks": 200, "tokens": { "totalTokens": 25911163, "inputTokens": 25854481, "cachedInputTokens": 25184768, "cacheWriteInputTokens": 0, "outputTokens": 56682, "reasoningOutputTokens": 28230 } }, { "id": "gpt__classify_objects_by_language__standard__g0__l3", "case_id": "classify_objects_by_language__standard__g0__l3", "method": "gpt", "task": "classify_objects_by_language", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 3, "reset_seed": 3, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0, "native_complete": true, "steps": 1100, "limit": 1100, "corrected_steps": 0, "chunks": 283, "tokens": { "totalTokens": 91730207, "inputTokens": 91554993, "cachedInputTokens": 89229440, "cacheWriteInputTokens": 0, "outputTokens": 175214, "reasoningOutputTokens": 32922 } }, { "id": "gpt__imitate_sorting_sequence__standard__g0__l3", "case_id": "imitate_sorting_sequence__standard__g0__l3", "method": "gpt", "task": "imitate_sorting_sequence", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 3, "reset_seed": 3, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0, "native_complete": true, "steps": 54, "limit": 1600, "corrected_steps": 0, "chunks": 11, "tokens": { "totalTokens": 1534875, "inputTokens": 1523692, "cachedInputTokens": 1454080, "cacheWriteInputTokens": 0, "outputTokens": 11183, "reasoningOutputTokens": 3378 } }, { "id": "gpt__arrange_largest_number__random__g0__l1", "case_id": "arrange_largest_number__random__g0__l1", "method": "gpt", "task": "arrange_largest_number", "variant": "random", "seeds": { "eval_seed": 0, "layout_id": 1, "reset_seed": 1, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0.25, "native_complete": true, "steps": 1050, "limit": 1050, "corrected_steps": 0, "chunks": 210, "tokens": { "totalTokens": 29387089, "inputTokens": 29334705, "cachedInputTokens": 28822400, "cacheWriteInputTokens": 0, "outputTokens": 52384, "reasoningOutputTokens": 29522 } }, { "id": "gpt__pack_objects_into_box__random__g0__l1", "case_id": "pack_objects_into_box__random__g0__l1", "method": "gpt", "task": "pack_objects_into_box", "variant": "random", "seeds": { "eval_seed": 0, "layout_id": 1, "reset_seed": 1, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0, "native_complete": true, "steps": 1300, "limit": 1300, "corrected_steps": 0, "chunks": 260, "tokens": { "totalTokens": 26493089, "inputTokens": 26456098, "cachedInputTokens": 25949184, "cacheWriteInputTokens": 0, "outputTokens": 36991, "reasoningOutputTokens": 6492 } }, { "id": "gpt__classify_objects__standard__g0__l3", "case_id": "classify_objects__standard__g0__l3", "method": "gpt", "task": "classify_objects", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 3, "reset_seed": 3, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "success", "success": true, "score": 1, "native_complete": true, "steps": 615, "limit": 1100, "corrected_steps": 0, "chunks": 123, "tokens": { "totalTokens": 21475317, "inputTokens": 21444911, "cachedInputTokens": 20970368, "cacheWriteInputTokens": 0, "outputTokens": 30406, "reasoningOutputTokens": 12352 } }, { "id": "gpt__build_tower__standard__g0__l3", "case_id": "build_tower__standard__g0__l3", "method": "gpt", "task": "build_tower", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 3, "reset_seed": 3, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0.1, "native_complete": true, "steps": 1050, "limit": 1050, "corrected_steps": 0, "chunks": 211, "tokens": { "totalTokens": 27872337, "inputTokens": 27801274, "cachedInputTokens": 26970112, "cacheWriteInputTokens": 0, "outputTokens": 71063, "reasoningOutputTokens": 46009 } }, { "id": "gpt__make_kong__standard__g0__l3", "case_id": "make_kong__standard__g0__l3", "method": "gpt", "task": "make_kong", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 3, "reset_seed": 3, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0, "native_complete": true, "steps": 600, "limit": 600, "corrected_steps": 0, "chunks": 120, "tokens": { "totalTokens": 32221117, "inputTokens": 32156050, "cachedInputTokens": 31367168, "cacheWriteInputTokens": 0, "outputTokens": 65067, "reasoningOutputTokens": 16462 } }, { "id": "gpt__fold_clothes__random__g0__l1", "case_id": "fold_clothes__random__g0__l1", "method": "gpt", "task": "fold_clothes", "variant": "random", "seeds": { "eval_seed": 0, "layout_id": 1, "reset_seed": 1, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0, "native_complete": true, "steps": 500, "limit": 500, "corrected_steps": 0, "chunks": 104, "tokens": { "totalTokens": 20311410, "inputTokens": 20254823, "cachedInputTokens": 19831680, "cacheWriteInputTokens": 0, "outputTokens": 56587, "reasoningOutputTokens": 7563 } }, { "id": "gpt__put_bottles_into_dustbin__standard__g0__l3", "case_id": "put_bottles_into_dustbin__standard__g0__l3", "method": "gpt", "task": "put_bottles_into_dustbin", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 3, "reset_seed": 3, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0, "native_complete": true, "steps": 700, "limit": 700, "corrected_steps": 0, "chunks": 140, "tokens": { "totalTokens": 19415991, "inputTokens": 19365042, "cachedInputTokens": 18906368, "cacheWriteInputTokens": 0, "outputTokens": 50949, "reasoningOutputTokens": 9847 } }, { "id": "gpt__organize_table__standard__g0__l4", "case_id": "organize_table__standard__g0__l4", "method": "gpt", "task": "organize_table", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 4, "reset_seed": 4, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0.25, "native_complete": true, "steps": 1000, "limit": 1000, "corrected_steps": 0, "chunks": 200, "tokens": { "totalTokens": 26318755, "inputTokens": 26276032, "cachedInputTokens": 25859200, "cacheWriteInputTokens": 0, "outputTokens": 42723, "reasoningOutputTokens": 21459 } }, { "id": "gpt__classify_objects_by_language__standard__g0__l4", "case_id": "classify_objects_by_language__standard__g0__l4", "method": "gpt", "task": "classify_objects_by_language", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 4, "reset_seed": 4, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "success", "success": true, "score": 1, "native_complete": true, "steps": 978, "limit": 1100, "corrected_steps": 0, "chunks": 196, "tokens": { "totalTokens": 25767060, "inputTokens": 25725504, "cachedInputTokens": 25368448, "cacheWriteInputTokens": 0, "outputTokens": 41556, "reasoningOutputTokens": 20463 } }, { "id": "gpt__imitate_sorting_sequence__standard__g0__l4", "case_id": "imitate_sorting_sequence__standard__g0__l4", "method": "gpt", "task": "imitate_sorting_sequence", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 4, "reset_seed": 4, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0, "native_complete": true, "steps": 98, "limit": 1600, "corrected_steps": 0, "chunks": 21, "tokens": { "totalTokens": 3944534, "inputTokens": 3926058, "cachedInputTokens": 3775872, "cacheWriteInputTokens": 0, "outputTokens": 18476, "reasoningOutputTokens": 5349 } }, { "id": "gpt__arrange_largest_number__random__g0__l2", "case_id": "arrange_largest_number__random__g0__l2", "method": "gpt", "task": "arrange_largest_number", "variant": "random", "seeds": { "eval_seed": 0, "layout_id": 2, "reset_seed": 2, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0.3, "native_complete": true, "steps": 1050, "limit": 1050, "corrected_steps": 0, "chunks": 210, "tokens": { "totalTokens": 30983023, "inputTokens": 30941810, "cachedInputTokens": 30515328, "cacheWriteInputTokens": 0, "outputTokens": 41213, "reasoningOutputTokens": 16627 } }, { "id": "gpt__classify_objects__standard__g0__l4", "case_id": "classify_objects__standard__g0__l4", "method": "gpt", "task": "classify_objects", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 4, "reset_seed": 4, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "success", "success": true, "score": 1, "native_complete": true, "steps": 598, "limit": 1100, "corrected_steps": 0, "chunks": 121, "tokens": { "totalTokens": 17409149, "inputTokens": 17387815, "cachedInputTokens": 17161344, "cacheWriteInputTokens": 0, "outputTokens": 21334, "reasoningOutputTokens": 7976 } }, { "id": "gpt__build_tower__standard__g0__l4", "case_id": "build_tower__standard__g0__l4", "method": "gpt", "task": "build_tower", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 4, "reset_seed": 4, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0.1, "native_complete": true, "steps": 1050, "limit": 1050, "corrected_steps": 0, "chunks": 210, "tokens": { "totalTokens": 27198496, "inputTokens": 27154839, "cachedInputTokens": 26703488, "cacheWriteInputTokens": 0, "outputTokens": 43657, "reasoningOutputTokens": 23633 } }, { "id": "gpt__make_kong__standard__g0__l4", "case_id": "make_kong__standard__g0__l4", "method": "gpt", "task": "make_kong", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 4, "reset_seed": 4, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0, "native_complete": true, "steps": 600, "limit": 600, "corrected_steps": 0, "chunks": 120, "tokens": { "totalTokens": 16898608, "inputTokens": 16862202, "cachedInputTokens": 16164608, "cacheWriteInputTokens": 0, "outputTokens": 36406, "reasoningOutputTokens": 20811 } }, { "id": "gpt__fold_clothes__random__g0__l2", "case_id": "fold_clothes__random__g0__l2", "method": "gpt", "task": "fold_clothes", "variant": "random", "seeds": { "eval_seed": 0, "layout_id": 2, "reset_seed": 2, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "success", "success": true, "score": 1, "native_complete": true, "steps": 257, "limit": 500, "corrected_steps": 0, "chunks": 52, "tokens": { "totalTokens": 5832845, "inputTokens": 5821613, "cachedInputTokens": 5563776, "cacheWriteInputTokens": 0, "outputTokens": 11232, "reasoningOutputTokens": 4406 } }, { "id": "gpt__put_bottles_into_dustbin__standard__g0__l4", "case_id": "put_bottles_into_dustbin__standard__g0__l4", "method": "gpt", "task": "put_bottles_into_dustbin", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 4, "reset_seed": 4, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "failure", "success": false, "score": 0.4, "native_complete": true, "steps": 700, "limit": 700, "corrected_steps": 0, "chunks": 140, "tokens": { "totalTokens": 19218893, "inputTokens": 19184350, "cachedInputTokens": 18881280, "cacheWriteInputTokens": 0, "outputTokens": 34543, "reasoningOutputTokens": 18448 } }, { "id": "gpt__classify_objects_by_language__standard__g0__l1", "case_id": "classify_objects_by_language__standard__g0__l1", "method": "gpt", "task": "classify_objects_by_language", "variant": "standard", "seeds": { "eval_seed": 0, "layout_id": 1, "reset_seed": 1, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "idle", "success": false, "score": null, "native_complete": false, "steps": 257, "limit": 1100, "corrected_steps": 0, "chunks": 56, "tokens": { "totalTokens": 16434295, "inputTokens": 16378419, "cachedInputTokens": 15473536, "cacheWriteInputTokens": 0, "outputTokens": 55876, "reasoningOutputTokens": 13186 } }, { "id": "gpt__pack_objects_into_box__random__g0__l2", "case_id": "pack_objects_into_box__random__g0__l2", "method": "gpt", "task": "pack_objects_into_box", "variant": "random", "seeds": { "eval_seed": 0, "layout_id": 2, "reset_seed": 2, "simulator_initial_seed": 0, "policy_rng_seed": 0 }, "status": "idle", "success": false, "score": null, "native_complete": false, "steps": 960, "limit": 1300, "corrected_steps": 0, "chunks": 192, "tokens": { "totalTokens": 26124680, "inputTokens": 26073391, "cachedInputTokens": 25659648, "cacheWriteInputTokens": 0, "outputTokens": 51289, "reasoningOutputTokens": 29758 } } ] }