{ "schemaVersion": 1, "generatedAt": "2026-09-21T05:18:50.179889+00:00", "before": { "n": 400, "benchSuccess": 322, "benchFailure": 78, "benchSuccessRate": 0.805, "mismatchCount": 47, "ias": 0.8825000000000001, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 47 }, "failure": { "benchSuccess": 0, "benchFailure": 26 }, "unknown": { "benchSuccess": 322, "benchFailure": 5 } }, "agentCounts": { "success": 47, "failure": 26, "unknown": 327 }, "selfAssessmentCoverage": 0.1825, "sourceStageCounts": { "baseline": 400 } }, "round1": { "n": 400, "benchSuccess": 345, "benchFailure": 55, "benchSuccessRate": 0.8625, "mismatchCount": 19, "ias": 0.9525, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 19 }, "failure": { "benchSuccess": 0, "benchFailure": 28 }, "unknown": { "benchSuccess": 345, "benchFailure": 8 } }, "agentCounts": { "success": 19, "failure": 28, "unknown": 353 }, "selfAssessmentCoverage": 0.1175, "sourceStageCounts": { "baseline": 310, "round1": 90 } }, "after": { "n": 400, "benchSuccess": 357, "benchFailure": 43, "benchSuccessRate": 0.8925, "mismatchCount": 1, "ias": 0.9975, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 1 }, "failure": { "benchSuccess": 0, "benchFailure": 33 }, "unknown": { "benchSuccess": 357, "benchFailure": 9 } }, "agentCounts": { "success": 1, "failure": 33, "unknown": 366 }, "selfAssessmentCoverage": 0.085, "sourceStageCounts": { "baseline": 310, "round1": 70, "round2": 20 } }, "rerunSubset": { "n": 90, "before": { "n": 90, "benchSuccess": 40, "benchFailure": 50, "benchSuccessRate": 0.4444444444444444, "mismatchCount": 47, "ias": 0.47777777777777775, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 47 }, "failure": { "benchSuccess": 0, "benchFailure": 3 }, "unknown": { "benchSuccess": 40, "benchFailure": 0 } }, "agentCounts": { "success": 47, "failure": 3, "unknown": 40 }, "selfAssessmentCoverage": 0.5555555555555556, "sourceStageCounts": { "baseline": 90 } }, "round1": { "n": 90, "benchSuccess": 63, "benchFailure": 27, "benchSuccessRate": 0.7, "mismatchCount": 19, "ias": 0.7888888888888889, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 19 }, "failure": { "benchSuccess": 0, "benchFailure": 5 }, "unknown": { "benchSuccess": 63, "benchFailure": 3 } }, "agentCounts": { "success": 19, "failure": 5, "unknown": 66 }, "selfAssessmentCoverage": 0.26666666666666666, "sourceStageCounts": { "round1": 90 } }, "after": { "n": 90, "benchSuccess": 75, "benchFailure": 15, "benchSuccessRate": 0.8333333333333334, "mismatchCount": 1, "ias": 0.9888888888888889, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 1 }, "failure": { "benchSuccess": 0, "benchFailure": 10 }, "unknown": { "benchSuccess": 75, "benchFailure": 4 } }, "agentCounts": { "success": 1, "failure": 10, "unknown": 79 }, "selfAssessmentCoverage": 0.12222222222222222, "sourceStageCounts": { "round1": 70, "round2": 20 } } }, "finalRoundSubset": { "n": 20, "before": { "n": 20, "benchSuccess": 0, "benchFailure": 20, "benchSuccessRate": 0.0, "mismatchCount": 19, "ias": 0.050000000000000044, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 19 }, "failure": { "benchSuccess": 0, "benchFailure": 1 }, "unknown": { "benchSuccess": 0, "benchFailure": 0 } }, "agentCounts": { "success": 19, "failure": 1, "unknown": 0 }, "selfAssessmentCoverage": 1.0, "sourceStageCounts": { "baseline": 20 } }, "round1": { "n": 20, "benchSuccess": 0, "benchFailure": 20, "benchSuccessRate": 0.0, "mismatchCount": 19, "ias": 0.050000000000000044, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 19 }, "failure": { "benchSuccess": 0, "benchFailure": 1 }, "unknown": { "benchSuccess": 0, "benchFailure": 0 } }, "agentCounts": { "success": 19, "failure": 1, "unknown": 0 }, "selfAssessmentCoverage": 1.0, "sourceStageCounts": { "round1": 20 } }, "after": { "n": 20, "benchSuccess": 12, "benchFailure": 8, "benchSuccessRate": 0.6, "mismatchCount": 1, "ias": 0.95, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 1 }, "failure": { "benchSuccess": 0, "benchFailure": 6 }, "unknown": { "benchSuccess": 12, "benchFailure": 1 } }, "agentCounts": { "success": 1, "failure": 6, "unknown": 13 }, "selfAssessmentCoverage": 0.35, "sourceStageCounts": { "round2": 20 } } }, "tasks": [ { "id": "libero_10_t00", "suite": "libero_10", "taskId": 0, "taskName": "LIVING_ROOM_SCENE2_put_both_the_alphabet_soup_and_the_tomato_sauce_in_the_basket", "instructionBefore": "put both the alphabet soup and the tomato sauce in the basket", "instructionRound1": "put both the alphabet soup and the tomato sauce in the basket", "instructionAfter": "put both the alphabet soup and the tomato sauce in the basket", "sourceStage": "baseline", "changed": false, "before": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "round1": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "after": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "transitions": { "successToSuccess": 10, "successToFailure": 0, "failureToSuccess": 0, "failureToFailure": 0, "n": 10, "netAdditionalSuccesses": 0 }, "episodeKeys": [ "libero_10_t00_r00", "libero_10_t00_r01", "libero_10_t00_r02", "libero_10_t00_r03", "libero_10_t00_r04", "libero_10_t00_r05", "libero_10_t00_r06", "libero_10_t00_r07", "libero_10_t00_r08", "libero_10_t00_r09" ] }, { "id": "libero_10_t01", "suite": "libero_10", "taskId": 1, "taskName": "LIVING_ROOM_SCENE2_put_both_the_cream_cheese_box_and_the_butter_in_the_basket", "instructionBefore": "put both the cream cheese box and the butter in the basket", "instructionRound1": "put both the cream cheese box and the butter in the basket", "instructionAfter": "put both the cream cheese box and the butter in the basket", "sourceStage": "baseline", "changed": false, "before": { "n": 10, "benchSuccess": 9, "benchFailure": 1, "benchSuccessRate": 0.9, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 1 }, "unknown": { "benchSuccess": 9, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 1, "unknown": 9 }, "selfAssessmentCoverage": 0.1, "sourceStageCounts": { "baseline": 10 } }, "round1": { "n": 10, "benchSuccess": 9, "benchFailure": 1, "benchSuccessRate": 0.9, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 1 }, "unknown": { "benchSuccess": 9, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 1, "unknown": 9 }, "selfAssessmentCoverage": 0.1, "sourceStageCounts": { "baseline": 10 } }, "after": { "n": 10, "benchSuccess": 9, "benchFailure": 1, "benchSuccessRate": 0.9, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 1 }, "unknown": { "benchSuccess": 9, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 1, "unknown": 9 }, "selfAssessmentCoverage": 0.1, "sourceStageCounts": { "baseline": 10 } }, "transitions": { "successToSuccess": 9, "successToFailure": 0, "failureToSuccess": 0, "failureToFailure": 1, "n": 10, "netAdditionalSuccesses": 0 }, "episodeKeys": [ "libero_10_t01_r00", "libero_10_t01_r01", "libero_10_t01_r02", "libero_10_t01_r03", "libero_10_t01_r04", "libero_10_t01_r05", "libero_10_t01_r06", "libero_10_t01_r07", "libero_10_t01_r08", "libero_10_t01_r09" ] }, { "id": "libero_10_t02", "suite": "libero_10", "taskId": 2, "taskName": "KITCHEN_SCENE3_turn_on_the_stove_and_put_the_moka_pot_on_it", "instructionBefore": "turn on the stove and put the moka pot on it", "instructionRound1": "turn on the stove and put the moka pot on it", "instructionAfter": "turn on the stove and put the moka pot on it", "sourceStage": "baseline", "changed": false, "before": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "round1": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "after": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "transitions": { "successToSuccess": 10, "successToFailure": 0, "failureToSuccess": 0, "failureToFailure": 0, "n": 10, "netAdditionalSuccesses": 0 }, "episodeKeys": [ "libero_10_t02_r00", "libero_10_t02_r01", "libero_10_t02_r02", "libero_10_t02_r03", "libero_10_t02_r04", "libero_10_t02_r05", "libero_10_t02_r06", "libero_10_t02_r07", "libero_10_t02_r08", "libero_10_t02_r09" ] }, { "id": "libero_10_t03", "suite": "libero_10", "taskId": 3, "taskName": "KITCHEN_SCENE4_put_the_black_bowl_in_the_bottom_drawer_of_the_cabinet_and_close_it", "instructionBefore": "put the black bowl in the bottom drawer of the cabinet and close it", "instructionRound1": "put the black bowl in the bottom drawer of the cabinet and close it", "instructionAfter": "put the black bowl in the bottom drawer of the cabinet and close it", "sourceStage": "baseline", "changed": false, "before": { "n": 10, "benchSuccess": 0, "benchFailure": 10, "benchSuccessRate": 0.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 7 }, "unknown": { "benchSuccess": 0, "benchFailure": 3 } }, "agentCounts": { "success": 0, "failure": 7, "unknown": 3 }, "selfAssessmentCoverage": 0.7, "sourceStageCounts": { "baseline": 10 } }, "round1": { "n": 10, "benchSuccess": 0, "benchFailure": 10, "benchSuccessRate": 0.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 7 }, "unknown": { "benchSuccess": 0, "benchFailure": 3 } }, "agentCounts": { "success": 0, "failure": 7, "unknown": 3 }, "selfAssessmentCoverage": 0.7, "sourceStageCounts": { "baseline": 10 } }, "after": { "n": 10, "benchSuccess": 0, "benchFailure": 10, "benchSuccessRate": 0.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 7 }, "unknown": { "benchSuccess": 0, "benchFailure": 3 } }, "agentCounts": { "success": 0, "failure": 7, "unknown": 3 }, "selfAssessmentCoverage": 0.7, "sourceStageCounts": { "baseline": 10 } }, "transitions": { "successToSuccess": 0, "successToFailure": 0, "failureToSuccess": 0, "failureToFailure": 10, "n": 10, "netAdditionalSuccesses": 0 }, "episodeKeys": [ "libero_10_t03_r00", "libero_10_t03_r01", "libero_10_t03_r02", "libero_10_t03_r03", "libero_10_t03_r04", "libero_10_t03_r05", "libero_10_t03_r06", "libero_10_t03_r07", "libero_10_t03_r08", "libero_10_t03_r09" ] }, { "id": "libero_10_t04", "suite": "libero_10", "taskId": 4, "taskName": "LIVING_ROOM_SCENE5_put_the_white_mug_on_the_left_plate_and_put_the_yellow_and_white_mug_on_the_right_plate", "instructionBefore": "put the white mug on the left plate and put the yellow and white mug on the right plate", "instructionRound1": "put the white mug on the left plate and put the yellow and white mug on the right plate", "instructionAfter": "put the white mug on the left plate and put the yellow and white mug on the right plate", "sourceStage": "baseline", "changed": false, "before": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "round1": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "after": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "transitions": { "successToSuccess": 10, "successToFailure": 0, "failureToSuccess": 0, "failureToFailure": 0, "n": 10, "netAdditionalSuccesses": 0 }, "episodeKeys": [ "libero_10_t04_r00", "libero_10_t04_r01", "libero_10_t04_r02", "libero_10_t04_r03", "libero_10_t04_r04", "libero_10_t04_r05", "libero_10_t04_r06", "libero_10_t04_r07", "libero_10_t04_r08", "libero_10_t04_r09" ] }, { "id": "libero_10_t05", "suite": "libero_10", "taskId": 5, "taskName": "STUDY_SCENE1_pick_up_the_book_and_place_it_in_the_back_compartment_of_the_caddy", "instructionBefore": "pick up the book and place it in the back compartment of the caddy", "instructionRound1": "pick up the book and place it in the narrow back compartment of the caddy", "instructionAfter": "pick up the book and place it in the back compartment of the caddy, between the two large side compartments", "sourceStage": "round2", "changed": true, "before": { "n": 10, "benchSuccess": 0, "benchFailure": 10, "benchSuccessRate": 0.0, "mismatchCount": 10, "ias": 0.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 10 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 0, "benchFailure": 0 } }, "agentCounts": { "success": 10, "failure": 0, "unknown": 0 }, "selfAssessmentCoverage": 1.0, "sourceStageCounts": { "baseline": 10 } }, "round1": { "n": 10, "benchSuccess": 0, "benchFailure": 10, "benchSuccessRate": 0.0, "mismatchCount": 9, "ias": 0.09999999999999998, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 9 }, "failure": { "benchSuccess": 0, "benchFailure": 1 }, "unknown": { "benchSuccess": 0, "benchFailure": 0 } }, "agentCounts": { "success": 9, "failure": 1, "unknown": 0 }, "selfAssessmentCoverage": 1.0, "sourceStageCounts": { "round1": 10 } }, "after": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "round2": 10 } }, "transitions": { "successToSuccess": 0, "successToFailure": 0, "failureToSuccess": 10, "failureToFailure": 0, "n": 10, "netAdditionalSuccesses": 10 }, "episodeKeys": [ "libero_10_t05_r00", "libero_10_t05_r01", "libero_10_t05_r02", "libero_10_t05_r03", "libero_10_t05_r04", "libero_10_t05_r05", "libero_10_t05_r06", "libero_10_t05_r07", "libero_10_t05_r08", "libero_10_t05_r09" ] }, { "id": "libero_10_t06", "suite": "libero_10", "taskId": 6, "taskName": "LIVING_ROOM_SCENE6_put_the_white_mug_on_the_plate_and_put_the_chocolate_pudding_to_the_right_of_the_plate", "instructionBefore": "put the white mug on the plate and put the chocolate pudding to the right of the plate", "instructionRound1": "put the white mug in the center of the plate and put the chocolate pudding immediately to the right of the plate", "instructionAfter": "put the white mug in the center of the plate and put the chocolate pudding immediately to the right of the plate", "sourceStage": "round1", "changed": true, "before": { "n": 10, "benchSuccess": 1, "benchFailure": 9, "benchSuccessRate": 0.1, "mismatchCount": 9, "ias": 0.09999999999999998, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 9 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 1, "benchFailure": 0 } }, "agentCounts": { "success": 9, "failure": 0, "unknown": 1 }, "selfAssessmentCoverage": 0.9, "sourceStageCounts": { "baseline": 10 } }, "round1": { "n": 10, "benchSuccess": 9, "benchFailure": 1, "benchSuccessRate": 0.9, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 9, "benchFailure": 1 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "round1": 10 } }, "after": { "n": 10, "benchSuccess": 9, "benchFailure": 1, "benchSuccessRate": 0.9, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 9, "benchFailure": 1 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "round1": 10 } }, "transitions": { "successToSuccess": 1, "successToFailure": 0, "failureToSuccess": 8, "failureToFailure": 1, "n": 10, "netAdditionalSuccesses": 8 }, "episodeKeys": [ "libero_10_t06_r00", "libero_10_t06_r01", "libero_10_t06_r02", "libero_10_t06_r03", "libero_10_t06_r04", "libero_10_t06_r05", "libero_10_t06_r06", "libero_10_t06_r07", "libero_10_t06_r08", "libero_10_t06_r09" ] }, { "id": "libero_10_t07", "suite": "libero_10", "taskId": 7, "taskName": "LIVING_ROOM_SCENE1_put_both_the_alphabet_soup_and_the_cream_cheese_box_in_the_basket", "instructionBefore": "put both the alphabet soup and the cream cheese box in the basket", "instructionRound1": "put both the alphabet soup and the cream cheese box fully inside the basket", "instructionAfter": "put both the alphabet soup and the cream cheese box fully inside the basket", "sourceStage": "round1", "changed": true, "before": { "n": 10, "benchSuccess": 9, "benchFailure": 1, "benchSuccessRate": 0.9, "mismatchCount": 1, "ias": 0.9, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 1 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 9, "benchFailure": 0 } }, "agentCounts": { "success": 1, "failure": 0, "unknown": 9 }, "selfAssessmentCoverage": 0.1, "sourceStageCounts": { "baseline": 10 } }, "round1": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "round1": 10 } }, "after": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "round1": 10 } }, "transitions": { "successToSuccess": 9, "successToFailure": 0, "failureToSuccess": 1, "failureToFailure": 0, "n": 10, "netAdditionalSuccesses": 1 }, "episodeKeys": [ "libero_10_t07_r00", "libero_10_t07_r01", "libero_10_t07_r02", "libero_10_t07_r03", "libero_10_t07_r04", "libero_10_t07_r05", "libero_10_t07_r06", "libero_10_t07_r07", "libero_10_t07_r08", "libero_10_t07_r09" ] }, { "id": "libero_10_t08", "suite": "libero_10", "taskId": 8, "taskName": "KITCHEN_SCENE8_put_both_moka_pots_on_the_stove", "instructionBefore": "put both moka pots on the stove", "instructionRound1": "put both moka pots on the stove", "instructionAfter": "put both moka pots on the stove", "sourceStage": "baseline", "changed": false, "before": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "round1": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "after": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "transitions": { "successToSuccess": 10, "successToFailure": 0, "failureToSuccess": 0, "failureToFailure": 0, "n": 10, "netAdditionalSuccesses": 0 }, "episodeKeys": [ "libero_10_t08_r00", "libero_10_t08_r01", "libero_10_t08_r02", "libero_10_t08_r03", "libero_10_t08_r04", "libero_10_t08_r05", "libero_10_t08_r06", "libero_10_t08_r07", "libero_10_t08_r08", "libero_10_t08_r09" ] }, { "id": "libero_10_t09", "suite": "libero_10", "taskId": 9, "taskName": "KITCHEN_SCENE6_put_the_yellow_and_white_mug_in_the_microwave_and_close_it", "instructionBefore": "put the yellow and white mug in the microwave and close it", "instructionRound1": "put the yellow and white mug in the microwave and close it", "instructionAfter": "put the yellow and white mug in the microwave and close it", "sourceStage": "baseline", "changed": false, "before": { "n": 10, "benchSuccess": 0, "benchFailure": 10, "benchSuccessRate": 0.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 9 }, "unknown": { "benchSuccess": 0, "benchFailure": 1 } }, "agentCounts": { "success": 0, "failure": 9, "unknown": 1 }, "selfAssessmentCoverage": 0.9, "sourceStageCounts": { "baseline": 10 } }, "round1": { "n": 10, "benchSuccess": 0, "benchFailure": 10, "benchSuccessRate": 0.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 9 }, "unknown": { "benchSuccess": 0, "benchFailure": 1 } }, "agentCounts": { "success": 0, "failure": 9, "unknown": 1 }, "selfAssessmentCoverage": 0.9, "sourceStageCounts": { "baseline": 10 } }, "after": { "n": 10, "benchSuccess": 0, "benchFailure": 10, "benchSuccessRate": 0.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 9 }, "unknown": { "benchSuccess": 0, "benchFailure": 1 } }, "agentCounts": { "success": 0, "failure": 9, "unknown": 1 }, "selfAssessmentCoverage": 0.9, "sourceStageCounts": { "baseline": 10 } }, "transitions": { "successToSuccess": 0, "successToFailure": 0, "failureToSuccess": 0, "failureToFailure": 10, "n": 10, "netAdditionalSuccesses": 0 }, "episodeKeys": [ "libero_10_t09_r00", "libero_10_t09_r01", "libero_10_t09_r02", "libero_10_t09_r03", "libero_10_t09_r04", "libero_10_t09_r05", "libero_10_t09_r06", "libero_10_t09_r07", "libero_10_t09_r08", "libero_10_t09_r09" ] }, { "id": "libero_goal_t00", "suite": "libero_goal", "taskId": 0, "taskName": "open_the_middle_drawer_of_the_cabinet", "instructionBefore": "open the middle drawer of the cabinet", "instructionRound1": "fully open the middle drawer of the cabinet", "instructionAfter": "fully open the middle drawer of the cabinet", "sourceStage": "round1", "changed": true, "before": { "n": 10, "benchSuccess": 8, "benchFailure": 2, "benchSuccessRate": 0.8, "mismatchCount": 2, "ias": 0.8, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 2 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 8, "benchFailure": 0 } }, "agentCounts": { "success": 2, "failure": 0, "unknown": 8 }, "selfAssessmentCoverage": 0.2, "sourceStageCounts": { "baseline": 10 } }, "round1": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "round1": 10 } }, "after": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "round1": 10 } }, "transitions": { "successToSuccess": 8, "successToFailure": 0, "failureToSuccess": 2, "failureToFailure": 0, "n": 10, "netAdditionalSuccesses": 2 }, "episodeKeys": [ "libero_goal_t00_r00", "libero_goal_t00_r01", "libero_goal_t00_r02", "libero_goal_t00_r03", "libero_goal_t00_r04", "libero_goal_t00_r05", "libero_goal_t00_r06", "libero_goal_t00_r07", "libero_goal_t00_r08", "libero_goal_t00_r09" ] }, { "id": "libero_goal_t01", "suite": "libero_goal", "taskId": 1, "taskName": "put_the_bowl_on_the_stove", "instructionBefore": "put the bowl on the stove", "instructionRound1": "put the bowl on the stove", "instructionAfter": "put the bowl on the stove", "sourceStage": "baseline", "changed": false, "before": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "round1": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "after": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "transitions": { "successToSuccess": 10, "successToFailure": 0, "failureToSuccess": 0, "failureToFailure": 0, "n": 10, "netAdditionalSuccesses": 0 }, "episodeKeys": [ "libero_goal_t01_r00", "libero_goal_t01_r01", "libero_goal_t01_r02", "libero_goal_t01_r03", "libero_goal_t01_r04", "libero_goal_t01_r05", "libero_goal_t01_r06", "libero_goal_t01_r07", "libero_goal_t01_r08", "libero_goal_t01_r09" ] }, { "id": "libero_goal_t02", "suite": "libero_goal", "taskId": 2, "taskName": "put_the_wine_bottle_on_top_of_the_cabinet", "instructionBefore": "put the wine bottle on top of the cabinet", "instructionRound1": "put the wine bottle on top of the cabinet", "instructionAfter": "put the wine bottle on top of the cabinet", "sourceStage": "baseline", "changed": false, "before": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "round1": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "after": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "transitions": { "successToSuccess": 10, "successToFailure": 0, "failureToSuccess": 0, "failureToFailure": 0, "n": 10, "netAdditionalSuccesses": 0 }, "episodeKeys": [ "libero_goal_t02_r00", "libero_goal_t02_r01", "libero_goal_t02_r02", "libero_goal_t02_r03", "libero_goal_t02_r04", "libero_goal_t02_r05", "libero_goal_t02_r06", "libero_goal_t02_r07", "libero_goal_t02_r08", "libero_goal_t02_r09" ] }, { "id": "libero_goal_t03", "suite": "libero_goal", "taskId": 3, "taskName": "open_the_top_drawer_and_put_the_bowl_inside", "instructionBefore": "open the top drawer and put the bowl inside", "instructionRound1": "open the top drawer and put the bowl inside", "instructionAfter": "open the top drawer and put the bowl inside", "sourceStage": "baseline", "changed": false, "before": { "n": 10, "benchSuccess": 4, "benchFailure": 6, "benchSuccessRate": 0.4, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 5 }, "unknown": { "benchSuccess": 4, "benchFailure": 1 } }, "agentCounts": { "success": 0, "failure": 5, "unknown": 5 }, "selfAssessmentCoverage": 0.5, "sourceStageCounts": { "baseline": 10 } }, "round1": { "n": 10, "benchSuccess": 4, "benchFailure": 6, "benchSuccessRate": 0.4, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 5 }, "unknown": { "benchSuccess": 4, "benchFailure": 1 } }, "agentCounts": { "success": 0, "failure": 5, "unknown": 5 }, "selfAssessmentCoverage": 0.5, "sourceStageCounts": { "baseline": 10 } }, "after": { "n": 10, "benchSuccess": 4, "benchFailure": 6, "benchSuccessRate": 0.4, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 5 }, "unknown": { "benchSuccess": 4, "benchFailure": 1 } }, "agentCounts": { "success": 0, "failure": 5, "unknown": 5 }, "selfAssessmentCoverage": 0.5, "sourceStageCounts": { "baseline": 10 } }, "transitions": { "successToSuccess": 4, "successToFailure": 0, "failureToSuccess": 0, "failureToFailure": 6, "n": 10, "netAdditionalSuccesses": 0 }, "episodeKeys": [ "libero_goal_t03_r00", "libero_goal_t03_r01", "libero_goal_t03_r02", "libero_goal_t03_r03", "libero_goal_t03_r04", "libero_goal_t03_r05", "libero_goal_t03_r06", "libero_goal_t03_r07", "libero_goal_t03_r08", "libero_goal_t03_r09" ] }, { "id": "libero_goal_t04", "suite": "libero_goal", "taskId": 4, "taskName": "put_the_bowl_on_top_of_the_cabinet", "instructionBefore": "put the bowl on top of the cabinet", "instructionRound1": "put the bowl on top of the cabinet", "instructionAfter": "put the bowl on top of the cabinet", "sourceStage": "baseline", "changed": false, "before": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "round1": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "after": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "transitions": { "successToSuccess": 10, "successToFailure": 0, "failureToSuccess": 0, "failureToFailure": 0, "n": 10, "netAdditionalSuccesses": 0 }, "episodeKeys": [ "libero_goal_t04_r00", "libero_goal_t04_r01", "libero_goal_t04_r02", "libero_goal_t04_r03", "libero_goal_t04_r04", "libero_goal_t04_r05", "libero_goal_t04_r06", "libero_goal_t04_r07", "libero_goal_t04_r08", "libero_goal_t04_r09" ] }, { "id": "libero_goal_t05", "suite": "libero_goal", "taskId": 5, "taskName": "push_the_plate_to_the_front_of_the_stove", "instructionBefore": "push the plate to the front of the stove", "instructionRound1": "push the plate directly in front of the stove, about 35 cm center to center", "instructionAfter": "push the plate to the front of the stove, close to its front edge", "sourceStage": "round2", "changed": true, "before": { "n": 10, "benchSuccess": 0, "benchFailure": 10, "benchSuccessRate": 0.0, "mismatchCount": 9, "ias": 0.09999999999999998, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 9 }, "failure": { "benchSuccess": 0, "benchFailure": 1 }, "unknown": { "benchSuccess": 0, "benchFailure": 0 } }, "agentCounts": { "success": 9, "failure": 1, "unknown": 0 }, "selfAssessmentCoverage": 1.0, "sourceStageCounts": { "baseline": 10 } }, "round1": { "n": 10, "benchSuccess": 0, "benchFailure": 10, "benchSuccessRate": 0.0, "mismatchCount": 10, "ias": 0.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 10 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 0, "benchFailure": 0 } }, "agentCounts": { "success": 10, "failure": 0, "unknown": 0 }, "selfAssessmentCoverage": 1.0, "sourceStageCounts": { "round1": 10 } }, "after": { "n": 10, "benchSuccess": 2, "benchFailure": 8, "benchSuccessRate": 0.2, "mismatchCount": 1, "ias": 0.9, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 1 }, "failure": { "benchSuccess": 0, "benchFailure": 6 }, "unknown": { "benchSuccess": 2, "benchFailure": 1 } }, "agentCounts": { "success": 1, "failure": 6, "unknown": 3 }, "selfAssessmentCoverage": 0.7, "sourceStageCounts": { "round2": 10 } }, "transitions": { "successToSuccess": 0, "successToFailure": 0, "failureToSuccess": 2, "failureToFailure": 8, "n": 10, "netAdditionalSuccesses": 2 }, "episodeKeys": [ "libero_goal_t05_r00", "libero_goal_t05_r01", "libero_goal_t05_r02", "libero_goal_t05_r03", "libero_goal_t05_r04", "libero_goal_t05_r05", "libero_goal_t05_r06", "libero_goal_t05_r07", "libero_goal_t05_r08", "libero_goal_t05_r09" ] }, { "id": "libero_goal_t06", "suite": "libero_goal", "taskId": 6, "taskName": "put_the_cream_cheese_in_the_bowl", "instructionBefore": "put the cream cheese in the bowl", "instructionRound1": "put the cream cheese in the bowl", "instructionAfter": "put the cream cheese in the bowl", "sourceStage": "baseline", "changed": false, "before": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "round1": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "after": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "transitions": { "successToSuccess": 10, "successToFailure": 0, "failureToSuccess": 0, "failureToFailure": 0, "n": 10, "netAdditionalSuccesses": 0 }, "episodeKeys": [ "libero_goal_t06_r00", "libero_goal_t06_r01", "libero_goal_t06_r02", "libero_goal_t06_r03", "libero_goal_t06_r04", "libero_goal_t06_r05", "libero_goal_t06_r06", "libero_goal_t06_r07", "libero_goal_t06_r08", "libero_goal_t06_r09" ] }, { "id": "libero_goal_t07", "suite": "libero_goal", "taskId": 7, "taskName": "turn_on_the_stove", "instructionBefore": "turn on the stove", "instructionRound1": "turn on the stove", "instructionAfter": "turn on the stove", "sourceStage": "baseline", "changed": false, "before": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "round1": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "after": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "transitions": { "successToSuccess": 10, "successToFailure": 0, "failureToSuccess": 0, "failureToFailure": 0, "n": 10, "netAdditionalSuccesses": 0 }, "episodeKeys": [ "libero_goal_t07_r00", "libero_goal_t07_r01", "libero_goal_t07_r02", "libero_goal_t07_r03", "libero_goal_t07_r04", "libero_goal_t07_r05", "libero_goal_t07_r06", "libero_goal_t07_r07", "libero_goal_t07_r08", "libero_goal_t07_r09" ] }, { "id": "libero_goal_t08", "suite": "libero_goal", "taskId": 8, "taskName": "put_the_bowl_on_the_plate", "instructionBefore": "put the bowl on the plate", "instructionRound1": "put the bowl on the plate", "instructionAfter": "put the bowl on the plate", "sourceStage": "baseline", "changed": false, "before": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "round1": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "after": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "transitions": { "successToSuccess": 10, "successToFailure": 0, "failureToSuccess": 0, "failureToFailure": 0, "n": 10, "netAdditionalSuccesses": 0 }, "episodeKeys": [ "libero_goal_t08_r00", "libero_goal_t08_r01", "libero_goal_t08_r02", "libero_goal_t08_r03", "libero_goal_t08_r04", "libero_goal_t08_r05", "libero_goal_t08_r06", "libero_goal_t08_r07", "libero_goal_t08_r08", "libero_goal_t08_r09" ] }, { "id": "libero_goal_t09", "suite": "libero_goal", "taskId": 9, "taskName": "put_the_wine_bottle_on_the_rack", "instructionBefore": "put the wine bottle on the rack", "instructionRound1": "put the wine bottle on the upper rack, with its base against the lower rail", "instructionAfter": "put the wine bottle on the upper rack, with its base against the lower rail", "sourceStage": "round1", "changed": true, "before": { "n": 10, "benchSuccess": 7, "benchFailure": 3, "benchSuccessRate": 0.7, "mismatchCount": 3, "ias": 0.7, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 3 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 7, "benchFailure": 0 } }, "agentCounts": { "success": 3, "failure": 0, "unknown": 7 }, "selfAssessmentCoverage": 0.3, "sourceStageCounts": { "baseline": 10 } }, "round1": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "round1": 10 } }, "after": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "round1": 10 } }, "transitions": { "successToSuccess": 7, "successToFailure": 0, "failureToSuccess": 3, "failureToFailure": 0, "n": 10, "netAdditionalSuccesses": 3 }, "episodeKeys": [ "libero_goal_t09_r00", "libero_goal_t09_r01", "libero_goal_t09_r02", "libero_goal_t09_r03", "libero_goal_t09_r04", "libero_goal_t09_r05", "libero_goal_t09_r06", "libero_goal_t09_r07", "libero_goal_t09_r08", "libero_goal_t09_r09" ] }, { "id": "libero_object_t00", "suite": "libero_object", "taskId": 0, "taskName": "pick_up_the_alphabet_soup_and_place_it_in_the_basket", "instructionBefore": "pick up the alphabet soup and place it in the basket", "instructionRound1": "pick up the blue-and-yellow alphabet soup can and place it in the basket", "instructionAfter": "pick up the blue-and-yellow alphabet soup can and place it in the basket", "sourceStage": "round1", "changed": true, "before": { "n": 10, "benchSuccess": 2, "benchFailure": 8, "benchSuccessRate": 0.2, "mismatchCount": 8, "ias": 0.19999999999999996, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 8 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 2, "benchFailure": 0 } }, "agentCounts": { "success": 8, "failure": 0, "unknown": 2 }, "selfAssessmentCoverage": 0.8, "sourceStageCounts": { "baseline": 10 } }, "round1": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "round1": 10 } }, "after": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "round1": 10 } }, "transitions": { "successToSuccess": 2, "successToFailure": 0, "failureToSuccess": 8, "failureToFailure": 0, "n": 10, "netAdditionalSuccesses": 8 }, "episodeKeys": [ "libero_object_t00_r00", "libero_object_t00_r01", "libero_object_t00_r02", "libero_object_t00_r03", "libero_object_t00_r04", "libero_object_t00_r05", "libero_object_t00_r06", "libero_object_t00_r07", "libero_object_t00_r08", "libero_object_t00_r09" ] }, { "id": "libero_object_t01", "suite": "libero_object", "taskId": 1, "taskName": "pick_up_the_cream_cheese_and_place_it_in_the_basket", "instructionBefore": "pick up the cream cheese and place it in the basket", "instructionRound1": "pick up the cream cheese and place it in the basket", "instructionAfter": "pick up the cream cheese and place it in the basket", "sourceStage": "baseline", "changed": false, "before": { "n": 10, "benchSuccess": 9, "benchFailure": 1, "benchSuccessRate": 0.9, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 1 }, "unknown": { "benchSuccess": 9, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 1, "unknown": 9 }, "selfAssessmentCoverage": 0.1, "sourceStageCounts": { "baseline": 10 } }, "round1": { "n": 10, "benchSuccess": 9, "benchFailure": 1, "benchSuccessRate": 0.9, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 1 }, "unknown": { "benchSuccess": 9, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 1, "unknown": 9 }, "selfAssessmentCoverage": 0.1, "sourceStageCounts": { "baseline": 10 } }, "after": { "n": 10, "benchSuccess": 9, "benchFailure": 1, "benchSuccessRate": 0.9, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 1 }, "unknown": { "benchSuccess": 9, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 1, "unknown": 9 }, "selfAssessmentCoverage": 0.1, "sourceStageCounts": { "baseline": 10 } }, "transitions": { "successToSuccess": 9, "successToFailure": 0, "failureToSuccess": 0, "failureToFailure": 1, "n": 10, "netAdditionalSuccesses": 0 }, "episodeKeys": [ "libero_object_t01_r00", "libero_object_t01_r01", "libero_object_t01_r02", "libero_object_t01_r03", "libero_object_t01_r04", "libero_object_t01_r05", "libero_object_t01_r06", "libero_object_t01_r07", "libero_object_t01_r08", "libero_object_t01_r09" ] }, { "id": "libero_object_t02", "suite": "libero_object", "taskId": 2, "taskName": "pick_up_the_salad_dressing_and_place_it_in_the_basket", "instructionBefore": "pick up the salad dressing and place it in the basket", "instructionRound1": "pick up the salad dressing and place it in the basket", "instructionAfter": "pick up the salad dressing and place it in the basket", "sourceStage": "baseline", "changed": false, "before": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "round1": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "after": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "transitions": { "successToSuccess": 10, "successToFailure": 0, "failureToSuccess": 0, "failureToFailure": 0, "n": 10, "netAdditionalSuccesses": 0 }, "episodeKeys": [ "libero_object_t02_r00", "libero_object_t02_r01", "libero_object_t02_r02", "libero_object_t02_r03", "libero_object_t02_r04", "libero_object_t02_r05", "libero_object_t02_r06", "libero_object_t02_r07", "libero_object_t02_r08", "libero_object_t02_r09" ] }, { "id": "libero_object_t03", "suite": "libero_object", "taskId": 3, "taskName": "pick_up_the_bbq_sauce_and_place_it_in_the_basket", "instructionBefore": "pick up the bbq sauce and place it in the basket", "instructionRound1": "pick up the bbq sauce and place it in the basket", "instructionAfter": "pick up the bbq sauce and place it in the basket", "sourceStage": "baseline", "changed": false, "before": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "round1": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "after": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "transitions": { "successToSuccess": 10, "successToFailure": 0, "failureToSuccess": 0, "failureToFailure": 0, "n": 10, "netAdditionalSuccesses": 0 }, "episodeKeys": [ "libero_object_t03_r00", "libero_object_t03_r01", "libero_object_t03_r02", "libero_object_t03_r03", "libero_object_t03_r04", "libero_object_t03_r05", "libero_object_t03_r06", "libero_object_t03_r07", "libero_object_t03_r08", "libero_object_t03_r09" ] }, { "id": "libero_object_t04", "suite": "libero_object", "taskId": 4, "taskName": "pick_up_the_ketchup_and_place_it_in_the_basket", "instructionBefore": "pick up the ketchup and place it in the basket", "instructionRound1": "pick up the white-capped ketchup bottle and place it in the basket", "instructionAfter": "pick up the white-capped ketchup bottle and place it in the basket", "sourceStage": "round1", "changed": true, "before": { "n": 10, "benchSuccess": 8, "benchFailure": 2, "benchSuccessRate": 0.8, "mismatchCount": 2, "ias": 0.8, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 2 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 8, "benchFailure": 0 } }, "agentCounts": { "success": 2, "failure": 0, "unknown": 8 }, "selfAssessmentCoverage": 0.2, "sourceStageCounts": { "baseline": 10 } }, "round1": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "round1": 10 } }, "after": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "round1": 10 } }, "transitions": { "successToSuccess": 8, "successToFailure": 0, "failureToSuccess": 2, "failureToFailure": 0, "n": 10, "netAdditionalSuccesses": 2 }, "episodeKeys": [ "libero_object_t04_r00", "libero_object_t04_r01", "libero_object_t04_r02", "libero_object_t04_r03", "libero_object_t04_r04", "libero_object_t04_r05", "libero_object_t04_r06", "libero_object_t04_r07", "libero_object_t04_r08", "libero_object_t04_r09" ] }, { "id": "libero_object_t05", "suite": "libero_object", "taskId": 5, "taskName": "pick_up_the_tomato_sauce_and_place_it_in_the_basket", "instructionBefore": "pick up the tomato sauce and place it in the basket", "instructionRound1": "pick up the tomato sauce and place it in the basket", "instructionAfter": "pick up the tomato sauce and place it in the basket", "sourceStage": "baseline", "changed": false, "before": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "round1": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "after": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "transitions": { "successToSuccess": 10, "successToFailure": 0, "failureToSuccess": 0, "failureToFailure": 0, "n": 10, "netAdditionalSuccesses": 0 }, "episodeKeys": [ "libero_object_t05_r00", "libero_object_t05_r01", "libero_object_t05_r02", "libero_object_t05_r03", "libero_object_t05_r04", "libero_object_t05_r05", "libero_object_t05_r06", "libero_object_t05_r07", "libero_object_t05_r08", "libero_object_t05_r09" ] }, { "id": "libero_object_t06", "suite": "libero_object", "taskId": 6, "taskName": "pick_up_the_butter_and_place_it_in_the_basket", "instructionBefore": "pick up the butter and place it in the basket", "instructionRound1": "pick up the butter and place it in the basket", "instructionAfter": "pick up the butter and place it in the basket", "sourceStage": "baseline", "changed": false, "before": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "round1": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "after": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "transitions": { "successToSuccess": 10, "successToFailure": 0, "failureToSuccess": 0, "failureToFailure": 0, "n": 10, "netAdditionalSuccesses": 0 }, "episodeKeys": [ "libero_object_t06_r00", "libero_object_t06_r01", "libero_object_t06_r02", "libero_object_t06_r03", "libero_object_t06_r04", "libero_object_t06_r05", "libero_object_t06_r06", "libero_object_t06_r07", "libero_object_t06_r08", "libero_object_t06_r09" ] }, { "id": "libero_object_t07", "suite": "libero_object", "taskId": 7, "taskName": "pick_up_the_milk_and_place_it_in_the_basket", "instructionBefore": "pick up the milk and place it in the basket", "instructionRound1": "pick up the milk and place it in the basket", "instructionAfter": "pick up the milk and place it in the basket", "sourceStage": "baseline", "changed": false, "before": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "round1": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "after": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "transitions": { "successToSuccess": 10, "successToFailure": 0, "failureToSuccess": 0, "failureToFailure": 0, "n": 10, "netAdditionalSuccesses": 0 }, "episodeKeys": [ "libero_object_t07_r00", "libero_object_t07_r01", "libero_object_t07_r02", "libero_object_t07_r03", "libero_object_t07_r04", "libero_object_t07_r05", "libero_object_t07_r06", "libero_object_t07_r07", "libero_object_t07_r08", "libero_object_t07_r09" ] }, { "id": "libero_object_t08", "suite": "libero_object", "taskId": 8, "taskName": "pick_up_the_chocolate_pudding_and_place_it_in_the_basket", "instructionBefore": "pick up the chocolate pudding and place it in the basket", "instructionRound1": "pick up the chocolate pudding and place it in the basket", "instructionAfter": "pick up the chocolate pudding and place it in the basket", "sourceStage": "baseline", "changed": false, "before": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "round1": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "after": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "transitions": { "successToSuccess": 10, "successToFailure": 0, "failureToSuccess": 0, "failureToFailure": 0, "n": 10, "netAdditionalSuccesses": 0 }, "episodeKeys": [ "libero_object_t08_r00", "libero_object_t08_r01", "libero_object_t08_r02", "libero_object_t08_r03", "libero_object_t08_r04", "libero_object_t08_r05", "libero_object_t08_r06", "libero_object_t08_r07", "libero_object_t08_r08", "libero_object_t08_r09" ] }, { "id": "libero_object_t09", "suite": "libero_object", "taskId": 9, "taskName": "pick_up_the_orange_juice_and_place_it_in_the_basket", "instructionBefore": "pick up the orange juice and place it in the basket", "instructionRound1": "pick up the orange juice and place it in the basket", "instructionAfter": "pick up the orange juice and place it in the basket", "sourceStage": "baseline", "changed": false, "before": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "round1": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "after": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "transitions": { "successToSuccess": 10, "successToFailure": 0, "failureToSuccess": 0, "failureToFailure": 0, "n": 10, "netAdditionalSuccesses": 0 }, "episodeKeys": [ "libero_object_t09_r00", "libero_object_t09_r01", "libero_object_t09_r02", "libero_object_t09_r03", "libero_object_t09_r04", "libero_object_t09_r05", "libero_object_t09_r06", "libero_object_t09_r07", "libero_object_t09_r08", "libero_object_t09_r09" ] }, { "id": "libero_spatial_t00", "suite": "libero_spatial", "taskId": 0, "taskName": "pick_up_the_black_bowl_between_the_plate_and_the_ramekin_and_place_it_on_the_plate", "instructionBefore": "pick up the black bowl between the plate and the ramekin and place it on the plate", "instructionRound1": "pick up the black bowl between the plate and the ramekin and place it on the plate", "instructionAfter": "pick up the black bowl between the plate and the ramekin and place it on the plate", "sourceStage": "baseline", "changed": false, "before": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "round1": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "after": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "transitions": { "successToSuccess": 10, "successToFailure": 0, "failureToSuccess": 0, "failureToFailure": 0, "n": 10, "netAdditionalSuccesses": 0 }, "episodeKeys": [ "libero_spatial_t00_r00", "libero_spatial_t00_r01", "libero_spatial_t00_r02", "libero_spatial_t00_r03", "libero_spatial_t00_r04", "libero_spatial_t00_r05", "libero_spatial_t00_r06", "libero_spatial_t00_r07", "libero_spatial_t00_r08", "libero_spatial_t00_r09" ] }, { "id": "libero_spatial_t01", "suite": "libero_spatial", "taskId": 1, "taskName": "pick_up_the_black_bowl_next_to_the_ramekin_and_place_it_on_the_plate", "instructionBefore": "pick up the black bowl next to the ramekin and place it on the plate", "instructionRound1": "pick up the black bowl next to the ramekin and place it on the plate", "instructionAfter": "pick up the black bowl next to the ramekin and place it on the plate", "sourceStage": "baseline", "changed": false, "before": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "round1": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "after": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "transitions": { "successToSuccess": 10, "successToFailure": 0, "failureToSuccess": 0, "failureToFailure": 0, "n": 10, "netAdditionalSuccesses": 0 }, "episodeKeys": [ "libero_spatial_t01_r00", "libero_spatial_t01_r01", "libero_spatial_t01_r02", "libero_spatial_t01_r03", "libero_spatial_t01_r04", "libero_spatial_t01_r05", "libero_spatial_t01_r06", "libero_spatial_t01_r07", "libero_spatial_t01_r08", "libero_spatial_t01_r09" ] }, { "id": "libero_spatial_t02", "suite": "libero_spatial", "taskId": 2, "taskName": "pick_up_the_black_bowl_from_table_center_and_place_it_on_the_plate", "instructionBefore": "pick up the black bowl from table center and place it on the plate", "instructionRound1": "pick up the black bowl from table center and place it on the plate", "instructionAfter": "pick up the black bowl from table center and place it on the plate", "sourceStage": "baseline", "changed": false, "before": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "round1": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "after": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "transitions": { "successToSuccess": 10, "successToFailure": 0, "failureToSuccess": 0, "failureToFailure": 0, "n": 10, "netAdditionalSuccesses": 0 }, "episodeKeys": [ "libero_spatial_t02_r00", "libero_spatial_t02_r01", "libero_spatial_t02_r02", "libero_spatial_t02_r03", "libero_spatial_t02_r04", "libero_spatial_t02_r05", "libero_spatial_t02_r06", "libero_spatial_t02_r07", "libero_spatial_t02_r08", "libero_spatial_t02_r09" ] }, { "id": "libero_spatial_t03", "suite": "libero_spatial", "taskId": 3, "taskName": "pick_up_the_black_bowl_on_the_cookie_box_and_place_it_on_the_plate", "instructionBefore": "pick up the black bowl on the cookie box and place it on the plate", "instructionRound1": "pick up the black bowl on the cookie box and place it on the plate", "instructionAfter": "pick up the black bowl on the cookie box and place it on the plate", "sourceStage": "baseline", "changed": false, "before": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "round1": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "after": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "transitions": { "successToSuccess": 10, "successToFailure": 0, "failureToSuccess": 0, "failureToFailure": 0, "n": 10, "netAdditionalSuccesses": 0 }, "episodeKeys": [ "libero_spatial_t03_r00", "libero_spatial_t03_r01", "libero_spatial_t03_r02", "libero_spatial_t03_r03", "libero_spatial_t03_r04", "libero_spatial_t03_r05", "libero_spatial_t03_r06", "libero_spatial_t03_r07", "libero_spatial_t03_r08", "libero_spatial_t03_r09" ] }, { "id": "libero_spatial_t04", "suite": "libero_spatial", "taskId": 4, "taskName": "pick_up_the_black_bowl_in_the_top_drawer_of_the_wooden_cabinet_and_place_it_on_the_plate", "instructionBefore": "pick up the black bowl in the top drawer of the wooden cabinet and place it on the plate", "instructionRound1": "pick up the black bowl in the top drawer of the wooden cabinet and place it in the center of the plate", "instructionAfter": "pick up the black bowl in the top drawer of the wooden cabinet and place it in the center of the plate", "sourceStage": "round1", "changed": true, "before": { "n": 10, "benchSuccess": 5, "benchFailure": 5, "benchSuccessRate": 0.5, "mismatchCount": 3, "ias": 0.7, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 3 }, "failure": { "benchSuccess": 0, "benchFailure": 2 }, "unknown": { "benchSuccess": 5, "benchFailure": 0 } }, "agentCounts": { "success": 3, "failure": 2, "unknown": 5 }, "selfAssessmentCoverage": 0.5, "sourceStageCounts": { "baseline": 10 } }, "round1": { "n": 10, "benchSuccess": 4, "benchFailure": 6, "benchSuccessRate": 0.4, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 4 }, "unknown": { "benchSuccess": 4, "benchFailure": 2 } }, "agentCounts": { "success": 0, "failure": 4, "unknown": 6 }, "selfAssessmentCoverage": 0.4, "sourceStageCounts": { "round1": 10 } }, "after": { "n": 10, "benchSuccess": 4, "benchFailure": 6, "benchSuccessRate": 0.4, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 4 }, "unknown": { "benchSuccess": 4, "benchFailure": 2 } }, "agentCounts": { "success": 0, "failure": 4, "unknown": 6 }, "selfAssessmentCoverage": 0.4, "sourceStageCounts": { "round1": 10 } }, "transitions": { "successToSuccess": 2, "successToFailure": 3, "failureToSuccess": 2, "failureToFailure": 3, "n": 10, "netAdditionalSuccesses": -1 }, "episodeKeys": [ "libero_spatial_t04_r00", "libero_spatial_t04_r01", "libero_spatial_t04_r02", "libero_spatial_t04_r03", "libero_spatial_t04_r04", "libero_spatial_t04_r05", "libero_spatial_t04_r06", "libero_spatial_t04_r07", "libero_spatial_t04_r08", "libero_spatial_t04_r09" ] }, { "id": "libero_spatial_t05", "suite": "libero_spatial", "taskId": 5, "taskName": "pick_up_the_black_bowl_on_the_ramekin_and_place_it_on_the_plate", "instructionBefore": "pick up the black bowl on the ramekin and place it on the plate", "instructionRound1": "pick up the black bowl on the ramekin and place it on the plate", "instructionAfter": "pick up the black bowl on the ramekin and place it on the plate", "sourceStage": "baseline", "changed": false, "before": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "round1": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "after": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "transitions": { "successToSuccess": 10, "successToFailure": 0, "failureToSuccess": 0, "failureToFailure": 0, "n": 10, "netAdditionalSuccesses": 0 }, "episodeKeys": [ "libero_spatial_t05_r00", "libero_spatial_t05_r01", "libero_spatial_t05_r02", "libero_spatial_t05_r03", "libero_spatial_t05_r04", "libero_spatial_t05_r05", "libero_spatial_t05_r06", "libero_spatial_t05_r07", "libero_spatial_t05_r08", "libero_spatial_t05_r09" ] }, { "id": "libero_spatial_t06", "suite": "libero_spatial", "taskId": 6, "taskName": "pick_up_the_black_bowl_next_to_the_cookie_box_and_place_it_on_the_plate", "instructionBefore": "pick up the black bowl next to the cookie box and place it on the plate", "instructionRound1": "pick up the black bowl next to the cookie box and place it on the plate", "instructionAfter": "pick up the black bowl next to the cookie box and place it on the plate", "sourceStage": "baseline", "changed": false, "before": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "round1": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "after": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "transitions": { "successToSuccess": 10, "successToFailure": 0, "failureToSuccess": 0, "failureToFailure": 0, "n": 10, "netAdditionalSuccesses": 0 }, "episodeKeys": [ "libero_spatial_t06_r00", "libero_spatial_t06_r01", "libero_spatial_t06_r02", "libero_spatial_t06_r03", "libero_spatial_t06_r04", "libero_spatial_t06_r05", "libero_spatial_t06_r06", "libero_spatial_t06_r07", "libero_spatial_t06_r08", "libero_spatial_t06_r09" ] }, { "id": "libero_spatial_t07", "suite": "libero_spatial", "taskId": 7, "taskName": "pick_up_the_black_bowl_on_the_stove_and_place_it_on_the_plate", "instructionBefore": "pick up the black bowl on the stove and place it on the plate", "instructionRound1": "pick up the black bowl on the stove and place it on the plate", "instructionAfter": "pick up the black bowl on the stove and place it on the plate", "sourceStage": "baseline", "changed": false, "before": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "round1": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "after": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "transitions": { "successToSuccess": 10, "successToFailure": 0, "failureToSuccess": 0, "failureToFailure": 0, "n": 10, "netAdditionalSuccesses": 0 }, "episodeKeys": [ "libero_spatial_t07_r00", "libero_spatial_t07_r01", "libero_spatial_t07_r02", "libero_spatial_t07_r03", "libero_spatial_t07_r04", "libero_spatial_t07_r05", "libero_spatial_t07_r06", "libero_spatial_t07_r07", "libero_spatial_t07_r08", "libero_spatial_t07_r09" ] }, { "id": "libero_spatial_t08", "suite": "libero_spatial", "taskId": 8, "taskName": "pick_up_the_black_bowl_next_to_the_plate_and_place_it_on_the_plate", "instructionBefore": "pick up the black bowl next to the plate and place it on the plate", "instructionRound1": "pick up the black bowl next to the plate and place it on the plate", "instructionAfter": "pick up the black bowl next to the plate and place it on the plate", "sourceStage": "baseline", "changed": false, "before": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "round1": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "after": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "transitions": { "successToSuccess": 10, "successToFailure": 0, "failureToSuccess": 0, "failureToFailure": 0, "n": 10, "netAdditionalSuccesses": 0 }, "episodeKeys": [ "libero_spatial_t08_r00", "libero_spatial_t08_r01", "libero_spatial_t08_r02", "libero_spatial_t08_r03", "libero_spatial_t08_r04", "libero_spatial_t08_r05", "libero_spatial_t08_r06", "libero_spatial_t08_r07", "libero_spatial_t08_r08", "libero_spatial_t08_r09" ] }, { "id": "libero_spatial_t09", "suite": "libero_spatial", "taskId": 9, "taskName": "pick_up_the_black_bowl_on_the_wooden_cabinet_and_place_it_on_the_plate", "instructionBefore": "pick up the black bowl on the wooden cabinet and place it on the plate", "instructionRound1": "pick up the black bowl on the wooden cabinet and place it on the plate", "instructionAfter": "pick up the black bowl on the wooden cabinet and place it on the plate", "sourceStage": "baseline", "changed": false, "before": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "round1": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "after": { "n": 10, "benchSuccess": 10, "benchFailure": 0, "benchSuccessRate": 1.0, "mismatchCount": 0, "ias": 1.0, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 0 }, "failure": { "benchSuccess": 0, "benchFailure": 0 }, "unknown": { "benchSuccess": 10, "benchFailure": 0 } }, "agentCounts": { "success": 0, "failure": 0, "unknown": 10 }, "selfAssessmentCoverage": 0.0, "sourceStageCounts": { "baseline": 10 } }, "transitions": { "successToSuccess": 10, "successToFailure": 0, "failureToSuccess": 0, "failureToFailure": 0, "n": 10, "netAdditionalSuccesses": 0 }, "episodeKeys": [ "libero_spatial_t09_r00", "libero_spatial_t09_r01", "libero_spatial_t09_r02", "libero_spatial_t09_r03", "libero_spatial_t09_r04", "libero_spatial_t09_r05", "libero_spatial_t09_r06", "libero_spatial_t09_r07", "libero_spatial_t09_r08", "libero_spatial_t09_r09" ] } ], "provenance": [ { "stage": "baseline", "runName": "astra_libero_v5_calibrated_parallel10_r4", "manifestSha256": "df08694520ff7ea3e9bfee061a29223e46c4db76bc54a07e637cbfcbe93f42f8", "episodeCount": 400, "taskCount": 40, "statistics": { "n": 400, "benchSuccess": 322, "benchFailure": 78, "benchSuccessRate": 0.805, "mismatchCount": 47, "ias": 0.8825000000000001, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 47 }, "failure": { "benchSuccess": 0, "benchFailure": 26 }, "unknown": { "benchSuccess": 322, "benchFailure": 5 } }, "agentCounts": { "success": 47, "failure": 26, "unknown": 327 }, "selfAssessmentCoverage": 0.1825, "sourceStageCounts": { "baseline": 400 } }, "totalAttempts": 404, "excludedInfrastructureAttempts": [ { "episodeKey": "libero_spatial_t02_r00", "attempt": "attempt_001", "status": "error", "reason": "codex_error" }, { "episodeKey": "libero_spatial_t02_r02", "attempt": "attempt_001", "status": "error", "reason": "codex_error" }, { "episodeKey": "libero_10_t03_r03", "attempt": "attempt_001", "status": "error", "reason": "codex_error" }, { "episodeKey": "libero_10_t03_r04", "attempt": "attempt_001", "status": "error", "reason": "codex_error" } ], "requestedModel": "gpt-6-astra", "reasoningEffort": "high", "maxSteps": { "libero_spatial": 500, "libero_goal": 500, "libero_object": 500, "libero_10": 500 }, "maxToolCalls": 750, "imageSize": 512, "workers": 10, "gpuIds": [ 4, 5, 6 ], "liberoCommit": "8f1084e3132a39270c3a13ebe37270a43ece2a01", "liberoTrackedDiffSha256": "12ae32cb1ec02d01eda3581b127c1fee3b0dc53572ed6baf239721a03d82e126", "genericPromptTemplateSha256": "760dc0260ca2ba00a79512c10ce4961245c7c7e8653bb0cdc9095deb05dc0635", "nativeRuntimeCodeSha256": { "robot_agent/session.py": "8b2bc1ee4d7272ba2331416c09dd70c66de71604d55f94604ef31e1038b54a3e", "robot_agent/calibration.py": "c130e6ebfb50b762afd25bb53c93292891f9d13e15168e9d9e38699c2274c21d", "robot_agent/control.py": "43dee71c90634fbe17a4b49be56a2ca4df91f765d888789e691f97c43a5677d6", "robot_agent/observations.py": "2a9930eb31c684008bc568a0ef65a44d188529cffcc206ec0a903717b056eb94", "robot_agent/video.py": "a3fb254e6a0f8db6311e8fda0733e3a625ffe4c855b9b77e7527ab1b5abaaf6b", "robot_agent/evaluation/codex.py": "b4ddc9fb07c5a87ba289eee27a05718e2d10e056bb8c2cdf4b487a7ba203bdee", "robot_agent/evaluation/common.py": "499bc5083243b457ac6e1f57793b97c9f0ba59883e741c19d431b0e3f7505379", "robot_agent/evaluation/runner.py": "179441ebbdc62340c3af50ecbbbed524b53dfa7146ae63e8cc4eb94aeff769d9" }, "publicInstructionObservationsChecked": 41251, "finalGatePassed": true, "firstStartedAt": "2026-09-17T06:51:26.371587+00:00", "lastFinishedAt": "2026-09-17T18:36:26.389149+00:00", "modelVerification": "Requested model and reasoning effort match manifest/result/launch; server-reported actual model is not independently exposed." }, { "stage": "round1", "runName": "astra_libero_v5_instruction_refined_parallel8_r1", "manifestSha256": "1fe4a3d8fadb450d032d237c5e77f0da1e5275f7452de9988b8fa5c9f28ce269", "episodeCount": 90, "taskCount": 9, "statistics": { "n": 90, "benchSuccess": 63, "benchFailure": 27, "benchSuccessRate": 0.7, "mismatchCount": 19, "ias": 0.7888888888888889, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 19 }, "failure": { "benchSuccess": 0, "benchFailure": 5 }, "unknown": { "benchSuccess": 63, "benchFailure": 3 } }, "agentCounts": { "success": 19, "failure": 5, "unknown": 66 }, "selfAssessmentCoverage": 0.26666666666666666, "sourceStageCounts": { "round1": 90 } }, "totalAttempts": 90, "excludedInfrastructureAttempts": [], "requestedModel": "gpt-6-astra", "reasoningEffort": "high", "maxSteps": { "libero_spatial": 500, "libero_goal": 500, "libero_object": 500, "libero_10": 500 }, "maxToolCalls": 750, "imageSize": 512, "workers": 8, "gpuIds": [ 4, 5, 6 ], "liberoCommit": "8f1084e3132a39270c3a13ebe37270a43ece2a01", "liberoTrackedDiffSha256": "12ae32cb1ec02d01eda3581b127c1fee3b0dc53572ed6baf239721a03d82e126", "genericPromptTemplateSha256": "760dc0260ca2ba00a79512c10ce4961245c7c7e8653bb0cdc9095deb05dc0635", "nativeRuntimeCodeSha256": { "robot_agent/session.py": "8b2bc1ee4d7272ba2331416c09dd70c66de71604d55f94604ef31e1038b54a3e", "robot_agent/calibration.py": "c130e6ebfb50b762afd25bb53c93292891f9d13e15168e9d9e38699c2274c21d", "robot_agent/control.py": "43dee71c90634fbe17a4b49be56a2ca4df91f765d888789e691f97c43a5677d6", "robot_agent/observations.py": "2a9930eb31c684008bc568a0ef65a44d188529cffcc206ec0a903717b056eb94", "robot_agent/video.py": "a3fb254e6a0f8db6311e8fda0733e3a625ffe4c855b9b77e7527ab1b5abaaf6b", "robot_agent/evaluation/codex.py": "b4ddc9fb07c5a87ba289eee27a05718e2d10e056bb8c2cdf4b487a7ba203bdee", "robot_agent/evaluation/common.py": "499bc5083243b457ac6e1f57793b97c9f0ba59883e741c19d431b0e3f7505379", "robot_agent/evaluation/runner.py": "179441ebbdc62340c3af50ecbbbed524b53dfa7146ae63e8cc4eb94aeff769d9" }, "publicInstructionObservationsChecked": 10127, "finalGatePassed": true, "firstStartedAt": "2026-09-20T20:14:01.763980+00:00", "lastFinishedAt": "2026-09-20T23:38:15.690382+00:00", "modelVerification": "Requested model and reasoning effort match manifest/result/launch; server-reported actual model is not independently exposed." }, { "stage": "round2", "runName": "astra_libero_v5_instruction_minimal_parallel4_r2", "manifestSha256": "08e43c9fb96ffebf033f0e93fd5371a8ef36b41566eadfa89db80b4958381847", "episodeCount": 20, "taskCount": 2, "statistics": { "n": 20, "benchSuccess": 12, "benchFailure": 8, "benchSuccessRate": 0.6, "mismatchCount": 1, "ias": 0.95, "matrix": { "success": { "benchSuccess": 0, "benchFailure": 1 }, "failure": { "benchSuccess": 0, "benchFailure": 6 }, "unknown": { "benchSuccess": 12, "benchFailure": 1 } }, "agentCounts": { "success": 1, "failure": 6, "unknown": 13 }, "selfAssessmentCoverage": 0.35, "sourceStageCounts": { "round2": 20 } }, "totalAttempts": 20, "excludedInfrastructureAttempts": [], "requestedModel": "gpt-6-astra", "reasoningEffort": "high", "maxSteps": { "libero_spatial": 500, "libero_goal": 500, "libero_object": 500, "libero_10": 500 }, "maxToolCalls": 750, "imageSize": 512, "workers": 4, "gpuIds": [ 0, 1, 2, 3 ], "liberoCommit": "8f1084e3132a39270c3a13ebe37270a43ece2a01", "liberoTrackedDiffSha256": "12ae32cb1ec02d01eda3581b127c1fee3b0dc53572ed6baf239721a03d82e126", "genericPromptTemplateSha256": "760dc0260ca2ba00a79512c10ce4961245c7c7e8653bb0cdc9095deb05dc0635", "nativeRuntimeCodeSha256": { "robot_agent/session.py": "8b2bc1ee4d7272ba2331416c09dd70c66de71604d55f94604ef31e1038b54a3e", "robot_agent/calibration.py": "c130e6ebfb50b762afd25bb53c93292891f9d13e15168e9d9e38699c2274c21d", "robot_agent/control.py": "43dee71c90634fbe17a4b49be56a2ca4df91f765d888789e691f97c43a5677d6", "robot_agent/observations.py": "2a9930eb31c684008bc568a0ef65a44d188529cffcc206ec0a903717b056eb94", "robot_agent/video.py": "a3fb254e6a0f8db6311e8fda0733e3a625ffe4c855b9b77e7527ab1b5abaaf6b", "robot_agent/evaluation/codex.py": "b4ddc9fb07c5a87ba289eee27a05718e2d10e056bb8c2cdf4b487a7ba203bdee", "robot_agent/evaluation/common.py": "499bc5083243b457ac6e1f57793b97c9f0ba59883e741c19d431b0e3f7505379", "robot_agent/evaluation/runner.py": "179441ebbdc62340c3af50ecbbbed524b53dfa7146ae63e8cc4eb94aeff769d9" }, "publicInstructionObservationsChecked": 2284, "finalGatePassed": true, "firstStartedAt": "2026-09-21T02:56:43.379104+00:00", "lastFinishedAt": "2026-09-21T04:32:07.052888+00:00", "modelVerification": "Requested model and reasoning effort match manifest/result/launch; server-reported actual model is not independently exposed." } ], "method": { "iasName": "Instinct-alignment score (operational definition)", "iasFormula": "1 - count(agent explicitly visually_complete AND native benchmark false) / N", "agentSuccess": "finish_assessment.json outcome == visually_complete", "agentFailure": "finish_assessment.json outcome == unable_to_continue", "agentUnknown": "No finish_assessment.json; never imputed from benchmark labels or free-form final prose.", "benchmark": "result.json success cross-checked against env_result.json, private state success/env_done, and final recorded physical-step success.", "selection": "Baseline 400, replace all 90 slots of the 9 round-one tasks, then replace all 20 slots of the 2 final-round tasks. Each slot has exactly one valid policy attempt; only four baseline infrastructure-error attempts were excluded.", "matrixRows": [ "success", "failure", "unknown" ], "matrixColumns": [ "benchSuccess", "benchFailure" ], "pairedFields": [ "episode_key", "suite", "task_id", "task_name", "rollout_id", "init_state_id", "seed", "task_order_index", "available_init_states", "bddl_sha256", "init_sha256", "max_steps", "max_tool_calls", "image_size" ], "limitations": [ "After is a 400-slot composite, not a fresh 400-episode rerun: 310 unchanged baseline outcomes, 70 round-one outcomes, and 20 final-round outcomes.", "Tasks were selected after observing baseline disagreements; final wording was refined using the same task/seed panel. This is adaptive development-set analysis, not a held-out causal estimate.", "There is no seed-level or best-of-round selection: each revised task contributes every episode from its latest specified executed instruction, including failures and regressions.", "Only task language changed in the policy inputs; requested model, generic prompt, seed/init state, native BDDL/init hashes, control/scoring runtime and budgets match. Worker counts changed 10 to 8 to 4, and final-round GPU scheduling also changed.", "An absent explicit finish is unknown, not agent success or agent failure. Native success often stops the episode before a self-assessment, so self-report missingness is outcome-dependent.", "IAS = 1 - explicit visually_complete AND benchmark-false / N is an observed false-completion rate complement for this policy and stopping protocol. It is not an intrinsic benchmark metric or a human judgment of semantic task correctness.", "IAS can rise because false completion declarations become unable_to_continue or unknown, even when native success does not improve. Native success rate, the full matrix, and unknown counts must be shown alongside IAS.", "Agent visual completion claims are not independent ground truth. The study does not measure false negatives reliably, compare human raters, or establish that every disagreement is an instruction defect.", "Ten episodes per task with a stochastic policy and no multiple independent reruns cannot isolate instruction causality or establish generalization to other policies or seeds.", "No post-training experiment was performed. Misaligned language/reward could be a training risk hypothesis; these results do not demonstrate post-training damage." ], "validation": { "passed": true, "validPolicyAttemptsAudited": 510, "instructionPairsChecked": 110, "nativeInputFilesHashed": 80, "publicInstructionObservationsChecked": 53662, "completeTaskGroupSelection": true, "nativeHashesModelSeedsBudgetsMatch": true, "nativeFinalScoreAgreement": true, "manifestIntegrity": true, "historicalRunGatesPassed": true } }, "transitions": { "beforeToRound1": { "successToSuccess": 319, "successToFailure": 3, "failureToSuccess": 26, "failureToFailure": 52, "n": 400, "netAdditionalSuccesses": 23 }, "beforeToAfter": { "successToSuccess": 319, "successToFailure": 3, "failureToSuccess": 38, "failureToFailure": 40, "n": 400, "netAdditionalSuccesses": 35 }, "round1ToAfter": { "successToSuccess": 345, "successToFailure": 0, "failureToSuccess": 12, "failureToFailure": 43, "n": 400, "netAdditionalSuccesses": 12 } } }