{
  "description": "Saved P2 evaluation and query-accounting summaries used by this post. Machine-specific paths are omitted; no robot evaluations were rerun.",
  "evaluation_protocol": {
    "episodes_per_task": 64,
    "reset_seed_range": [
      10000,
      10063
    ],
    "student_training_seed": 1
  },
  "a1": {
    "230k": {
      "source_file": "results/data/a1/a1_raw_230k_final_eval.json",
      "step": 230000,
      "final": {
        "ode": {
          "square": {
            "success": 0.421875,
            "k": 27,
            "n": 64,
            "wilson95": [
              0.3086959366145158,
              0.5439018794747275
            ],
            "ret": -333.796875
          },
          "toolhang": {
            "success": 0.3125,
            "k": 20,
            "n": 64,
            "wilson95": [
              0.21230909502265743,
              0.43392566359152623
            ],
            "ret": -870.703125
          }
        },
        "sde": {
          "square": {
            "success": 0.65625,
            "k": 42,
            "n": 64,
            "wilson95": [
              0.5340341181762038,
              0.7607702496453097
            ],
            "ret": -287.453125
          },
          "toolhang": {
            "success": 0.53125,
            "k": 34,
            "n": 64,
            "wilson95": [
              0.41071875000475133,
              0.6482421235595515
            ],
            "ret": -729.859375
          }
        }
      }
    },
    "500k": {
      "source_file": "results/data/a1/a1_raw_500k_final_eval.json",
      "step": 500000,
      "final": {
        "ode": {
          "square": {
            "success": 0.296875,
            "k": 19,
            "n": 64,
            "wilson95": [
              0.19904999589075245,
              0.4177043259412798
            ],
            "ret": -346.875
          },
          "toolhang": {
            "success": 0.765625,
            "k": 49,
            "n": 64,
            "wilson95": [
              0.6486650974929234,
              0.8525023278036498
            ],
            "ret": -581.921875
          }
        },
        "sde": {
          "square": {
            "success": 0.296875,
            "k": 19,
            "n": 64,
            "wilson95": [
              0.19904999589075245,
              0.4177043259412798
            ],
            "ret": -354.53125
          },
          "toolhang": {
            "success": 0.90625,
            "k": 58,
            "n": 64,
            "wilson95": [
              0.8101690141672233,
              0.9563223421687121
            ],
            "ret": -514.0625
          }
        }
      }
    }
  },
  "a2": {
    "square": {
      "source_file": "results/data/a2/a2_sq_specialist_eval.json",
      "step": 1000000,
      "results": {
        "ode": {
          "mode": "ode",
          "description": "Deterministic ODE (mode i)",
          "success": 0.875,
          "k": 56,
          "n": 64,
          "wilson95": [
            0.7722521366978408,
            0.9352783460737919
          ],
          "return": -184.796875
        },
        "sde": {
          "mode": "sde",
          "description": "Single-sample SDE (mode ii)",
          "success": 0.890625,
          "k": 57,
          "n": 64,
          "wilson95": [
            0.7910113670806147,
            0.9459995524731695
          ],
          "return": -170.421875
        },
        "bon": {
          "mode": "bon",
          "description": "Best-of-8 + Critic argmax (mode iii)",
          "success": 0.96875,
          "k": 62,
          "n": 64,
          "wilson95": [
            0.8930250611770575,
            0.9913880422874833
          ],
          "return": -148.015625
        }
      }
    },
    "toolhang": {
      "source_file": "results/data/a2/a2_th_specialist_eval.json",
      "step": 1000000,
      "results": {
        "ode": {
          "mode": "ode",
          "description": "Deterministic ODE (mode i)",
          "success": 0.90625,
          "k": 58,
          "n": 64,
          "wilson95": [
            0.8101690141672233,
            0.9563223421687121
          ],
          "return": -377.765625
        },
        "sde": {
          "mode": "sde",
          "description": "Single-sample SDE (mode ii)",
          "success": 0.90625,
          "k": 58,
          "n": 64,
          "wilson95": [
            0.8101690141672233,
            0.9563223421687121
          ],
          "return": -377.21875
        },
        "bon": {
          "mode": "bon",
          "description": "Best-of-8 + Critic argmax (mode iii)",
          "success": 0.890625,
          "k": 57,
          "n": 64,
          "wilson95": [
            0.7910113670806147,
            0.9459995524731695
          ],
          "return": -382.703125
        }
      }
    }
  },
  "a3": {
    "Q1": {
      "source_file": "results/data/a3/a3_Q1_final_eval.json",
      "training": {
        "seed": 1,
        "steps": 20000,
        "batch": 256,
        "lr": 0.0003,
        "clip_grad": 1000.0,
        "tau": 0.005
      },
      "final": {
        "ode": {
          "square": {
            "success": 0.578125,
            "k": 37,
            "n": 64,
            "wilson95": [
              0.4560981205252726,
              0.6913040633854843
            ],
            "ret": -247.140625
          },
          "toolhang": {
            "success": 0.4375,
            "k": 28,
            "n": 64,
            "wilson95": [
              0.32293817065427327,
              0.5591400822171212
            ],
            "ret": -705.515625
          }
        },
        "sde": {
          "square": {
            "success": 0.671875,
            "k": 43,
            "n": 64,
            "wilson95": [
              0.5499887622745426,
              0.7742960423291224
            ],
            "ret": -221.640625
          },
          "toolhang": {
            "success": 0.484375,
            "k": 31,
            "n": 64,
            "wilson95": [
              0.36633348565783896,
              0.6041860775600096
            ],
            "ret": -669.109375
          }
        }
      },
      "episodes_used": "all",
      "data": {
        "square": {
          "pairs_in_prefix": 1998,
          "episodes_in_prefix": 50,
          "pairs_used": 1998,
          "success_pair_fraction": 0.7997997997997998
        },
        "toolhang": {
          "pairs_in_prefix": 1983,
          "episodes_in_prefix": 43,
          "pairs_used": 1983,
          "success_pair_fraction": 0.81089258698941
        }
      }
    },
    "Q2": {
      "source_file": "results/data/a3/a3_Q2_final_eval.json",
      "training": {
        "seed": 1,
        "steps": 20000,
        "batch": 256,
        "lr": 0.0003,
        "clip_grad": 1000.0,
        "tau": 0.005
      },
      "final": {
        "ode": {
          "square": {
            "success": 0.59375,
            "k": 38,
            "n": 64,
            "wilson95": [
              0.471449865045497,
              0.7054327556474111
            ],
            "ret": -254.359375
          },
          "toolhang": {
            "success": 0.40625,
            "k": 26,
            "n": 64,
            "wilson95": [
              0.29456724435258885,
              0.528550134954503
            ],
            "ret": -786.984375
          }
        },
        "sde": {
          "square": {
            "success": 0.8125,
            "k": 52,
            "n": 64,
            "wilson95": [
              0.7002540844522482,
              0.889354651190779
            ],
            "ret": -184.3125
          },
          "toolhang": {
            "success": 0.71875,
            "k": 46,
            "n": 64,
            "wilson95": [
              0.598658376004965,
              0.8140677389451542
            ],
            "ret": -520.375
          }
        }
      },
      "episodes_used": "all",
      "data": {
        "square": {
          "pairs_in_prefix": 7998,
          "episodes_in_prefix": 191,
          "pairs_used": 7998,
          "success_pair_fraction": 0.7749437359339835
        },
        "toolhang": {
          "pairs_in_prefix": 7969,
          "episodes_in_prefix": 175,
          "pairs_used": 7969,
          "success_pair_fraction": 0.8117706111180826
        }
      }
    },
    "Q3": {
      "source_file": "results/data/a3/a3_Q3_final_eval.json",
      "training": {
        "seed": 1,
        "steps": 20000,
        "batch": 256,
        "lr": 0.0003,
        "clip_grad": 1000.0,
        "tau": 0.005
      },
      "final": {
        "ode": {
          "square": {
            "success": 0.6875,
            "k": 44,
            "n": 64,
            "wilson95": [
              0.5660743364084737,
              0.7876909049773426
            ],
            "ret": -226.515625
          },
          "toolhang": {
            "success": 0.671875,
            "k": 43,
            "n": 64,
            "wilson95": [
              0.5499887622745426,
              0.7742960423291224
            ],
            "ret": -562.5
          }
        },
        "sde": {
          "square": {
            "success": 0.796875,
            "k": 51,
            "n": 64,
            "wilson95": [
              0.6828613244878341,
              0.8772669743730419
            ],
            "ret": -188.953125
          },
          "toolhang": {
            "success": 0.671875,
            "k": 43,
            "n": 64,
            "wilson95": [
              0.5499887622745426,
              0.7742960423291224
            ],
            "ret": -566.71875
          }
        }
      },
      "episodes_used": "all",
      "data": {
        "square": {
          "pairs_in_prefix": 31980,
          "episodes_in_prefix": 776,
          "pairs_used": 31980,
          "success_pair_fraction": 0.8123827392120075
        },
        "toolhang": {
          "pairs_in_prefix": 31994,
          "episodes_in_prefix": 672,
          "pairs_used": 31994,
          "success_pair_fraction": 0.7382321685315997
        }
      }
    }
  },
  "a4": {
    "Q1": {
      "source_file": "results/data/a4/a4_Q1_final_eval.json",
      "training": {
        "seed": 1,
        "steps": 20000,
        "batch": 256,
        "lr": 0.0003,
        "clip_grad": 1000.0,
        "tau": 0.005
      },
      "final": {
        "ode": {
          "square": {
            "success": 0.296875,
            "k": 19,
            "n": 64,
            "wilson95": [
              0.19904999589075245,
              0.4177043259412798
            ],
            "ret": -332.421875
          },
          "toolhang": {
            "success": 0.296875,
            "k": 19,
            "n": 64,
            "wilson95": [
              0.19904999589075245,
              0.4177043259412798
            ],
            "ret": -799.453125
          }
        },
        "sde": {
          "square": {
            "success": 0.234375,
            "k": 15,
            "n": 64,
            "wilson95": [
              0.1474976721963502,
              0.35133490250707655
            ],
            "ret": -346.03125
          },
          "toolhang": {
            "success": 0.59375,
            "k": 38,
            "n": 64,
            "wilson95": [
              0.471449865045497,
              0.7054327556474111
            ],
            "ret": -605.4375
          }
        }
      },
      "budget": {
        "square": 1998,
        "toolhang": 1983
      },
      "rounds": [
        {
          "round": 0,
          "start_step": 0,
          "tasks": {
            "square": {
              "queries": 499,
              "finished_episodes": 6,
              "student_successes": 5,
              "student_success_rate": 0.8333333333333334,
              "mean_abs_diff_student_vs_teacher": 0.11370225250720978
            },
            "toolhang": {
              "queries": 495,
              "finished_episodes": 4,
              "student_successes": 2,
              "student_success_rate": 0.5,
              "mean_abs_diff_student_vs_teacher": 0.19009292125701904
            }
          },
          "end_step": 5000,
          "aggregate_pairs": {
            "square": 499,
            "toolhang": 495
          },
          "eval": {
            "sde": {
              "square": {
                "success": 0.0625,
                "k": 4,
                "n": 64,
                "wilson95": [
                  0.024570803674792932,
                  0.14997696642496888
                ],
                "ret": -389.484375
              },
              "toolhang": {
                "success": 0.0,
                "k": 0,
                "n": 64,
                "wilson95": [
                  0.0,
                  0.056626022971156334
                ],
                "ret": -1000.0
              }
            }
          }
        },
        {
          "round": 1,
          "start_step": 5000,
          "tasks": {
            "square": {
              "queries": 499,
              "finished_episodes": 4,
              "student_successes": 0,
              "student_success_rate": 0.0,
              "mean_abs_diff_student_vs_teacher": 0.10459084063768387
            },
            "toolhang": {
              "queries": 495,
              "finished_episodes": 3,
              "student_successes": 0,
              "student_success_rate": 0.0,
              "mean_abs_diff_student_vs_teacher": 0.3425605893135071
            }
          },
          "end_step": 10000,
          "aggregate_pairs": {
            "square": 998,
            "toolhang": 990
          },
          "eval": {
            "sde": {
              "square": {
                "success": 0.140625,
                "k": 9,
                "n": 64,
                "wilson95": [
                  0.07578487006414436,
                  0.24616508394637426
                ],
                "ret": -375.1875
              },
              "toolhang": {
                "success": 0.21875,
                "k": 14,
                "n": 64,
                "wilson95": [
                  0.13502197461349755,
                  0.3343301633077779
                ],
                "ret": -861.53125
              }
            }
          }
        },
        {
          "round": 2,
          "start_step": 10000,
          "tasks": {
            "square": {
              "queries": 499,
              "finished_episodes": 7,
              "student_successes": 5,
              "student_success_rate": 0.7142857142857143,
              "mean_abs_diff_student_vs_teacher": 0.0981919914484024
            },
            "toolhang": {
              "queries": 495,
              "finished_episodes": 4,
              "student_successes": 2,
              "student_success_rate": 0.5,
              "mean_abs_diff_student_vs_teacher": 0.291080504655838
            }
          },
          "end_step": 15000,
          "aggregate_pairs": {
            "square": 1497,
            "toolhang": 1485
          },
          "eval": {
            "sde": {
              "square": {
                "success": 0.234375,
                "k": 15,
                "n": 64,
                "wilson95": [
                  0.1474976721963502,
                  0.35133490250707655
                ],
                "ret": -348.1875
              },
              "toolhang": {
                "success": 0.578125,
                "k": 37,
                "n": 64,
                "wilson95": [
                  0.4560981205252726,
                  0.6913040633854843
                ],
                "ret": -619.125
              }
            }
          }
        },
        {
          "round": 3,
          "start_step": 15000,
          "tasks": {
            "square": {
              "queries": 501,
              "finished_episodes": 5,
              "student_successes": 0,
              "student_success_rate": 0.0,
              "mean_abs_diff_student_vs_teacher": 0.07752062380313873
            },
            "toolhang": {
              "queries": 498,
              "finished_episodes": 5,
              "student_successes": 2,
              "student_success_rate": 0.4,
              "mean_abs_diff_student_vs_teacher": 0.17375096678733826
            }
          },
          "end_step": 20000,
          "aggregate_pairs": {
            "square": 1998,
            "toolhang": 1983
          },
          "eval": {
            "ode": {
              "square": {
                "success": 0.296875,
                "k": 19,
                "n": 64,
                "wilson95": [
                  0.19904999589075245,
                  0.4177043259412798
                ],
                "ret": -332.421875
              },
              "toolhang": {
                "success": 0.296875,
                "k": 19,
                "n": 64,
                "wilson95": [
                  0.19904999589075245,
                  0.4177043259412798
                ],
                "ret": -799.453125
              }
            },
            "sde": {
              "square": {
                "success": 0.234375,
                "k": 15,
                "n": 64,
                "wilson95": [
                  0.1474976721963502,
                  0.35133490250707655
                ],
                "ret": -346.03125
              },
              "toolhang": {
                "success": 0.59375,
                "k": 38,
                "n": 64,
                "wilson95": [
                  0.471449865045497,
                  0.7054327556474111
                ],
                "ret": -605.4375
              }
            }
          }
        }
      ]
    },
    "Q2": {
      "source_file": "results/data/a4/a4_Q2_final_eval.json",
      "training": {
        "seed": 1,
        "steps": 20000,
        "batch": 256,
        "lr": 0.0003,
        "clip_grad": 1000.0,
        "tau": 0.005
      },
      "final": {
        "ode": {
          "square": {
            "success": 0.640625,
            "k": 41,
            "n": 64,
            "wilson95": [
              0.5182062363615084,
              0.7471176946778538
            ],
            "ret": -231.375
          },
          "toolhang": {
            "success": 0.265625,
            "k": 17,
            "n": 64,
            "wilson95": [
              0.17296238569537764,
              0.38483106257235195
            ],
            "ret": -842.984375
          }
        },
        "sde": {
          "square": {
            "success": 0.734375,
            "k": 47,
            "n": 64,
            "wilson95": [
              0.615168937427648,
              0.8270376143046224
            ],
            "ret": -214.21875
          },
          "toolhang": {
            "success": 0.65625,
            "k": 42,
            "n": 64,
            "wilson95": [
              0.5340341181762038,
              0.7607702496453097
            ],
            "ret": -596.203125
          }
        }
      },
      "budget": {
        "square": 7998,
        "toolhang": 7969
      },
      "rounds": [
        {
          "round": 0,
          "start_step": 0,
          "tasks": {
            "square": {
              "queries": 1999,
              "finished_episodes": 26,
              "student_successes": 18,
              "student_success_rate": 0.6923076923076923,
              "mean_abs_diff_student_vs_teacher": 0.10895252972841263
            },
            "toolhang": {
              "queries": 1992,
              "finished_episodes": 18,
              "student_successes": 6,
              "student_success_rate": 0.3333333333333333,
              "mean_abs_diff_student_vs_teacher": 0.15530845522880554
            }
          },
          "end_step": 5000,
          "aggregate_pairs": {
            "square": 1999,
            "toolhang": 1992
          },
          "eval": {
            "sde": {
              "square": {
                "success": 0.421875,
                "k": 27,
                "n": 64,
                "wilson95": [
                  0.3086959366145158,
                  0.5439018794747275
                ],
                "ret": -314.046875
              },
              "toolhang": {
                "success": 0.015625,
                "k": 1,
                "n": 64,
                "wilson95": [
                  0.0027634707504014047,
                  0.08334298900290629
                ],
                "ret": -992.640625
              }
            }
          }
        },
        {
          "round": 1,
          "start_step": 5000,
          "tasks": {
            "square": {
              "queries": 1999,
              "finished_episodes": 25,
              "student_successes": 10,
              "student_success_rate": 0.4,
              "mean_abs_diff_student_vs_teacher": 0.0789341926574707
            },
            "toolhang": {
              "queries": 1992,
              "finished_episodes": 15,
              "student_successes": 0,
              "student_success_rate": 0.0,
              "mean_abs_diff_student_vs_teacher": 0.1794830560684204
            }
          },
          "end_step": 10000,
          "aggregate_pairs": {
            "square": 3998,
            "toolhang": 3984
          },
          "eval": {
            "sde": {
              "square": {
                "success": 0.59375,
                "k": 38,
                "n": 64,
                "wilson95": [
                  0.471449865045497,
                  0.7054327556474111
                ],
                "ret": -256.734375
              },
              "toolhang": {
                "success": 0.296875,
                "k": 19,
                "n": 64,
                "wilson95": [
                  0.19904999589075245,
                  0.4177043259412798
                ],
                "ret": -811.484375
              }
            }
          }
        },
        {
          "round": 2,
          "start_step": 10000,
          "tasks": {
            "square": {
              "queries": 1999,
              "finished_episodes": 30,
              "student_successes": 17,
              "student_success_rate": 0.5666666666666667,
              "mean_abs_diff_student_vs_teacher": 0.07590032368898392
            },
            "toolhang": {
              "queries": 1992,
              "finished_episodes": 20,
              "student_successes": 7,
              "student_success_rate": 0.35,
              "mean_abs_diff_student_vs_teacher": 0.16650931537151337
            }
          },
          "end_step": 15000,
          "aggregate_pairs": {
            "square": 5997,
            "toolhang": 5976
          },
          "eval": {
            "sde": {
              "square": {
                "success": 0.734375,
                "k": 47,
                "n": 64,
                "wilson95": [
                  0.615168937427648,
                  0.8270376143046224
                ],
                "ret": -210.265625
              },
              "toolhang": {
                "success": 0.546875,
                "k": 35,
                "n": 64,
                "wilson95": [
                  0.42573382520679776,
                  0.6627074851396563
                ],
                "ret": -662.1875
              }
            }
          }
        },
        {
          "round": 3,
          "start_step": 15000,
          "tasks": {
            "square": {
              "queries": 2001,
              "finished_episodes": 28,
              "student_successes": 13,
              "student_success_rate": 0.4642857142857143,
              "mean_abs_diff_student_vs_teacher": 0.07269197702407837
            },
            "toolhang": {
              "queries": 1993,
              "finished_episodes": 20,
              "student_successes": 7,
              "student_success_rate": 0.35,
              "mean_abs_diff_student_vs_teacher": 0.12709759175777435
            }
          },
          "end_step": 20000,
          "aggregate_pairs": {
            "square": 7998,
            "toolhang": 7969
          },
          "eval": {
            "ode": {
              "square": {
                "success": 0.640625,
                "k": 41,
                "n": 64,
                "wilson95": [
                  0.5182062363615084,
                  0.7471176946778538
                ],
                "ret": -231.375
              },
              "toolhang": {
                "success": 0.265625,
                "k": 17,
                "n": 64,
                "wilson95": [
                  0.17296238569537764,
                  0.38483106257235195
                ],
                "ret": -842.984375
              }
            },
            "sde": {
              "square": {
                "success": 0.734375,
                "k": 47,
                "n": 64,
                "wilson95": [
                  0.615168937427648,
                  0.8270376143046224
                ],
                "ret": -214.21875
              },
              "toolhang": {
                "success": 0.65625,
                "k": 42,
                "n": 64,
                "wilson95": [
                  0.5340341181762038,
                  0.7607702496453097
                ],
                "ret": -596.203125
              }
            }
          }
        }
      ]
    },
    "Q3": {
      "source_file": "results/data/a4/a4_Q3_final_eval.json",
      "training": {
        "seed": 1,
        "steps": 20000,
        "batch": 256,
        "lr": 0.0003,
        "clip_grad": 1000.0,
        "tau": 0.005
      },
      "final": {
        "ode": {
          "square": {
            "success": 0.875,
            "k": 56,
            "n": 64,
            "wilson95": [
              0.7722521366978408,
              0.9352783460737919
            ],
            "ret": -190.4375
          },
          "toolhang": {
            "success": 0.734375,
            "k": 47,
            "n": 64,
            "wilson95": [
              0.615168937427648,
              0.8270376143046224
            ],
            "ret": -546.421875
          }
        },
        "sde": {
          "square": {
            "success": 0.875,
            "k": 56,
            "n": 64,
            "wilson95": [
              0.7722521366978408,
              0.9352783460737919
            ],
            "ret": -178.390625
          },
          "toolhang": {
            "success": 0.765625,
            "k": 49,
            "n": 64,
            "wilson95": [
              0.6486650974929234,
              0.8525023278036498
            ],
            "ret": -516.171875
          }
        }
      },
      "budget": {
        "square": 31980,
        "toolhang": 31994
      },
      "rounds": [
        {
          "round": 0,
          "start_step": 0,
          "tasks": {
            "square": {
              "queries": 7995,
              "finished_episodes": 103,
              "student_successes": 59,
              "student_success_rate": 0.5728155339805825,
              "mean_abs_diff_student_vs_teacher": 0.10824926197528839
            },
            "toolhang": {
              "queries": 7998,
              "finished_episodes": 78,
              "student_successes": 34,
              "student_success_rate": 0.4358974358974359,
              "mean_abs_diff_student_vs_teacher": 0.15369290113449097
            }
          },
          "end_step": 5000,
          "aggregate_pairs": {
            "square": 7995,
            "toolhang": 7998
          },
          "eval": {
            "sde": {
              "square": {
                "success": 0.734375,
                "k": 47,
                "n": 64,
                "wilson95": [
                  0.615168937427648,
                  0.8270376143046224
                ],
                "ret": -224.515625
              },
              "toolhang": {
                "success": 0.21875,
                "k": 14,
                "n": 64,
                "wilson95": [
                  0.13502197461349755,
                  0.3343301633077779
                ],
                "ret": -891.34375
              }
            }
          }
        },
        {
          "round": 1,
          "start_step": 5000,
          "tasks": {
            "square": {
              "queries": 7995,
              "finished_episodes": 120,
              "student_successes": 74,
              "student_success_rate": 0.6166666666666667,
              "mean_abs_diff_student_vs_teacher": 0.06258440762758255
            },
            "toolhang": {
              "queries": 7998,
              "finished_episodes": 69,
              "student_successes": 9,
              "student_success_rate": 0.13043478260869565,
              "mean_abs_diff_student_vs_teacher": 0.24080012738704681
            }
          },
          "end_step": 10000,
          "aggregate_pairs": {
            "square": 15990,
            "toolhang": 15996
          },
          "eval": {
            "sde": {
              "square": {
                "success": 0.75,
                "k": 48,
                "n": 64,
                "wilson95": [
                  0.6318349082561628,
                  0.8398520802582591
                ],
                "ret": -211.328125
              },
              "toolhang": {
                "success": 0.578125,
                "k": 37,
                "n": 64,
                "wilson95": [
                  0.4560981205252726,
                  0.6913040633854843
                ],
                "ret": -646.671875
              }
            }
          }
        },
        {
          "round": 2,
          "start_step": 10000,
          "tasks": {
            "square": {
              "queries": 7995,
              "finished_episodes": 144,
              "student_successes": 105,
              "student_success_rate": 0.7291666666666666,
              "mean_abs_diff_student_vs_teacher": 0.041726190596818924
            },
            "toolhang": {
              "queries": 7998,
              "finished_episodes": 90,
              "student_successes": 44,
              "student_success_rate": 0.4888888888888889,
              "mean_abs_diff_student_vs_teacher": 0.11913039535284042
            }
          },
          "end_step": 15000,
          "aggregate_pairs": {
            "square": 23985,
            "toolhang": 23994
          },
          "eval": {
            "sde": {
              "square": {
                "success": 0.8125,
                "k": 52,
                "n": 64,
                "wilson95": [
                  0.7002540844522482,
                  0.889354651190779
                ],
                "ret": -191.640625
              },
              "toolhang": {
                "success": 0.640625,
                "k": 41,
                "n": 64,
                "wilson95": [
                  0.5182062363615084,
                  0.7471176946778538
                ],
                "ret": -598.65625
              }
            }
          }
        },
        {
          "round": 3,
          "start_step": 15000,
          "tasks": {
            "square": {
              "queries": 7995,
              "finished_episodes": 159,
              "student_successes": 130,
              "student_success_rate": 0.8176100628930818,
              "mean_abs_diff_student_vs_teacher": 0.042857006192207336
            },
            "toolhang": {
              "queries": 8000,
              "finished_episodes": 106,
              "student_successes": 67,
              "student_success_rate": 0.6320754716981132,
              "mean_abs_diff_student_vs_teacher": 0.09501417726278305
            }
          },
          "end_step": 20000,
          "aggregate_pairs": {
            "square": 31980,
            "toolhang": 31994
          },
          "eval": {
            "ode": {
              "square": {
                "success": 0.875,
                "k": 56,
                "n": 64,
                "wilson95": [
                  0.7722521366978408,
                  0.9352783460737919
                ],
                "ret": -190.4375
              },
              "toolhang": {
                "success": 0.734375,
                "k": 47,
                "n": 64,
                "wilson95": [
                  0.615168937427648,
                  0.8270376143046224
                ],
                "ret": -546.421875
              }
            },
            "sde": {
              "square": {
                "success": 0.875,
                "k": 56,
                "n": 64,
                "wilson95": [
                  0.7722521366978408,
                  0.9352783460737919
                ],
                "ret": -178.390625
              },
              "toolhang": {
                "success": 0.765625,
                "k": 49,
                "n": 64,
                "wilson95": [
                  0.6486650974929234,
                  0.8525023278036498
                ],
                "ret": -516.171875
              }
            }
          }
        }
      ]
    }
  }
}
