{
  "global_step": 8868,
  "aggregate_epoch": 4.0,
  "model_path": "/ko_edu_snowflake_final_train/checkpoints/step_08868",
  "validation": {
    "global_step": 8868,
    "aggregate_epoch": 4.0,
    "content_macro_f1": 0.7691223696148852,
    "content_f1_class2_3_mean": 0.7002175340967471
  },
  "tests": {
    "public_test": {
      "aggregate_equal_context_weight": {
        "selection_loss": 0.5996472404267125,
        "content_mae": 0.36416186395414174,
        "content_macro_mae": 0.39134996560001856,
        "content_exact_accuracy": 0.7229817708333334,
        "content_spearman": 0.9219387450855417,
        "content_gaussian_nll": -0.525285516083209,
        "content_mean_predicted_variance": 0.2302397398710588,
        "integrity_accuracy": 0.9256184895833334,
        "integrity_macro_f1": 0.5145415702399313,
        "critical_average_precision": 0.1668109006932243,
        "critical_roc_auc": 0.924077158723946,
        "critical_f1_at_0_5": 0.20155038759689922,
        "critical_brier": 0.007695191131648721,
        "final_mae": 0.3742708417087128,
        "final_macro_mae": 0.40425132911669354,
        "final_exact_accuracy": 0.7154134114583334,
        "final_spearman": 0.9196958171483666,
        "integrity_confusion_matrix": [
          [
            11192,
            412,
            45
          ],
          [
            385,
            156,
            17
          ],
          [
            42,
            13,
            26
          ]
        ],
        "content_distribution": {
          "0": 3088,
          "1": 2782,
          "2": 2964,
          "3": 2007,
          "4": 1447
        },
        "final_distribution": {
          "0": 3088,
          "1": 2782,
          "2": 3029,
          "3": 2086,
          "4": 1303
        },
        "content_per_class": {
          "0": {
            "precision": 0.9783031088082902,
            "recall": 0.923006416131989,
            "f1": 0.9498506524131426,
            "support": 3273
          },
          "1": {
            "precision": 0.6294033069734004,
            "recall": 0.8106481481481481,
            "f1": 0.7086199919061109,
            "support": 2160
          },
          "2": {
            "precision": 0.6271929824561403,
            "recall": 0.6441441441441441,
            "f1": 0.6355555555555555,
            "support": 2886
          },
          "3": {
            "precision": 0.5371200797209765,
            "recall": 0.5502807554874937,
            "f1": 0.5436207766011094,
            "support": 1959
          },
          "4": {
            "precision": 0.8120248790601244,
            "recall": 0.5845771144278606,
            "f1": 0.6797801562048018,
            "support": 2010
          }
        },
        "content_macro_f1": 0.7034854265361441,
        "content_f1_class2_3_mean": 0.5895881660783324,
        "content_mean_variance_correct": 0.16486197243390396,
        "content_mean_variance_wrong": 0.4008672621717884,
        "content_wrong_to_correct_variance_ratio": 2.431532610302252,
        "content_variance_wrong_auroc": 0.799274570902363,
        "content_variance_wrong_average_precision": 0.5623532341417358,
        "content_one_sigma_accuracy": 0.7645670572916666,
        "content_fixed_half_point_accuracy": 0.7229817708333334,
        "content_mean_predicted_sigma": 0.4059100152568453,
        "content_mean_absolute_calibration_gap": 0.23573810502832473
      },
      "by_length": {
        "512": {
          "selection_loss": 0.6475959000101105,
          "content_mae": 0.403041226476617,
          "content_macro_mae": 0.43819375073195327,
          "content_exact_accuracy": 0.69873046875,
          "content_spearman": 0.9094162283707627,
          "content_gaussian_nll": -0.33558642342258216,
          "content_mean_predicted_variance": 0.25569876981790673,
          "integrity_accuracy": 0.92724609375,
          "integrity_macro_f1": 0.5196996608761314,
          "critical_average_precision": 0.16239140288737114,
          "critical_roc_auc": 0.9300401409027608,
          "critical_f1_at_0_5": 0.24489795918367346,
          "critical_brier": 0.008444094912288811,
          "final_mae": 0.4134857682716224,
          "final_macro_mae": 0.4519832393323484,
          "final_exact_accuracy": 0.689453125,
          "final_spearman": 0.9069972446950932,
          "integrity_confusion_matrix": [
            [
              3741,
              122,
              20
            ],
            [
              133,
              46,
              7
            ],
            [
              13,
              3,
              11
            ]
          ],
          "content_distribution": {
            "0": 1031,
            "1": 997,
            "2": 1037,
            "3": 666,
            "4": 365
          },
          "final_distribution": {
            "0": 1031,
            "1": 997,
            "2": 1058,
            "3": 695,
            "4": 315
          },
          "content_per_class": {
            "0": {
              "precision": 0.9747817652764307,
              "recall": 0.921173235563703,
              "f1": 0.9472196041470311,
              "support": 1091
            },
            "1": {
              "precision": 0.5927783350050151,
              "recall": 0.8208333333333333,
              "f1": 0.6884100174723354,
              "support": 720
            },
            "2": {
              "precision": 0.5756991321118612,
              "recall": 0.6205821205821206,
              "f1": 0.5972986493246624,
              "support": 962
            },
            "3": {
              "precision": 0.5225225225225225,
              "recall": 0.5329249617151608,
              "f1": 0.5276724791508719,
              "support": 653
            },
            "4": {
              "precision": 0.8794520547945206,
              "recall": 0.4791044776119403,
              "f1": 0.6202898550724638,
              "support": 670
            }
          },
          "content_macro_f1": 0.6761781210334729,
          "content_f1_class2_3_mean": 0.5624855642377671,
          "content_mean_variance_correct": 0.17687892520976323,
          "content_mean_variance_wrong": 0.4385046006675881,
          "content_wrong_to_correct_variance_ratio": 2.4791229376114723,
          "content_variance_wrong_auroc": 0.8046483457862315,
          "content_variance_wrong_average_precision": 0.5972072289789552,
          "content_one_sigma_accuracy": 0.746337890625,
          "content_fixed_half_point_accuracy": 0.69873046875,
          "content_mean_predicted_sigma": 0.4275034727898084,
          "content_mean_absolute_calibration_gap": 0.2608774770819382
        },
        "1024": {
          "selection_loss": 0.5793806554162239,
          "content_mae": 0.35502210318509775,
          "content_macro_mae": 0.3809260249430456,
          "content_exact_accuracy": 0.729248046875,
          "content_spearman": 0.9258773309917829,
          "content_gaussian_nll": -0.5862765162921657,
          "content_mean_predicted_variance": 0.21921119402604106,
          "integrity_accuracy": 0.931396484375,
          "integrity_macro_f1": 0.5242311737226991,
          "critical_average_precision": 0.20618147810728718,
          "critical_roc_auc": 0.9281923850613947,
          "critical_f1_at_0_5": 0.2,
          "critical_brier": 0.007129219811954048,
          "final_mae": 0.36424440595520335,
          "final_macro_mae": 0.39263800988537184,
          "final_exact_accuracy": 0.72119140625,
          "final_spearman": 0.9237399815073253,
          "integrity_confusion_matrix": [
            [
              3758,
              112,
              13
            ],
            [
              133,
              48,
              5
            ],
            [
              14,
              4,
              9
            ]
          ],
          "content_distribution": {
            "0": 1027,
            "1": 920,
            "2": 989,
            "3": 669,
            "4": 491
          },
          "final_distribution": {
            "0": 1027,
            "1": 920,
            "2": 1014,
            "3": 688,
            "4": 447
          },
          "content_per_class": {
            "0": {
              "precision": 0.9795520934761441,
              "recall": 0.922089825847846,
              "f1": 0.9499527856468366,
              "support": 1091
            },
            "1": {
              "precision": 0.6326086956521739,
              "recall": 0.8083333333333333,
              "f1": 0.7097560975609756,
              "support": 720
            },
            "2": {
              "precision": 0.6339737108190091,
              "recall": 0.6517671517671517,
              "f1": 0.6427473090722706,
              "support": 962
            },
            "3": {
              "precision": 0.5500747384155455,
              "recall": 0.5635528330781011,
              "f1": 0.556732223903177,
              "support": 653
            },
            "4": {
              "precision": 0.8228105906313645,
              "recall": 0.6029850746268657,
              "f1": 0.6959517657192076,
              "support": 670
            }
          },
          "content_macro_f1": 0.7110280363804935,
          "content_f1_class2_3_mean": 0.5997397664877238,
          "content_mean_variance_correct": 0.15720720139904037,
          "content_mean_variance_wrong": 0.3862138324181519,
          "content_wrong_to_correct_variance_ratio": 2.456718451706433,
          "content_variance_wrong_auroc": 0.8013581848364252,
          "content_variance_wrong_average_precision": 0.5641231749105367,
          "content_one_sigma_accuracy": 0.767822265625,
          "content_fixed_half_point_accuracy": 0.729248046875,
          "content_mean_predicted_sigma": 0.39683885099409644,
          "content_mean_absolute_calibration_gap": 0.22794024060909682
        },
        "2048": {
          "selection_loss": 0.563851957099979,
          "content_mae": 0.33442226220071036,
          "content_macro_mae": 0.3549301211250569,
          "content_exact_accuracy": 0.740966796875,
          "content_spearman": 0.9327042422925155,
          "content_gaussian_nll": -0.653993608534879,
          "content_mean_predicted_variance": 0.21580925576922855,
          "integrity_accuracy": 0.918212890625,
          "integrity_macro_f1": 0.49500456357784506,
          "critical_average_precision": 0.16431265610031154,
          "critical_roc_auc": 0.9154173834685017,
          "critical_f1_at_0_5": 0.15,
          "critical_brier": 0.007512258670703306,
          "final_mae": 0.3450823508993126,
          "final_macro_mae": 0.3681327381323604,
          "final_exact_accuracy": 0.735595703125,
          "final_spearman": 0.9304844155402512,
          "integrity_confusion_matrix": [
            [
              3693,
              178,
              12
            ],
            [
              119,
              62,
              5
            ],
            [
              15,
              6,
              6
            ]
          ],
          "content_distribution": {
            "0": 1030,
            "1": 865,
            "2": 938,
            "3": 672,
            "4": 591
          },
          "final_distribution": {
            "0": 1030,
            "1": 865,
            "2": 957,
            "3": 703,
            "4": 541
          },
          "content_per_class": {
            "0": {
              "precision": 0.9805825242718447,
              "recall": 0.925756186984418,
              "f1": 0.9523809523809523,
              "support": 1091
            },
            "1": {
              "precision": 0.6682080924855491,
              "recall": 0.8027777777777778,
              "f1": 0.7293375394321766,
              "support": 720
            },
            "2": {
              "precision": 0.6769722814498934,
              "recall": 0.66008316008316,
              "f1": 0.6684210526315789,
              "support": 962
            },
            "3": {
              "precision": 0.5386904761904762,
              "recall": 0.554364471669219,
              "f1": 0.5464150943396227,
              "support": 653
            },
            "4": {
              "precision": 0.7614213197969543,
              "recall": 0.6716417910447762,
              "f1": 0.7137192704203014,
              "support": 670
            }
          },
          "content_macro_f1": 0.7220547818409264,
          "content_f1_class2_3_mean": 0.6074180734856007,
          "content_mean_variance_correct": 0.16106371287430865,
          "content_mean_variance_wrong": 0.372409371401728,
          "content_wrong_to_correct_variance_ratio": 2.3121866791457233,
          "content_variance_wrong_auroc": 0.7882823235671796,
          "content_variance_wrong_average_precision": 0.5170919064835311,
          "content_one_sigma_accuracy": 0.779541015625,
          "content_fixed_half_point_accuracy": 0.740966796875,
          "content_mean_predicted_sigma": 0.39338772198663097,
          "content_mean_absolute_calibration_gap": 0.2183965973939393
        }
      }
    },
    "private_ood_test": {
      "aggregate_equal_context_weight": {
        "selection_loss": 0.5928699060725598,
        "content_mae": 0.3732402183663052,
        "content_macro_mae": 0.37847784761947306,
        "content_exact_accuracy": 0.7152506510416666,
        "content_spearman": 0.8885748177452322,
        "content_gaussian_nll": -0.16548063811710678,
        "content_mean_predicted_variance": 0.2825092818944354,
        "integrity_accuracy": 0.9122721354166666,
        "integrity_macro_f1": 0.5191581195438396,
        "critical_average_precision": 0.14243176618765288,
        "critical_roc_auc": 0.8837826503394979,
        "critical_f1_at_0_5": 0.14414414414414414,
        "critical_brier": 0.014604646909798027,
        "final_mae": 0.3995507869677533,
        "final_macro_mae": 0.39592509330157394,
        "final_exact_accuracy": 0.7035319010416666,
        "final_spearman": 0.8782610375325385,
        "integrity_confusion_matrix": [
          [
            10816,
            269,
            21
          ],
          [
            611,
            374,
            11
          ],
          [
            76,
            90,
            20
          ]
        ],
        "content_distribution": {
          "0": 708,
          "1": 2758,
          "2": 3859,
          "3": 1847,
          "4": 3116
        },
        "final_distribution": {
          "0": 708,
          "1": 2758,
          "2": 3894,
          "3": 1990,
          "4": 2938
        },
        "content_per_class": {
          "0": {
            "precision": 0.8728813559322034,
            "recall": 0.8174603174603174,
            "f1": 0.8442622950819673,
            "support": 756
          },
          "1": {
            "precision": 0.7962291515591008,
            "recall": 0.6547406082289803,
            "f1": 0.7185863874345549,
            "support": 3354
          },
          "2": {
            "precision": 0.6351386369525784,
            "recall": 0.7006861063464837,
            "f1": 0.666304200081555,
            "support": 3498
          },
          "3": {
            "precision": 0.4672441797509475,
            "recall": 0.5100472813238771,
            "f1": 0.4877083922011868,
            "support": 1692
          },
          "4": {
            "precision": 0.8539794608472401,
            "recall": 0.8905622489959839,
            "f1": 0.8718872870249017,
            "support": 2988
          }
        },
        "content_macro_f1": 0.7177497123648331,
        "content_f1_class2_3_mean": 0.5770062961413709,
        "content_mean_variance_correct": 0.22792565066763823,
        "content_mean_variance_wrong": 0.4196157508433695,
        "content_wrong_to_correct_variance_ratio": 1.8410203047100402,
        "content_variance_wrong_auroc": 0.7425900435249433,
        "content_variance_wrong_average_precision": 0.468111215636207,
        "content_one_sigma_accuracy": 0.7752278645833334,
        "content_fixed_half_point_accuracy": 0.7152506510416666,
        "content_mean_predicted_sigma": 0.468670747090409,
        "content_mean_absolute_calibration_gap": 0.2627904350268082
      },
      "by_length": {
        "512": {
          "selection_loss": 0.6120346124297937,
          "content_mae": 0.3830975716779529,
          "content_macro_mae": 0.39077505134591706,
          "content_exact_accuracy": 0.706298828125,
          "content_spearman": 0.8809364124482335,
          "content_gaussian_nll": -0.17671267232550236,
          "content_mean_predicted_variance": 0.30312733544612275,
          "integrity_accuracy": 0.90771484375,
          "integrity_macro_f1": 0.5140433335549824,
          "critical_average_precision": 0.11496175566449311,
          "critical_roc_auc": 0.8559302381371248,
          "critical_f1_at_0_5": 0.15,
          "critical_brier": 0.015920309394525538,
          "final_mae": 0.41359218036217615,
          "final_macro_mae": 0.4119320265168561,
          "final_exact_accuracy": 0.689208984375,
          "final_spearman": 0.8698473259231785,
          "integrity_confusion_matrix": [
            [
              3585,
              104,
              13
            ],
            [
              201,
              126,
              5
            ],
            [
              27,
              28,
              7
            ]
          ],
          "content_distribution": {
            "0": 230,
            "1": 1043,
            "2": 1237,
            "3": 591,
            "4": 995
          },
          "final_distribution": {
            "0": 230,
            "1": 1043,
            "2": 1249,
            "3": 666,
            "4": 908
          },
          "content_per_class": {
            "0": {
              "precision": 0.8782608695652174,
              "recall": 0.8015873015873016,
              "f1": 0.8381742738589212,
              "support": 252
            },
            "1": {
              "precision": 0.7392138063279002,
              "recall": 0.6896243291592129,
              "f1": 0.7135585377140213,
              "support": 1118
            },
            "2": {
              "precision": 0.6313662085691188,
              "recall": 0.6698113207547169,
              "f1": 0.6500208073241781,
              "support": 1166
            },
            "3": {
              "precision": 0.4754653130287648,
              "recall": 0.49822695035460995,
              "f1": 0.48658008658008656,
              "support": 564
            },
            "4": {
              "precision": 0.8623115577889447,
              "recall": 0.8614457831325302,
              "f1": 0.861878453038674,
              "support": 996
            }
          },
          "content_macro_f1": 0.7100424317031763,
          "content_f1_class2_3_mean": 0.5683004469521323,
          "content_mean_variance_correct": 0.24633350799266054,
          "content_mean_variance_wrong": 0.43970634028641054,
          "content_wrong_to_correct_variance_ratio": 1.7850041753130537,
          "content_variance_wrong_auroc": 0.736118857137603,
          "content_variance_wrong_average_precision": 0.46183723218564027,
          "content_one_sigma_accuracy": 0.783447265625,
          "content_fixed_half_point_accuracy": 0.706298828125,
          "content_mean_predicted_sigma": 0.4868545213487623,
          "content_mean_absolute_calibration_gap": 0.2706309738992609
        },
        "1024": {
          "selection_loss": 0.5862670943393096,
          "content_mae": 0.3789426484455323,
          "content_macro_mae": 0.38048229162949315,
          "content_exact_accuracy": 0.7060546875,
          "content_spearman": 0.8906623521795864,
          "content_gaussian_nll": -0.12287645864569759,
          "content_mean_predicted_variance": 0.2687230434777398,
          "integrity_accuracy": 0.914794921875,
          "integrity_macro_f1": 0.5244504767937112,
          "critical_average_precision": 0.17686078916073436,
          "critical_roc_auc": 0.8947194811841285,
          "critical_f1_at_0_5": 0.1891891891891892,
          "critical_brier": 0.013953239596104013,
          "final_mae": 0.4034991220801931,
          "final_macro_mae": 0.39627136522476464,
          "final_exact_accuracy": 0.6962890625,
          "final_spearman": 0.8806180293419044,
          "integrity_confusion_matrix": [
            [
              3629,
              69,
              4
            ],
            [
              219,
              110,
              3
            ],
            [
              25,
              29,
              8
            ]
          ],
          "content_distribution": {
            "0": 242,
            "1": 869,
            "2": 1264,
            "3": 665,
            "4": 1056
          },
          "final_distribution": {
            "0": 242,
            "1": 869,
            "2": 1272,
            "3": 700,
            "4": 1013
          },
          "content_per_class": {
            "0": {
              "precision": 0.8636363636363636,
              "recall": 0.8293650793650794,
              "f1": 0.8461538461538461,
              "support": 252
            },
            "1": {
              "precision": 0.8193325661680092,
              "recall": 0.6368515205724508,
              "f1": 0.7166582788122798,
              "support": 1118
            },
            "2": {
              "precision": 0.6186708860759493,
              "recall": 0.6706689536878216,
              "f1": 0.6436213991769547,
              "support": 1166
            },
            "3": {
              "precision": 0.43909774436090226,
              "recall": 0.5177304964539007,
              "f1": 0.4751830756712775,
              "support": 564
            },
            "4": {
              "precision": 0.8494318181818182,
              "recall": 0.9006024096385542,
              "f1": 0.8742690058479532,
              "support": 996
            }
          },
          "content_macro_f1": 0.7111771211324622,
          "content_f1_class2_3_mean": 0.5594022374241161,
          "content_mean_variance_correct": 0.21197317897944154,
          "content_mean_variance_wrong": 0.40503584092714057,
          "content_wrong_to_correct_variance_ratio": 1.9107881613948123,
          "content_variance_wrong_auroc": 0.7573059258442353,
          "content_variance_wrong_average_precision": 0.4909850888338813,
          "content_one_sigma_accuracy": 0.76025390625,
          "content_fixed_half_point_accuracy": 0.7060546875,
          "content_mean_predicted_sigma": 0.4568070278293196,
          "content_mean_absolute_calibration_gap": 0.2568249294043655
        },
        "2048": {
          "selection_loss": 0.5701281481916703,
          "content_mae": 0.35768043497543045,
          "content_macro_mae": 0.364176199883009,
          "content_exact_accuracy": 0.7333984375,
          "content_spearman": 0.8949834476665999,
          "content_gaussian_nll": -0.1968527833801204,
          "content_mean_predicted_variance": 0.2756774667594437,
          "integrity_accuracy": 0.914306640625,
          "integrity_macro_f1": 0.5180030709490746,
          "critical_average_precision": 0.1761922067653546,
          "critical_roc_auc": 0.9030278919506773,
          "critical_f1_at_0_5": 0.08823529411764706,
          "critical_brier": 0.013940391738764532,
          "final_mae": 0.3815610584608907,
          "final_macro_mae": 0.37957188816310106,
          "final_exact_accuracy": 0.72509765625,
          "final_spearman": 0.885492549972223,
          "integrity_confusion_matrix": [
            [
              3602,
              96,
              4
            ],
            [
              191,
              138,
              3
            ],
            [
              24,
              33,
              5
            ]
          ],
          "content_distribution": {
            "0": 236,
            "1": 846,
            "2": 1358,
            "3": 591,
            "4": 1065
          },
          "final_distribution": {
            "0": 236,
            "1": 846,
            "2": 1373,
            "3": 624,
            "4": 1017
          },
          "content_per_class": {
            "0": {
              "precision": 0.8771186440677966,
              "recall": 0.8214285714285714,
              "f1": 0.8483606557377049,
              "support": 252
            },
            "1": {
              "precision": 0.8427895981087471,
              "recall": 0.6377459749552773,
              "f1": 0.7260692464358453,
              "support": 1118
            },
            "2": {
              "precision": 0.6539027982326951,
              "recall": 0.7615780445969125,
              "f1": 0.7036450079239303,
              "support": 1166
            },
            "3": {
              "precision": 0.4906937394247039,
              "recall": 0.5141843971631206,
              "f1": 0.5021645021645021,
              "support": 564
            },
            "4": {
              "precision": 0.8507042253521127,
              "recall": 0.9096385542168675,
              "f1": 0.8791848617176128,
              "support": 996
            }
          },
          "content_macro_f1": 0.731884854795919,
          "content_f1_class2_3_mean": 0.6029047550442161,
          "content_mean_variance_correct": 0.22555568291829578,
          "content_mean_variance_wrong": 0.4135582713920519,
          "content_wrong_to_correct_variance_ratio": 1.8335085422869053,
          "content_variance_wrong_auroc": 0.7346610197392488,
          "content_variance_wrong_average_precision": 0.4611742547263893,
          "content_one_sigma_accuracy": 0.781982421875,
          "content_fixed_half_point_accuracy": 0.7333984375,
          "content_mean_predicted_sigma": 0.4623506920931452,
          "content_mean_absolute_calibration_gap": 0.2609154017767981
        }
      }
    }
  },
  "prediction_manifest": {
    "private_ood_test_ctx1024.parquet": {
      "rows": 4096,
      "bytes": 317328,
      "sha256": "f2c54290a754bdf608e39e706c9948868cd341d1b3a6cce07f6ae7c018c07858"
    },
    "private_ood_test_ctx2048.parquet": {
      "rows": 4096,
      "bytes": 314219,
      "sha256": "7e409a5182ebf42043873173d5617fea04e406cd6730417bab6e02adea919bf2"
    },
    "private_ood_test_ctx512.parquet": {
      "rows": 4096,
      "bytes": 318598,
      "sha256": "9f63b8d7299fc5f1e395e39b0185edc493899782f330802bf1bf499f0947dd98"
    },
    "public_test_ctx1024.parquet": {
      "rows": 4096,
      "bytes": 313111,
      "sha256": "e930d8592489f8d78670345f98d7573396a9cbe953e6ef1c873c09e8a0dd5d75"
    },
    "public_test_ctx2048.parquet": {
      "rows": 4096,
      "bytes": 310365,
      "sha256": "83a8a7d417be256de469edbc65c7c0104dbd5879505952df888c6cbcdb0b202b"
    },
    "public_test_ctx512.parquet": {
      "rows": 4096,
      "bytes": 314796,
      "sha256": "0ab3099b4b1caa5b4b33ce4c0bf89892317984a21fbed0e649d495a39b4cb935"
    }
  }
}