| { | |
| "ref_ppl": { | |
| "chat": 8.5587, | |
| "code": 5.6556, | |
| "long-book": 12.1535, | |
| "short-all": 7.1382 | |
| }, | |
| "gsm8k_bf16_cpu_acc": 0.98, | |
| "rows": [ | |
| { | |
| "config": "bf16", | |
| "plan": "plans/bf16.json", | |
| "weights": "IT", | |
| "read_GB": 23.8446, | |
| "floor_ms@128": 46.66, | |
| "chat": { | |
| "ppl_delta_pct": -0.77, | |
| "dnll": -0.0077, | |
| "se": 0.0014, | |
| "top1": 0.975, | |
| "kl": 0.0095, | |
| "kl_p99": 0.103 | |
| }, | |
| "code": { | |
| "ppl_delta_pct": -0.05, | |
| "dnll": -0.0005, | |
| "se": 0.0014, | |
| "top1": 0.9843, | |
| "kl": 0.0139, | |
| "kl_p99": 0.061 | |
| }, | |
| "long-book": { | |
| "ppl_delta_pct": -0.14, | |
| "dnll": -0.0014, | |
| "se": 0.0013, | |
| "top1": 0.9764, | |
| "kl": 0.0048, | |
| "kl_p99": 0.053 | |
| }, | |
| "short-all": { | |
| "ppl_delta_pct": -0.45, | |
| "dnll": -0.0045, | |
| "se": 0.001, | |
| "top1": 0.9791, | |
| "kl": 0.0114, | |
| "kl_p99": 0.081 | |
| } | |
| }, | |
| { | |
| "config": "a-bfp8", | |
| "plan": "plans/a-bfp8.json", | |
| "weights": "IT", | |
| "read_GB": 12.6675, | |
| "floor_ms@128": 24.83, | |
| "chat": { | |
| "ppl_delta_pct": -0.29, | |
| "dnll": -0.0029, | |
| "se": 0.0015, | |
| "top1": 0.9713, | |
| "kl": 0.0122, | |
| "kl_p99": 0.139 | |
| }, | |
| "code": { | |
| "ppl_delta_pct": 1.01, | |
| "dnll": 0.01, | |
| "se": 0.0014, | |
| "top1": 0.9827, | |
| "kl": 0.0057, | |
| "kl_p99": 0.08 | |
| }, | |
| "long-book": { | |
| "ppl_delta_pct": 1.61, | |
| "dnll": 0.016, | |
| "se": 0.0015, | |
| "top1": 0.9723, | |
| "kl": 0.007, | |
| "kl_p99": 0.068 | |
| }, | |
| "short-all": { | |
| "ppl_delta_pct": 0.28, | |
| "dnll": 0.0028, | |
| "se": 0.001, | |
| "top1": 0.9763, | |
| "kl": 0.0093, | |
| "kl_p99": 0.105 | |
| } | |
| }, | |
| { | |
| "config": "a-lm4", | |
| "plan": "plans/a-lm4.json", | |
| "weights": "IT", | |
| "read_GB": 12.1641, | |
| "floor_ms@128": 23.84, | |
| "chat": { | |
| "ppl_delta_pct": 0.61, | |
| "dnll": 0.0061, | |
| "se": 0.0018, | |
| "top1": 0.955, | |
| "kl": 0.0208, | |
| "kl_p99": 0.244 | |
| }, | |
| "code": { | |
| "ppl_delta_pct": 1.98, | |
| "dnll": 0.0196, | |
| "se": 0.0022, | |
| "top1": 0.9703, | |
| "kl": 0.0169, | |
| "kl_p99": 0.289 | |
| }, | |
| "long-book": { | |
| "ppl_delta_pct": 2.55, | |
| "dnll": 0.0252, | |
| "se": 0.0023, | |
| "top1": 0.9605, | |
| "kl": 0.0149, | |
| "kl_p99": 0.128 | |
| }, | |
| "short-all": { | |
| "ppl_delta_pct": 1.21, | |
| "dnll": 0.012, | |
| "se": 0.0014, | |
| "top1": 0.9617, | |
| "kl": 0.0191, | |
| "kl_p99": 0.264 | |
| } | |
| }, | |
| { | |
| "config": "f-dn4", | |
| "plan": "plans/f-dn4.json", | |
| "weights": "IT", | |
| "read_GB": 11.2519, | |
| "floor_ms@128": 22.06, | |
| "chat": { | |
| "ppl_delta_pct": 2.61, | |
| "dnll": 0.0257, | |
| "se": 0.004, | |
| "top1": 0.9102, | |
| "kl": 0.0942, | |
| "kl_p99": 1.456 | |
| }, | |
| "code": { | |
| "ppl_delta_pct": 2.43, | |
| "dnll": 0.024, | |
| "se": 0.005, | |
| "top1": 0.9333, | |
| "kl": 0.101, | |
| "kl_p99": 1.949 | |
| }, | |
| "long-book": { | |
| "ppl_delta_pct": -0.64, | |
| "dnll": -0.0065, | |
| "se": 0.0049, | |
| "top1": 0.912, | |
| "kl": 0.0732, | |
| "kl_p99": 0.819 | |
| }, | |
| "short-all": { | |
| "ppl_delta_pct": 2.53, | |
| "dnll": 0.025, | |
| "se": 0.0031, | |
| "top1": 0.9203, | |
| "kl": 0.0972, | |
| "kl_p99": 1.65 | |
| } | |
| }, | |
| { | |
| "config": "f-gu4", | |
| "plan": "plans/f-gu4.json", | |
| "weights": "IT", | |
| "read_GB": 9.8363, | |
| "floor_ms@128": 19.3, | |
| "chat": { | |
| "ppl_delta_pct": 1.56, | |
| "dnll": 0.0155, | |
| "se": 0.004, | |
| "top1": 0.9021, | |
| "kl": 0.0962, | |
| "kl_p99": 1.461 | |
| }, | |
| "code": { | |
| "ppl_delta_pct": 2.43, | |
| "dnll": 0.024, | |
| "se": 0.0051, | |
| "top1": 0.9286, | |
| "kl": 0.1111, | |
| "kl_p99": 2.099 | |
| }, | |
| "long-book": { | |
| "ppl_delta_pct": 9.73, | |
| "dnll": 0.0929, | |
| "se": 0.0053, | |
| "top1": 0.9053, | |
| "kl": 0.0848, | |
| "kl_p99": 0.863 | |
| }, | |
| "short-all": { | |
| "ppl_delta_pct": 1.94, | |
| "dnll": 0.0192, | |
| "se": 0.0032, | |
| "top1": 0.9137, | |
| "kl": 0.1028, | |
| "kl_p99": 1.702 | |
| } | |
| }, | |
| { | |
| "config": "g-rank3072", | |
| "plan": "plans/g-rank3072.json", | |
| "weights": "IT", | |
| "read_GB": 11.6058, | |
| "floor_ms@128": 22.75, | |
| "chat": { | |
| "ppl_delta_pct": -0.04, | |
| "dnll": -0.0004, | |
| "se": 0.0031, | |
| "top1": 0.9308, | |
| "kl": 0.0548, | |
| "kl_p99": 0.837 | |
| }, | |
| "code": { | |
| "ppl_delta_pct": 2.05, | |
| "dnll": 0.0203, | |
| "se": 0.0041, | |
| "top1": 0.9478, | |
| "kl": 0.0688, | |
| "kl_p99": 1.243 | |
| }, | |
| "long-book": { | |
| "ppl_delta_pct": 7.3, | |
| "dnll": 0.0704, | |
| "se": 0.0037, | |
| "top1": 0.9422, | |
| "kl": 0.036, | |
| "kl_p99": 0.353 | |
| }, | |
| "short-all": { | |
| "ppl_delta_pct": 0.87, | |
| "dnll": 0.0087, | |
| "se": 0.0025, | |
| "top1": 0.9382, | |
| "kl": 0.061, | |
| "kl_p99": 1.024 | |
| } | |
| }, | |
| { | |
| "config": "g-rank2048", | |
| "plan": "plans/g-rank2048.json", | |
| "weights": "IT", | |
| "read_GB": 10.5441, | |
| "floor_ms@128": 20.68, | |
| "chat": { | |
| "ppl_delta_pct": 1.25, | |
| "dnll": 0.0125, | |
| "se": 0.0037, | |
| "top1": 0.9126, | |
| "kl": 0.082, | |
| "kl_p99": 1.28 | |
| }, | |
| "code": { | |
| "ppl_delta_pct": 1.82, | |
| "dnll": 0.018, | |
| "se": 0.0048, | |
| "top1": 0.9349, | |
| "kl": 0.0971, | |
| "kl_p99": 1.813 | |
| }, | |
| "long-book": { | |
| "ppl_delta_pct": 8.86, | |
| "dnll": 0.0849, | |
| "se": 0.0048, | |
| "top1": 0.9169, | |
| "kl": 0.0671, | |
| "kl_p99": 0.713 | |
| }, | |
| "short-all": { | |
| "ppl_delta_pct": 1.5, | |
| "dnll": 0.0149, | |
| "se": 0.003, | |
| "top1": 0.9224, | |
| "kl": 0.0886, | |
| "kl_p99": 1.542 | |
| } | |
| }, | |
| { | |
| "config": "b-mlp4", | |
| "plan": "plans/b-mlp4.json", | |
| "weights": "IT", | |
| "read_GB": 8.4207, | |
| "floor_ms@128": 16.53, | |
| "chat": { | |
| "ppl_delta_pct": 5.17, | |
| "dnll": 0.0504, | |
| "se": 0.0052, | |
| "top1": 0.8706, | |
| "kl": 0.1747, | |
| "kl_p99": 2.743 | |
| }, | |
| "code": { | |
| "ppl_delta_pct": 3.91, | |
| "dnll": 0.0383, | |
| "se": 0.0066, | |
| "top1": 0.9083, | |
| "kl": 0.1758, | |
| "kl_p99": 3.069 | |
| }, | |
| "long-book": { | |
| "ppl_delta_pct": 9.91, | |
| "dnll": 0.0945, | |
| "se": 0.007, | |
| "top1": 0.8769, | |
| "kl": 0.1502, | |
| "kl_p99": 1.589 | |
| }, | |
| "short-all": { | |
| "ppl_delta_pct": 4.61, | |
| "dnll": 0.0451, | |
| "se": 0.0041, | |
| "top1": 0.8871, | |
| "kl": 0.1752, | |
| "kl_p99": 2.848 | |
| } | |
| }, | |
| { | |
| "config": "c-mlp4-lm4", | |
| "plan": "plans/c-mlp4-lm4.json", | |
| "weights": "IT", | |
| "read_GB": 7.9174, | |
| "floor_ms@128": 15.55, | |
| "chat": { | |
| "ppl_delta_pct": 6.21, | |
| "dnll": 0.0603, | |
| "se": 0.0054, | |
| "top1": 0.8667, | |
| "kl": 0.186, | |
| "kl_p99": 2.85 | |
| }, | |
| "code": { | |
| "ppl_delta_pct": 4.99, | |
| "dnll": 0.0487, | |
| "se": 0.0069, | |
| "top1": 0.9057, | |
| "kl": 0.1882, | |
| "kl_p99": 3.352 | |
| }, | |
| "long-book": { | |
| "ppl_delta_pct": 10.8, | |
| "dnll": 0.1026, | |
| "se": 0.0072, | |
| "top1": 0.874, | |
| "kl": 0.1582, | |
| "kl_p99": 1.699 | |
| }, | |
| "short-all": { | |
| "ppl_delta_pct": 5.67, | |
| "dnll": 0.0552, | |
| "se": 0.0043, | |
| "top1": 0.8838, | |
| "kl": 0.1869, | |
| "kl_p99": 3.089 | |
| } | |
| }, | |
| { | |
| "config": "e-all4", | |
| "plan": "plans/e-all4.json", | |
| "weights": "IT", | |
| "read_GB": 7.2096, | |
| "floor_ms@128": 14.17, | |
| "chat": { | |
| "ppl_delta_pct": 7.24, | |
| "dnll": 0.0699, | |
| "se": 0.0068, | |
| "top1": 0.8266, | |
| "kl": 0.2946, | |
| "kl_p99": 4.066 | |
| }, | |
| "code": { | |
| "ppl_delta_pct": 8.32, | |
| "dnll": 0.0799, | |
| "se": 0.0085, | |
| "top1": 0.8736, | |
| "kl": 0.2975, | |
| "kl_p99": 4.776 | |
| }, | |
| "long-book": { | |
| "ppl_delta_pct": 19.16, | |
| "dnll": 0.1753, | |
| "se": 0.0107, | |
| "top1": 0.8009, | |
| "kl": 0.3806, | |
| "kl_p99": 3.648 | |
| }, | |
| "short-all": { | |
| "ppl_delta_pct": 7.71, | |
| "dnll": 0.0743, | |
| "se": 0.0053, | |
| "top1": 0.8472, | |
| "kl": 0.2959, | |
| "kl_p99": 4.394 | |
| } | |
| }, | |
| { | |
| "config": "e-all4-lm4", | |
| "plan": "plans/e-all4-lm4.json", | |
| "weights": "IT", | |
| "read_GB": 6.7063, | |
| "floor_ms@128": 13.18, | |
| "chat": { | |
| "ppl_delta_pct": 8.48, | |
| "dnll": 0.0814, | |
| "se": 0.0069, | |
| "top1": 0.824, | |
| "kl": 0.3089, | |
| "kl_p99": 4.273 | |
| }, | |
| "code": { | |
| "ppl_delta_pct": 9.76, | |
| "dnll": 0.0931, | |
| "se": 0.0087, | |
| "top1": 0.8709, | |
| "kl": 0.3114, | |
| "kl_p99": 5.03 | |
| }, | |
| "long-book": { | |
| "ppl_delta_pct": 19.92, | |
| "dnll": 0.1816, | |
| "se": 0.0108, | |
| "top1": 0.7996, | |
| "kl": 0.3899, | |
| "kl_p99": 3.703 | |
| }, | |
| "short-all": { | |
| "ppl_delta_pct": 9.04, | |
| "dnll": 0.0865, | |
| "se": 0.0055, | |
| "top1": 0.8446, | |
| "kl": 0.31, | |
| "kl_p99": 4.618 | |
| } | |
| }, | |
| { | |
| "config": "d-qat-a", | |
| "plan": "plans/a-bfp8.json", | |
| "weights": "QAT", | |
| "read_GB": 12.6675, | |
| "floor_ms@128": 24.83, | |
| "chat": { | |
| "ppl_delta_pct": -2.86, | |
| "dnll": -0.029, | |
| "se": 0.0053, | |
| "top1": 0.9074, | |
| "kl": 0.1289, | |
| "kl_p99": 2.267 | |
| }, | |
| "code": { | |
| "ppl_delta_pct": -8.9, | |
| "dnll": -0.0932, | |
| "se": 0.0059, | |
| "top1": 0.928, | |
| "kl": 0.1018, | |
| "kl_p99": 1.497 | |
| }, | |
| "long-book": { | |
| "ppl_delta_pct": -4.87, | |
| "dnll": -0.0499, | |
| "se": 0.0054, | |
| "top1": 0.9055, | |
| "kl": 0.0771, | |
| "kl_p99": 0.782 | |
| }, | |
| "short-all": { | |
| "ppl_delta_pct": -5.55, | |
| "dnll": -0.0571, | |
| "se": 0.0039, | |
| "top1": 0.9164, | |
| "kl": 0.117, | |
| "kl_p99": 2.004 | |
| } | |
| }, | |
| { | |
| "config": "d-qat-b", | |
| "plan": "plans/b-mlp4.json", | |
| "weights": "QAT", | |
| "read_GB": 8.4207, | |
| "floor_ms@128": 16.53, | |
| "chat": { | |
| "ppl_delta_pct": 0.98, | |
| "dnll": 0.0098, | |
| "se": 0.0062, | |
| "top1": 0.8658, | |
| "kl": 0.2104, | |
| "kl_p99": 3.608 | |
| }, | |
| "code": { | |
| "ppl_delta_pct": -0.85, | |
| "dnll": -0.0086, | |
| "se": 0.0075, | |
| "top1": 0.8987, | |
| "kl": 0.1855, | |
| "kl_p99": 3.064 | |
| }, | |
| "long-book": { | |
| "ppl_delta_pct": 6.68, | |
| "dnll": 0.0647, | |
| "se": 0.0074, | |
| "top1": 0.8719, | |
| "kl": 0.1512, | |
| "kl_p99": 1.517 | |
| }, | |
| "short-all": { | |
| "ppl_delta_pct": 0.18, | |
| "dnll": 0.0018, | |
| "se": 0.0048, | |
| "top1": 0.8802, | |
| "kl": 0.1995, | |
| "kl_p99": 3.406 | |
| } | |
| } | |
| ], | |
| "gsm8k_50": { | |
| "a-lm4": { | |
| "acc": 0.98, | |
| "agree": 0.96 | |
| }, | |
| "a-bfp8": { | |
| "acc": 1.0, | |
| "agree": 0.98 | |
| }, | |
| "b-mlp4": { | |
| "acc": 0.98, | |
| "agree": 0.96 | |
| } | |
| }, | |
| "pick": "a-lm4" | |
| } |