| { |
| "frontiers": [ |
| 2048, |
| 4096, |
| 6144, |
| 8192, |
| 10240, |
| 12288, |
| 14336, |
| 16384, |
| 18432, |
| 20480, |
| 22528, |
| 24576, |
| 26624, |
| 28672, |
| 30720, |
| 32768, |
| 34816, |
| 36864, |
| 38912, |
| 40960, |
| 43008, |
| 45056, |
| 47104, |
| 49152, |
| 51200, |
| 53248, |
| 55296, |
| 57344, |
| 59392, |
| 61440, |
| 63488, |
| 65536 |
| ], |
| "note": "ds4-dfm-rs \u00b7 MTP/DFlash off \u00b7 GPU 300\u20132200 MHz (observed busy 2184\u20132190 MHz)", |
| "plotted_metric": "both", |
| "series": [ |
| { |
| "prefix": "baseline", |
| "label": "Baseline \u00b7 7a78fcd8", |
| "sources": [ |
| { |
| "file": "baseline-1.csv", |
| "sha256": "f372c54f30db87637e26f3fc0ae3f55094a9c584bbd2779e78a76c371e7d5e1a" |
| }, |
| { |
| "file": "baseline-2.csv", |
| "sha256": "ef677f2d5b9ed6fc96b9abd9a5250666c10e116b79fa13dde3ac1be92fa37b5e" |
| }, |
| { |
| "file": "baseline-3.csv", |
| "sha256": "4b378fe907edb702d6edd5eedae8489a14802f8e23cb689ef77b2c27c1e008ed" |
| } |
| ], |
| "metrics": { |
| "prefill_tps": { |
| "median": [ |
| 1097.73, |
| 1124.92, |
| 1090.59, |
| 1067.27, |
| 1053.47, |
| 1035.85, |
| 1011.32, |
| 981.52, |
| 965.36, |
| 952.13, |
| 933.37, |
| 907.66, |
| 892.87, |
| 870.6, |
| 855.81, |
| 846.59, |
| 829.28, |
| 815.37, |
| 806.09, |
| 792.69, |
| 778.95, |
| 765.42, |
| 753.16, |
| 736.32, |
| 729.82, |
| 718.32, |
| 704.18, |
| 701.02, |
| 690.35, |
| 675.98, |
| 668.73, |
| 658.99 |
| ], |
| "min": [ |
| 1079.12, |
| 1123.37, |
| 1089.58, |
| 1064.56, |
| 1050.71, |
| 1033.62, |
| 1010.57, |
| 981.24, |
| 965.34, |
| 949.06, |
| 929.79, |
| 907.48, |
| 892.4, |
| 869.23, |
| 854.66, |
| 845.78, |
| 828.87, |
| 811.55, |
| 805.99, |
| 792.19, |
| 777.26, |
| 763.97, |
| 749.65, |
| 736.12, |
| 728.79, |
| 717.49, |
| 703.4, |
| 700.14, |
| 689.12, |
| 674.48, |
| 667.45, |
| 657.9 |
| ], |
| "max": [ |
| 1098.49, |
| 1126.11, |
| 1094.8, |
| 1068.44, |
| 1055.75, |
| 1036.29, |
| 1012.34, |
| 985.73, |
| 968.18, |
| 952.78, |
| 933.52, |
| 910.45, |
| 893.38, |
| 872.3, |
| 858.15, |
| 848.19, |
| 830.49, |
| 815.7, |
| 807.25, |
| 794.9, |
| 780.02, |
| 765.92, |
| 754.67, |
| 737.09, |
| 731.39, |
| 719.42, |
| 706.98, |
| 701.17, |
| 691.61, |
| 676.85, |
| 669.7, |
| 659.64 |
| ], |
| "retention_64k_over_2k": 0.6003206617292048 |
| }, |
| "gen_tps": { |
| "median": [ |
| 25.28, |
| 25.39, |
| 24.95, |
| 24.63, |
| 24.32, |
| 24.14, |
| 23.82, |
| 23.49, |
| 23.18, |
| 22.86, |
| 22.63, |
| 22.28, |
| 22.0, |
| 21.8, |
| 21.47, |
| 21.34, |
| 21.0, |
| 20.71, |
| 20.57, |
| 20.25, |
| 20.02, |
| 19.86, |
| 19.59, |
| 19.29, |
| 19.25, |
| 18.99, |
| 18.75, |
| 18.52, |
| 18.3, |
| 18.06, |
| 18.02, |
| 17.86 |
| ], |
| "min": [ |
| 25.28, |
| 25.37, |
| 24.89, |
| 24.52, |
| 24.3, |
| 24.05, |
| 23.81, |
| 23.47, |
| 23.12, |
| 22.85, |
| 22.61, |
| 22.27, |
| 21.94, |
| 21.79, |
| 21.46, |
| 21.27, |
| 20.98, |
| 20.7, |
| 20.48, |
| 20.24, |
| 20.0, |
| 19.83, |
| 19.56, |
| 19.29, |
| 19.21, |
| 18.98, |
| 18.75, |
| 18.5, |
| 18.24, |
| 18.05, |
| 18.01, |
| 17.82 |
| ], |
| "max": [ |
| 25.37, |
| 25.41, |
| 25.04, |
| 24.78, |
| 24.43, |
| 24.18, |
| 23.83, |
| 23.49, |
| 23.19, |
| 22.89, |
| 22.65, |
| 22.29, |
| 22.03, |
| 21.8, |
| 21.5, |
| 21.35, |
| 21.03, |
| 20.72, |
| 20.57, |
| 20.27, |
| 20.05, |
| 19.86, |
| 19.61, |
| 19.34, |
| 19.27, |
| 19.01, |
| 18.76, |
| 18.52, |
| 18.33, |
| 18.09, |
| 18.05, |
| 17.86 |
| ], |
| "retention_64k_over_2k": 0.7064873417721519 |
| } |
| } |
| }, |
| { |
| "prefix": "final", |
| "label": "Retained Prefill \u00b7 2db2fc38 (#68)", |
| "sources": [ |
| { |
| "file": "final-1.csv", |
| "sha256": "03c782ec76f24eb21c6661c92a42f7b014be0f7a3d0363afbfa953479337aa50" |
| }, |
| { |
| "file": "final-2.csv", |
| "sha256": "7b588b853d607f4f3f3e7caddc60d76fe95f58b57462f686404ce127bfdac824" |
| }, |
| { |
| "file": "final-3.csv", |
| "sha256": "8c69c8922b75ac927aa73657488c35125221dfccdd6646cfce2085a154fad81a" |
| } |
| ], |
| "metrics": { |
| "prefill_tps": { |
| "median": [ |
| 1110.75, |
| 1141.99, |
| 1107.4, |
| 1083.37, |
| 1069.19, |
| 1048.54, |
| 1023.62, |
| 997.48, |
| 977.01, |
| 963.58, |
| 940.53, |
| 919.24, |
| 904.81, |
| 882.32, |
| 864.19, |
| 854.75, |
| 839.2, |
| 822.29, |
| 812.51, |
| 800.71, |
| 785.28, |
| 771.86, |
| 759.52, |
| 742.56, |
| 737.22, |
| 723.81, |
| 712.99, |
| 707.97, |
| 693.08, |
| 681.86, |
| 674.66, |
| 663.48 |
| ], |
| "min": [ |
| 1110.46, |
| 1141.25, |
| 1101.87, |
| 1077.69, |
| 1068.95, |
| 1044.02, |
| 1022.83, |
| 996.64, |
| 971.73, |
| 962.01, |
| 940.25, |
| 918.37, |
| 904.47, |
| 879.79, |
| 864.06, |
| 852.38, |
| 838.28, |
| 819.41, |
| 809.62, |
| 800.66, |
| 784.88, |
| 770.67, |
| 757.28, |
| 741.99, |
| 735.72, |
| 723.18, |
| 712.47, |
| 706.81, |
| 692.52, |
| 680.99, |
| 672.5, |
| 663.33 |
| ], |
| "max": [ |
| 1113.59, |
| 1146.34, |
| 1110.0, |
| 1083.59, |
| 1073.62, |
| 1051.37, |
| 1025.58, |
| 997.85, |
| 977.3, |
| 963.75, |
| 941.93, |
| 922.26, |
| 906.53, |
| 884.27, |
| 866.4, |
| 855.11, |
| 839.29, |
| 822.56, |
| 815.12, |
| 802.29, |
| 787.48, |
| 772.82, |
| 763.14, |
| 745.53, |
| 737.76, |
| 724.78, |
| 713.04, |
| 708.31, |
| 693.27, |
| 682.45, |
| 674.82, |
| 665.55 |
| ], |
| "retention_64k_over_2k": 0.5973261309925726 |
| }, |
| "gen_tps": { |
| "median": [ |
| 25.31, |
| 25.38, |
| 24.98, |
| 24.7, |
| 24.2, |
| 24.11, |
| 23.83, |
| 23.53, |
| 23.21, |
| 22.87, |
| 22.6, |
| 22.31, |
| 22.0, |
| 21.76, |
| 21.52, |
| 21.32, |
| 21.02, |
| 20.72, |
| 20.53, |
| 20.28, |
| 20.06, |
| 19.88, |
| 19.58, |
| 19.28, |
| 19.25, |
| 18.97, |
| 18.75, |
| 18.51, |
| 18.31, |
| 18.06, |
| 18.02, |
| 17.8 |
| ], |
| "min": [ |
| 25.24, |
| 25.35, |
| 24.93, |
| 24.47, |
| 24.14, |
| 24.11, |
| 23.79, |
| 23.48, |
| 23.17, |
| 22.82, |
| 22.57, |
| 22.26, |
| 22.0, |
| 21.73, |
| 21.5, |
| 21.32, |
| 21.01, |
| 20.71, |
| 20.51, |
| 20.27, |
| 19.99, |
| 19.82, |
| 19.58, |
| 19.26, |
| 19.23, |
| 18.96, |
| 18.73, |
| 18.5, |
| 18.28, |
| 18.05, |
| 18.01, |
| 17.76 |
| ], |
| "max": [ |
| 25.44, |
| 25.42, |
| 25.06, |
| 24.8, |
| 24.37, |
| 24.13, |
| 23.84, |
| 23.53, |
| 23.22, |
| 22.92, |
| 22.61, |
| 22.34, |
| 22.02, |
| 21.8, |
| 21.55, |
| 21.34, |
| 21.03, |
| 20.74, |
| 20.55, |
| 20.29, |
| 20.1, |
| 19.89, |
| 19.62, |
| 19.29, |
| 19.25, |
| 19.01, |
| 18.81, |
| 18.56, |
| 18.32, |
| 18.07, |
| 18.05, |
| 17.85 |
| ], |
| "retention_64k_over_2k": 0.7032793362307389 |
| } |
| } |
| } |
| ], |
| "final_vs_first": { |
| "prefill_tps": { |
| "final_vs_first_gain_pct": [ |
| 1.1860840097291714, |
| 1.5174412402659598, |
| 1.5413675166653151, |
| 1.508521742389446, |
| 1.4922114535772302, |
| 1.2250808514746359, |
| 1.21623225091958, |
| 1.6260493927785502, |
| 1.2068036794563586, |
| 1.2025668763719244, |
| 0.7671127205716832, |
| 1.2758081219840012, |
| 1.33726074344529, |
| 1.3461980243510308, |
| 0.9791893060375756, |
| 0.9638668068368439, |
| 1.1962184063283976, |
| 0.8486944577308364, |
| 0.796437122405691, |
| 1.011744818276994, |
| 0.812632389755441, |
| 0.8413681377544391, |
| 0.8444420840193256, |
| 0.8474576271186196, |
| 1.013948644871343, |
| 0.7642833277647698, |
| 1.251100570876762, |
| 0.9914125131950557, |
| 0.39545158253060375, |
| 0.8698482203615576, |
| 0.8867554917530107, |
| 0.681345695685831 |
| ], |
| "retention_change_percentage_points": -0.29945307366322504 |
| }, |
| "gen_tps": { |
| "final_vs_first_gain_pct": [ |
| 0.11867088607593335, |
| -0.03938558487593946, |
| 0.12024048096193063, |
| 0.2842062525375555, |
| -0.49342105263158187, |
| -0.12427506213753547, |
| 0.04198152812762146, |
| 0.1702852277565059, |
| 0.129421915444361, |
| 0.04374453193352057, |
| -0.1325673884224332, |
| 0.13464991023337536, |
| 0.0, |
| -0.18348623853210455, |
| 0.23288309268747476, |
| -0.09372071227741197, |
| 0.09523809523810378, |
| 0.048285852245277816, |
| -0.1944579484686404, |
| 0.1481481481481639, |
| 0.19980019980019303, |
| 0.10070493454179541, |
| -0.05104645227157478, |
| -0.051840331778107984, |
| 0.0, |
| -0.10531858873090716, |
| 0.0, |
| -0.053995680345564345, |
| 0.054644808743153916, |
| 0.0, |
| 0.0, |
| -0.33594624860021627 |
| ], |
| "retention_change_percentage_points": -0.3208005541412984 |
| } |
| }, |
| "protocol": "protocol.json", |
| "validation": { |
| "completed_fresh_processes": 6, |
| "frontiers_per_process": 32, |
| "total_frontiers": 192, |
| "finite_logits_per_frontier": 152576, |
| "greedy_tokens_per_frontier": 128, |
| "all_frontier_logits_and_token_streams_byte_exact": true, |
| "frontier_reference_sha256": { |
| "2048": { |
| "logits_sha256": "1af18fb3d66fd0dfe1fbab1f583066ceb544f4a8af6782d1389541dafe8cff78", |
| "tokens_sha256": "65612f5748827f3f85e7d8849eb3e3b073eae82ca100640001c29df10e87c833" |
| }, |
| "4096": { |
| "logits_sha256": "9f0be7f878c5379c1273bb30aff76600119f49abfed33c37b2fe2f53c5eebdc7", |
| "tokens_sha256": "859ce8f5c9d2e406bc1097df91fa21790471232142a80efd703074ae1ce5ac03" |
| }, |
| "6144": { |
| "logits_sha256": "1537e4bcc24754cc4db4ebe73e9eed2e2d59862c34def21f01d34e11a690ba43", |
| "tokens_sha256": "8bdcfc95af3d0d85fc0b5b27bd3ab3b4cdd2b9ed594c28bb0b1513daa95be975" |
| }, |
| "8192": { |
| "logits_sha256": "e43a995b4691dacca4c88ecd07f049f930ea82ae49b78ab27550baf47a0a75b9", |
| "tokens_sha256": "183d91523ce2589f38f969e2df810eef7435f1f1c34cc750602513e396b6bfdf" |
| }, |
| "10240": { |
| "logits_sha256": "b8138ba9b226ffc42d56a9f11c70d192582310e066037fe5c41ed1e00d261949", |
| "tokens_sha256": "f4cc23ae4bb373abda2b06f84f0d230c4d0ea53c4b14141eb2233a7526a786d2" |
| }, |
| "12288": { |
| "logits_sha256": "28ee223f07038f39f5b3a2def7400dca875ee97feaa9f9fc227e30b01e24d0e2", |
| "tokens_sha256": "94f7b4a8ec3cc2f85cc346e07b8972efa82eeaf1bd382d6cf9f5198d7472d72e" |
| }, |
| "14336": { |
| "logits_sha256": "a6bb6adda7587b93ce0b028132add9eede84d9e97b8a82da28f434e0af389629", |
| "tokens_sha256": "4409345fc60b53c4443ace4baca42cc2d55beac001fa61911c95e83b52a4ff93" |
| }, |
| "16384": { |
| "logits_sha256": "8a413a2d4a7441a3d9af0c0e7d17c564ecc526c06bb3be326ccfd471110bddf9", |
| "tokens_sha256": "e12baf8a3452453629c9600e03e33dd85dbd249c3ac36a0d322fd8ae670f08b8" |
| }, |
| "18432": { |
| "logits_sha256": "ce5eade20d8b9e71ef7326b8b4f1947b970e55ecb9938cd1061bb5dbc79248e7", |
| "tokens_sha256": "7f13d9661f28c1d62a2296640245893a154b21be896b1a9e35651eb1c29a2e3c" |
| }, |
| "20480": { |
| "logits_sha256": "646d543ce9dba6dfc68fa83d50fb76185b54177e63e3937ea52c7f760a49c05d", |
| "tokens_sha256": "00866a0ccffd8897de1e0494b024b62239f313542a8bfbd0f7e0660ff915c663" |
| }, |
| "22528": { |
| "logits_sha256": "c9ac614cbcb2c5d97b62c21e96105a34656db1e4732f5a81aca1b4e0093183cd", |
| "tokens_sha256": "a8b73913bc0a8544a2c98f966f055a109b2e95367ce0431930ac6a36d2bd3bcc" |
| }, |
| "24576": { |
| "logits_sha256": "1d62c99bda2aeaee72bcb48955271711371ffa09659ef6b46b5d3b966240a9fd", |
| "tokens_sha256": "dc05d1d0b0a241511d25a36de429592498c82f352aaa944258a94bf97416ff76" |
| }, |
| "26624": { |
| "logits_sha256": "5042a484e30f8a671a6bb2883328f461b5e0bea55684a6a4ee2c15e4d897215d", |
| "tokens_sha256": "82ffbd2335f735bfe50850e821853a61c2ff9491de90fc0f6f76f4f4bb47f6de" |
| }, |
| "28672": { |
| "logits_sha256": "90a95603d4fe00722b2b68b627e347d70517dc7c89ca581e84f636de3b47ff7a", |
| "tokens_sha256": "b7861287989a6e9ae1df091515eef2710d3c16ac41b09da448952e12dd10a810" |
| }, |
| "30720": { |
| "logits_sha256": "1eb9444e4894d085b404d5549e9738eefa1d09467fa1c8126efaeb1f1e908e88", |
| "tokens_sha256": "0f4984acfefbd6f1414a29cb2be9555f0d7e64ae487e2ea857d2f9fdb272a67f" |
| }, |
| "32768": { |
| "logits_sha256": "4ad2c099a6b5e1fb303b84375a2ce551084e4356419dd254856493a1424da2a2", |
| "tokens_sha256": "911d500295889a1bf5dda0b62f5765b4a9fdde28c55cb8dda137be1e10361cbc" |
| }, |
| "34816": { |
| "logits_sha256": "4bb65d221a6f6454837f763d8b1703af5e6c60b7e37b635a3d5fe3a9f9265e8a", |
| "tokens_sha256": "3fb5163d382e58965126876ff4b6e8c275ce4cb3406ae32f18bda96a51d9f24a" |
| }, |
| "36864": { |
| "logits_sha256": "22459f508e673d6f7ff5203e0fa2c52700873725e8a5dc225d868d4717753ce1", |
| "tokens_sha256": "ef617c0a1edfd2757b7f19355c0af947291e1abb1682078254706419f4682be5" |
| }, |
| "38912": { |
| "logits_sha256": "9d2aed1ea304489b58c30f87a8b8db15d2a038a10982bb5e5a30c879f404fc37", |
| "tokens_sha256": "c004ae086b61fa74e1c0456e9897960e17e830bb0b0fdea16d7372e97e450a2b" |
| }, |
| "40960": { |
| "logits_sha256": "4177bbc969fed82a4bef4377cd9ad4bd4b34c9e2ed38217ac82a54a73f6ce38e", |
| "tokens_sha256": "7b9db185b8ce38ce7c8b72414ccf196780cbd6e44ed4577df79e860ef2e4e2e2" |
| }, |
| "43008": { |
| "logits_sha256": "5f0232ef9f488013332f872b9e055f1256c0f2754a734072c82900094a269bed", |
| "tokens_sha256": "7c0cc0c717f15104b192df5c3de553f8f0b909e678c0878dd83996bf931f0adb" |
| }, |
| "45056": { |
| "logits_sha256": "e4ceafc7575446290179684ec62091eba5db3d8089696e717ecc34773009fc63", |
| "tokens_sha256": "9678bd457bbc28fe67519f1c374a65e8f851c4f0516b03acf2112e70edd4f384" |
| }, |
| "47104": { |
| "logits_sha256": "a3c46f54fc3e228de5b9b6aa4350db547071e94ba88ec453b41a91519fbe7ff2", |
| "tokens_sha256": "1e750dda1ec69038f5696a174881ba8134a98c6011bdc9f191f5836b4b10a8c6" |
| }, |
| "49152": { |
| "logits_sha256": "d0d9b4293dacfbe0a0a482b0f0da70f1d85faea58b1270d22cc643e6b2f1bac9", |
| "tokens_sha256": "fb5a3a5545544f645248f00cc507fca17a0da818c5778f056954d08a2338e65d" |
| }, |
| "51200": { |
| "logits_sha256": "c3c0db726767aaa3a57c828246b62835404790069ef87df64713252b067e4703", |
| "tokens_sha256": "d239db777c9c28dcb9612ee8c7739d5e9a1a24fa3ba63b0ab05f8e019bdc7205" |
| }, |
| "53248": { |
| "logits_sha256": "49e8cceda1d176698ab1bf89020dfca858c0bc3e2bff287a603245d0e3422a52", |
| "tokens_sha256": "737fdbc24bd1704933789ebfcd8ac28f03fc353076df1ab3d8705962cc815a29" |
| }, |
| "55296": { |
| "logits_sha256": "ccae6d33debe68b15ef9f8e0e888f16a85a77a92fe39ff3a5fa02a2c740842a0", |
| "tokens_sha256": "682c42b3eb788118183cedd462c4977e8ff401ebf4ad95f1860bcf4ee6f5dd91" |
| }, |
| "57344": { |
| "logits_sha256": "89ce5fa02b199c2608777c8f51542421a4a9bbddee589929aa8a20faf204f0bd", |
| "tokens_sha256": "948e706e0a746238a48634a5b27250e6f43e6e0038c43b577f1d8ab09076607a" |
| }, |
| "59392": { |
| "logits_sha256": "901f593d48571c1fc81e530fac87c738514df8aef8197fb33b78d2aee39414f5", |
| "tokens_sha256": "0aa117055ccc8d07ca5d5fb0ffb57cbaee602266b43a41ebd827d918a44507ee" |
| }, |
| "61440": { |
| "logits_sha256": "c80b984c64c4541ad09261367ba17d0af8fa96a2d9ec56ecbab18ef2d1b8ee43", |
| "tokens_sha256": "0a44b0935b2077ac154d4f941db305623d06a0a6e07c2b7a775c47e821d1d93b" |
| }, |
| "63488": { |
| "logits_sha256": "2af06d515744a994909f93a0327faca096876daaf6c384e159db9794ca96f2b2", |
| "tokens_sha256": "d615c1e8680c96afb9d165adeabb046d82017108917d6cf772a899803b7e70a3" |
| }, |
| "65536": { |
| "logits_sha256": "f06950f6f0b4479044142a8880eea0902f4f9dbe231dd5023bd6426c46532c0f", |
| "tokens_sha256": "9f2e8d2a450f58091ae5702d0809e3698bcf5f3bff40f41e47e002d9077dc94d" |
| } |
| } |
| } |
| } |
|
|
