| ctx | p3d-pc-gate-s1 spec greedy tok/s (tok/cycle) | p3d-pc-gate-s1 spec recommended tok/s (tok/cycle) | p3d-pc-gate-s1 spec coding tok/s (tok/cycle) | p3d-pc-gate-s1 spec penalized tok/s (tok/cycle) | p3d-pc-gate-s1 nospec greedy | p3d-pc-gate-s1 nospec recommended | p3d-pc-gate-s1 nospec coding | p3d-pc-gate-s1 nospec penalized | old greedy | old recommended | old coding | old penalized |
|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 128 | 55.97 (2.81) | 56.42 (2.95) | 54.59 (2.89) | 59.15 (3.13) | 22.77 | 26.24 | 26.31 | 26.33 | 55.96 | 17.87 | 17.59 | 17.48 |
| 2048 | 56.74 (3.1) | 52.3 (2.91) | 53.89 (3.07) | 51.95 (2.95) | 22.17 | 25.51 | 25.59 | 25.58 | 57.11 | 17.24 | 17.26 | 16.92 |
| 32768 | 49.76 (2.75) | 50.01 (2.81) | 47.75 (2.82) | 46.83 (2.81) | 21.67 | 24.63 | 24.69 | 24.72 | 49.4 | 17.04 | 16.91 | 16.51 |
| 261632 | 28.29 (2.33) | 29.58 (2.6) | 27.49 (2.56) | 25.91 (2.66) | 17.57 | 19.6 | 19.67 | 19.67 | 28.32 | 14.32 | 14.24 | 13.86 |
| { | ||||||||||||
| "p3d-pc-gate-s1": { | ||||||||||||
| "spec_bench": { | ||||||||||||
| "128/greedy": { |
"tok_s": 55.97,
"ttft_s": 0.09,
"acceptance": 0.362,
"tokens_per_cycle": 2.81,
"requests": 3,
"distinct_texts": 1
}, "128/recommended": { "tok_s": 56.42, "ttft_s": 0.11, "acceptance": 0.391, "tokens_per_cycle": 2.95, "requests": 3, "distinct_texts": 3 }, "128/coding": { "tok_s": 54.59, "ttft_s": 0.1, "acceptance": 0.378, "tokens_per_cycle": 2.89, "requests": 3, "distinct_texts": 3 }, "128/penalized": { "tok_s": 59.15, "ttft_s": 0.11, "acceptance": 0.426, "tokens_per_cycle": 3.13, "requests": 3, "distinct_texts": 3 }, "2048/greedy": { "tok_s": 56.74, "ttft_s": 0.65, "acceptance": 0.419, "tokens_per_cycle": 3.1, "requests": 3, "distinct_texts": 1 }, "2048/recommended": { "tok_s": 52.3, "ttft_s": 0.66, "acceptance": 0.382, "tokens_per_cycle": 2.91, "requests": 3, "distinct_texts": 3 }, "2048/coding": { "tok_s": 53.89, "ttft_s": 0.66, "acceptance": 0.414, "tokens_per_cycle": 3.07, "requests": 3, "distinct_texts": 3 }, "2048/penalized": { "tok_s": 51.95, "ttft_s": 0.66, "acceptance": 0.391, "tokens_per_cycle": 2.95, "requests": 3, "distinct_texts": 3 }, "32768/greedy": { "tok_s": 49.76, "ttft_s": 9.94, "acceptance": 0.349, "tokens_per_cycle": 2.75, "requests": 2, "distinct_texts": 1 }, "32768/recommended": { "tok_s": 50.01, "ttft_s": 9.95, "acceptance": 0.362, "tokens_per_cycle": 2.81, "requests": 2, "distinct_texts": 2 }, "32768/coding": { "tok_s": 47.75, "ttft_s": 9.95, "acceptance": 0.365, "tokens_per_cycle": 2.82, "requests": 2, "distinct_texts": 2 }, "32768/penalized": { "tok_s": 46.83, "ttft_s": 9.96, "acceptance": 0.362, "tokens_per_cycle": 2.81, "requests": 2, "distinct_texts": 2 }, "261632/greedy": { "tok_s": 28.29, "ttft_s": 42.11, "acceptance": 0.268, "tokens_per_cycle": 2.33, "requests": 1, "distinct_texts": 1 }, "261632/recommended": { "tok_s": 29.58, "ttft_s": 42.11, "acceptance": 0.322, "tokens_per_cycle": 2.6, "requests": 1, "distinct_texts": 1 }, "261632/coding": { "tok_s": 27.49, "ttft_s": 42.11, "acceptance": 0.316, "tokens_per_cycle": 2.56, "requests": 1, "distinct_texts": 1 }, "261632/penalized": { "tok_s": 25.91, "ttft_s": 42.11, "acceptance": 0.334, "tokens_per_cycle": 2.66, "requests": 1, "distinct_texts": 1 }, "cold_ttft_s": { "128": 0.11, "2048": 0.64, "32768": 17.08, "261632": 392.41 } }, "spec_default": { "128/default": { "tok_s": 54.72, "ttft_s": 0.1, "acceptance": 0.392, "tokens_per_cycle": 2.96, "requests": 2, "distinct_texts": 2 }, "cold_ttft_s": { "128": 0.09 } }, "nospec_bench": { "128/greedy": { "tok_s": 22.77, "ttft_s": 0.09, "acceptance": null, "tokens_per_cycle": null, "requests": 2, "distinct_texts": 1 }, "128/recommended": { "tok_s": 26.24, "ttft_s": 0.11, "acceptance": null, "tokens_per_cycle": null, "requests": 2, "distinct_texts": 2 }, "128/coding": { "tok_s": 26.31, "ttft_s": 0.1, "acceptance": null, "tokens_per_cycle": null, "requests": 2, "distinct_texts": 2 }, "128/penalized": { "tok_s": 26.33, "ttft_s": 0.11, "acceptance": null, "tokens_per_cycle": null, "requests": 2, "distinct_texts": 2 }, "2048/greedy": { "tok_s": 22.17, "ttft_s": 0.65, "acceptance": null, "tokens_per_cycle": null, "requests": 2, "distinct_texts": 1 }, "2048/recommended": { "tok_s": 25.51, "ttft_s": 0.66, "acceptance": null, "tokens_per_cycle": null,
Skipped by parent request (not run, not passed)
- non-spec coding gsm8k x3 (p3d-pc-gate-s1-ns-gsm2): stopped mid-run at 38/100 rows; partial file renamed nospec-gsm/gsm-coding-s1.jsonl.partial-skipped, excluded
- any further old-host-path gsm8k or repeat runs
Seeded reproducibility fix (src-g2, commit d6f9202; patch gemma-sampling-s2.patch)
- tests/test_seed_repro_g.py: g1 FAIL (runs/seedtest-g1.log), g2 PASS (runs/seedtest-g2.log); HTTP smoke dev2 + spec-a SEED_REPRO True