| { |
| "model": "orca-gufo", |
| "base_url": "http://127.0.0.1:8082", |
| "stream": true, |
| "cases": [ |
| "mixed-short", |
| "mixed-medium", |
| "mixed-long", |
| "repetitive-short" |
| ], |
| "repeats": 3, |
| "rows": [ |
| { |
| "case": "mixed-short", |
| "style": "mixed", |
| "words": 256, |
| "nonce": 2, |
| "wall_s": 3.811, |
| "ttft_s": 0.593, |
| "finish_reason": "length", |
| "response_text": "Based on the provided text, which emphasizes that local inference performance is a complex function of prompt length, memory traffic, attention state, expert routing, speculative acceptance, and scheduling, here are 20 distinct observations regarding the design of a reliable local inference benchmark.\n\n1. **Prompt Length Stratification**: Benchmarks must not rely on a single prompt length. Instead, they should employ a stratified sampling approach covering short (e.g., <1k tokens), medium (1k\u20138k tokens), and long-context (32k+ tokens) scenarios. This is critical because attention mechanisms scale quadratically (or linearly", |
| "usage": { |
| "prompt_tokens": 371, |
| "completion_tokens": 128, |
| "total_tokens": 499, |
| "prompt_tokens_details": { |
| "cached_tokens": 0 |
| }, |
| "cached_tokens": 0, |
| "prompt_tokens_per_second": 654.4588707113202, |
| "completion_tokens_per_second": 40.08264799014515, |
| "draft_tokens": 121, |
| "draft_tokens_accepted": 80, |
| "gufo": { |
| "cache_hit": false, |
| "cache_miss_reason": "prefix_changed", |
| "cache_common_prefix_tokens": 10, |
| "cache_checkpoint_tokens": 364, |
| "cache_restore_bytes": 0, |
| "cache_snapshot_bytes": 263178012, |
| "cache_disk_queued_bytes": 0, |
| "cache_shared_bytes": 0, |
| "cache_restore_ms": 0, |
| "cache_snapshot_ms": 40.999477, |
| "cache_disk_enqueue_ms": 0, |
| "cache_disk_hit": false, |
| "cache_shared_prefix_snapshots": 0, |
| "cache_shared_prefix_bytes": 0, |
| "cache_shared_prefix_ms": 0, |
| "prefill_tokens": 371, |
| "prefill_chunks": 2, |
| "active_decode_prefill_chunks": 0, |
| "max_prefill_chunk_tokens": 364, |
| "queue_depth_at_submit": 1, |
| "client_queue_depth_at_submit": 1, |
| "resident_requests_at_admission": 1, |
| "requested_logical_concurrency": 1, |
| "physical_execution_width": 1, |
| "queue_ms": 6.33606, |
| "prefill_ms": 566.880543, |
| "decode_ms": 3193.401794, |
| "ttft_ms": 592.136816, |
| "mean_inter_token_ms": 25.318874622047254, |
| "max_inter_token_ms": 120.004104, |
| "execution_plan": "serial-c1" |
| } |
| }, |
| "timings": { |
| "prompt_n": 371, |
| "prompt_ms": 566.880543, |
| "prompt_per_token_ms": 1.5279799002695418, |
| "prompt_per_second": 654.4588707113202, |
| "predicted_n": 128, |
| "predicted_ms": 3193.401794, |
| "predicted_per_token_ms": 24.948451515625, |
| "predicted_per_second": 40.08264799014515, |
| "cache_n": 0, |
| "cache_restore_ms": 0, |
| "cache_snapshot_ms": 40.999477, |
| "cache_disk_enqueue_ms": 0, |
| "draft_n": 121, |
| "draft_n_accepted": 80 |
| }, |
| "server_timing": null |
| }, |
| { |
| "case": "mixed-short", |
| "style": "mixed", |
| "words": 256, |
| "nonce": 3, |
| "wall_s": 7.907, |
| "ttft_s": 1.32, |
| "finish_reason": "length", |
| "response_text": "1. **Isolate Hardware Variables**: To ensure reproducibility, the benchmark must strictly control for hardware thermal throttling and power state fluctuations. Local inference performance is heavily dependent on sustained clock speeds; therefore, the testing environment should include active cooling monitoring and power limit enforcement to prevent thermal variance from skewing latency measurements across different prompt lengths.\n\n2. **Standardize Prompt Tokenization**: Since performance depends on prompt length, the benchmark must use a fixed tokenizer version and vocabulary. Different tokenizers can produce varying token counts for the same text, leading to inconsistent memory traffic and attention state calculations. All inputs must be pre-tokenized and stored in", |
| "usage": { |
| "prompt_tokens": 371, |
| "completion_tokens": 128, |
| "total_tokens": 499, |
| "prompt_tokens_details": { |
| "cached_tokens": 0 |
| }, |
| "cached_tokens": 0, |
| "prompt_tokens_per_second": 290.10629052678826, |
| "completion_tokens_per_second": 19.49985618665634, |
| "draft_tokens": 67, |
| "draft_tokens_accepted": 31, |
| "gufo": { |
| "cache_hit": false, |
| "cache_miss_reason": "prefix_changed", |
| "cache_common_prefix_tokens": 10, |
| "cache_checkpoint_tokens": 364, |
| "cache_restore_bytes": 0, |
| "cache_snapshot_bytes": 263178012, |
| "cache_disk_queued_bytes": 0, |
| "cache_shared_bytes": 0, |
| "cache_restore_ms": 0, |
| "cache_snapshot_ms": 52.840072000000006, |
| "cache_disk_enqueue_ms": 0, |
| "cache_disk_hit": false, |
| "cache_shared_prefix_snapshots": 0, |
| "cache_shared_prefix_bytes": 0, |
| "cache_shared_prefix_ms": 0, |
| "prefill_tokens": 371, |
| "prefill_chunks": 2, |
| "active_decode_prefill_chunks": 0, |
| "max_prefill_chunk_tokens": 364, |
| "queue_depth_at_submit": 1, |
| "client_queue_depth_at_submit": 1, |
| "resident_requests_at_admission": 1, |
| "requested_logical_concurrency": 1, |
| "physical_execution_width": 1, |
| "queue_ms": 7.560011, |
| "prefill_ms": 1278.841625, |
| "decode_ms": 6564.150975, |
| "ttft_ms": 1319.193592, |
| "mean_inter_token_ms": 51.84411945669291, |
| "max_inter_token_ms": 936.646994, |
| "execution_plan": "serial-c1" |
| } |
| }, |
| "timings": { |
| "prompt_n": 371, |
| "prompt_ms": 1278.841625, |
| "prompt_per_token_ms": 3.4470124663072776, |
| "prompt_per_second": 290.10629052678826, |
| "predicted_n": 128, |
| "predicted_ms": 6564.150975, |
| "predicted_per_token_ms": 51.2824294921875, |
| "predicted_per_second": 19.49985618665634, |
| "cache_n": 0, |
| "cache_restore_ms": 0, |
| "cache_snapshot_ms": 52.840072000000006, |
| "cache_disk_enqueue_ms": 0, |
| "draft_n": 67, |
| "draft_n_accepted": 31 |
| }, |
| "server_timing": null |
| }, |
| { |
| "case": "mixed-short", |
| "style": "mixed", |
| "words": 256, |
| "nonce": 4, |
| "wall_s": 5.654, |
| "ttft_s": 0.599, |
| "finish_reason": "length", |
| "response_text": "1. **Isolate Hardware Variables**: To ensure reproducibility, the benchmark must strictly control for hardware thermal throttling and power state fluctuations. Local inference performance is heavily dependent on sustained clock speeds; therefore, monitoring GPU/CPU temperatures and enforcing a cooldown period between trials is critical to prevent data skew caused by thermal degradation.\n\n2. **Standardize Prompt Lengths**: Since the prompt explicitly notes that performance depends on prompt length, the benchmark must define specific token counts (e.g., 128, 1k, 4k, 32k) for input. This allows for the analysis of how prefill time", |
| "usage": { |
| "prompt_tokens": 371, |
| "completion_tokens": 128, |
| "total_tokens": 499, |
| "prompt_tokens_details": { |
| "cached_tokens": 0 |
| }, |
| "cached_tokens": 0, |
| "prompt_tokens_per_second": 651.7683363900976, |
| "completion_tokens_per_second": 25.429608733758133, |
| "draft_tokens": 83, |
| "draft_tokens_accepted": 31, |
| "gufo": { |
| "cache_hit": false, |
| "cache_miss_reason": "prefix_changed", |
| "cache_common_prefix_tokens": 10, |
| "cache_checkpoint_tokens": 364, |
| "cache_restore_bytes": 0, |
| "cache_snapshot_bytes": 263178012, |
| "cache_disk_queued_bytes": 0, |
| "cache_shared_bytes": 0, |
| "cache_restore_ms": 0, |
| "cache_snapshot_ms": 39.952989, |
| "cache_disk_enqueue_ms": 0, |
| "cache_disk_hit": false, |
| "cache_shared_prefix_snapshots": 0, |
| "cache_shared_prefix_bytes": 0, |
| "cache_shared_prefix_ms": 0, |
| "prefill_tokens": 371, |
| "prefill_chunks": 2, |
| "active_decode_prefill_chunks": 0, |
| "max_prefill_chunk_tokens": 364, |
| "queue_depth_at_submit": 1, |
| "client_queue_depth_at_submit": 1, |
| "resident_requests_at_admission": 1, |
| "requested_logical_concurrency": 1, |
| "physical_execution_width": 1, |
| "queue_ms": 8.073455, |
| "prefill_ms": 569.220656, |
| "decode_ms": 5033.502533999997, |
| "ttft_ms": 598.387415, |
| "mean_inter_token_ms": 39.78323293700788, |
| "max_inter_token_ms": 132.553107, |
| "execution_plan": "serial-c1" |
| } |
| }, |
| "timings": { |
| "prompt_n": 371, |
| "prompt_ms": 569.220656, |
| "prompt_per_token_ms": 1.5342874824797843, |
| "prompt_per_second": 651.7683363900976, |
| "predicted_n": 128, |
| "predicted_ms": 5033.502533999997, |
| "predicted_per_token_ms": 39.32423854687497, |
| "predicted_per_second": 25.429608733758133, |
| "cache_n": 0, |
| "cache_restore_ms": 0, |
| "cache_snapshot_ms": 39.952989, |
| "cache_disk_enqueue_ms": 0, |
| "draft_n": 83, |
| "draft_n_accepted": 31 |
| }, |
| "server_timing": null |
| }, |
| { |
| "case": "mixed-medium", |
| "style": "mixed", |
| "words": 2048, |
| "nonce": 6, |
| "wall_s": 6.971, |
| "ttft_s": 2.396, |
| "finish_reason": "length", |
| "response_text": "Based on the repetitive stress test provided in \"Trial 000006,\" which highlights the critical factors of prompt length, memory traffic, attention state, expert routing, speculative acceptance, and scheduling, here are 20 distinct, detailed observations for designing a reliable local inference benchmark.\n\n### 1. Isolate Prompt Length Scaling from Context Window Limits\nBenchmarks must explicitly separate the impact of prompt length on prefill latency (processing the input) from decode latency (generating output). As prompt length increases, the KV cache grows linearly, increasing memory bandwidth pressure. A reliable benchmark should test across a gradient of", |
| "usage": { |
| "prompt_tokens": 2678, |
| "completion_tokens": 128, |
| "total_tokens": 2806, |
| "prompt_tokens_details": { |
| "cached_tokens": 0 |
| }, |
| "cached_tokens": 0, |
| "prompt_tokens_per_second": 1139.9697722499218, |
| "completion_tokens_per_second": 28.174137778392815, |
| "draft_tokens": 70, |
| "draft_tokens_accepted": 40, |
| "gufo": { |
| "cache_hit": false, |
| "cache_miss_reason": "prefix_changed", |
| "cache_common_prefix_tokens": 10, |
| "cache_checkpoint_tokens": 2671, |
| "cache_restore_bytes": 0, |
| "cache_snapshot_bytes": 385626932, |
| "cache_disk_queued_bytes": 0, |
| "cache_shared_bytes": 0, |
| "cache_restore_ms": 0, |
| "cache_snapshot_ms": 66.929748, |
| "cache_disk_enqueue_ms": 0, |
| "cache_disk_hit": false, |
| "cache_shared_prefix_snapshots": 0, |
| "cache_shared_prefix_bytes": 0, |
| "cache_shared_prefix_ms": 0, |
| "prefill_tokens": 2678, |
| "prefill_chunks": 3, |
| "active_decode_prefill_chunks": 0, |
| "max_prefill_chunk_tokens": 2048, |
| "queue_depth_at_submit": 1, |
| "client_queue_depth_at_submit": 1, |
| "resident_requests_at_admission": 1, |
| "requested_logical_concurrency": 1, |
| "physical_execution_width": 1, |
| "queue_ms": 5.884644, |
| "prefill_ms": 2349.1850969999996, |
| "decode_ms": 4543.173637000001, |
| "ttft_ms": 2394.370212, |
| "mean_inter_token_ms": 35.99080907874016, |
| "max_inter_token_ms": 149.981815, |
| "execution_plan": "serial-c1" |
| } |
| }, |
| "timings": { |
| "prompt_n": 2678, |
| "prompt_ms": 2349.1850969999996, |
| "prompt_per_token_ms": 0.8772162423450335, |
| "prompt_per_second": 1139.9697722499218, |
| "predicted_n": 128, |
| "predicted_ms": 4543.173637000001, |
| "predicted_per_token_ms": 35.493544039062506, |
| "predicted_per_second": 28.174137778392815, |
| "cache_n": 0, |
| "cache_restore_ms": 0, |
| "cache_snapshot_ms": 66.929748, |
| "cache_disk_enqueue_ms": 0, |
| "draft_n": 70, |
| "draft_n_accepted": 40 |
| }, |
| "server_timing": null |
| }, |
| { |
| "case": "mixed-medium", |
| "style": "mixed", |
| "words": 2048, |
| "nonce": 7, |
| "wall_s": 6.521, |
| "ttft_s": 2.383, |
| "finish_reason": "length", |
| "response_text": "Based on the repetitive prompt provided, which highlights critical variables in local inference performance (prompt length, memory traffic, attention state, expert routing, speculative acceptance, and scheduling), here are 20 distinct, detailed observations for designing a reliable local inference benchmark.\n\n### 1. Isolate Memory Bandwidth vs. Compute Bound Scenarios\nLocal inference is rarely purely compute-bound; it is often memory-bandwidth-bound, especially during the prefill phase of long prompts. A reliable benchmark must separate these phases. Measure **Time to First Token (TTFT)** to assess prefill performance (compute-heavy) and **Time Per Output Token (", |
| "usage": { |
| "prompt_tokens": 2678, |
| "completion_tokens": 128, |
| "total_tokens": 2806, |
| "prompt_tokens_details": { |
| "cached_tokens": 0 |
| }, |
| "cached_tokens": 0, |
| "prompt_tokens_per_second": 1149.6524412424196, |
| "completion_tokens_per_second": 31.143162779423605, |
| "draft_tokens": 97, |
| "draft_tokens_accepted": 70, |
| "gufo": { |
| "cache_hit": false, |
| "cache_miss_reason": "prefix_changed", |
| "cache_common_prefix_tokens": 10, |
| "cache_checkpoint_tokens": 2671, |
| "cache_restore_bytes": 0, |
| "cache_snapshot_bytes": 385626932, |
| "cache_disk_queued_bytes": 0, |
| "cache_shared_bytes": 0, |
| "cache_restore_ms": 0, |
| "cache_snapshot_ms": 66.576988, |
| "cache_disk_enqueue_ms": 0, |
| "cache_disk_hit": false, |
| "cache_shared_prefix_snapshots": 0, |
| "cache_shared_prefix_bytes": 0, |
| "cache_shared_prefix_ms": 0, |
| "prefill_tokens": 2678, |
| "prefill_chunks": 3, |
| "active_decode_prefill_chunks": 0, |
| "max_prefill_chunk_tokens": 2048, |
| "queue_depth_at_submit": 1, |
| "client_queue_depth_at_submit": 1, |
| "resident_requests_at_admission": 1, |
| "requested_logical_concurrency": 1, |
| "physical_execution_width": 1, |
| "queue_ms": 9.417863, |
| "prefill_ms": 2329.399655, |
| "decode_ms": 4110.051406999999, |
| "ttft_ms": 2381.137262, |
| "mean_inter_token_ms": 32.55401520472441, |
| "max_inter_token_ms": 268.795415, |
| "execution_plan": "serial-c1" |
| } |
| }, |
| "timings": { |
| "prompt_n": 2678, |
| "prompt_ms": 2329.399655, |
| "prompt_per_token_ms": 0.8698281011949216, |
| "prompt_per_second": 1149.6524412424196, |
| "predicted_n": 128, |
| "predicted_ms": 4110.051406999999, |
| "predicted_per_token_ms": 32.10977661718749, |
| "predicted_per_second": 31.14316277942361, |
| "cache_n": 0, |
| "cache_restore_ms": 0, |
| "cache_snapshot_ms": 66.576988, |
| "cache_disk_enqueue_ms": 0, |
| "draft_n": 97, |
| "draft_n_accepted": 70 |
| }, |
| "server_timing": null |
| }, |
| { |
| "case": "mixed-medium", |
| "style": "mixed", |
| "words": 2048, |
| "nonce": 8, |
| "wall_s": 6.426, |
| "ttft_s": 2.402, |
| "finish_reason": "length", |
| "response_text": "Based on the repetitive stress test provided in \"Trial 000008,\" which highlights the critical factors of prompt length, memory traffic, attention state, expert routing, speculative acceptance, and scheduling, here are 20 distinct, detailed observations for designing a reliable local inference benchmark.\n\n### 1. Isolate Memory Bandwidth from Compute Throughput\nLocal inference is often memory-bound rather than compute-bound, especially for large models. A reliable benchmark must distinguish between tokens generated per second (TPS) during the prefill phase (compute-heavy) and the decode phase (memory-bandwidth-heavy). If a benchmark only reports", |
| "usage": { |
| "prompt_tokens": 2678, |
| "completion_tokens": 128, |
| "total_tokens": 2806, |
| "prompt_tokens_details": { |
| "cached_tokens": 0 |
| }, |
| "cached_tokens": 0, |
| "prompt_tokens_per_second": 1137.5872053781195, |
| "completion_tokens_per_second": 32.07869667768016, |
| "draft_tokens": 107, |
| "draft_tokens_accepted": 69, |
| "gufo": { |
| "cache_hit": false, |
| "cache_miss_reason": "prefix_changed", |
| "cache_common_prefix_tokens": 10, |
| "cache_checkpoint_tokens": 2671, |
| "cache_restore_bytes": 0, |
| "cache_snapshot_bytes": 385626932, |
| "cache_disk_queued_bytes": 0, |
| "cache_shared_bytes": 0, |
| "cache_restore_ms": 0, |
| "cache_snapshot_ms": 63.366051, |
| "cache_disk_enqueue_ms": 0, |
| "cache_disk_hit": false, |
| "cache_shared_prefix_snapshots": 0, |
| "cache_shared_prefix_bytes": 0, |
| "cache_shared_prefix_ms": 0, |
| "prefill_tokens": 2678, |
| "prefill_chunks": 3, |
| "active_decode_prefill_chunks": 0, |
| "max_prefill_chunk_tokens": 2048, |
| "queue_depth_at_submit": 1, |
| "client_queue_depth_at_submit": 1, |
| "resident_requests_at_admission": 1, |
| "requested_logical_concurrency": 1, |
| "physical_execution_width": 1, |
| "queue_ms": 10.573636, |
| "prefill_ms": 2354.105239, |
| "decode_ms": 3990.1870479999993, |
| "ttft_ms": 2400.792925, |
| "mean_inter_token_ms": 31.634204779527558, |
| "max_inter_token_ms": 167.349836, |
| "execution_plan": "serial-c1" |
| } |
| }, |
| "timings": { |
| "prompt_n": 2678, |
| "prompt_ms": 2354.105239, |
| "prompt_per_token_ms": 0.8790534873039582, |
| "prompt_per_second": 1137.5872053781195, |
| "predicted_n": 128, |
| "predicted_ms": 3990.1870479999993, |
| "predicted_per_token_ms": 31.173336312499995, |
| "predicted_per_second": 32.07869667768016, |
| "cache_n": 0, |
| "cache_restore_ms": 0, |
| "cache_snapshot_ms": 63.366051, |
| "cache_disk_enqueue_ms": 0, |
| "draft_n": 107, |
| "draft_n_accepted": 69 |
| }, |
| "server_timing": null |
| }, |
| { |
| "case": "mixed-long", |
| "style": "mixed", |
| "words": 8192, |
| "nonce": 10, |
| "wall_s": 12.607, |
| "ttft_s": 8.653, |
| "finish_reason": "length", |
| "response_text": "Based on the repetitive prompt provided, which highlights the critical variables affecting local inference performance (prompt length, memory traffic, attention state, expert routing, speculative acceptance, and scheduling), here are 20 distinct, detailed observations for designing a reliable local inference benchmark.\n\n### 1. Isolate Memory Bandwidth vs. Compute Bound Scenarios\nLocal inference is rarely purely compute-bound; it is often memory-bandwidth-bound, especially during the prefill phase of long prompts. A reliable benchmark must separate these phases. Measure **Time to First Token (TTFT)** to assess prefill performance (compute-heavy) and **Time Per Output Token", |
| "usage": { |
| "prompt_tokens": 10583, |
| "completion_tokens": 128, |
| "total_tokens": 10711, |
| "prompt_tokens_details": { |
| "cached_tokens": 0 |
| }, |
| "cached_tokens": 0, |
| "prompt_tokens_per_second": 1232.9338655646895, |
| "completion_tokens_per_second": 32.751178022388736, |
| "draft_tokens": 103, |
| "draft_tokens_accepted": 69, |
| "gufo": { |
| "cache_hit": false, |
| "cache_miss_reason": "prefix_changed", |
| "cache_common_prefix_tokens": 9, |
| "cache_checkpoint_tokens": 10576, |
| "cache_restore_bytes": 0, |
| "cache_snapshot_bytes": 819759676, |
| "cache_disk_queued_bytes": 0, |
| "cache_shared_bytes": 0, |
| "cache_restore_ms": 0, |
| "cache_snapshot_ms": 91.95378600000001, |
| "cache_disk_enqueue_ms": 0, |
| "cache_disk_hit": false, |
| "cache_shared_prefix_snapshots": 0, |
| "cache_shared_prefix_bytes": 0, |
| "cache_shared_prefix_ms": 0, |
| "prefill_tokens": 10583, |
| "prefill_chunks": 7, |
| "active_decode_prefill_chunks": 0, |
| "max_prefill_chunk_tokens": 2048, |
| "queue_depth_at_submit": 1, |
| "client_queue_depth_at_submit": 1, |
| "resident_requests_at_admission": 1, |
| "requested_logical_concurrency": 1, |
| "physical_execution_width": 1, |
| "queue_ms": 14.053646, |
| "prefill_ms": 8583.590974, |
| "decode_ms": 3908.2563659999983, |
| "ttft_ms": 8650.661426, |
| "mean_inter_token_ms": 31.08107268503936, |
| "max_inter_token_ms": 156.642943, |
| "execution_plan": "serial-c1" |
| } |
| }, |
| "timings": { |
| "prompt_n": 10583, |
| "prompt_ms": 8583.590974, |
| "prompt_per_token_ms": 0.8110735116696589, |
| "prompt_per_second": 1232.9338655646895, |
| "predicted_n": 128, |
| "predicted_ms": 3908.2563659999983, |
| "predicted_per_token_ms": 30.533252859374986, |
| "predicted_per_second": 32.751178022388736, |
| "cache_n": 0, |
| "cache_restore_ms": 0, |
| "cache_snapshot_ms": 91.95378600000001, |
| "cache_disk_enqueue_ms": 0, |
| "draft_n": 103, |
| "draft_n_accepted": 69 |
| }, |
| "server_timing": null |
| }, |
| { |
| "case": "mixed-long", |
| "style": "mixed", |
| "words": 8192, |
| "nonce": 11, |
| "wall_s": 12.128, |
| "ttft_s": 8.607, |
| "finish_reason": "length", |
| "response_text": "Based on the factors listed in the prompt (prompt length, memory traffic, attention state, expert routing, speculative acceptance, and scheduling), here are 20 distinct, detailed observations regarding the design of a reliable local inference benchmark.\n\n1. **Prompt Length Sensitivity and KV Cache Scaling**\n Benchmarks must explicitly test across a wide spectrum of prompt lengths (e.g., 1k, 8k, 32k, 128k tokens) because Time-to-First-Token (TTFT) scales non-linearly with prompt size due to the quadratic complexity of self-attention mechanisms during the pre", |
| "usage": { |
| "prompt_tokens": 10583, |
| "completion_tokens": 128, |
| "total_tokens": 10711, |
| "prompt_tokens_details": { |
| "cached_tokens": 0 |
| }, |
| "cached_tokens": 0, |
| "prompt_tokens_per_second": 1240.0517719540815, |
| "completion_tokens_per_second": 36.88155447748826, |
| "draft_tokens": 105, |
| "draft_tokens_accepted": 76, |
| "gufo": { |
| "cache_hit": false, |
| "cache_miss_reason": "prefix_changed", |
| "cache_common_prefix_tokens": 10, |
| "cache_checkpoint_tokens": 10576, |
| "cache_restore_bytes": 0, |
| "cache_snapshot_bytes": 819759676, |
| "cache_disk_queued_bytes": 0, |
| "cache_shared_bytes": 0, |
| "cache_restore_ms": 0, |
| "cache_snapshot_ms": 100.480586, |
| "cache_disk_enqueue_ms": 0, |
| "cache_disk_hit": false, |
| "cache_shared_prefix_snapshots": 0, |
| "cache_shared_prefix_bytes": 0, |
| "cache_shared_prefix_ms": 0, |
| "prefill_tokens": 10583, |
| "prefill_chunks": 7, |
| "active_decode_prefill_chunks": 0, |
| "max_prefill_chunk_tokens": 2048, |
| "queue_depth_at_submit": 1, |
| "client_queue_depth_at_submit": 1, |
| "resident_requests_at_admission": 1, |
| "requested_logical_concurrency": 1, |
| "physical_execution_width": 1, |
| "queue_ms": 14.809892, |
| "prefill_ms": 8534.321097999999, |
| "decode_ms": 3470.5695519999995, |
| "ttft_ms": 8606.40134, |
| "mean_inter_token_ms": 27.668167275590534, |
| "max_inter_token_ms": 134.150973, |
| "execution_plan": "serial-c1" |
| } |
| }, |
| "timings": { |
| "prompt_n": 10583, |
| "prompt_ms": 8534.321097999999, |
| "prompt_per_token_ms": 0.8064179436832655, |
| "prompt_per_second": 1240.0517719540815, |
| "predicted_n": 128, |
| "predicted_ms": 3470.5695519999995, |
| "predicted_per_token_ms": 27.113824624999996, |
| "predicted_per_second": 36.88155447748826, |
| "cache_n": 0, |
| "cache_restore_ms": 0, |
| "cache_snapshot_ms": 100.480586, |
| "cache_disk_enqueue_ms": 0, |
| "draft_n": 105, |
| "draft_n_accepted": 76 |
| }, |
| "server_timing": null |
| }, |
| { |
| "case": "mixed-long", |
| "style": "mixed", |
| "words": 8192, |
| "nonce": 12, |
| "wall_s": 12.199, |
| "ttft_s": 8.719, |
| "finish_reason": "length", |
| "response_text": "Based on the factors listed in the prompt (prompt length, memory traffic, attention state, expert routing, speculative acceptance, and scheduling), here are 20 distinct, detailed observations regarding the design of a reliable local inference benchmark.\n\n### 1. Decoupling Prefill and Decode Latencies\nA reliable benchmark must strictly separate Time-to-First-Token (TTFT) from Inter-Token Latency (ITL). TTFT is dominated by the prefill phase, which is compute-bound and heavily influenced by prompt length and attention state initialization. ITL is dominated by the decode phase, which is memory-bandwidth-bound. Averaging", |
| "usage": { |
| "prompt_tokens": 10583, |
| "completion_tokens": 128, |
| "total_tokens": 10711, |
| "prompt_tokens_details": { |
| "cached_tokens": 0 |
| }, |
| "cached_tokens": 0, |
| "prompt_tokens_per_second": 1225.5564521500717, |
| "completion_tokens_per_second": 37.196986338526244, |
| "draft_tokens": 99, |
| "draft_tokens_accepted": 72, |
| "gufo": { |
| "cache_hit": false, |
| "cache_miss_reason": "prefix_changed", |
| "cache_common_prefix_tokens": 10, |
| "cache_checkpoint_tokens": 10576, |
| "cache_restore_bytes": 0, |
| "cache_snapshot_bytes": 819759676, |
| "cache_disk_queued_bytes": 0, |
| "cache_shared_bytes": 0, |
| "cache_restore_ms": 0, |
| "cache_snapshot_ms": 88.621681, |
| "cache_disk_enqueue_ms": 0, |
| "cache_disk_hit": false, |
| "cache_shared_prefix_snapshots": 0, |
| "cache_shared_prefix_bytes": 0, |
| "cache_shared_prefix_ms": 0, |
| "prefill_tokens": 10583, |
| "prefill_chunks": 7, |
| "active_decode_prefill_chunks": 0, |
| "max_prefill_chunk_tokens": 2048, |
| "queue_depth_at_submit": 1, |
| "client_queue_depth_at_submit": 1, |
| "resident_requests_at_admission": 1, |
| "requested_logical_concurrency": 1, |
| "physical_execution_width": 1, |
| "queue_ms": 26.821613, |
| "prefill_ms": 8635.261134999999, |
| "decode_ms": 3441.1389900000004, |
| "ttft_ms": 8717.024983, |
| "mean_inter_token_ms": 27.36114006299213, |
| "max_inter_token_ms": 123.481909, |
| "execution_plan": "serial-c1" |
| } |
| }, |
| "timings": { |
| "prompt_n": 10583, |
| "prompt_ms": 8635.261134999999, |
| "prompt_per_token_ms": 0.8159558853822166, |
| "prompt_per_second": 1225.5564521500717, |
| "predicted_n": 128, |
| "predicted_ms": 3441.1389900000004, |
| "predicted_per_token_ms": 26.883898359375003, |
| "predicted_per_second": 37.19698633852624, |
| "cache_n": 0, |
| "cache_restore_ms": 0, |
| "cache_snapshot_ms": 88.621681, |
| "cache_disk_enqueue_ms": 0, |
| "draft_n": 99, |
| "draft_n_accepted": 72 |
| }, |
| "server_timing": null |
| }, |
| { |
| "case": "repetitive-short", |
| "style": "repetitive", |
| "words": 256, |
| "nonce": 14, |
| "wall_s": 2.493, |
| "ttft_s": 0.648, |
| "finish_reason": "stop", |
| "response_text": "The quick brown fox jumps over the lazy dog.\nThe quick brown fox jumps over the lazy dog.\nThe quick brown fox jumps over the lazy dog.\nThe quick brown fox jumps over the lazy dog.\nThe quick brown fox jumps over the lazy dog.\nThe quick brown fox jumps over the lazy dog.\nThe quick brown fox jumps over the lazy dog.\nThe quick brown fox jumps over the lazy dog.\nThe quick brown fox jumps over the lazy dog.\nThe quick brown fox jumps over the lazy dog.", |
| "usage": { |
| "prompt_tokens": 369, |
| "completion_tokens": 109, |
| "total_tokens": 478, |
| "prompt_tokens_details": { |
| "cached_tokens": 0 |
| }, |
| "cached_tokens": 0, |
| "prompt_tokens_per_second": 613.8400704108163, |
| "completion_tokens_per_second": 60.102040284568076, |
| "draft_tokens": 93, |
| "draft_tokens_accepted": 90, |
| "gufo": { |
| "cache_hit": false, |
| "cache_miss_reason": "prefix_changed", |
| "cache_common_prefix_tokens": 10, |
| "cache_checkpoint_tokens": 362, |
| "cache_restore_bytes": 0, |
| "cache_snapshot_bytes": 263044876, |
| "cache_disk_queued_bytes": 0, |
| "cache_shared_bytes": 0, |
| "cache_restore_ms": 0, |
| "cache_snapshot_ms": 55.329412000000005, |
| "cache_disk_enqueue_ms": 0, |
| "cache_disk_hit": false, |
| "cache_shared_prefix_snapshots": 0, |
| "cache_shared_prefix_bytes": 0, |
| "cache_shared_prefix_ms": 0, |
| "prefill_tokens": 369, |
| "prefill_chunks": 2, |
| "active_decode_prefill_chunks": 0, |
| "max_prefill_chunk_tokens": 362, |
| "queue_depth_at_submit": 1, |
| "client_queue_depth_at_submit": 1, |
| "resident_requests_at_admission": 1, |
| "requested_logical_concurrency": 1, |
| "physical_execution_width": 1, |
| "queue_ms": 16.849518, |
| "prefill_ms": 601.133777, |
| "decode_ms": 1813.582359, |
| "ttft_ms": 646.693018, |
| "mean_inter_token_ms": 17.038821842592604, |
| "max_inter_token_ms": 200.873314, |
| "execution_plan": "serial-c1" |
| } |
| }, |
| "timings": { |
| "prompt_n": 369, |
| "prompt_ms": 601.133777, |
| "prompt_per_token_ms": 1.6290888265582657, |
| "prompt_per_second": 613.8400704108163, |
| "predicted_n": 109, |
| "predicted_ms": 1813.582359, |
| "predicted_per_token_ms": 16.638370266055045, |
| "predicted_per_second": 60.102040284568076, |
| "cache_n": 0, |
| "cache_restore_ms": 0, |
| "cache_snapshot_ms": 55.329412000000005, |
| "cache_disk_enqueue_ms": 0, |
| "draft_n": 93, |
| "draft_n_accepted": 90 |
| }, |
| "server_timing": null |
| }, |
| { |
| "case": "repetitive-short", |
| "style": "repetitive", |
| "words": 256, |
| "nonce": 15, |
| "wall_s": 2.499, |
| "ttft_s": 0.651, |
| "finish_reason": "stop", |
| "response_text": "The quick brown fox jumps over the lazy dog.\nThe quick brown fox jumps over the lazy dog.\nThe quick brown fox jumps over the lazy dog.\nThe quick brown fox jumps over the lazy dog.\nThe quick brown fox jumps over the lazy dog.\nThe quick brown fox jumps over the lazy dog.\nThe quick brown fox jumps over the lazy dog.\nThe quick brown fox jumps over the lazy dog.\nThe quick brown fox jumps over the lazy dog.\nThe quick brown fox jumps over the lazy dog.", |
| "usage": { |
| "prompt_tokens": 369, |
| "completion_tokens": 109, |
| "total_tokens": 478, |
| "prompt_tokens_details": { |
| "cached_tokens": 0 |
| }, |
| "cached_tokens": 0, |
| "prompt_tokens_per_second": 616.4785745427549, |
| "completion_tokens_per_second": 59.92816720683945, |
| "draft_tokens": 101, |
| "draft_tokens_accepted": 90, |
| "gufo": { |
| "cache_hit": false, |
| "cache_miss_reason": "prefix_changed", |
| "cache_common_prefix_tokens": 10, |
| "cache_checkpoint_tokens": 362, |
| "cache_restore_bytes": 0, |
| "cache_snapshot_bytes": 263044876, |
| "cache_disk_queued_bytes": 0, |
| "cache_shared_bytes": 0, |
| "cache_restore_ms": 0, |
| "cache_snapshot_ms": 60.25607600000001, |
| "cache_disk_enqueue_ms": 0, |
| "cache_disk_hit": false, |
| "cache_shared_prefix_snapshots": 0, |
| "cache_shared_prefix_bytes": 0, |
| "cache_shared_prefix_ms": 0, |
| "prefill_tokens": 369, |
| "prefill_chunks": 2, |
| "active_decode_prefill_chunks": 0, |
| "max_prefill_chunk_tokens": 362, |
| "queue_depth_at_submit": 1, |
| "client_queue_depth_at_submit": 1, |
| "resident_requests_at_admission": 1, |
| "requested_logical_concurrency": 1, |
| "physical_execution_width": 1, |
| "queue_ms": 16.707863, |
| "prefill_ms": 598.5609479999999, |
| "decode_ms": 1818.8442110000005, |
| "ttft_ms": 650.312493, |
| "mean_inter_token_ms": 17.07431117592594, |
| "max_inter_token_ms": 107.754056, |
| "execution_plan": "serial-c1" |
| } |
| }, |
| "timings": { |
| "prompt_n": 369, |
| "prompt_ms": 598.5609479999999, |
| "prompt_per_token_ms": 1.6221163902439022, |
| "prompt_per_second": 616.4785745427549, |
| "predicted_n": 109, |
| "predicted_ms": 1818.8442110000005, |
| "predicted_per_token_ms": 16.686644137614685, |
| "predicted_per_second": 59.92816720683945, |
| "cache_n": 0, |
| "cache_restore_ms": 0, |
| "cache_snapshot_ms": 60.25607600000001, |
| "cache_disk_enqueue_ms": 0, |
| "draft_n": 101, |
| "draft_n_accepted": 90 |
| }, |
| "server_timing": null |
| }, |
| { |
| "case": "repetitive-short", |
| "style": "repetitive", |
| "words": 256, |
| "nonce": 16, |
| "wall_s": 2.489, |
| "ttft_s": 0.648, |
| "finish_reason": "stop", |
| "response_text": "The quick brown fox jumps over the lazy dog.\nThe quick brown fox jumps over the lazy dog.\nThe quick brown fox jumps over the lazy dog.\nThe quick brown fox jumps over the lazy dog.\nThe quick brown fox jumps over the lazy dog.\nThe quick brown fox jumps over the lazy dog.\nThe quick brown fox jumps over the lazy dog.\nThe quick brown fox jumps over the lazy dog.\nThe quick brown fox jumps over the lazy dog.\nThe quick brown fox jumps over the lazy dog.", |
| "usage": { |
| "prompt_tokens": 369, |
| "completion_tokens": 109, |
| "total_tokens": 478, |
| "prompt_tokens_details": { |
| "cached_tokens": 0 |
| }, |
| "cached_tokens": 0, |
| "prompt_tokens_per_second": 614.1311630548279, |
| "completion_tokens_per_second": 60.25200961319156, |
| "draft_tokens": 98, |
| "draft_tokens_accepted": 89, |
| "gufo": { |
| "cache_hit": false, |
| "cache_miss_reason": "prefix_changed", |
| "cache_common_prefix_tokens": 10, |
| "cache_checkpoint_tokens": 362, |
| "cache_restore_bytes": 0, |
| "cache_snapshot_bytes": 263044876, |
| "cache_disk_queued_bytes": 0, |
| "cache_shared_bytes": 0, |
| "cache_restore_ms": 0, |
| "cache_snapshot_ms": 54.033383, |
| "cache_disk_enqueue_ms": 0, |
| "cache_disk_hit": false, |
| "cache_shared_prefix_snapshots": 0, |
| "cache_shared_prefix_bytes": 0, |
| "cache_shared_prefix_ms": 0, |
| "prefill_tokens": 369, |
| "prefill_chunks": 2, |
| "active_decode_prefill_chunks": 0, |
| "max_prefill_chunk_tokens": 362, |
| "queue_depth_at_submit": 1, |
| "client_queue_depth_at_submit": 1, |
| "resident_requests_at_admission": 1, |
| "requested_logical_concurrency": 1, |
| "physical_execution_width": 1, |
| "queue_ms": 17.431679, |
| "prefill_ms": 600.848845, |
| "decode_ms": 1809.0682899999997, |
| "ttft_ms": 647.005877, |
| "mean_inter_token_ms": 16.984669064814828, |
| "max_inter_token_ms": 103.259397, |
| "execution_plan": "serial-c1" |
| } |
| }, |
| "timings": { |
| "prompt_n": 369, |
| "prompt_ms": 600.848845, |
| "prompt_per_token_ms": 1.6283166531165312, |
| "prompt_per_second": 614.1311630548279, |
| "predicted_n": 109, |
| "predicted_ms": 1809.0682899999997, |
| "predicted_per_token_ms": 16.596956788990823, |
| "predicted_per_second": 60.25200961319156, |
| "cache_n": 0, |
| "cache_restore_ms": 0, |
| "cache_snapshot_ms": 54.033383, |
| "cache_disk_enqueue_ms": 0, |
| "draft_n": 98, |
| "draft_n_accepted": 89 |
| }, |
| "server_timing": null |
| } |
| ] |
| } |
