{ "schema_version": "5.1", "name": "ornith-1.5-35b-a3b", "tt_metal_version": "0.65.2.dev10123", "arch": "blackhole", "device_count": 2, "producer": { "tt_kernel_version": "0.1.0", "created_at": "2026-10-28T16:24:18.987830+00:00", "hostname": "qb2-120-p06t08" }, "weights": { "repo_id": "ornith-ai/Ornith-1.5-35B-A3B", "revision": "10fbf86fed7ecee4a061f8b499a618f46001cac1", "allow_patterns": null, "ignore_patterns": null, "repo_type": "model" }, "auxiliary_weights": [], "mesh": null, "entrypoint": null, "resources": null, "capabilities": null, "env": {}, "bundled": null, "deps": null, "container": { "image": { "registry": "hf", "repository": "ornith-1.5-35b-a3b", "tag": "tt-model/ornith-1.5-35b-a3b:9f0546abcdb8", "digest": "sha256:9f0546abcdb8140fc1aac8e20141b19039f4e4a73f1805cbea62d51936ce4649" }, "kind": "vllm-plugin", "runtime": { "vllm": { "version": "0.26.0" }, "plugin": { "path": "/home/raahem/vllm-tt-plugin-pub", "sha": "e37ccd87d88f87628ffb63e24bcd3d2f16350936", "dirty": false }, "extra_models_dir": "models/autoports/ornith_ai_ornith_1_5_35b_a3b/vllm_bundle", "overrides": [ "torch==2.11.0+cpu", "torchvision==0.26.0+cpu", "transformers==5.12.1", "tokenizers==0.22.2" ], "lock": "requirements.lock" }, "serve": { "hardware": null, "mesh_device": null, "port": 8000, "max_model_len": 262144, "max_num_seqs": 32, "block_size": 64, "server_timeout": null, "capabilities": { "tool_parser": "qwen3_xml", "reasoning_parser": "qwen3" }, "additional_config": { "tt": { "sample_on_device_mode": "all", "trace_region_size": 805306368, "l1_small_size": 24576, "fabric_config": "FABRIC_1D" } }, "args": [ "--enable-auto-tool-choice", "--enable-prefix-caching", "--default-chat-template-kwargs", "{\"enable_thinking\": true}", "--limit-mm-per-prompt", "{\"image\": 4, \"video\": 0}" ], "env": { "ARCH_NAME": "blackhole", "HF_MODEL": "ornith-ai/Ornith-1.5-35B-A3B", "QWEN35_TRACE_REGION_MIB": "768", "QWEN35_PROGRAM_GUARD": "1", "QWEN35_VISION": "1", "QWEN35_VISION_MAX_IMAGES": "4", "QWEN35_VISION_MAX_PIXELS": "4194304", "QWEN35_BATCHED_PREFILL": "0", "VLLM_ENGINE_READY_TIMEOUT_S": "3600", "QWEN35_MOE_FUSED_DECODE": "1", "QWEN35_DECODE_REPLICATED": "1", "QWEN35_GDN_DECODE_OPS": "1", "QWEN35_ATTN_DECODE_OPS": "1", "QWEN35_MTP": "0", "QWEN35_ROWS_TOPK": "1" } }, "serve_profiles": [ { "hardware": "p300x2", "mesh_device": "P300x2", "port": null, "max_model_len": null, "max_num_seqs": null, "block_size": null, "server_timeout": null, "capabilities": null, "additional_config": {}, "args": [], "env": { "QWEN35_FABRIC_BOUNDARY": "1", "QWEN35_FB_MAX_ROWS": "4", "QWEN35_FB_CONSUMER": "1", "QWEN35_FB_ROUTER": "1", "QWEN35_MOE_SPLIT_PAIR": "1" }, "name": "p300x2", "description": "Two p300 boards (4 dies, tensor parallel 4), 32 sequences, 262144 context, bfp8 routed experts." }, { "hardware": "p150x4", "mesh_device": "P150x4", "port": null, "max_model_len": null, "max_num_seqs": null, "block_size": null, "server_timeout": null, "capabilities": null, "additional_config": {}, "args": [], "env": { "QWEN35_FABRIC_BOUNDARY": "1", "QWEN35_FB_MAX_ROWS": "4", "QWEN35_FB_CONSUMER": "1", "QWEN35_FB_ROUTER": "1", "QWEN35_MOE_SPLIT_PAIR": "1" }, "name": "p150x4", "description": "Four p150 boards (4 dies), same configuration as p300x2. Not run on hardware (the identical (1,4) mesh)." }, { "hardware": "p300", "mesh_device": "P300", "port": null, "max_model_len": null, "max_num_seqs": null, "block_size": null, "server_timeout": null, "capabilities": null, "additional_config": {}, "args": [], "env": { "TT_VISIBLE_DEVICES": "0,1" }, "name": "p300", "description": "One p300 board (2 dies, tensor parallel 2), 32 sequences, 262144 context, bfp8 routed experts." }, { "hardware": "p150x2", "mesh_device": "P150x2", "port": null, "max_model_len": null, "max_num_seqs": null, "block_size": null, "server_timeout": null, "capabilities": null, "additional_config": {}, "args": [], "env": { "TT_VISIBLE_DEVICES": "0,1" }, "name": "p150x2", "description": "Two p150 boards (2 dies), same configuration as p300. Not run on hardware (the identical (1,2) mesh)." }, { "hardware": "p150", "mesh_device": "P150", "port": null, "max_model_len": null, "max_num_seqs": null, "block_size": null, "server_timeout": null, "capabilities": null, "additional_config": { "tt": { "fabric_config": "DISABLED" } }, "args": [ "--enable-auto-tool-choice", "--enable-prefix-caching", "--default-chat-template-kwargs", "{\"enable_thinking\": true}", "--limit-mm-per-prompt", "{\"image\": 0, \"video\": 0}" ], "env": { "QWEN35_VISION": "0", "QWEN35_GDN_DECODE_OPS": "0", "QWEN35_ATTN_DECODE_OPS": "0", "QWEN35_MTP": "3" }, "name": "p150", "description": "One p150 (1 die), 32 sequences, bfp4 routed experts (bfp8 does not fit one die), text only, no fabric. One die holds 4 up|gate columns per DRAM bank and expert, which the fused MoE kernels do not read yet: this profile keeps the v2 MoE (with the fused router top-k)." }, { "hardware": "p100", "mesh_device": "P150", "port": null, "max_model_len": null, "max_num_seqs": null, "block_size": null, "server_timeout": null, "capabilities": null, "additional_config": { "tt": { "fabric_config": "DISABLED" } }, "args": [ "--enable-auto-tool-choice", "--enable-prefix-caching", "--default-chat-template-kwargs", "{\"enable_thinking\": true}", "--limit-mm-per-prompt", "{\"image\": 0, \"video\": 0}" ], "env": { "QWEN35_VISION": "0", "QWEN35_GDN_DECODE_OPS": "0", "QWEN35_ATTN_DECODE_OPS": "0", "QWEN35_MTP": "3" }, "name": "p100", "description": "One p100 (1 die, 7 DRAM banks): the p150 profile unchanged - 32 sequences, bfp4 routed experts, text only, no fabric. Like p150 it keeps the v2 MoE (7 DRAM banks, 4 columns per bank)." } ], "default_profile": "p300", "code_dir": "code", "verify": [ "import models.autoports.ornith_ai_ornith_1_5_35b_a3b.tt.generator_vllm as g; assert g.Qwen35MoeForConditionalGeneration", "import models.autoports.ornith_ai_ornith_1_5_35b_a3b.tt.decode_buckets as b; assert b.widths(32) == [1, 2, 4, 8, 16, 32]" ], "built": { "image": "tt-model/ornith-1.5-35b-a3b:9f0546abcdb8", "repo": "raahemnabeel/ornith-1.5-35b-a3b", "tt_model_version": "0.1.0", "created_at": "2026-10-28T16:16:04+00:00", "tt_metal": { "sha": "d9ed49c49330298e207d9532d173c9d6a09ff81a", "describe": "v0.79.0-dev20260909-320-gd9ed49c493", "dirty": false, "scm_version": "0.65.2.dev10123", "mode": "local", "remote": "https://github.com/rnabeelTT/tt-metal.git", "branch": "ornith_fusion", "pushed": true }, "code_sha256": "aead6d7885da18440e9e5afc4c6d4426389c310cf33a34bd4fb3d0da5cc67bce", "plugin": { "sha": "e37ccd87d88f87628ffb63e24bcd3d2f16350936", "path": "/home/raahem/vllm-tt-plugin-pub", "dirty": false }, "image_digest": "sha256:9f0546abcdb8140fc1aac8e20141b19039f4e4a73f1805cbea62d51936ce4649" }, "card": { "description": "Ornith-1.5-35B-A3B (35B total / 3B active MoE: 256 experts, top-8 + shared expert, 40 layers of Gated DeltaNet + gated attention, vision tower, MTP head) on Blackhole through vLLM and the Tenstorrent plugin, with a fused decode step: the v1 bundle (raahemnabeel/ornith-1.5-35b-a3b-v1) plus fused MoE, Gated DeltaNet, attention, layer-boundary and sampler decode ops for 1-32 concurrent users. Same weights, profiles and serving recipe; on four dies single-user decode 45 -> 220 tok/s and 32-user output 346 -> 591 tok/s, prefill unchanged or faster.\n", "attribution": null, "prerequisites": null, "quickstart": "### What changed since v1\nThis is the v1 bundle ([raahemnabeel/ornith-1.5-35b-a3b-v1](https://huggingface.co/raahemnabeel/ornith-1.5-35b-a3b-v1)) with its decode step\nrebuilt around fused kernels. Same checkpoint, same vLLM plugin build, same profiles and serving recipe; prefill is the\nv1's (TTFT equal or lower). The changes, in the order they were made (each measured, kept only if it won):\n\n1. **Router.** One 1D matmul (was DRAM-sharded on 8 cores); top-8 + softmax + shared-gate sigmoid on the RISC cores in\n one op instead of a 7-op chain.\n2. **Fused MoE decode.** Routed + shared experts in three ops (route, gate|up + SwiGLU, down) that stream each selected\n expert's weights in place, for every decode width from 1 to 32 rows (several users' experts as one union, each weight\n column read once for all rows). On four dies the gate and up halves of an expert column are split across a pair of\n cores, with readers placed off the congested NOC rows.\n3. **Replicated-residual layer boundary.** The decode residual stays replicated on every die: each mixer / MoE output is\n one all-reduce + add + local RMSNorm.\n4. **Gated DeltaNet decode mixer** as two programs (causal conv; gates + norms + recurrent state update + gated norm),\n with each head's value dimension split across up to four cores.\n5. **Attention decode ops** around the QKV matmul, cache update and SDPA decode (q / k norm + RoPE + gate in one prep op,\n output gating in one post op).\n6. **Fabric layer boundary** (four dies, 1-4 users): the all-reduce, residual add and RMSNorm run as one op over the\n fabric, and the next layer's input projection (or the MoE router) runs inside the same program, streaming its weights\n while the boundary computes.\n7. **Sampler top-k over the live rows only** (1-8 users), instead of all 32 slot rows.\n\nMTP speculative decoding is off on the multi-die profiles: its verify pass runs the unfused path, and the fused plain\ndecode step is faster on real text (four dies: 218 vs 86 tok/s).\n\nNumerics: the routed and shared expert weights are stored rounded to 5 significant bits (exactly what the LoFi expert\nmatmul reads), activations / routing weights rounded to the 7 bits the matmul's other operand reads; the fused boundary\nand DeltaNet ops sum in a different order than the op chain they replace. Output quality is checked below.\n\nOn one die (`p150`, `p100`) the full expert width does not fit the fused kernels' bank layout, so those profiles keep the\nv1 MoE and gain only the fused router top-k (1-row steps), with MTP on as in v1.\n\n### Use with your client\nPoint an OpenAI-compatible client at `http://127.0.0.1:20000` (or the port `tt-model serve` printed) with model\n`ornith-ai/Ornith-1.5-35B-A3B`. Thinking is ON by default (reasoning in the message's `reasoning` field, answer in\n`content`); turn it off per request with `\"chat_template_kwargs\": {\"enable_thinking\": false}`. Give thinking requests a\nlarge `max_tokens`. Tool calls use the `qwen3_xml` parser. Images (up to 4 per request, resized to at most 4 MP) as\n`image_url` parts on the multi-die profiles.\n\n### Prefix caching\nOn by default (multi-die profiles). When a prompt starts with the same 2,048 or more tokens as a recent one (a document sent\nagain, the next turn of a chat, a long shared system prompt), the server resumes from the last 2,048-token boundary\nthe two share instead of prefilling from the start: vLLM reuses the attention KV blocks, and the server restores the\nGated DeltaNet state it saved at that boundary (32 snapshots on four dies, 16 on two; each snapshot 22.5 MiB per die on four dies, 0.7 GiB per die in all, taken from the KV pool, least recently used evicted). A prompt seen for the first time\nruns exactly the prefill it ran without caching. Turn it off with `tt-model serve ... -- --no-enable-prefix-caching`.\n\n### Profiles\n`tt-model serve raahemnabeel/ornith-1.5-35b-a3b --profile `:\n- `p300` (default) / `p150x2` - two dies, bfp8 routed experts, text + images (no fabric boundary: its receive buffers do\n not fit next to the two-die prefill).\n- `p300x2` / `p150x4` - four dies, bfp8 routed experts, text + images: the fastest profile.\n- `p150` / `p100` - one die, bfp4 routed experts, text only (v1 MoE, MTP on).\n\nEvery profile: 32 concurrent sequences, 262144-token context.\n", "architecture": null, "status": null, "intended_use": null, "out_of_scope_use": null, "usage": null, "performance": "### Accuracy\nBenchmark evals through the served API (p300 profile, thinking off, one sample per question, seed 42, temperature 0.6 /\ntop_p 0.95 / top_k 20, lm-evaluation-harness), the same settings and box as the v1 bundle's run:\n\n| Eval | v1 | current |\n| --- | ---: | ---: |\n| GPQA-Diamond (40 questions) | 0.775 | 0.675 |\n| IFEval prompt-level strict (100) | 0.76 | 0.82 |\n| LiveCodeBench v6 (50) | 0.36 | 0.40 |\n\nAll within the error bars of these subset sizes (GPQA +-0.07, IFEval +-0.04, LiveCodeBench +-0.07): no measurable\nquality change. The whole GPQA-Diamond set (198 questions, same settings) scores 0.707 +-0.032 on this bundle; the\nsame build scored 0.75 on the first 40 questions in that run (0.675 in the subset run above), the spread one sample\nper question gives at n = 40. Layer-level PCC against Hugging Face (real weights): decode >= 0.9999 at 1 user,\nper-user minimum >= 0.9999 at 32 users.\n\nTeacher forcing against the Hugging Face bf16 model (v1 bundle; 100 greedy tokens, top-1 / top-5 / top-100):\n\n| profile | code chat prompt | 2048-token book prompt |\n| --- | --- | --- |\n| 4 dies (bfp8 experts) | 0.97 / 1.00 / 1.00 | 0.89 / 0.98 / 1.00 |\n| 2 dies (bfp8 experts) | 0.96 / 1.00 / 1.00 | 0.91 / 0.97 / 1.00 |\n| 1 die (bfp4 experts) | 0.96 / 1.00 / 1.00 | 0.84 / 0.94 / 0.99 |\n\n### Performance\nMeasured 2026-10-25 on one QuietBox 2 (4 Blackhole p300c dies; p300 and p150 rows on a subset of it), both bundles\nserved exactly as shipped (`tt-model serve --profile

`), one server at a time: random-token prompts of\nexactly the input length, output pinned with ignore_eos, streamed; users are concurrent requests in a closed loop;\nTTFT = time to first streamed token; TPOT = (E2EL - TTFT) / (output - 1); one unrecorded warm-up per row. v1\nruns MTP speculative decoding while one request is live (its single-user rows include it); this bundle runs plain\ndecode on the multi-die profiles. The real-text row (12 chat prompts, 384 output tokens, temperature 0) shows\nsingle-user speed on predictable text, where MTP helps v1 most. Each cell is `v1 -> current`.\n\n| Profile | ISL | OSL | Concurrency | N | TTFT (ms) | Prefill tok/s/u | Decode tok/s/u | E2EL (s) |\n| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |\n| p300x2 | 128 | 128 | 1 | 4 | 118 | 1,083 | 220.0 | 0.70 |\n| p300x2 | 1024 | 256 | 32 | 128 | 7,265 | 141 | 42.1 | 13.40 |\n| p300 | 128 | 128 | 1 | 4 | 133 | 962 | 126.3 | 1.14 |\n| p300 | 1024 | 256 | 32 | 128 | 8,628 | 119 | 25.4 | 18.69 |\n| p150 | 1024 | 256 | 32 | 128 | 11,879 | 86 | 8.4 | 42.06 |\n\n#### p300x2\n\nFour-die rows measured on the previous revision, which differs only in the sampler top-k's index width (16 -> 32 bits, for two dies). This revision, same setup: 1 user 4.57 ms TPOT, real text 217.4 tok/s, 8 users 9.71 ms, 32 users 1024/256 24.2 ms.\n\nLatency, batch 1 (single user): v1 -> current\n\n| Input tokens | Output tokens | TTFT (ms) | Prefill tok/s | TPOT (ms) | Decode tok/s | E2EL (s) |\n| ---: | ---: | ---: | ---: | ---: | ---: | ---: |\n| 128 | 128 | 179 -> 118 | 717 -> 1,083 | 22.4 -> 4.5 | 44.5 -> **220.0** (+394 %) | 3.04 -> 0.70 |\n| 1,024 | 256 | 287 -> 223 | 3,572 -> 4,584 | 20.3 -> 4.7 | 49.2 -> **214.5** (+336 %) | 5.48 -> 1.41 |\n| 4,096 | 256 | 728 -> 646 | 5,625 -> 6,340 | 22.0 -> 4.8 | 45.4 -> **207.9** (+358 %) | 6.34 -> 1.87 |\n| 16,384 | 256 | 2,760 -> 2,636 | 5,936 -> 6,216 | 24.4 -> 5.0 | 41.0 -> **200.0** (+388 %) | 8.98 -> 3.91 |\n| 32,768 | 256 | 5,716 -> 5,495 | 5,733 -> 5,963 | 25.4 -> 5.3 | 39.3 -> **190.1** (+384 %) | 12.20 -> 6.84 |\n| 65,536 | 256 | 12,607 -> 12,177 | 5,198 -> 5,382 | 23.3 -> 5.8 | 43.0 -> **173.4** (+304 %) | 18.54 -> 13.65 |\n| 130,816 | 256 | 30,102 -> 28,766 | 4,346 -> 4,548 | 26.1 -> 6.6 | 38.4 -> **150.5** (+292 %) | 36.75 -> 30.46 |\n\nReal text, single user (12 chat prompts x 384 output tokens, temperature 0; median per prompt): 58.5 -> **217.9 tok/s** (+272 %).\n\nConcurrency, 128 input / 128 output tokens: v1 -> current\n\n| Users | N | TTFT (ms) | TPOT (ms) | Decode tok/s/user | Total out tok/s | E2EL (s) |\n| ---: | ---: | ---: | ---: | ---: | ---: | ---: |\n| 2 | 8 | 340 -> 332 | 24.8 -> 6.7 | 40.3 -> 149.6 | 73.3 -> **215.8** (+194 %) | 3.49 -> 1.19 |\n| 4 | 16 | 680 -> 660 | 28.4 -> 7.2 | 35.2 -> 139.3 | 118.9 -> **323.7** (+172 %) | 4.30 -> 1.59 |\n| 8 | 32 | 1,135 -> 1,125 | 37.0 -> 9.4 | 27.0 -> 106.0 | 175.0 -> **440.3** (+152 %) | 5.85 -> 2.32 |\n| 16 | 64 | 2,112 -> 2,052 | 55.1 -> 13.7 | 18.2 -> 73.2 | 225.0 -> **541.5** (+141 %) | 9.11 -> 3.77 |\n| 32 | 128 | 3,975 -> 3,927 | 62.0 -> 24.0 | 16.1 -> 41.7 | 345.8 -> **591.1** (+71 %) | 11.84 -> 6.96 |\n\nConcurrency, 1024 input / 256 output tokens: v1 -> current\n\n| Users | N | TTFT (ms) | TPOT (ms) | Decode tok/s/user | Total out tok/s | E2EL (s) |\n| ---: | ---: | ---: | ---: | ---: | ---: | ---: |\n| 2 | 8 | 561 -> 547 | 24.3 -> 6.4 | 41.2 -> 157.3 | 75.8 -> **235.3** (+210 %) | 6.75 -> 2.17 |\n| 4 | 16 | 1,119 -> 1,086 | 28.4 -> 7.2 | 35.2 -> 138.5 | 122.5 -> **349.9** (+186 %) | 8.36 -> 2.91 |\n| 8 | 32 | 1,996 -> 1,986 | 37.0 -> 9.6 | 27.0 -> 103.8 | 178.9 -> **464.0** (+159 %) | 11.44 -> 4.42 |\n| 16 | 64 | 3,814 -> 3,718 | 55.3 -> 13.5 | 18.1 -> 74.0 | 228.6 -> **570.1** (+149 %) | 17.93 -> 7.18 |\n| 32 | 128 | 7,347 -> 7,265 | 61.8 -> 23.8 | 16.2 -> 42.1 | 353.8 -> **614.0** (+74 %) | 23.12 -> 13.40 |\n\n#### p300\n\nLatency, batch 1 (single user): v1 -> current\n\n| Input tokens | Output tokens | TTFT (ms) | Prefill tok/s | TPOT (ms) | Decode tok/s | E2EL (s) |\n| ---: | ---: | ---: | ---: | ---: | ---: | ---: |\n| 128 | 128 | 202 -> 133 | 634 -> 962 | 23.3 -> 7.9 | 42.9 -> **126.3** (+194 %) | 3.16 -> 1.14 |\n| 1,024 | 256 | 346 -> 270 | 2,960 -> 3,787 | 25.0 -> 8.0 | 40.0 -> **124.4** (+211 %) | 6.72 -> 2.32 |\n| 4,096 | 256 | 891 -> 807 | 4,595 -> 5,077 | 23.3 -> 8.2 | 42.9 -> **122.1** (+185 %) | 6.84 -> 2.90 |\n| 16,384 | 256 | 3,413 -> 3,267 | 4,801 -> 5,015 | 32.9 -> 8.4 | 30.4 -> **119.2** (+292 %) | 11.79 -> 5.41 |\n| 32,768 | 256 | 7,059 -> 6,801 | 4,642 -> 4,818 | 22.6 -> 8.6 | 44.3 -> **115.8** (+162 %) | 12.82 -> 9.00 |\n| 65,536 | 256 | 15,296 -> 14,765 | 4,284 -> 4,439 | 21.5 -> 9.1 | 46.5 -> **109.6** (+136 %) | 20.78 -> 17.09 |\n| 130,816 | 256 | 35,579 -> 34,008 | 3,677 -> 3,847 | 37.4 -> 10.1 | 26.8 -> **99.3** (+271 %) | 45.11 -> 36.58 |\n\nReal text, single user (12 chat prompts x 384 output tokens, temperature 0; median per prompt): 52.6 -> **125.6 tok/s** (+139 %).\n\nConcurrency, 128 input / 128 output tokens: v1 -> current\n\n| Users | N | TTFT (ms) | TPOT (ms) | Decode tok/s/user | Total out tok/s | E2EL (s) |\n| ---: | ---: | ---: | ---: | ---: | ---: | ---: |\n| 2 | 8 | 355 -> 352 | 28.5 -> 10.4 | 35.1 -> 96.2 | 64.4 -> **153.2** (+138 %) | 3.97 -> 1.67 |\n| 4 | 16 | 704 -> 689 | 34.5 -> 12.6 | 28.9 -> 79.3 | 100.5 -> **225.5** (+124 %) | 5.09 -> 2.28 |\n| 8 | 32 | 1,225 -> 1,239 | 47.8 -> 17.4 | 20.9 -> 57.6 | 140.0 -> **299.7** (+114 %) | 7.30 -> 3.43 |\n| 16 | 64 | 2,264 -> 2,288 | 75.3 -> 24.7 | 13.3 -> 40.5 | 172.8 -> **378.2** (+119 %) | 11.85 -> 5.40 |\n| 32 | 128 | 4,315 -> 4,226 | 84.8 -> 40.6 | 11.8 -> 24.6 | 270.8 -> **437.8** (+62 %) | 15.07 -> 9.36 |\n\nConcurrency, 1024 input / 256 output tokens: v1 -> current\n\n| Users | N | TTFT (ms) | TPOT (ms) | Decode tok/s/user | Total out tok/s | E2EL (s) |\n| ---: | ---: | ---: | ---: | ---: | ---: | ---: |\n| 2 | 8 | 642 -> 623 | 28.0 -> 10.1 | 35.7 -> 99.0 | 65.7 -> **159.5** (+143 %) | 7.78 -> 3.21 |\n| 4 | 16 | 1,286 -> 1,237 | 34.5 -> 12.7 | 29.0 -> 78.9 | 101.7 -> **232.6** (+129 %) | 10.07 -> 4.42 |\n| 8 | 32 | 2,358 -> 2,288 | 47.8 -> 17.6 | 20.9 -> 56.9 | 140.6 -> **303.5** (+116 %) | 14.56 -> 6.75 |\n| 16 | 64 | 4,498 -> 4,408 | 75.6 -> 24.2 | 13.2 -> 41.2 | 172.2 -> **385.6** (+124 %) | 23.77 -> 10.61 |\n| 32 | 128 | 8,802 -> 8,628 | 84.0 -> 39.3 | 11.9 -> 25.4 | 271.1 -> **436.4** (+61 %) | 30.16 -> 18.69 |\n\n#### p150\n\nOne die keeps the v1 decode path (the fused MoE kernel needs at most two up|gate columns per DRAM bank, i.e. two or more dies, and the replicated boundary needs a mesh): p150 decode is the v1 MoE plus the fused router top-k on 1-row steps, MTP on, and the tables below show parity. The single-user random-token latency table is left out: with adaptive MTP on random tokens it does not repeat on one die (15.9, 52.4 and 27.0 ms TPOT at 128/128 in three runs of v1); the v1 card's p150 latency rows still describe this bundle.\n\nReal text, single user (12 chat prompts x 384 output tokens, temperature 0; median per prompt): 42.6 -> **44.1 tok/s** (+4 %).\n\nConcurrency, 128 input / 128 output tokens: v1 -> current\n\n| Users | N | TTFT (ms) | TPOT (ms) | Decode tok/s/user | Total out tok/s | E2EL (s) |\n| ---: | ---: | ---: | ---: | ---: | ---: | ---: |\n| 2 | 8 | 412 -> 414 | 36.8 -> 36.8 | 27.2 -> 27.2 | 50.4 -> **50.4** (+0 %) | 5.08 -> 5.08 |\n| 4 | 16 | 814 -> 816 | 45.6 -> 45.6 | 21.9 -> 21.9 | 77.4 -> **77.4** (-0 %) | 6.61 -> 6.61 |\n| 8 | 32 | 1,455 -> 1,445 | 63.9 -> 63.9 | 15.6 -> 15.6 | 106.9 -> **107.0** (+0 %) | 9.58 -> 9.57 |\n| 16 | 64 | 2,715 -> 2,711 | 100.4 -> 100.4 | 10.0 -> 10.0 | 132.4 -> **132.4** (+0 %) | 15.47 -> 15.47 |\n| 32 | 128 | 5,244 -> 5,219 | 120.2 -> 120.2 | 8.3 -> 8.3 | 200.0 -> **200.0** (+0 %) | 20.51 -> 20.47 |\n\nConcurrency, 1024 input / 256 output tokens: v1 -> current\n\n| Users | N | TTFT (ms) | TPOT (ms) | Decode tok/s/user | Total out tok/s | E2EL (s) |\n| ---: | ---: | ---: | ---: | ---: | ---: | ---: |\n| 2 | 8 | 829 -> 829 | 36.3 -> 36.3 | 27.5 -> 27.5 | 50.7 -> **50.7** (+0 %) | 10.09 -> 10.09 |\n| 4 | 16 | 1,658 -> 1,642 | 45.6 -> 45.6 | 21.9 -> 21.9 | 77.1 -> **77.2** (+0 %) | 13.29 -> 13.27 |\n| 8 | 32 | 3,108 -> 3,108 | 64.1 -> 64.1 | 15.6 -> 15.6 | 105.3 -> **105.3** (+0 %) | 19.46 -> 19.46 |\n| 16 | 64 | 6,033 -> 6,032 | 100.9 -> 100.9 | 9.9 -> 9.9 | 128.9 -> **128.9** (-0 %) | 31.77 -> 31.77 |\n| 32 | 128 | 11,885 -> 11,879 | 119.4 -> 118.3 | 8.4 -> 8.4 | 193.0 -> **194.7** (+1 %) | 42.33 -> 42.06 |\n\n### Prefix caching\nMeasured 2026-10-28 on p300x2, one user, temperature 0: the same build served with caching off and on. TTFT of a\nprompt (Python source from this repo) sent a second time, and of each turn of a chat that opens with an 8k-token code\ndocument and resends the conversation so far (as every chat client does):\n\n| Request | caching off | caching on |\n| --- | ---: | ---: |\n| 7,652-token prompt, sent again | 1.30 s | 0.37 s (3.5x) |\n| 24,224-token prompt, sent again | 4.02 s | 0.42 s (9.6x) |\n| chat turns 2-6 (7.9k-8.5k tokens) | 1.29-1.48 s | 0.21-0.51 s |\n| chat with thinking on, turns 2-4 (4.1k tokens) | 0.75-0.76 s | 0.14-0.44 s |\n\nPrompts seen for the first time are unchanged: no-hit rows at 128 / 2,048 / 8,192 input tokens (1 user) and 128/128 at\n32 users, TTFT and TPOT within 1 % (1 user 128/128: TTFT 120 / 120 ms, TPOT 4.55 / 4.58 ms). Two dies (p300): a 27,176-token prompt sent again 5.78 -> 0.37 s, chat turns 2-4 0.45-0.55 s (turn 1 1.62 s). Greedy output with caching on matched caching off on every resent prompt and on\n9 of 10 chat turns; the one that differs diverges at a near-tie token 71 characters in (the reused KV blocks were computed in a different prefill pass).\n", "limitations": "- MTP speculative decoding is off on the multi-die profiles: the MTP verify pass still runs the unfused rows path, and\n with it on, single-user real-text decode is slower than the fused plain step (four dies: 86 vs 218 tok/s). A fused\n multi-token verify is the way back to MTP.\n- Two dies (`p300`, `p150x2`) run without the fabric layer boundary: its persistent receive buffers do not fit next to\n the two-die prefill's circular buffers.\n- One die (`p150`, `p100`) keeps the v1 MoE: the fused kernels read 1 or 2 up|gate columns per DRAM bank and\n expert, one die holds 4 (and p100 has 7 banks).\n- Prefix caching resumes only at 2,048-token boundaries: a prompt sharing fewer than 2,048 tokens with an earlier one\n gets no benefit, and the shared tokens past the last boundary are prefilled again. Requests with images are not\n cached. The one-die profiles (`p150`, `p100`) keep MTP on and serve without prefix caching.\n- The p150x4 and p150x2 profiles run the same meshes as p300x2 / p300 but were not run on p150 boards.\n- The device sampler supports `top_k` up to 32.\n- Benchmark evals ran on subsets (40 / 100 / 50 questions, plus the full 198-question GPQA-Diamond for this build) on the\n p300 profile only.\n", "risks": null, "licensing": null, "related": "The v1 bundle this builds on: [raahemnabeel/ornith-1.5-35b-a3b-v1](https://huggingface.co/raahemnabeel/ornith-1.5-35b-a3b-v1). Engineering log with every measurement: `models/autoports/ornith_ai_ornith_1_5_35b_a3b/fusion/HANDOFF.md` on the `ornith_fusion` branch of github.com/rnabeelTT/tt-metal.\n", "license": null, "pipeline_tag": null, "base_model": null } } }