{
"manifest_schema": "0.1.3",
"repo": "litert-community/Qwen3-4B-Thinking-2507",
"generated": "2026-10-13",
"generator": "make_manifest.py",
"model": {
"display_name": "Qwen3-4B-Thinking-2507",
"base_model": "Qwen/Qwen3-4B-Thinking-2507",
"architecture": "qwen3-dense (36 layers)",
"parameters_b": 4.0,
"license": "apache-2.0",
"session_defaults": {
"max_output_tokens_min": 2048,
"notes": "Reasoning model: it emits a ... chain before the answer. At a short output budget it is cut off mid-thought and never answers."
},
"context_length": 4096,
"capabilities": {
"vision": false,
"audio": false,
"thinking": {
"declared": true,
"channel": {
"start": "\n",
"end": "\n"
},
"control": "always"
},
"channels": [
{
"name": "thought",
"start": "\n",
"end": "\n"
}
]
}
},
"variants": [
{
"file": "Qwen3_4b_thinking_dynamic_wi4b32_afp32.litertlm",
"sha256": "110098cfd2766cc46a5138bb03b44f0350b7608052cb48321a47b9f0ebdb89d9",
"size_bytes": 2274193168,
"sections": [
{
"type": "LlmMetadataProto",
"size_bytes": 3707
},
{
"type": "ExecutorMetadataProto",
"size_bytes": 4967
},
{
"type": "HF_Tokenizer_Zlib",
"size_bytes": 2142162
},
{
"type": "TFLiteModel",
"size_bytes": 2271997712,
"model_type": "tf_lite_prefill_decode"
}
],
"quantization": "dynamic int4 block-32 weights, fp32 activations; GPU graph optimizations (RoPE/fused-QKV/fused-GateUp composites), static prefill allocation",
"backends": [
"cpu",
"gpu"
],
"measured": [
{
"device": "Galaxy S26 (SM-S942Q, Qualcomm SM8850, Adreno)",
"backend": "gpu",
"runtime": "litert_lm_advanced_main v0.16.0, LiteRT CL delegate",
"prompt_tokens": 202,
"decode_tokens": 3894,
"prefill_tps": 225.3,
"decode_tps": 15.29,
"ttft_s": 0.96,
"load_s": 11.0,
"peak_memory_mb": 1627,
"runs": 1,
"date": "2026-08-24",
"source": "S4 GPU gate, 205-token benchmark, single run; full delegation (3764 ops across 3 subgraphs); generation gate answered the probe correctly. No same-device CPU control was taken, so this is a GPU speed, not a measured win over CPU."
}
],
"default_backend": "cpu",
"recommended": [
{
"platform": "android",
"device_class": "flagship",
"backend": "gpu",
"reason": "the Android path verified on a Galaxy S26 (S4 gate 2026-08-24): full delegation (3764 ops across 3 subgraphs) and a correct answer to the probe; 15.3 tok/s decode, engine init 11.0 s, peak 1627 MB; the repo's other S26-verified file (model.litertlm) decodes 9.0 tok/s on the same handset. No same-device CPU control was taken, so this is the verified choice for the class, not a measured win over CPU"
}
],
"min_runtime_version": "0.15.0",
"known_issues": [
"Block-32 int4 corrupts GPU output beyond iPhone: macOS Metal also degenerates (question-echo loops, truncated reasoning; measured 2026-08-31 before and after the metadata repair, pre-existing). Use this file on CPU; it delegated and generated on the Galaxy S26 Android GPU gate (2026-08-24). The block-128 file is the GPU-safe choice",
"Thinking cannot be switched off on litert-lm 0.18.0: ThinkingConfig(enable_thinking=False) has no effect on this file (with a 400-token cap the thought channel used the whole cap and no answer text appeared; measured 2026-10-13); set thinking_token_budget (300 answered 17 × 23 under a 700-token cap) and a cap above it."
]
},
{
"file": "model.litertlm",
"sha256": "8a8a6a419a5570b3f4814d358fb795c0dcb6931206c7dae29480f47564c3f04d",
"size_bytes": 2477192112,
"sections": [
{
"type": "LlmMetadataProto",
"size_bytes": 191
},
{
"type": "HF_Tokenizer_Zlib",
"size_bytes": 2142162
},
{
"type": "TFLiteModel",
"size_bytes": 2082392624,
"model_type": "tf_lite_prefill_decode"
},
{
"type": "TFLiteModel",
"size_bytes": 392606640,
"model_type": "tf_lite_embedder"
}
],
"quantization": "int4 block-128 symmetric weights + OCTAV optimal clipping; int8 embeddings (externalized section)",
"backends": [
"cpu",
"gpu"
],
"default_backend": "gpu",
"min_runtime_version": "0.15.0",
"recommended": [
{
"platform": "android",
"device_class": "flagship",
"backend": "gpu",
"reason": "Galaxy S26 (SM-S942Q, Android 16), litert_lm_advanced_main from a LiteRT-LM main build of 2026-09-18: prefill_128 (1707/1707 nodes), prefill_16 (1707/1707) and decode (1631/1631) run fully on the OpenCL GPU delegate, one partition each, and the file answered 5 of 5 gate prompts correctly. At 1024 prompt / 256 decode tokens the GPU decodes 13.9-20.2 tok/s and prefills 130-264 tok/s (3 iterations in one process, cold to warm), against 7.2-7.4 and 47-65 tok/s on CPU (XNNPACK, 4 threads, weight cache, 2 iterations) on the same phone; peak memory (VmHWM) is 1.17 GB on GPU against 4.08 GB on CPU"
},
{
"platform": "macos",
"backend": "gpu",
"reason": "Mac Studio M4 Max, litert-lm 0.17.1 benchmark --cache no, 256 prompt / 256 decode tokens: GPU (WebGPU on Metal, fp16 activations) decodes 92.1 tok/s and prefills 1100 tok/s, against 18.4 and 113 tok/s on CPU (XNNPACK) for the same file (median of 3 and 2 processes); TTFT for a 16-token prompt is 0.037 s on GPU against 0.478 s on CPU"
},
{
"platform": "ios",
"backend": "gpu",
"reason": "block-128 is the iPhone-safe int4 recipe for this family; prefer this file over the block-32 variant on iPhone GPU. This file's iPhone run (iPhone 17 Pro) was on the previous model.litertlm, before the 2026-10-06 GPU graph rewrite, and was not repeated on the current file"
}
],
"requirements": {
"platform_notes": [
"4B-class file (2.48 GB): budget device RAM accordingly; on Android import via AI Edge Gallery and enable the GPU toggle at import",
"iPhone 17 Pro Metal decode was observed at ~14 tok/s in an earlier on-device note; its run log is not retained, so it is not listed as a measured row"
]
},
"measured": [
{
"device": "Apple M4 Max",
"os": "macOS 27.0",
"backend": "gpu",
"runtime": "litert-lm 0.17.1 (pip CLI benchmark), GPU fp16 activations (WebGPU on Metal)",
"prompt_tokens": 256,
"decode_tokens": 256,
"prefill_tps": 1099.8,
"decode_tps": 92.1,
"ttft_s": 0.244,
"load_s": 2.17,
"cache": "no",
"runs": 3,
"date": "2026-10-05",
"source": "GPU-graph re-ship card bench: 3 separate benchmark processes, median; the previous file measured in the same window: prefill 984.0 tok/s, decode 66.6 tok/s, TTFT 0.282 s"
},
{
"device": "Apple M4 Max",
"os": "macOS 27.0",
"backend": "cpu",
"runtime": "litert-lm 0.17.1 (pip CLI benchmark) (XNNPACK)",
"prompt_tokens": 256,
"decode_tokens": 256,
"prefill_tps": 113.3,
"decode_tps": 18.4,
"ttft_s": 2.567,
"load_s": 2.11,
"cache": "no",
"runs": 2,
"date": "2026-10-05",
"source": "GPU-graph re-ship card bench: 2 separate benchmark processes, median; the previous file measured in the same window: prefill 113.0 tok/s, decode 18.5 tok/s, TTFT 2.570 s"
},
{
"device": "Apple M4 Max",
"os": "macOS 27.0",
"backend": "gpu",
"runtime": "litert-lm 0.17.1 (pip CLI benchmark), GPU fp16 activations (WebGPU on Metal)",
"prompt_tokens": 16,
"decode_tokens": 32,
"prefill_tps": 695.5,
"decode_tps": 89.7,
"ttft_s": 0.037,
"load_s": 2.16,
"cache": "no",
"runs": 3,
"date": "2026-10-05",
"source": "GPU-graph re-ship card bench (short-prompt cell, TTFT is the figure of interest): 3 separate benchmark processes, median; the previous file measured in the same window: prefill 122.4 tok/s, decode 64.7 tok/s, TTFT 0.152 s"
},
{
"device": "Apple M4 Max",
"os": "macOS 27.0",
"backend": "cpu",
"runtime": "litert-lm 0.17.1 (pip CLI benchmark) (XNNPACK)",
"prompt_tokens": 16,
"decode_tokens": 32,
"prefill_tps": 96.2,
"decode_tps": 16.0,
"ttft_s": 0.478,
"load_s": 2.09,
"cache": "no",
"runs": 2,
"date": "2026-10-05",
"source": "GPU-graph re-ship card bench (short-prompt cell, TTFT is the figure of interest): 2 separate benchmark processes, median; the previous file measured in the same window: prefill 14.2 tok/s, decode 15.6 tok/s, TTFT 1.444 s"
},
{
"device": "Galaxy S26 (SM-S942Q, Qualcomm SM8850, Adreno)",
"os": "Android 16",
"backend": "gpu",
"runtime": "litert_lm_advanced_main (LiteRT-LM main, 2026-09-18 build), LiteRT OpenCL delegate, fp16 activations",
"prompt_tokens": 1024,
"decode_tokens": 256,
"prefill_tps": "129.9-264.1",
"decode_tps": "13.87-20.23",
"ttft_s": "3.93-7.95",
"load_s": 7.92,
"max_num_tokens": 1280,
"cache": "no",
"runs": 3,
"date": "2026-10-05",
"source": "GPU-graph re-ship device leg (round 16, turn T2): 3 iterations in one process; the handset throttles after the first iteration, so the range spans cold to warm; cold engine init (--disable_cache=true)",
"peak_memory_mb": 1118.7
},
{
"device": "Galaxy S26 (SM-S942Q, Qualcomm SM8850, Adreno)",
"os": "Android 16",
"backend": "gpu",
"runtime": "litert_lm_advanced_main (LiteRT-LM main, 2026-09-18 build), LiteRT OpenCL delegate, fp16 activations",
"prompt_tokens": 16,
"decode_tokens": 32,
"prefill_tps": "165.2-175.6",
"decode_tps": "17.35-21.01",
"ttft_s": "0.14-0.15",
"load_s": 7.88,
"max_num_tokens": 1280,
"cache": "no",
"runs": 3,
"date": "2026-10-05",
"source": "GPU-graph re-ship device leg (round 16, turn T2): 3 iterations in one process; cold engine init (--disable_cache=true)",
"peak_memory_mb": 1115.5
},
{
"device": "Galaxy S26 (SM-S942Q, Qualcomm SM8850, Adreno)",
"os": "Android 16",
"backend": "cpu",
"runtime": "litert_lm_advanced_main (LiteRT-LM main, 2026-09-18 build), XNNPACK 4 threads, weight cache",
"prompt_tokens": 1024,
"decode_tokens": 256,
"prefill_tps": "46.6-65.5",
"decode_tps": "7.2-7.39",
"ttft_s": "15.78-22.13",
"load_s": 2.81,
"max_num_tokens": 1280,
"cache": "disk (XNNPACK weight cache)",
"runs": 2,
"date": "2026-10-05",
"source": "GPU-graph re-ship device leg (round 16, turn T2): 2 iterations in one process; init includes writing the weight cache",
"peak_memory_mb": 3893.6
},
{
"device": "Galaxy S26 (SM-S942Q, Qualcomm SM8850, Adreno)",
"os": "Android 16",
"backend": "cpu",
"runtime": "litert_lm_advanced_main (LiteRT-LM main, 2026-09-18 build), XNNPACK 4 threads, weight cache",
"prompt_tokens": 16,
"decode_tokens": 32,
"prefill_tps": "52.8-56.4",
"decode_tps": "7.66-8.78",
"ttft_s": "0.4-0.43",
"load_s": 2.82,
"max_num_tokens": 1280,
"cache": "disk (XNNPACK weight cache)",
"runs": 3,
"date": "2026-10-05",
"source": "GPU-graph re-ship device leg (round 16, turn T2): 3 iterations in one process; the first iteration writes the XNNPACK weight cache, so the values are iterations 2-3; init includes writing the weight cache",
"peak_memory_mb": 3502.3
}
],
"known_issues": [
"Pixel 8a (8 GB), measured on the previous file (before the 2026-10-06 GPU graph rewrite; not re-measured on the current file): the prefill graph delegated 1674/1674 nodes on OpenCL but execution OOM-rebooted the phone (Android GPU re-gate 2026-08-17, transfer verified byte-for-byte); use CPU or a 12 GB-class device for GPU",
"Thinking cannot be switched off on litert-lm 0.18.0: ThinkingConfig(enable_thinking=False) has no effect on this file (with a 400-token cap the thought channel used the whole cap and no answer text appeared; measured 2026-10-13); set thinking_token_budget (300 answered 17 × 23 under a 700-token cap) and a cap above it."
]
}
]
}