# SPDX-License-Identifier: Apache-2.0
# tt-model-manager container manifest (schema 5.1) for SAM3 on a Blackhole p150a.
#
# Run every tt-model command from THIS directory (`root: code` below is CWD-relative):
#   tt-model package --container tt-model.yaml --out <outdir>    # amd64 + Docker>=25, hours cold
#   tt-model serve   <outdir>/sam3-p150/tt_kernel_manifest.json
#   tt-model push    <outdir>/sam3-p150 --publish
schema: "5.1"

repo: changh95/sam3-p150
name: sam3-p150

# A POINTER, never baked into the image. facebook/sam3 is GATED (Meta SAM License): the
# host needs `hf auth login` with an approved account. Only the 3.45 GB sam3.pt (+ the
# 25 KB config.json) is pulled; the app loads sam3.pt via checkpoint_path so this pin is
# what actually runs (see serve.env.TT_WEIGHTS_REVISION - the launcher passes no revision).
weights:
  repo: facebook/sam3
  revision: 3c879f39826c281e95690f02c7821c4de09afae7
  allow_patterns: ["sam3.pt", "config.json"]

kind: tt-dit-server
arch: blackhole

source:
  tt_metal: /home/deepgadget/experiments/gbp-tt/tt-metal   # v0.78.0-dev20260820-25, torch 2.11.0
  # The port imports nothing from tt-metal's models/ tree; the schema needs >= 1 entry.
  code:
    - models/common/lightweightmodule.py
  extra_code:
    - root: code
      paths:
        - tt_sam3          # the port (tt/, reference/, tests/, server/, assets/bpe vocab)
        - scripts
        - conftest.py
  ubuntu: "22.04"
  python: "3.12"

runtime:
  app: tt_sam3.server.app:app
  mesh_shape_env: TT_MESH_SHAPE
  # On top of the kind's fastapi/uvicorn/pydantic/pillow and the auto-pinned torch==2.11.0+cpu.
  # sam3 0.1.4 pulls triton (sam3/model/edt.py), psutil (sam3_video_predictor) and
  # torchvision.ops (decoder RoIAlign) at import, so they are listed explicitly; torchvision
  # and triton are pinned to the versions that match torch 2.11.0.
  packages:
    - "numpy>=1.26,<2"
    - "sam3==0.1.4"
    - "torchvision==0.26.0"
    - "timm>=1.0.17"
    - huggingface_hub
    - iopath
    - psutil
    - "triton==3.6.0"
    - einops
    - pycocotools
    - ftfy
    - regex
    - tqdm
    - typing_extensions

serve:
  port: 20000
  hardware: p150
  mesh_device: P150
  env:
    TT_WEIGHTS_REVISION: "3c879f39826c281e95690f02c7821c4de09afae7"
    TT_METAL_VISIBLE_DEVICES: "0"
    SAM3_BPE_PATH: "/opt/tt-metal/tt_sam3/assets/bpe_simple_vocab_16e6.txt.gz"
    SAM3_TORCH_COMPILE: "0"     # the runtime image has no C++ compiler for inductor
    SAM3_WARMUP_RUNS: "2"
    # Fused device paths (megakernel pass, device-validated 2026-09-13, DEVICE_VALIDATION.md "Results"):
    # encoder as one metal trace + fused ops, window-order residency, minimal_matmul / fused-residual
    # matmuls, on-device patch embed. "1" is also the code default; "0" = the legacy device graph.
    TT_FUSED: "1"

# Build-time assertions, run INSIDE the finished image as uid 1000 with no device/weights.
verify:
  - "import tt_sam3.server.app as a; assert a.app"
  - "import sam3.model_builder, sam3.model.data_misc, sam3.model.vitdet, sam3.model.encoder, sam3.model.box_ops"
  - "import triton, psutil, torchvision.ops, timm, iopath, huggingface_hub, einops, pycocotools, ftfy, regex, tqdm"
  - "import torchvision; assert torchvision.__version__.startswith('0.26.0'), torchvision.__version__"
  - "import numpy; assert numpy.__version__.startswith('1.'), numpy.__version__"
  - "from pathlib import Path; assert Path('/opt/tt-metal/tt_sam3/assets/bpe_simple_vocab_16e6.txt.gz').is_file()"
  - "from sam3.model.tokenizer_ve import SimpleTokenizer; t = SimpleTokenizer(bpe_path='/opt/tt-metal/tt_sam3/assets/bpe_simple_vocab_16e6.txt.gz'); assert t.encode('a red car')"
  - "import tt_sam3.tt.ttnn_sam3_vitdet, tt_sam3.tt.ttnn_sam3_encoder, tt_sam3.tt.ttnn_sam3_model"

card:
  description: >
    Meta's SAM3 (Segment Anything with Concepts, 840.5M params) for text-prompted open-vocabulary instance segmentation, with the ViT-H/14 backbone and fusion encoder running on one Tenstorrent Blackhole p150a via tt-nn: image plus noun phrases in, per-instance masks, boxes and scores out.

    Weights: [facebook/sam3](https://huggingface.co/facebook/sam3) ·
    Paper: [arXiv:2511.16719](https://arxiv.org/abs/2511.16719) ·
    Upstream code: [facebookresearch/sam3](https://github.com/facebookresearch/sam3) ·
    Port: [changh95/tt-sam3](https://github.com/changh95/tt-sam3)
  quickstart: |
    ### Run with tt-cli

    ```bash
    tt serve changh95/sam3-p150
    printf '{"image":"%s","prompts":["car"]}' "$(base64 -w0 media/source_1.png)" > req.json
    curl -s localhost:20000/predict -H 'Content-Type: application/json' -d @req.json
    tt model stop changh95/sam3-p150
    ```

    - `POST /predict`: `image` (base64 PNG/JPEG), `prompts` (1–4 noun phrases); optional `threshold` (0.5), `max_instances` (50), `mask_format` (`png` | `rle` | `none`).
    - `GET /health`, `GET /info`.

    ### Response

    ```json
    {"width": 1920, "height": 1080, "input_size": 1008, "threshold": 0.5, "mask_format": "png",
     "results": [{"prompt": "car", "presence": 0.995, "num_instances": 3, "instances": [
        {"score": 0.9727, "box_xyxy": [545.8, 447.9, 1159.1, 854.4], "mask_area": 175127, "mask_png": "..."},
        {"score": 0.818, "...": "..."}, {"score": 0.788, "...": "..."}]}],
     "timing_ms": {"decode": 26.5, "preprocess": 7.5, "forward": 489.8, "forward_per_prompt": 489.8, "postprocess": 20.6, "total": 544.4}}
    ```

    - One `results` entry per prompt; `instances` are sorted by score, `box_xyxy` is `[x0, y0, x1, y1]` in original image pixels.
    - `mask_png` is a base64 8-bit 0/255 PNG at the original H×W; `rle` gives COCO uncompressed column-major `{size, counts}`, `none` omits masks.

    ### Demo

    Synthetic street scene, prompt `"car"`.

    | Input (`media/source_1.png`) | Masks on p150a (`media/target_1_ttnn.png`) |
    |:---:|:---:|
    | ![](media/source_1.png) | ![](media/target_1_ttnn.png) |

    ### Accuracy and speed

    | Metric | Value |
    |---|---:|
    | Top-mask IoU vs torch CPU fp32 reference (demo image, `"car"`) | 0.999 (top score 0.973 vs 0.972) |
    | End-to-end mask PCC vs torch fp32 (demo image, 200 queries) · ViT backbone PCC (demo activations) | 0.984 · 0.993 |
    | Inference, served over HTTP (warm, batch 1, one prompt, 1920×1080; medians of 4×50 requests over two sessions) | 472–490 ms forward (~480) · ~535 ms end-to-end (~1.9 req/s) |
    | Same, legacy device graph (`TT_FUSED=0`) | 540–578 ms forward (~560) · ~615 ms end-to-end |
    | Same forward on an RTX 5090 (same host, port's torch reference, eager PyTorch, batch 1, incl. H2D/D2H; bf16 / fp16 autocast) | full network 55.3 / 55.7 ms → GPU 8.8× faster than the p150a's 487 ms `forward`, of which ~380 ms is the CPU neck/decoder/segmentation head; like-for-like device islands (backbone + fusion encoder, p150a 106 ms): GPU bf16 39.7 ms (2.7×), fp32-strict 123 ms (p150a 1.16× faster); fp32-strict full network 149 ms (3.3×); best `torch.compile` 48.0 ms |

    ### Caveats

    - Every image is squashed to 1008×1008; one image per request, batch 1; text prompts only (no box/point prompts), and each of the 1–4 prompts costs a full forward.
    - `facebook/sam3` is gated under the Meta SAM License: request access and `hf auth login` on the host before `serve` (3.45 GB `sam3.pt`).
    - Only the ViT backbone and fusion encoder run on device (bfloat8_b/bf16); the FPN neck, text encoder, decoder and mask head run eagerly on the host CPU and dominate served latency.
    - The device graphs run the fused paths by default (`TT_FUSED=1`: encoder as one metal trace with fused ops, window-order residency, `minimal_matmul` / fused-residual matmuls, on-device patch embed); `TT_FUSED=0` restores the legacy graph (same masks within IoU 0.997+, ~85 ms slower).
    - Not an OpenAI-compatible API; `GET /v1/models` is a stub so the tt-model ready card does not 404.
    - Validated on tt-metal `v0.78.0-dev20260820` (main `8b98410e730`), single p150a only.
    - GPU comparison: the p150a's `forward` time is ~78 % CPU (neck, transformer decoder, segmentation head); on the device islands the p150a is 1.16× faster than fp32-strict GPU and 2.7× slower than GPU bf16. RTX 5090 rows (2026-09-14): same host, the port's own torch reference (same weights) run eagerly in PyTorch 2.11 cu128 with fp32 weights + autocast unless stated, no TensorRT; medians of 50 iterations after warm-up, H2D/D2H included; the p150a rows are the served bf16 fused path incl. upload/readback. p150a power was not measured, so no efficiency comparison is made. Full table: [`GPU_COMPARISON.md`](GPU_COMPARISON.md).

    ### Licensing

    - Weights: [facebook/sam3](https://huggingface.co/facebook/sam3), [Meta SAM License](https://github.com/facebookresearch/sam3/blob/main/LICENSE) (gated, not OSI).
    - Port and serving code (`code/`): from [changh95/tt-sam3](https://github.com/changh95/tt-sam3), distributed under the same SAM License terms; `code/tt_sam3/assets/bpe_simple_vocab_16e6.txt.gz` is the MIT-licensed CLIP BPE vocabulary vendored from the upstream `sam3` repo.