"""Assemble and verify MTP45 sidecars and retain only its unchanged carrier tensors.""" import hashlib import json import math import os import shutil import time from pathlib import Path import torch from safetensors import safe_open from safetensors.torch import load_file, save_file from build_checkpoint import build_rank, sha256_file, tensor_sha256, CODEC from p8_coupled_scales import SCALE_NAMES, validate_coupled_component out = Path(os.environ["CV2_OUT"]) root = Path(os.environ["CV2_ROOT"]) while not (out / "status/mtp-encoded.ready").exists(): time.sleep(10) inputs = out / "mtp-inputs" input_info = json.loads((inputs / "inputs.json").read_text()) core_sha = sha256_file(root / "code/codecv2/encode_layer_v2.py") wrapper_sha = sha256_file(root / "ops/encode-mtp-v2.py") chunks, metrics = {}, [] for path in sorted((out / "mtp-chunks").glob("*.safetensors")): with safe_open(str(path), framework="pt", device="cpu") as src: meta = src.metadata() assert meta["schema"] == "trellismx-codec-v2-mtp-chunk.v1" assert meta["layer"] == "45" and meta["bits"] == "4" assert meta["encoder_sha256"] == core_sha and meta["mtp_wrapper_sha256"] == wrapper_sha assert meta["mtp_inputs_sha256"] == sha256_file(inputs / "inputs.json") assert float(meta["beta"]) == 0.5 and float(meta["percdamp"]) == 0.3 start, end = map(int, meta["expert_range"].split(":")) for expert in range(start, end): assert expert not in chunks chunks[expert] = {p: (src.get_tensor(f"{expert}.{p}.trellis"), src.get_tensor(f"{expert}.{p}.scale_ue8m0")) for p in ("gate", "up", "down")} metrics.extend(json.loads(path.with_suffix(".metrics.json").read_text())["rows"]) assert set(chunks) == set(range(288)) assert {r["expert"] for r in metrics} == set(range(288)) and len(metrics) == 288 assert all(math.isfinite(v) for row in metrics for v in row.values()) dest = out / "mtp-sidecars" (dest / "sidecars").mkdir(parents=True, exist_ok=True) (dest / "design").mkdir(exist_ok=True) design = {"schema": "trellismx-codecv2-mtp-design.v1", "layer": 45, "bits": 4, "encoder_core_sha256": core_sha, "mtp_wrapper_sha256": wrapper_sha, "beta": 0.5, "percdamp": 0.3, "coupled_sign_draw": 0, "scale_policy": input_info["scale_policy"], "scale_sha256": input_info["scale_sha256"], "capture_sha256": input_info["source_capture_sha256"], "calibration": "64 fit windows; 2047 valid MTP rows each; all routed plus every fourth all-token row", "main_model_scales_unchanged": True} design_path = dest / "design/design-2.json" design_path.write_text(json.dumps(design, indent=2) + "\n") design_sha = sha256_file(design_path) scale = load_file(str(inputs / "coupled-scales.safetensors")) transform_sha = sha256_file(out / "ckpt-B/design/transform.json") records, ranks = [], [] for rank in range(4): template = out / "ckpt-B/sidecars" / f"p8-layer-044-tp4-rank-{rank}.safetensors" tensors, meta = build_rank(template, chunks, rank, 4, design_sha) sl = slice(rank * 512, (rank + 1) * 512) tensors["gate_up_suh_fp16"] = scale["gate_up_suh"] tensors["down_svh_fp16"] = scale["down_svh"] tensors["intermediate_scales_fp16"] = torch.cat([scale[key][:, sl] for key in ("gate_svh", "up_svh", "down_suh")], dim=1).contiguous() meta.pop("exl3_scale_source_sha256", None) meta.update(layer="45", mtp_scale_source_sha256=input_info["scale_sha256"], mtp_scale_policy=input_info["scale_policy"], mtp_capture_sha256=input_info["source_capture_sha256"]) for name in SCALE_NAMES: meta["sha256_" + name] = tensor_sha256(tensors[name]) validate_coupled_component(meta, {name: tensors[name] for name in SCALE_NAMES}, layer=45, rank=rank, experts=288, hidden=4096, intermediate=512, expected_transform_sha256=transform_sha) name = f"sidecars/p8-layer-045-tp4-rank-{rank}.safetensors" path = dest / name save_file(tensors, str(path), metadata=meta) records.append({"layer": 45, "rank": rank, "bits": 4, "path": name, "bytes": path.stat().st_size, "sha256": sha256_file(path), "source_design_sha256": design_sha}) # Read the actual saved artifact for the full stream reassembly check. with safe_open(str(path), framework="pt", device="cpu") as src: ranks.append({key: src.get_tensor(key) for key in CODEC}) print(json.dumps({"saved": name, "bytes": path.stat().st_size}), flush=True) mismatches = 0 for expert in range(288): actual = { "gate": (torch.cat([t["w13_trellis"][0, expert] for t in ranks], 1), torch.cat([t["w13_scale_ue8m0"][expert, 512:] for t in ranks], 0)), "up": (torch.cat([t["w13_trellis"][1, expert] for t in ranks], 1), torch.cat([t["w13_scale_ue8m0"][expert, :512] for t in ranks], 0)), "down": (torch.cat([t["w2_trellis"][expert] for t in ranks], 0), torch.cat([t["w2_scale_ue8m0"][expert] for t in ranks], 1))} mismatches += not all(torch.equal(a, b) for key in actual for a, b in zip(actual[key], chunks[expert][key])) assert mismatches == 0 report = {"status": "PASS", "layer": 45, "experts": 288, "sidecars": 4, "expert_stream_mismatches": mismatches, "codec_closure": "PASS: all 864 projection encodes", "coupled_component_validation": "PASS", "routed_sidecar_bytes": sum(r["bytes"] for r in records), "mean_tw_nmse": {key: sum(r[key] for r in metrics) / 288 for key in ("gate_tw_nmse", "up_tw_nmse", "down_tw_nmse")}, "scale_policy": input_info["scale_policy"], "native_execution_tested": False} (dest / "MTP-VERIFY-REPORT.json").write_text(json.dumps(report, indent=2) + "\n") (dest / "mtp-manifest.json").write_text(json.dumps({"files": records, "design_sha256": design_sha}, indent=2) + "\n") # Remove only replaced MTP expert tensors; retain all 25 other MTP tensors bit for bit. old_carrier = out / "native-carrier" carrier = out / "native-carrier-mtp" carrier.mkdir(exist_ok=True) base_receipt = json.loads((out / "native-carrier-receipt.json").read_text()) kept_tensors = {} for record in base_receipt["files"]: name = record["path"] if name.startswith("model-mtp-"): with safe_open(str(old_carrier / name), framework="pt", device="cpu") as src: for key in src.keys(): if ".layers.45.mlp.experts." not in key: assert key not in kept_tensors kept_tensors[key] = src.get_tensor(key) elif name != "model.safetensors.index.json": if not (carrier / name).exists(): os.link(old_carrier / name, carrier / name) assert len(kept_tensors) == 25 mtp_file = "model-mtp-nonexpert.safetensors" save_file(kept_tensors, str(carrier / mtp_file)) with safe_open(str(carrier / mtp_file), framework="pt", device="cpu") as check: assert set(check.keys()) == set(kept_tensors) assert all(torch.equal(check.get_tensor(key), value) for key, value in kept_tensors.items()) index = json.loads((old_carrier / "model.safetensors.index.json").read_text()) weight_map = {k: v for k, v in index["weight_map"].items() if ".layers.45.mlp.experts." not in k} for key in kept_tensors: weight_map[key] = mtp_file index["weight_map"] = weight_map index["metadata"]["total_size"] -= 7247757312 + 226492416 (carrier / "model.safetensors.index.json").write_text(json.dumps(index, indent=2) + "\n") known = {r["path"]: r for r in base_receipt["files"]} new_records = [] for path in sorted(carrier.iterdir()): assert path.is_file() digest = known[path.name]["sha256"] if path.name in known and path.name != "model.safetensors.index.json" else sha256_file(path) new_records.append({"path": path.name, "bytes": path.stat().st_size, "sha256": digest}) receipt = {**base_receipt, "files": new_records, "bytes": sum(r["bytes"] for r in new_records), "indexed_tensors": len(weight_map), "mtp_expert_tensors_replaced": 1728, "mtp_nonexpert_tensors_preserved": 25, "mtp_nonexpert_tensor_equality": "PASS", "derivation": "MTP45 FP8 experts and scales replaced by Trellis sidecars; other tensors unchanged"} (out / "native-carrier-mtp-receipt.json").write_text(json.dumps(receipt, indent=2) + "\n") (out / "status/mtp.ready").write_text(design_sha + "\n") print(json.dumps(report), flush=True)