@echo off
rem Example: Sakura MiMo-V2.6-Flash-MOPD P160 on a 64 GB Strix Halo (llama.cpp Vulkan build, see runtime\patches\BUILD.md).
rem Lives in runtime\ of the model repo; the GGUF is one folder up. Put llama-server.exe on PATH or next to this file.
rem   start_server_example.bat          fast mode: 16k context, ~9-10.5 tok/s with the desktop in use
rem   start_server_example.bat long     32k context (multi-step chats / agents), ~5.6-7.7 tok/s
rem ~35 GiB GPU; the experts of layers 14-33 (~20 GiB) are computed by the CPU from RAM.
rem -ub 512 is the best overall setting (decode 9.6 tok/s). -ub 1024/2048 make prefill 1.8-2.8x faster but nearly
rem halve decode (5.3-5.9 tok/s): the bigger GPU buffers spill into shared RAM and take it from the CPU-side experts.
rem Use a larger -ub only when the prompt is much longer than the answer (e.g. reading large files once).
setlocal
set PATH=%~dp0;%PATH%
set CTX=16384
set LLAMA_EXPERT_LOCK_LIST=
rem Long mode: optional (expert-lock patch only) - pin the hottest 30 %% of the CPU-side experts (~6 GiB) in RAM.
if /I "%~1"=="long" (
  set CTX=32768
  set LLAMA_EXPERT_LOCK_LIST=%~dp0hotlock_L14-33.txt
)
cd /d "%~dp0.."
llama-server.exe -m MiMo-V2.6-Flash-MOPD-P160-OnPolicy-v3la-IQ2_XS.gguf -ngl 99 --jinja -c %CTX% -fit off --expert-streaming -b 2048 -ub 512 ^
  -ot "blk\.(1[4-9]|2[0-9]|3[0-3])\.ffn_(gate|up|down)_exps\.weight=CPU" ^
  --temp 1.0 --top-p 0.95 --host 127.0.0.1 --port 8080