"""Score a System One request with Drex Max (text and/or images). python inference.py --request request.json python inference.py --model /path/to/checkpoint --8bit """ import argparse import json import sys from pathlib import Path HERE = Path(__file__).resolve().parent sys.path.insert(0, str(HERE / "code")) from drex_max.model import DecisionModel, answer # noqa: E402 def main() -> None: p = argparse.ArgumentParser() p.add_argument("--model", default=str(HERE), help="Local checkpoint directory (default: this folder)") p.add_argument("--request", default=str(HERE / "request.json"), help="JSON file with one System One request") p.add_argument("--max-length", type=int, default=8192, help="Longest request in tokens") p.add_argument("--8bit", dest="load_in_8bit", action="store_true", help="Load the backbone in 8-bit (CUDA)") p.add_argument("--device", default=None, help="Defaults to cuda when available") a = p.parse_args() req = json.loads(Path(a.request).read_text()) model = DecisionModel(a.model, device=a.device, load_in_8bit=a.load_in_8bit) images = req.get("images") or [] out = {} for name, q in req["questions"].items(): row = {"state": req["state"], "question": q} if images: row["images"] = images out[name] = answer(q, model.predict([row], max_length=a.max_length)[0]) print(json.dumps({"answers": out}, indent=2)) if __name__ == "__main__": main()