[
  {
    "name": "configuration",
    "url": "http://127.0.0.1:8000",
    "model": "google/gemma-4-12B-it",
    "contexts": [
      32768,
      131072,
      262016
    ],
    "tokens": 128,
    "native_max_len": 262144,
    "request_timeout_seconds": 3600,
    "cases": [
      {
        "input_tokens": 32768,
        "output_tokens": 128
      },
      {
        "input_tokens": 131072,
        "output_tokens": 128
      },
      {
        "input_tokens": 262016,
        "output_tokens": 128
      }
    ],
    "begin_key_token_region": [
      0,
      43
    ],
    "end_key_suffix_tokens": 65,
    "timing_basis": "Client first/last non-empty SSE text arrival; MTP may emit multiple tokens per event"
  },
  {
    "name": "short_before",
    "input_tokens": 25,
    "requested_output_tokens": 32,
    "requested_total_tokens": 57,
    "ttft_seconds": 0.09304460400016978,
    "decode_wall_seconds": 0.04858210598467849,
    "seconds": 0.18983703898265958,
    "decode_tokens_per_second": 144.08597276963692,
    "finish_reason": "stop",
    "sse_done": true,
    "response": {
      "text": "The capital of France is Paris.",
      "usage": {
        "prompt_tokens": 25,
        "total_tokens": 33,
        "completion_tokens": 8
      }
    },
    "expected_keys": {
      "known_answer": "Paris"
    },
    "retrieved_keys": {
      "known_answer": true
    },
    "accuracy": true
  },
  {
    "name": "layout_32768",
    "input_tokens": 32768,
    "begin_key_offset": 35,
    "middle_key_offset": 16021,
    "end_key_offset": 32707
  },
  {
    "name": "stream_32768_128",
    "input_tokens": 32768,
    "requested_output_tokens": 128,
    "requested_total_tokens": 32896,
    "ttft_seconds": 16.924654099013424,
    "decode_wall_seconds": 0.43388441897695884,
    "seconds": 17.35861225798726,
    "decode_tokens_per_second": 78.36188282623154,
    "finish_reason": "stop",
    "sse_done": true,
    "response": {
      "text": "CEDAR-7419-QUARTZ\nHARBOR-5096-VIOLET\nORBIT-2863-MARBLE",
      "usage": {
        "prompt_tokens": 32768,
        "total_tokens": 32803,
        "completion_tokens": 35
      }
    },
    "expected_keys": {
      "begin": "CEDAR-7419-QUARTZ",
      "middle": "HARBOR-5096-VIOLET",
      "end": "ORBIT-2863-MARBLE"
    },
    "retrieved_keys": {
      "begin": true,
      "middle": true,
      "end": true
    },
    "accuracy": true
  },
  {
    "name": "layout_131072",
    "input_tokens": 131072,
    "begin_key_offset": 35,
    "middle_key_offset": 65832,
    "end_key_offset": 131011
  },
  {
    "name": "stream_131072_128",
    "input_tokens": 131072,
    "requested_output_tokens": 128,
    "requested_total_tokens": 131200,
    "ttft_seconds": 123.8322498589987,
    "decode_wall_seconds": 0.5941902149934322,
    "seconds": 124.42653353299829,
    "decode_tokens_per_second": 57.22073360022567,
    "finish_reason": "stop",
    "sse_done": true,
    "response": {
      "text": "CEDAR-7419-QUARTZ\nHARBOR-5096-VIOLET\nORBIT-2863-MARBLE",
      "usage": {
        "prompt_tokens": 131072,
        "total_tokens": 131107,
        "completion_tokens": 35
      }
    },
    "expected_keys": {
      "begin": "CEDAR-7419-QUARTZ",
      "middle": "HARBOR-5096-VIOLET",
      "end": "ORBIT-2863-MARBLE"
    },
    "retrieved_keys": {
      "begin": true,
      "middle": true,
      "end": true
    },
    "accuracy": true
  },
  {
    "name": "layout_262016",
    "input_tokens": 262016,
    "begin_key_offset": 35,
    "middle_key_offset": 130632,
    "end_key_offset": 261955
  },
  {
    "name": "stream_262016_128",
    "input_tokens": 262016,
    "requested_output_tokens": 128,
    "requested_total_tokens": 262144,
    "ttft_seconds": 394.9077806400019,
    "decode_wall_seconds": 0.9461079349857755,
    "seconds": 395.9787403039809,
    "decode_tokens_per_second": 35.9367031421327,
    "finish_reason": "stop",
    "sse_done": true,
    "response": {
      "text": "CEDAR-7419-QUARTZ\nHARBOR-5096-VIOLET\nORBIT-2863-MARBLE",
      "usage": {
        "prompt_tokens": 262016,
        "total_tokens": 262051,
        "completion_tokens": 35
      }
    },
    "expected_keys": {
      "begin": "CEDAR-7419-QUARTZ",
      "middle": "HARBOR-5096-VIOLET",
      "end": "ORBIT-2863-MARBLE"
    },
    "retrieved_keys": {
      "begin": true,
      "middle": true,
      "end": true
    },
    "accuracy": true
  },
  {
    "name": "native_overflow_rejection",
    "input_tokens": 262016,
    "requested_output_tokens": 129,
    "requested_total_tokens": 262145,
    "native_max_len": 262144,
    "status": 400,
    "seconds": 0.014746526983799413,
    "rejected_before_generation": true,
    "response": {
      "error": {
        "message": "You passed 262016 input tokens and requested 129 output tokens. However, the model's context length is only 262144 tokens, resulting in a maximum input length of 262015 tokens. Please reduce the length of the input prompt. (parameter=input_tokens, value=262016)",
        "type": "BadRequestError",
        "param": "input_tokens",
        "code": 400
      }
    }
  },
  {
    "name": "mtp_acceptance",
    "accepted_tokens_before": 10881.0,
    "accepted_tokens_after": 10962.0,
    "accepted_tokens_delta": 81.0
  },
  {
    "name": "short_after",
    "input_tokens": 25,
    "requested_output_tokens": 32,
    "requested_total_tokens": 57,
    "ttft_seconds": 0.09321309201186523,
    "decode_wall_seconds": 0.04820338898571208,
    "seconds": 0.18969050902524032,
    "decode_tokens_per_second": 145.21800535798144,
    "finish_reason": "stop",
    "sse_done": true,
    "response": {
      "text": "The capital of France is Paris.",
      "usage": {
        "prompt_tokens": 25,
        "total_tokens": 33,
        "completion_tokens": 8
      }
    },
    "expected_keys": {
      "known_answer": "Paris"
    },
    "retrieved_keys": {
      "known_answer": true
    },
    "accuracy": true
  },
  {
    "name": "state_isolation",
    "exact_text_match": true
  }
]