| [ |
| { |
| "name": "configuration", |
| "url": "http://127.0.0.1:8000", |
| "model": "google/gemma-4-12B-it", |
| "contexts": [ |
| 32768, |
| 131072, |
| 262016 |
| ], |
| "tokens": 128, |
| "native_max_len": 262144, |
| "request_timeout_seconds": 3600, |
| "cases": [ |
| { |
| "input_tokens": 32768, |
| "output_tokens": 128 |
| }, |
| { |
| "input_tokens": 131072, |
| "output_tokens": 128 |
| }, |
| { |
| "input_tokens": 262016, |
| "output_tokens": 128 |
| } |
| ], |
| "begin_key_token_region": [ |
| 0, |
| 43 |
| ], |
| "end_key_suffix_tokens": 65, |
| "timing_basis": "Client first/last non-empty SSE text arrival; MTP may emit multiple tokens per event" |
| }, |
| { |
| "name": "short_before", |
| "input_tokens": 25, |
| "requested_output_tokens": 32, |
| "requested_total_tokens": 57, |
| "ttft_seconds": 0.09304460400016978, |
| "decode_wall_seconds": 0.04858210598467849, |
| "seconds": 0.18983703898265958, |
| "decode_tokens_per_second": 144.08597276963692, |
| "finish_reason": "stop", |
| "sse_done": true, |
| "response": { |
| "text": "The capital of France is Paris.", |
| "usage": { |
| "prompt_tokens": 25, |
| "total_tokens": 33, |
| "completion_tokens": 8 |
| } |
| }, |
| "expected_keys": { |
| "known_answer": "Paris" |
| }, |
| "retrieved_keys": { |
| "known_answer": true |
| }, |
| "accuracy": true |
| }, |
| { |
| "name": "layout_32768", |
| "input_tokens": 32768, |
| "begin_key_offset": 35, |
| "middle_key_offset": 16021, |
| "end_key_offset": 32707 |
| }, |
| { |
| "name": "stream_32768_128", |
| "input_tokens": 32768, |
| "requested_output_tokens": 128, |
| "requested_total_tokens": 32896, |
| "ttft_seconds": 16.924654099013424, |
| "decode_wall_seconds": 0.43388441897695884, |
| "seconds": 17.35861225798726, |
| "decode_tokens_per_second": 78.36188282623154, |
| "finish_reason": "stop", |
| "sse_done": true, |
| "response": { |
| "text": "CEDAR-7419-QUARTZ\nHARBOR-5096-VIOLET\nORBIT-2863-MARBLE", |
| "usage": { |
| "prompt_tokens": 32768, |
| "total_tokens": 32803, |
| "completion_tokens": 35 |
| } |
| }, |
| "expected_keys": { |
| "begin": "CEDAR-7419-QUARTZ", |
| "middle": "HARBOR-5096-VIOLET", |
| "end": "ORBIT-2863-MARBLE" |
| }, |
| "retrieved_keys": { |
| "begin": true, |
| "middle": true, |
| "end": true |
| }, |
| "accuracy": true |
| }, |
| { |
| "name": "layout_131072", |
| "input_tokens": 131072, |
| "begin_key_offset": 35, |
| "middle_key_offset": 65832, |
| "end_key_offset": 131011 |
| }, |
| { |
| "name": "stream_131072_128", |
| "input_tokens": 131072, |
| "requested_output_tokens": 128, |
| "requested_total_tokens": 131200, |
| "ttft_seconds": 123.8322498589987, |
| "decode_wall_seconds": 0.5941902149934322, |
| "seconds": 124.42653353299829, |
| "decode_tokens_per_second": 57.22073360022567, |
| "finish_reason": "stop", |
| "sse_done": true, |
| "response": { |
| "text": "CEDAR-7419-QUARTZ\nHARBOR-5096-VIOLET\nORBIT-2863-MARBLE", |
| "usage": { |
| "prompt_tokens": 131072, |
| "total_tokens": 131107, |
| "completion_tokens": 35 |
| } |
| }, |
| "expected_keys": { |
| "begin": "CEDAR-7419-QUARTZ", |
| "middle": "HARBOR-5096-VIOLET", |
| "end": "ORBIT-2863-MARBLE" |
| }, |
| "retrieved_keys": { |
| "begin": true, |
| "middle": true, |
| "end": true |
| }, |
| "accuracy": true |
| }, |
| { |
| "name": "layout_262016", |
| "input_tokens": 262016, |
| "begin_key_offset": 35, |
| "middle_key_offset": 130632, |
| "end_key_offset": 261955 |
| }, |
| { |
| "name": "stream_262016_128", |
| "input_tokens": 262016, |
| "requested_output_tokens": 128, |
| "requested_total_tokens": 262144, |
| "ttft_seconds": 394.9077806400019, |
| "decode_wall_seconds": 0.9461079349857755, |
| "seconds": 395.9787403039809, |
| "decode_tokens_per_second": 35.9367031421327, |
| "finish_reason": "stop", |
| "sse_done": true, |
| "response": { |
| "text": "CEDAR-7419-QUARTZ\nHARBOR-5096-VIOLET\nORBIT-2863-MARBLE", |
| "usage": { |
| "prompt_tokens": 262016, |
| "total_tokens": 262051, |
| "completion_tokens": 35 |
| } |
| }, |
| "expected_keys": { |
| "begin": "CEDAR-7419-QUARTZ", |
| "middle": "HARBOR-5096-VIOLET", |
| "end": "ORBIT-2863-MARBLE" |
| }, |
| "retrieved_keys": { |
| "begin": true, |
| "middle": true, |
| "end": true |
| }, |
| "accuracy": true |
| }, |
| { |
| "name": "native_overflow_rejection", |
| "input_tokens": 262016, |
| "requested_output_tokens": 129, |
| "requested_total_tokens": 262145, |
| "native_max_len": 262144, |
| "status": 400, |
| "seconds": 0.014746526983799413, |
| "rejected_before_generation": true, |
| "response": { |
| "error": { |
| "message": "You passed 262016 input tokens and requested 129 output tokens. However, the model's context length is only 262144 tokens, resulting in a maximum input length of 262015 tokens. Please reduce the length of the input prompt. (parameter=input_tokens, value=262016)", |
| "type": "BadRequestError", |
| "param": "input_tokens", |
| "code": 400 |
| } |
| } |
| }, |
| { |
| "name": "mtp_acceptance", |
| "accepted_tokens_before": 10881.0, |
| "accepted_tokens_after": 10962.0, |
| "accepted_tokens_delta": 81.0 |
| }, |
| { |
| "name": "short_after", |
| "input_tokens": 25, |
| "requested_output_tokens": 32, |
| "requested_total_tokens": 57, |
| "ttft_seconds": 0.09321309201186523, |
| "decode_wall_seconds": 0.04820338898571208, |
| "seconds": 0.18969050902524032, |
| "decode_tokens_per_second": 145.21800535798144, |
| "finish_reason": "stop", |
| "sse_done": true, |
| "response": { |
| "text": "The capital of France is Paris.", |
| "usage": { |
| "prompt_tokens": 25, |
| "total_tokens": 33, |
| "completion_tokens": 8 |
| } |
| }, |
| "expected_keys": { |
| "known_answer": "Paris" |
| }, |
| "retrieved_keys": { |
| "known_answer": true |
| }, |
| "accuracy": true |
| }, |
| { |
| "name": "state_isolation", |
| "exact_text_match": true |
| } |
| ] |
|
|
