diff --git a/benchmarks/benchmark_throughput.py b/benchmarks/benchmark_throughput.py index e4f0d2011133a..ddffe60d56cbd 100644 --- a/benchmarks/benchmark_throughput.py +++ b/benchmarks/benchmark_throughput.py @@ -5,6 +5,7 @@ import random import time from typing import List, Optional +from typing import List, Optional import torch import uvloop @@ -22,6 +23,23 @@ from vllm.utils import FlexibleArgumentParser, merge_async_iterators +@dataclasses.dataclass +class SampleRequest: + """A class representing a single inference request for benchmarking. + + Attributes: + prompt: The input text prompt for the model. + multi_modal_data: Optional dictionary containing multi-modal data (e.g. + images). + prompt_len: The length of the prompt in tokens. + expected_output_len: The expected length of the output in tokens. + """ + prompt: str + prompt_len: int + expected_output_len: int + multi_modal_data: Optional[MultiModalDataDict] = None + + @dataclasses.dataclass class SampleRequest: """A class representing a single inference request for benchmarking.