From 31f46a0d35da80118bac5f80c533019cd50ddd9a Mon Sep 17 00:00:00 2001 From: Nick Hill Date: Sun, 29 Sep 2024 10:43:14 +0100 Subject: [PATCH] [BugFix] Fix seeded random sampling with encoder-decoder models (#8870) Co-authored-by: Roger Wang --- vllm/worker/enc_dec_model_runner.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/vllm/worker/enc_dec_model_runner.py b/vllm/worker/enc_dec_model_runner.py index 3bb4e28c6e1b..0f8b4eeacde0 100644 --- a/vllm/worker/enc_dec_model_runner.py +++ b/vllm/worker/enc_dec_model_runner.py @@ -268,11 +268,13 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]): encoder_input_positions=encoder_input_positions_tensor, ) + generators = self.get_generators(finished_requests_ids) sampling_metadata = SamplingMetadata.prepare(seq_group_metadata_list, model_input.seq_lens, model_input.query_lens, self.device, - self.pin_memory) + self.pin_memory, + generators=generators) is_prompt = (seq_group_metadata_list[0].is_prompt if seq_group_metadata_list else None) return dataclasses.replace(model_input,