xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2025-12-13 17:25:38 +08:00
Code Issues Packages Projects Releases Wiki Activity

[BugFix] Fix seeded random sampling with encoder-decoder models (#8870)

Browse Source 
Co-authored-by: Roger Wang <ywang@roblox.com>
This commit is contained in:
Nick Hill 2024-09-29 10:43:14 +01:00 committed by GitHub
parent 3d49776bbb
commit 31f46a0d35
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 3 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
vllm/worker/enc_dec_model_runner.py
View File

@ -268,11 +268,13 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
encoder_input_positions=encoder_input_positions_tensor, encoder_input_positions=encoder_input_positions_tensor,
) )
generators = self.get_generators(finished_requests_ids)
sampling_metadata = SamplingMetadata.prepare(seq_group_metadata_list, sampling_metadata = SamplingMetadata.prepare(seq_group_metadata_list,
model_input.seq_lens, model_input.seq_lens,
model_input.query_lens, model_input.query_lens,
self.device, self.device,
self.pin_memory) self.pin_memory,
generators=generators)
is_prompt = (seq_group_metadata_list[0].is_prompt is_prompt = (seq_group_metadata_list[0].is_prompt
if seq_group_metadata_list else None) if seq_group_metadata_list else None)
return dataclasses.replace(model_input, return dataclasses.replace(model_input,