xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2025-12-22 17:15:47 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Bugfix] use right truncation for non-generative tasks (#12050)

Browse Source 
Signed-off-by: Joe Runde <Joseph.Runde@ibm.com>
This commit is contained in:
Joe Runde 2025-01-15 09:31:01 -07:00 committed by GitHub
parent 57e729e874
commit edce722eaa
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 13 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
tests/entrypoints/llm/test_encode.py
View File

@ -105,3 +105,10 @@ def test_multiple_pooling_params(llm: LLM):
# pooling_params is None, default params should be applied # pooling_params is None, default params should be applied
outputs = llm.encode(PROMPTS, pooling_params=None) outputs = llm.encode(PROMPTS, pooling_params=None)
assert len(PROMPTS) == len(outputs) assert len(PROMPTS) == len(outputs)
@pytest.mark.skip_global_cleanup
def test_right_side_truncation(llm: LLM):
# Embeddings models should truncate the end of the prompt
tokenizer = llm.get_tokenizer()
assert tokenizer.truncation_side == "right"

4
vllm/config.py
View File

@ -357,6 +357,10 @@ class ModelConfig:
supported_tasks, task = self._resolve_task(task, self.hf_config) supported_tasks, task = self._resolve_task(task, self.hf_config)
self.supported_tasks = supported_tasks self.supported_tasks = supported_tasks
self.task: Final = task self.task: Final = task
if self.task in ("draft", "generate"):
self.truncation_side = "left"
else:
self.truncation_side = "right"
self.pooler_config = self._init_pooler_config(override_pooler_config) self.pooler_config = self._init_pooler_config(override_pooler_config)
self.logits_processor_pattern = logits_processor_pattern self.logits_processor_pattern = logits_processor_pattern

3
vllm/transformers_utils/tokenizer_group/__init__.py
View File

@ -24,7 +24,8 @@ def init_tokenizer_from_configs(model_config: ModelConfig,
max_input_length=None, max_input_length=None,
tokenizer_mode=model_config.tokenizer_mode, tokenizer_mode=model_config.tokenizer_mode,
trust_remote_code=model_config.trust_remote_code, trust_remote_code=model_config.trust_remote_code,
revision=model_config.tokenizer_revision) revision=model_config.tokenizer_revision,
truncation_side=model_config.truncation_side)
return get_tokenizer_group(parallel_config.tokenizer_pool_config, return get_tokenizer_group(parallel_config.tokenizer_pool_config,
**init_kwargs) **init_kwargs)