From a4851cfe6890cf1cbe3af9176429a8b741c29929 Mon Sep 17 00:00:00 2001 From: Chauncey Date: Mon, 14 Jul 2025 19:06:45 +0800 Subject: [PATCH] [Bugfix]: Fix messy code when using logprobs (#20910) Signed-off-by: chaunceyjiang --- vllm/transformers_utils/detokenizer_utils.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/vllm/transformers_utils/detokenizer_utils.py b/vllm/transformers_utils/detokenizer_utils.py index 6812cda7110f5..be1040c3e0147 100644 --- a/vllm/transformers_utils/detokenizer_utils.py +++ b/vllm/transformers_utils/detokenizer_utils.py @@ -78,7 +78,6 @@ def convert_prompt_ids_to_tokens( def convert_ids_list_to_tokens( tokenizer: AnyTokenizer, token_ids: list[int], - skip_special_tokens: bool = False, ) -> list[str]: """Detokenize the input ids individually. @@ -92,10 +91,8 @@ def convert_ids_list_to_tokens( """ token_str_lst = [] for token_id in token_ids: - token_str = tokenizer.decode( - [token_id], - skip_special_tokens=skip_special_tokens, - ) + # use default skip_special_tokens. + token_str = tokenizer.decode([token_id]) if token_str is None: token_str = "" token_str_lst.append(token_str)