diff --git a/vllm/transformers_utils/detokenizer_utils.py b/vllm/transformers_utils/detokenizer_utils.py index 6812cda7110f5..be1040c3e0147 100644 --- a/vllm/transformers_utils/detokenizer_utils.py +++ b/vllm/transformers_utils/detokenizer_utils.py @@ -78,7 +78,6 @@ def convert_prompt_ids_to_tokens( def convert_ids_list_to_tokens( tokenizer: AnyTokenizer, token_ids: list[int], - skip_special_tokens: bool = False, ) -> list[str]: """Detokenize the input ids individually. @@ -92,10 +91,8 @@ def convert_ids_list_to_tokens( """ token_str_lst = [] for token_id in token_ids: - token_str = tokenizer.decode( - [token_id], - skip_special_tokens=skip_special_tokens, - ) + # use default skip_special_tokens. + token_str = tokenizer.decode([token_id]) if token_str is None: token_str = "" token_str_lst.append(token_str)