xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2025-12-10 20:45:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

Revert "[PERF] Use faster way of decode in tokenizer: avoid useless list-to-list conversion (#20000)" (#23396)

Browse Source 
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung 2025-08-23 12:16:48 +08:00 committed by GitHub
parent 308fa287a8
commit b4e9fd811f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 3 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
vllm/transformers_utils/tokenizer.py
View File

@ -49,12 +49,11 @@ def decode_tokens(
`skip_special_tokens=None` means to use the backend's default `skip_special_tokens=None` means to use the backend's default
settings. settings.
""" """
decode_method = getattr(tokenizer, "_decode", tokenizer.decode)
if skip_special_tokens is not None: if skip_special_tokens is not None:
return decode_method(token_ids, return tokenizer.decode(token_ids,
skip_special_tokens=skip_special_tokens) skip_special_tokens=skip_special_tokens)
return decode_method(token_ids) return tokenizer.decode(token_ids)
def encode_tokens( def encode_tokens(