xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2025-12-10 04:15:01 +08:00
Code Issues Packages Projects Releases Wiki Activity

Only turn on FastIncrementalDetokenizer when tokenizers >= 0.21.1 (#17158)

Browse Source
This commit is contained in:
Lu Fang 2025-04-25 02:10:32 -07:00 committed by GitHub
parent ef19e67d2c
commit fc966e9cc6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 5 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
vllm/v1/engine/detokenizer.py
View File

@ -2,6 +2,8 @@
from abc import ABC, abstractmethod
from typing import Optional
import tokenizers
from packaging import version
from tokenizers import Tokenizer
from tokenizers.decoders import DecodeStream
from transformers import PreTrainedTokenizerFast
@ -43,8 +45,10 @@ class IncrementalDetokenizer:
# No tokenizer => skipping detokenization.
return IncrementalDetokenizer()
if isinstance(tokenizer, PreTrainedTokenizerFast):
if (isinstance(tokenizer, PreTrainedTokenizerFast) and version.parse(
tokenizers.__version__) >= version.parse("0.21.1")):
# Fast tokenizer => use tokenizers library DecodeStream.
# And only tokenizers >= 0.21.1 supports Fast Detokenizer.
return FastIncrementalDetokenizer(tokenizer, request)
# Fall back to slow python-based incremental detokenization.