xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2026-05-03 11:37:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

[V1] Add missing tokenizer options for Detokenizer (#10288)

Browse Source 
Signed-off-by: Roger Wang <ywang@roblox.com>
This commit is contained in:
Roger Wang 2024-11-13 03:02:56 -08:00 committed by GitHub
parent d909acf9fe
commit bb7991aa29
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 15 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
vllm/v1/engine/detokenizer.py
View File

@ -192,10 +192,17 @@ class IncrementalDetokenizer:
class Detokenizer: class Detokenizer:
def __init__(self, tokenizer_name: str): def __init__(self,
tokenizer_name: str,
tokenizer_mode: str = "auto",
trust_remote_code: bool = False,
revision: Optional[str] = None):
# TODO: once we support LoRA, we should should pass the tokenizer # TODO: once we support LoRA, we should should pass the tokenizer
# here. We currently have two copies (this + in the LLMEngine). # here. We currently have two copies (this + in the LLMEngine).
self.tokenizer = get_tokenizer(tokenizer_name) self.tokenizer = get_tokenizer(tokenizer_name=tokenizer_name,
tokenizer_mode=tokenizer_mode,
trust_remote_code=trust_remote_code,
revision=revision)
# Request id -> IncrementalDetokenizer # Request id -> IncrementalDetokenizer
self.request_states: Dict[str, IncrementalDetokenizer] = {} self.request_states: Dict[str, IncrementalDetokenizer] = {}

7
vllm/v1/engine/llm_engine.py
View File

@ -53,7 +53,12 @@ class LLMEngine:
input_registry) input_registry)
# Detokenizer (converts EngineCoreOutputs --> RequestOutput) # Detokenizer (converts EngineCoreOutputs --> RequestOutput)
self.detokenizer = Detokenizer(vllm_config.model_config.tokenizer) self.detokenizer = Detokenizer(
tokenizer_name=vllm_config.model_config.tokenizer,
tokenizer_mode=vllm_config.model_config.tokenizer_mode,
trust_remote_code=vllm_config.model_config.trust_remote_code,
revision=vllm_config.model_config.tokenizer_revision,
)
# EngineCore (gets EngineCoreRequests and gives EngineCoreOutputs) # EngineCore (gets EngineCoreRequests and gives EngineCoreOutputs)
self.engine_core = EngineCoreClient.make_client( self.engine_core = EngineCoreClient.make_client(