xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2026-04-05 03:57:02 +08:00
Code Issues Packages Projects Releases Wiki Activity

[V1] Allow tokenizer_mode and trust_remote_code for Detokenizer (#10211)

Browse Source 
Signed-off-by: Roger Wang <ywang@roblox.com>
This commit is contained in:
Roger Wang 2024-11-11 02:01:18 -08:00 committed by GitHub
parent 36e4acd02a
commit 5fb1f935b0
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 19 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
vllm/v1/engine/llm_engine.py
View File

@ -125,7 +125,10 @@ class LLMEngine:
# Ping the tokenizer to ensure liveness if it runs in a
# different process.
self.tokenizer.ping()
self.detokenizer = Detokenizer(self.model_config.tokenizer)
self.detokenizer = Detokenizer(
tokenizer_name=self.model_config.tokenizer,
tokenizer_mode=self.model_config.tokenizer_mode,
trust_remote_code=self.model_config.trust_remote_code)
self.generation_config_fields = _load_generation_config_dict(
model_config)

19
vllm/v1/tokenizer/detokenizer.py
View File

@ -42,13 +42,17 @@ class DetokenizerOutputs(msgspec.Struct):
class Detokenizer:
def __init__(self, tokenizer_name: str):
def __init__(self, tokenizer_name: str, tokenizer_mode: str,
trust_remote_code: bool):
# FIXME(woosuk): Currently, the detokenizer is just a hacky prototype.
# For example, it does not terminate properly. We need to improve this.
self.push_port = get_open_port()
self.pull_port = get_open_port()
self.detokenizer = DetokenizerProc(tokenizer_name, self.push_port,
self.pull_port)
self.detokenizer = DetokenizerProc(tokenizer_name=tokenizer_name,
tokenizer_mode=tokenizer_mode,
trust_remote_code=trust_remote_code,
push_port=self.push_port,
pull_port=self.pull_port)
self.detokenizer.start()
self.zmq_context = zmq.Context()
@ -82,11 +86,15 @@ class DetokenizerProc(multiprocessing.Process):
def __init__(
self,
tokenizer_name: str,
tokenizer_mode: str,
trust_remote_code: bool,
pull_port: int,
push_port: int,
):
super().__init__()
self.tokenizer_name = tokenizer_name
self.tokenizer_mode = tokenizer_mode
self.trust_remote_code = trust_remote_code
# NOTE: The pull_port of the detokenizer should be the same as the
# push_port of the engine. Vice versa.
self.pull_port = pull_port
@ -97,7 +105,10 @@ class DetokenizerProc(multiprocessing.Process):
# not picklable.
self.msgpack_encoder = msgpack.Encoder()
self.msgpack_decoder = msgpack.Decoder(DetokenizerInputs)
self.tokenizer = get_tokenizer(self.tokenizer_name)
self.tokenizer = get_tokenizer(
tokenizer_name=self.tokenizer_name,
tokenizer_mode=self.tokenizer_mode,
trust_remote_code=self.trust_remote_code)
# req_id -> RequestState
self.request_states: Dict[str, RequestState] = {}