From d01f66b0394e62a11429c8f0afd9a56b7b2b7f0c Mon Sep 17 00:00:00 2001 From: zifeitong Date: Fri, 7 Feb 2025 23:04:34 -0800 Subject: [PATCH] [Bugfix] Fix multi-round chat error when mistral tokenizer is used (#12859) Signed-off-by: Zifei Tong Co-authored-by: Cyrus Leung --- vllm/transformers_utils/tokenizers/mistral.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/vllm/transformers_utils/tokenizers/mistral.py b/vllm/transformers_utils/tokenizers/mistral.py index 1550f978ed201..7a1dba424464e 100644 --- a/vllm/transformers_utils/tokenizers/mistral.py +++ b/vllm/transformers_utils/tokenizers/mistral.py @@ -291,6 +291,16 @@ class MistralTokenizer: from mistral_common.protocol.instruct.request import ( ChatCompletionRequest) + + # mistral-common requires AssistantMessage content to be string [1]. + # + # [1]: https://github.com/mistralai/mistral-common/blob/f4a06998b75ed78bbf5aaf569590b772ea26c9f6/src/mistral_common/protocol/instruct/messages.py#L80 + for message in messages: + if message.get("role") == "assistant": + content = message.get("content") + if isinstance(content, list): + content = "\n".join(chunk.get("text") for chunk in content) + message["content"] = content request = ChatCompletionRequest(messages=messages, tools=tools) # type: ignore[type-var] encoded = self.mistral.encode_chat_completion(request)