xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2025-12-22 00:16:03 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Bugfix] Handle <|tool_call|> token in granite tool parser (#11039)

Browse Source 
Signed-off-by: Travis Johnson <tsjohnso@us.ibm.com>
This commit is contained in:
Travis Johnson 2024-12-10 03:27:11 -07:00 committed by GitHub
parent fe2e10c71b
commit beb16b2c81
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 3 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
vllm/entrypoints/openai/tool_parsers/granite_tool_parser.py
View File

@ -35,11 +35,13 @@ class GraniteToolParser(ToolParser):
def __init__(self, tokenizer: AnyTokenizer): def __init__(self, tokenizer: AnyTokenizer):
super().__init__(tokenizer) super().__init__(tokenizer)
self.bot_token = "<|tool_call|>"
def extract_tool_calls( def extract_tool_calls(
self, model_output: str, self, model_output: str,
request: ChatCompletionRequest) -> ExtractedToolCallInformation: request: ChatCompletionRequest) -> ExtractedToolCallInformation:
stripped = model_output.strip() # remove whitespace and the BOT token if it exists
stripped = model_output.strip().removeprefix(self.bot_token).lstrip()
if not stripped or stripped[0] != '[': if not stripped or stripped[0] != '[':
return ExtractedToolCallInformation(tools_called=False, return ExtractedToolCallInformation(tools_called=False,
tool_calls=[], tool_calls=[],