xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2026-01-03 00:00:57 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fix streaming for granite tool call when <|tool_call|> is present (#11069)

Browse Source 
Signed-off-by: Max de Bayser <mbayser@br.ibm.com>
This commit is contained in:
Maximilien de Bayser 2024-12-11 01:51:40 -03:00 committed by GitHub
parent ffa48c9146
commit e39400a4b6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 5 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
vllm/entrypoints/openai/tool_parsers/granite_tool_parser.py
View File

@ -88,7 +88,11 @@ class GraniteToolParser(ToolParser):
) -> Union[DeltaMessage, None]:
start_idx = consume_space(0, current_text)
if not current_text or current_text[start_idx] != '[':
if current_text[start_idx:].startswith(self.bot_token):
start_idx = consume_space(start_idx + len(self.bot_token),
current_text)
if not current_text or start_idx >= len(current_text)\
or current_text[start_idx] != '[':
return DeltaMessage(content=delta_text)
# bit mask flags for partial JSON parsing. If the name hasn't been