xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2025-12-15 00:05:48 +08:00
Code Issues Packages Projects Releases Wiki Activity

implement Structural Tag with Guidance backend (#17333)

Browse Source 
Signed-off-by: Michal Moskal <michal@moskal.me>
This commit is contained in:
Michał Moskal 2025-04-28 19:21:32 -07:00 committed by GitHub
parent 506475de5f
commit 86d9fc29cb
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 32 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
tests/v1/entrypoints/llm/test_struct_output_generate.py
View File

@ -435,13 +435,10 @@ Given the previous instructions, what is the weather in New York City?
""" """
# Change this once other backends support structural_tag # Change this once other backends support structural_tag
if guided_decoding_backend.startswith("xgrammar"):
outputs = llm.generate(prompts=prompt, outputs = llm.generate(prompts=prompt,
sampling_params=sampling_params, sampling_params=sampling_params,
use_tqdm=True) use_tqdm=True)
assert outputs is not None assert outputs is not None
else:
outputs = []
for output in outputs: for output in outputs:
assert output is not None assert output is not None

31
vllm/v1/structured_output/backend_guidance.py
View File

@ -173,7 +173,8 @@ def serialize_guidance_grammar(
disable_any_whitespace: bool = False, disable_any_whitespace: bool = False,
no_additional_properties: bool = False, no_additional_properties: bool = False,
) -> str: ) -> str:
if request_type == StructuredOutputOptions.JSON:
def _process_schema(grammar_spec: Union[str, dict[str, Any]], ) -> str:
if no_additional_properties: if no_additional_properties:
grammar_spec = process_for_additional_properties(grammar_spec) grammar_spec = process_for_additional_properties(grammar_spec)
return llguidance.LLMatcher.grammar_from_json_schema( return llguidance.LLMatcher.grammar_from_json_schema(
@ -181,6 +182,9 @@ def serialize_guidance_grammar(
defaults={ defaults={
"whitespace_flexible": not disable_any_whitespace, "whitespace_flexible": not disable_any_whitespace,
}) })
if request_type == StructuredOutputOptions.JSON:
return _process_schema(grammar_spec)
elif request_type == StructuredOutputOptions.JSON_OBJECT: elif request_type == StructuredOutputOptions.JSON_OBJECT:
return llguidance.LLMatcher.grammar_from_json_schema( return llguidance.LLMatcher.grammar_from_json_schema(
'{"type": "object"}', '{"type": "object"}',
@ -195,8 +199,29 @@ def serialize_guidance_grammar(
elif request_type == StructuredOutputOptions.CHOICE: elif request_type == StructuredOutputOptions.CHOICE:
tp = "choice" tp = "choice"
elif request_type == StructuredOutputOptions.STRUCTURAL_TAG: elif request_type == StructuredOutputOptions.STRUCTURAL_TAG:
raise ValueError("Structural tag is not supported " if isinstance(grammar_spec, str):
"for guidance backend yet") s_tag = json.loads(grammar_spec)
else:
s_tag = grammar_spec
triggers: list[str] = s_tag["triggers"]
tags: list[llguidance.StructTag] = []
for s in s_tag["structures"]:
begin: str = s["begin"]
trig = next((t for t in triggers if begin.startswith(t)), None)
if trig is None:
raise ValueError(
f"Trigger {begin} not found in triggers {triggers}")
tags.append(
llguidance.StructTag(
trigger=trig,
begin=s["begin"],
grammar=_process_schema(s["schema"]),
end=s["end"],
))
if not tags:
raise ValueError(
"No structural tags found in the grammar spec.")
return llguidance.StructTag.to_grammar(tags)
else: else:
logger.error("Validation should have already occurred. " logger.error("Validation should have already occurred. "
"Please file an issue.") "Please file an issue.")