mirror of
https://git.datalinker.icu/vllm-project/vllm.git
synced 2025-12-10 05:04:58 +08:00
29283e8976
Signed-off-by: Aaron Pham <contact@aarnphm.xyz> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
142 lines
5.0 KiB
Python
142 lines
5.0 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
import json
|
|
from typing import Final
|
|
|
|
import pytest
|
|
import schemathesis
|
|
from hypothesis import settings
|
|
from schemathesis import GenerationConfig
|
|
|
|
from ...utils import RemoteOpenAIServer
|
|
|
|
schemathesis.experimental.OPEN_API_3_1.enable()
|
|
|
|
MODEL_NAME = "HuggingFaceTB/SmolVLM-256M-Instruct"
|
|
MAXIMUM_IMAGES = 2
|
|
DEFAULT_TIMEOUT_SECONDS: Final[int] = 10
|
|
LONG_TIMEOUT_SECONDS: Final[int] = 60
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def server():
|
|
args = [
|
|
"--runner",
|
|
"generate",
|
|
"--max-model-len",
|
|
"2048",
|
|
"--max-num-seqs",
|
|
"5",
|
|
"--enforce-eager",
|
|
"--trust-remote-code",
|
|
"--limit-mm-per-prompt",
|
|
json.dumps({"image": MAXIMUM_IMAGES}),
|
|
]
|
|
|
|
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
|
yield remote_server
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def get_schema(server):
|
|
# avoid generating null (\x00) bytes in strings during test case generation
|
|
return schemathesis.openapi.from_uri(
|
|
f"{server.url_root}/openapi.json",
|
|
generation_config=GenerationConfig(allow_x00=False),
|
|
)
|
|
|
|
|
|
schema = schemathesis.from_pytest_fixture("get_schema")
|
|
|
|
|
|
@schemathesis.hook
|
|
def before_generate_case(context: schemathesis.hooks.HookContext, strategy):
|
|
op = context.operation
|
|
assert op is not None
|
|
|
|
def no_invalid_types(case: schemathesis.models.Case):
|
|
"""
|
|
This filter skips test cases with invalid data that schemathesis
|
|
incorrectly generates due to permissive schema configurations.
|
|
|
|
1. Skips `POST /tokenize` endpoint cases with `"type": "file"` in
|
|
message content, which isn't implemented.
|
|
|
|
2. Skips tool_calls with `"type": "custom"` which schemathesis
|
|
incorrectly generates instead of the valid `"type": "function"`.
|
|
|
|
Example test cases that are skipped:
|
|
curl -X POST -H 'Content-Type: application/json' \
|
|
-d '{"messages": [{"content": [{"file": {}, "type": "file"}], "role": "user"}]}' \
|
|
http://localhost:8000/tokenize
|
|
|
|
curl -X POST -H 'Content-Type: application/json' \
|
|
-d '{"messages": [{"role": "assistant", "tool_calls": [{"custom": {"input": "", "name": ""}, "id": "", "type": "custom"}]}]}' \
|
|
http://localhost:8000/v1/chat/completions
|
|
""" # noqa: E501
|
|
if hasattr(case, "body") and isinstance(case.body, dict):
|
|
if ("messages" in case.body
|
|
and isinstance(case.body["messages"], list)
|
|
and len(case.body["messages"]) > 0):
|
|
|
|
for message in case.body["messages"]:
|
|
if not isinstance(message, dict):
|
|
continue
|
|
|
|
# Check for invalid file type in tokenize endpoint
|
|
if op.method.lower() == "post" and op.path == "/tokenize":
|
|
content = message.get("content", [])
|
|
if (isinstance(content, list) and len(content) > 0
|
|
and any(
|
|
item.get("type") == "file"
|
|
for item in content)):
|
|
return False
|
|
|
|
# Check for invalid tool_calls with non-function types
|
|
tool_calls = message.get("tool_calls", [])
|
|
if isinstance(tool_calls, list):
|
|
for tool_call in tool_calls:
|
|
if isinstance(tool_call, dict):
|
|
if tool_call.get("type") != "function":
|
|
return False
|
|
if "custom" in tool_call:
|
|
return False
|
|
|
|
# Sometimes structured_outputs.grammar is generated to be empty
|
|
# Causing a server error in EBNF grammar parsing
|
|
# https://github.com/vllm-project/vllm/pull/22587#issuecomment-3195253421
|
|
structured_outputs = case.body.get("structured_outputs", {})
|
|
grammar = structured_outputs.get("grammar") if isinstance(
|
|
structured_outputs, dict) else None
|
|
|
|
if grammar == '':
|
|
# Allow None (will be handled as no grammar)
|
|
# But skip empty strings
|
|
return False
|
|
|
|
return True
|
|
|
|
return strategy.filter(no_invalid_types)
|
|
|
|
|
|
@schema.parametrize()
|
|
@schema.override(headers={"Content-Type": "application/json"})
|
|
@settings(deadline=LONG_TIMEOUT_SECONDS * 1000)
|
|
def test_openapi_stateless(case: schemathesis.Case):
|
|
key = (
|
|
case.operation.method.upper(),
|
|
case.operation.path,
|
|
)
|
|
if case.operation.path.startswith("/v1/responses"):
|
|
# Skip responses API as it is meant to be stateful.
|
|
return
|
|
|
|
timeout = {
|
|
# requires a longer timeout
|
|
("POST", "/v1/chat/completions"):
|
|
LONG_TIMEOUT_SECONDS,
|
|
}.get(key, DEFAULT_TIMEOUT_SECONDS)
|
|
|
|
#No need to verify SSL certificate for localhost
|
|
case.call_and_validate(verify=False, timeout=timeout)
|