From 0bd2a573a5a88b6e6135145c5ef5abe3ac942f83 Mon Sep 17 00:00:00 2001 From: Michele Catalano Date: Mon, 3 Jul 2023 18:17:50 +0200 Subject: [PATCH] Allow send list of str for the Prompt on openai demo endpoint /v1/completions (#323) * allow str or List[str] for prompt * Update vllm/entrypoints/openai/api_server.py Co-authored-by: Zhuohan Li --------- Co-authored-by: Zhuohan Li --- vllm/entrypoints/openai/api_server.py | 6 +++++- vllm/entrypoints/openai/protocol.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/vllm/entrypoints/openai/api_server.py b/vllm/entrypoints/openai/api_server.py index 8a2573fe2b0e..43eaa5c60df1 100644 --- a/vllm/entrypoints/openai/api_server.py +++ b/vllm/entrypoints/openai/api_server.py @@ -357,7 +357,11 @@ async def create_completion(raw_request: Request): model_name = request.model request_id = f"cmpl-{random_uuid()}" - prompt = request.prompt + if isinstance(request.prompt, list): + assert len(request.prompt) == 1 + prompt = request.prompt[0] + else: + prompt = request.prompt created_time = int(time.time()) try: sampling_params = SamplingParams( diff --git a/vllm/entrypoints/openai/protocol.py b/vllm/entrypoints/openai/protocol.py index 3728241edc03..6c45b507329b 100644 --- a/vllm/entrypoints/openai/protocol.py +++ b/vllm/entrypoints/openai/protocol.py @@ -73,7 +73,7 @@ class ChatCompletionRequest(BaseModel): class CompletionRequest(BaseModel): model: str - prompt: str + prompt: Union[str, List[str]] suffix: Optional[str] = None max_tokens: Optional[int] = 16 temperature: Optional[float] = 1.0