xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2025-12-09 17:45:19 +08:00
Code Issues Packages Projects Releases Wiki Activity

Allow send list of str for the Prompt on openai demo endpoint /v1/completions (#323)

Browse Source 
* allow str or List[str] for prompt

* Update vllm/entrypoints/openai/api_server.py

Co-authored-by: Zhuohan Li <zhuohan123@gmail.com>

---------

Co-authored-by: Zhuohan Li <zhuohan123@gmail.com>
This commit is contained in:
Michele Catalano 2023-07-03 18:17:50 +02:00 committed by GitHub
parent 49b26e2cec
commit 0bd2a573a5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 6 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
vllm/entrypoints/openai/api_server.py
View File

@ -357,7 +357,11 @@ async def create_completion(raw_request: Request):
model_name = request.model
request_id = f"cmpl-{random_uuid()}"
prompt = request.prompt
if isinstance(request.prompt, list):
assert len(request.prompt) == 1
prompt = request.prompt[0]
else:
prompt = request.prompt
created_time = int(time.time())
try:
sampling_params = SamplingParams(

2
vllm/entrypoints/openai/protocol.py
View File

@ -73,7 +73,7 @@ class ChatCompletionRequest(BaseModel):
class CompletionRequest(BaseModel):
model: str
prompt: str
prompt: Union[str, List[str]]
suffix: Optional[str] = None
max_tokens: Optional[int] = 16
temperature: Optional[float] = 1.0