xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2025-12-13 16:55:48 +08:00
Code Issues Packages Projects Releases Wiki Activity

Align max_tokens behavior with openai (#852)

Browse Source
This commit is contained in:
Wen Sun 2023-09-24 09:10:13 +08:00 committed by GitHub
parent 9f6be8692e
commit bbbf86565f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 3 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
vllm/entrypoints/openai/api_server.py
View File

@ -130,6 +130,8 @@ async def check_length(
input_ids = tokenizer(prompt).input_ids input_ids = tokenizer(prompt).input_ids
token_num = len(input_ids) token_num = len(input_ids)
if request.max_tokens is None:
request.max_tokens = max_model_len - token_num
if token_num + request.max_tokens > max_model_len: if token_num + request.max_tokens > max_model_len:
return input_ids, create_error_response( return input_ids, create_error_response(
HTTPStatus.BAD_REQUEST, HTTPStatus.BAD_REQUEST,

2
vllm/entrypoints/openai/protocol.py
View File

@ -58,7 +58,7 @@ class ChatCompletionRequest(BaseModel):
temperature: Optional[float] = 0.7 temperature: Optional[float] = 0.7
top_p: Optional[float] = 1.0 top_p: Optional[float] = 1.0
n: Optional[int] = 1 n: Optional[int] = 1
max_tokens: Optional[int] = 16 max_tokens: Optional[int] = None
stop: Optional[Union[str, List[str]]] = Field(default_factory=list) stop: Optional[Union[str, List[str]]] = Field(default_factory=list)
stream: Optional[bool] = False stream: Optional[bool] = False
presence_penalty: Optional[float] = 0.0 presence_penalty: Optional[float] = 0.0