xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2026-05-30 17:07:09 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Bugfix][Benchmarks] Fix a benchmark of deepspeed-mii backend to use api_key (#17291)

Browse Source 
Signed-off-by: Teruaki Ishizaki <teruaki.ishizaki@ntt.com>
This commit is contained in:
Teruaki Ishizaki 2025-05-23 13:30:47 +09:00 committed by GitHub
parent ed5d408255
commit 4be2255c81
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 8 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
benchmarks/backend_request_func.py
View File

@ -194,6 +194,11 @@ async def async_request_deepspeed_mii(
request_func_input: RequestFuncInput, request_func_input: RequestFuncInput,
pbar: Optional[tqdm] = None, pbar: Optional[tqdm] = None,
) -> RequestFuncOutput: ) -> RequestFuncOutput:
api_url = request_func_input.api_url
assert api_url.endswith(("completions", "profile")), (
"OpenAI Completions API URL must end with 'completions' or 'profile'."
)
async with aiohttp.ClientSession( async with aiohttp.ClientSession(
trust_env=True, timeout=AIOHTTP_TIMEOUT trust_env=True, timeout=AIOHTTP_TIMEOUT
) as session: ) as session:
@ -204,6 +209,8 @@ async def async_request_deepspeed_mii(
"temperature": 0.01, # deepspeed-mii does not accept 0.0 temp. "temperature": 0.01, # deepspeed-mii does not accept 0.0 temp.
"top_p": 1.0, "top_p": 1.0,
} }
headers = {"Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}"}
output = RequestFuncOutput() output = RequestFuncOutput()
output.prompt_len = request_func_input.prompt_len output.prompt_len = request_func_input.prompt_len
@ -215,7 +222,7 @@ async def async_request_deepspeed_mii(
st = time.perf_counter() st = time.perf_counter()
try: try:
async with session.post( async with session.post(
url=request_func_input.api_url, json=payload url=api_url, json=payload, headers=headers
) as response: ) as response:
if response.status == 200: if response.status == 200:
parsed_resp = await response.json() parsed_resp = await response.json()