xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2025-12-10 00:45:01 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Bugfix] fix missing last itl in openai completions benchmark (#5926)

Browse Source
This commit is contained in:
mcalman 2024-06-28 22:34:42 -04:00 committed by GitHub
parent 7f83f40dee
commit c4bca740e8
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 5 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
benchmarks/backend_request_func.py
View File

@ -265,6 +265,9 @@ async def async_request_openai_completions(
else:
data = json.loads(chunk)
# NOTE: Some completion API might have a last
# usage summary response without a token so we
# want to check a token was generated
if data["choices"][0]["text"]:
timestamp = time.perf_counter()
# First token
@ -273,12 +276,8 @@ async def async_request_openai_completions(
output.ttft = ttft
# Decoding phase
# NOTE: Some completion API might have a last
# usage summary response without a token so we
# do not want to include as inter-token-latency
elif data.get("usage", None) is None:
output.itl.append(timestamp -
most_recent_timestamp)
output.itl.append(timestamp -
most_recent_timestamp)
most_recent_timestamp = timestamp
generated_text += data["choices"][0]["text"]