xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2026-06-01 17:27:04 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Doc] Add stream flag for chat completion example (#18524)

Browse Source 
Signed-off-by: calvin chen <120380290@qq.com>
This commit is contained in:
Calvin Chen 2025-05-22 22:07:10 +08:00 committed by GitHub
parent 4e04eceb58
commit 3f505233fd
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 21 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
examples/online_serving/openai_chat_completion_client.py
View File

@ -3,6 +3,9 @@
NOTE: start a supported chat completion model server with `vllm serve`, e.g. NOTE: start a supported chat completion model server with `vllm serve`, e.g.
vllm serve meta-llama/Llama-2-7b-chat-hf vllm serve meta-llama/Llama-2-7b-chat-hf
""" """
import argparse
from openai import OpenAI from openai import OpenAI
# Modify OpenAI's API key and API base to use vLLM's API server. # Modify OpenAI's API key and API base to use vLLM's API server.
@ -24,7 +27,15 @@ messages = [{
}] }]
def main(): def parse_args():
parser = argparse.ArgumentParser(description="Client for vLLM API server")
parser.add_argument("--stream",
action="store_true",
help="Enable streaming response")
return parser.parse_args()
def main(args):
client = OpenAI( client = OpenAI(
# defaults to os.environ.get("OPENAI_API_KEY") # defaults to os.environ.get("OPENAI_API_KEY")
api_key=openai_api_key, api_key=openai_api_key,
@ -34,16 +45,23 @@ def main():
models = client.models.list() models = client.models.list()
model = models.data[0].id model = models.data[0].id
# Chat Completion API
chat_completion = client.chat.completions.create( chat_completion = client.chat.completions.create(
messages=messages, messages=messages,
model=model, model=model,
stream=args.stream,
) )
print("-" * 50) print("-" * 50)
print("Chat completion results:") print("Chat completion results:")
print(chat_completion) if args.stream:
for c in chat_completion:
print(c)
else:
print(chat_completion)
print("-" * 50) print("-" * 50)
if __name__ == "__main__": if __name__ == "__main__":
main() args = parse_args()
main(args)