xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2025-12-10 06:55:01 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Misc] Refine ray_serve_deepseek example (#17204)

Browse Source 
Signed-off-by: Rui Qiao <ruisearch42@gmail.com>
This commit is contained in:
Rui Qiao 2025-04-25 16:06:59 -07:00 committed by GitHub
parent a0e619e62a
commit c53e0730cb
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 29 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
examples/online_serving/ray_serve_deepseek.py
View File

@ -8,37 +8,41 @@ Run `python3 ray_serve_deepseek.py` to deploy the model.
"""
from ray import serve
from ray.serve.llm import LLMConfig, LLMRouter, LLMServer
from ray.serve.llm import LLMConfig, build_openai_app
llm_config = LLMConfig(
model_loading_config=dict(
model_id="deepseek",
# Change to model download path
model_source="/path/to/the/model",
),
deployment_config=dict(autoscaling_config=dict(
min_replicas=1,
max_replicas=1,
)),
model_loading_config={
"model_id": "deepseek",
# Since DeepSeek model is huge, it is recommended to pre-download
# the model to local disk, say /path/to/the/model and specify:
# model_source="/path/to/the/model"
"model_source": "deepseek-ai/DeepSeek-R1",
},
deployment_config={
"autoscaling_config": {
"min_replicas": 1,
"max_replicas": 1,
}
},
# Change to the accelerator type of the node
accelerator_type="H100",
runtime_env=dict(env_vars=dict(VLLM_USE_V1="1")),
runtime_env={"env_vars": {
"VLLM_USE_V1": "1"
}},
# Customize engine arguments as needed (e.g. vLLM engine kwargs)
engine_kwargs=dict(
tensor_parallel_size=8,
pipeline_parallel_size=2,
gpu_memory_utilization=0.92,
dtype="auto",
max_num_seqs=40,
max_model_len=16384,
enable_chunked_prefill=True,
enable_prefix_caching=True,
trust_remote_code=True,
),
engine_kwargs={
"tensor_parallel_size": 8,
"pipeline_parallel_size": 2,
"gpu_memory_utilization": 0.92,
"dtype": "auto",
"max_num_seqs": 40,
"max_model_len": 16384,
"enable_chunked_prefill": True,
"enable_prefix_caching": True,
"trust_remote_code": True,
},
)
# Deploy the application
deployment = LLMServer.as_deployment(
llm_config.get_serve_options(name_prefix="vLLM:")).bind(llm_config)
llm_app = LLMRouter.as_deployment().bind([deployment])
llm_app = build_openai_app({"llm_configs": [llm_config]})
serve.run(llm_app)