diff --git a/docker/Dockerfile.rocm b/docker/Dockerfile.rocm
index 4c09808a14333..b7245e8067783 100644
--- a/docker/Dockerfile.rocm
+++ b/docker/Dockerfile.rocm
@@ -51,6 +51,8 @@ FROM fetch_vllm_${REMOTE_VLLM} AS fetch_vllm
 # -----------------------
 # vLLM build stages
 FROM fetch_vllm AS build_vllm
+# Set target device to ROCm for build
+ENV VLLM_TARGET_DEVICE=rocm
 # Build vLLM
 RUN cd vllm \
     && python3 -m pip install -r requirements/rocm.txt \