diff --git a/.buildkite/scripts/hardware_ci/run-cpu-test.sh b/.buildkite/scripts/hardware_ci/run-cpu-test.sh
index 7c7dbb461ce0..57a7bc4e5f5d 100644
--- a/.buildkite/scripts/hardware_ci/run-cpu-test.sh
+++ b/.buildkite/scripts/hardware_ci/run-cpu-test.sh
@@ -78,6 +78,12 @@ function cpu_tests() {
   #   VLLM_USE_V1=0 pytest -s -v \
   #   tests/quantization/test_ipex_quant.py"
 
+  # Run multi-lora tests
+  docker exec cpu-test-"$NUMA_NODE" bash -c "
+    set -e
+    pytest -s -v \
+    tests/lora/test_qwen2vl.py"
+
   # online serving
   docker exec cpu-test-"$NUMA_NODE" bash -c '
     set -e
@@ -89,12 +95,6 @@ function cpu_tests() {
       --model meta-llama/Llama-3.2-3B-Instruct \
       --num-prompts 20 \
       --endpoint /v1/completions'
-
-  # Run multi-lora tests
-  docker exec cpu-test-"$NUMA_NODE" bash -c "
-    set -e
-    pytest -s -v \
-    tests/lora/test_qwen2vl.py"
 }
 
 # All of CPU tests are expected to be finished less than 40 mins.
diff --git a/docker/Dockerfile.cpu b/docker/Dockerfile.cpu
index 5e49e87131ec..1a0981f8ea6d 100644
--- a/docker/Dockerfile.cpu
+++ b/docker/Dockerfile.cpu
@@ -19,16 +19,14 @@
 #   VLLM_CPU_AVX512VNNI=false (default)|true
 #
 
-######################### BASE IMAGE #########################
-FROM ubuntu:22.04 AS base
+######################### COMMON BASE IMAGE #########################
+FROM ubuntu:22.04 AS base-common
 
 WORKDIR /workspace/
 
 ARG PYTHON_VERSION=3.12
 ARG PIP_EXTRA_INDEX_URL="https://download.pytorch.org/whl/cpu"
 
-ENV LD_PRELOAD=""
-
 # Install minimal dependencies and uv
 RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
     --mount=type=cache,target=/var/lib/apt,sharing=locked \
@@ -63,17 +61,18 @@ RUN --mount=type=cache,target=/root/.cache/uv \
 ARG TARGETARCH
 ENV TARGETARCH=${TARGETARCH}
 
-RUN if [ "$TARGETARCH" = "arm64" ]; then \
-        PRELOAD_PATH="/usr/lib/aarch64-linux-gnu/libtcmalloc_minimal.so.4"; \
-    else \
-        PRELOAD_PATH="/usr/lib/x86_64-linux-gnu/libtcmalloc_minimal.so.4:/opt/venv/lib/libiomp5.so"; \
-    fi && \
-    echo "export LD_PRELOAD=$PRELOAD_PATH" >> ~/.bashrc
+######################### x86_64 BASE IMAGE #########################
+FROM base-common AS base-amd64
 
-# Ensure that the LD_PRELOAD environment variable for export is in effect.
-SHELL ["/bin/bash", "-c"]
+ENV LD_PRELOAD="/usr/lib/x86_64-linux-gnu/libtcmalloc_minimal.so.4:/opt/venv/lib/libiomp5.so"
 
-ENV LD_PRELOAD=${LD_PRELOAD}
+######################### arm64 BASE IMAGE #########################
+FROM base-common AS base-arm64
+
+ENV LD_PRELOAD="/usr/lib/aarch64-linux-gnu/libtcmalloc_minimal.so.4"
+
+######################### BASE IMAGE #########################
+FROM base-${TARGETARCH} AS base
 
 RUN echo 'ulimit -c 0' >> ~/.bashrc