Dockerfile: # The vLLM Dockerfile is used to construct vLLM image that can be directly used
  # to run the OpenAI compatible server.
  
  # Please update any changes made here to
  # docs/source/contributing/dockerfile/dockerfile.md and
  # docs/source/assets/contributing/dockerfile-stages-dependency.png
  
  ARG CUDA_VERSION=12.4.1
  #################### BASE BUILD IMAGE ####################
  # prepare basic build environment
  FROM nvidia/cuda:${CUDA_VERSION}-devel-ubuntu20.04 AS base
  ARG CUDA_VERSION=12.4.1
  ARG PYTHON_VERSION=3.12
  ARG TARGETPLATFORM
  ENV DEBIAN_FRONTEND=noninteractive
  
  # Install Python and other dependencies
  RUN echo 'tzdata tzdata/Areas select America' | debconf-set-selections \
      && echo 'tzdata tzdata/Zones/America select Los_Angeles' | debconf-set-selections \
      && apt-get update -y \
      && apt-get install -y ccache software-properties-common git curl sudo \
      && add-apt-repository ppa:deadsnakes/ppa \
      && apt-get update -y \
      && apt-get install -y python${PYTHON_VERSION} python${PYTHON_VERSION}-dev python${PYTHON_VERSION}-venv \
      && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1 \
      && update-alternatives --set python3 /usr/bin/python${PYTHON_VERSION} \
      && ln -sf /usr/bin/python${PYTHON_VERSION}-config /usr/bin/python3-config \
      && curl -sS https://bootstrap.pypa.io/get-pip.py | python${PYTHON_VERSION} \
      && python3 --version && python3 -m pip --version
  # Install uv for faster pip installs
  RUN --mount=type=cache,target=/root/.cache/uv \
      python3 -m pip install uv
  
  # This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
  # Reference: https://github.com/astral-sh/uv/pull/1694
  ENV UV_HTTP_TIMEOUT=500
  
  # Upgrade to GCC 10 to avoid https://gcc.gnu.org/bugzilla/show_bug.cgi?id=92519
  # as it was causing spam when compiling the CUTLASS kernels
  RUN apt-get install -y gcc-10 g++-10
  RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 110 --slave /usr/bin/g++ g++ /usr/bin/g++-10
  RUN <<EOF
  gcc --version
  EOF
  
  # Workaround for https://github.com/openai/triton/issues/2507 and
  # https://github.com/pytorch/pytorch/issues/107960 -- hopefully
  # this won't be needed for future versions of this docker image
  # or future versions of triton.
  RUN ldconfig /usr/local/cuda-$(echo $CUDA_VERSION | cut -d. -f1,2)/compat/
  
  WORKDIR /workspace
  
  # install build and runtime dependencies
  
  # arm64 (GH200) build follows the practice of "use existing pytorch" build,
  # we need to install torch and torchvision from the nightly builds first,
  # pytorch will not appear as a vLLM dependency in all of the following steps
  # after this step
  RUN --mount=type=cache,target=/root/.cache/uv \
      if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \
          uv pip install --system --index-url https://download.pytorch.org/whl/nightly/cu128 "torch==2.8.0.dev20250318+cu128" "torchvision==0.22.0.dev20250319";  \
          uv pip install --system --index-url https://download.pytorch.org/whl/nightly/cu128 --pre pytorch_triton==3.3.0+gitab727c40; \
      fi
  
  COPY requirements/common.txt requirements/common.txt
  COPY requirements/cuda.txt requirements/cuda.txt
  RUN --mount=type=cache,target=/root/.cache/uv \
      uv pip install --system -r requirements/cuda.txt
  
  # cuda arch list used by torch
  # can be useful for both `dev` and `test`
  # explicitly set the list to avoid issues with torch 2.2
  # see https://github.com/pytorch/pytorch/pull/123243
  ARG torch_cuda_arch_list='7.0 7.5 8.0 8.6 8.9 9.0+PTX'
  ENV TORCH_CUDA_ARCH_LIST=${torch_cuda_arch_list}
  # Override the arch list for flash-attn to reduce the binary size
  ARG vllm_fa_cmake_gpu_arches='80-real;90-real'
  ENV VLLM_FA_CMAKE_GPU_ARCHES=${vllm_fa_cmake_gpu_arches}
  #################### BASE BUILD IMAGE ####################
  
  #################### WHEEL BUILD IMAGE ####################
  FROM base AS build
  ARG TARGETPLATFORM
  
  # install build dependencies
  COPY requirements/build.txt requirements/build.txt
  
  # This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
  # Reference: https://github.com/astral-sh/uv/pull/1694
  ENV UV_HTTP_TIMEOUT=500
  
  RUN --mount=type=cache,target=/root/.cache/uv \
      uv pip install --system -r requirements/build.txt
  
  COPY . .
  ARG GIT_REPO_CHECK=0
  RUN --mount=type=bind,source=.git,target=.git \
      if [ "$GIT_REPO_CHECK" != "0" ]; then bash tools/check_repo.sh ; fi
  
  # max jobs used by Ninja to build extensions
  ARG max_jobs=2
  ENV MAX_JOBS=${max_jobs}
  # number of threads used by nvcc
  ARG nvcc_threads=8
  ENV NVCC_THREADS=$nvcc_threads
  
  ARG USE_SCCACHE
  ARG SCCACHE_BUCKET_NAME=vllm-build-sccache
  ARG SCCACHE_REGION_NAME=us-west-2
  ARG SCCACHE_S3_NO_CREDENTIALS=0
  # if USE_SCCACHE is set, use sccache to speed up compilation
  RUN --mount=type=cache,target=/root/.cache/uv \
      --mount=type=bind,source=.git,target=.git \
      if [ "$USE_SCCACHE" = "1" ]; then \
          echo "Installing sccache..." \
          && curl -L -o sccache.tar.gz https://github.com/mozilla/sccache/releases/download/v0.8.1/sccache-v0.8.1-x86_64-unknown-linux-musl.tar.gz \
          && tar -xzf sccache.tar.gz \
          && sudo mv sccache-v0.8.1-x86_64-unknown-linux-musl/sccache /usr/bin/sccache \
          && rm -rf sccache.tar.gz sccache-v0.8.1-x86_64-unknown-linux-musl \
          && export SCCACHE_BUCKET=${SCCACHE_BUCKET_NAME} \
          && export SCCACHE_REGION=${SCCACHE_REGION_NAME} \
          && export SCCACHE_S3_NO_CREDENTIALS=${SCCACHE_S3_NO_CREDENTIALS} \
          && export SCCACHE_IDLE_TIMEOUT=0 \
          && export CMAKE_BUILD_TYPE=Release \
          && sccache --show-stats \
          && python3 setup.py bdist_wheel --dist-dir=dist --py-limited-api=cp38 \
          && sccache --show-stats; \
      fi
  
  ENV CCACHE_DIR=/root/.cache/ccache
  RUN --mount=type=cache,target=/root/.cache/ccache \
      --mount=type=cache,target=/root/.cache/uv \
      --mount=type=bind,source=.git,target=.git  \
      if [ "$USE_SCCACHE" != "1" ]; then \
          # Clean any existing CMake artifacts
          rm -rf .deps && \
          mkdir -p .deps && \
          python3 setup.py bdist_wheel --dist-dir=dist --py-limited-api=cp38; \
      fi
  
  # Check the size of the wheel if RUN_WHEEL_CHECK is true
  COPY .buildkite/check-wheel-size.py check-wheel-size.py
  # sync the default value with .buildkite/check-wheel-size.py
  ARG VLLM_MAX_SIZE_MB=400
  ENV VLLM_MAX_SIZE_MB=$VLLM_MAX_SIZE_MB
  ARG RUN_WHEEL_CHECK=true
  RUN if [ "$RUN_WHEEL_CHECK" = "true" ]; then \
          python3 check-wheel-size.py dist; \
      else \
          echo "Skipping wheel size check."; \
      fi
  #################### EXTENSION Build IMAGE ####################
  
  #################### DEV IMAGE ####################
  FROM base as dev
  
  # This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
  # Reference: https://github.com/astral-sh/uv/pull/1694
  ENV UV_HTTP_TIMEOUT=500
  
  COPY requirements/lint.txt requirements/lint.txt
  COPY requirements/test.txt requirements/test.txt
  COPY requirements/dev.txt requirements/dev.txt
  RUN --mount=type=cache,target=/root/.cache/uv \
      uv pip install --system -r requirements/dev.txt
  #################### DEV IMAGE ####################
  
  #################### vLLM installation IMAGE ####################
  # image with vLLM installed
  # TODO: Restore to base image after FlashInfer AOT wheel fixed
  FROM nvidia/cuda:${CUDA_VERSION}-devel-ubuntu22.04 AS vllm-base
  ARG CUDA_VERSION=12.4.1
  ARG PYTHON_VERSION=3.12
  WORKDIR /vllm-workspace
  ENV DEBIAN_FRONTEND=noninteractive
  ARG TARGETPLATFORM
  
  RUN PYTHON_VERSION_STR=$(echo ${PYTHON_VERSION} | sed 's/\.//g') && \
      echo "export PYTHON_VERSION_STR=${PYTHON_VERSION_STR}" >> /etc/environment
  
  # Install Python and other dependencies
  RUN echo 'tzdata tzdata/Areas select America' | debconf-set-selections \
      && echo 'tzdata tzdata/Zones/America select Los_Angeles' | debconf-set-selections \
      && apt-get update -y \
      && apt-get install -y ccache software-properties-common git curl wget sudo vim python3-pip \
      && apt-get install -y ffmpeg libsm6 libxext6 libgl1 \
      && add-apt-repository ppa:deadsnakes/ppa \
      && apt-get update -y \
      && apt-get install -y python${PYTHON_VERSION} python${PYTHON_VERSION}-dev python${PYTHON_VERSION}-venv libibverbs-dev \
      && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1 \
      && update-alternatives --set python3 /usr/bin/python${PYTHON_VERSION} \
      && ln -sf /usr/bin/python${PYTHON_VERSION}-config /usr/bin/python3-config \
      && curl -sS https://bootstrap.pypa.io/get-pip.py | python${PYTHON_VERSION} \
      && python3 --version && python3 -m pip --version
  # Install uv for faster pip installs
  RUN --mount=type=cache,target=/root/.cache/uv \
      python3 -m pip install uv
  
  # This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
  # Reference: https://github.com/astral-sh/uv/pull/1694
  ENV UV_HTTP_TIMEOUT=500
  
  # Workaround for https://github.com/openai/triton/issues/2507 and
  # https://github.com/pytorch/pytorch/issues/107960 -- hopefully
  # this won't be needed for future versions of this docker image
  # or future versions of triton.
  RUN ldconfig /usr/local/cuda-$(echo $CUDA_VERSION | cut -d. -f1,2)/compat/
  
  # arm64 (GH200) build follows the practice of "use existing pytorch" build,
  # we need to install torch and torchvision from the nightly builds first,
  # pytorch will not appear as a vLLM dependency in all of the following steps
  # after this step
  RUN --mount=type=cache,target=/root/.cache/uv \
      if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \
          uv pip install --system --index-url https://download.pytorch.org/whl/nightly/cu128 "torch==2.8.0.dev20250318+cu128" "torchvision==0.22.0.dev20250319";  \
          uv pip install --system --index-url https://download.pytorch.org/whl/nightly/cu128 --pre pytorch_triton==3.3.0+gitab727c40; \
      fi
  
  # Install vllm wheel first, so that torch etc will be installed.
  RUN --mount=type=bind,from=build,src=/workspace/dist,target=/vllm-workspace/dist \
      --mount=type=cache,target=/root/.cache/uv \
      uv pip install --system dist/*.whl --verbose
  
  # If we need to build FlashInfer wheel before its release:
  # $ export FLASHINFER_ENABLE_AOT=1
  # $ # Note we remove 7.0 from the arch list compared to the list below, since FlashInfer only supports sm75+
  # $ export TORCH_CUDA_ARCH_LIST='7.5 8.0 8.6 8.9 9.0+PTX'
  # $ git clone https://github.com/flashinfer-ai/flashinfer.git --recursive
  # $ cd flashinfer
  # $ git checkout 524304395bd1d8cd7d07db083859523fcaa246a4
  # $ rm -rf build
  # $ python3 setup.py bdist_wheel --dist-dir=dist --verbose
  # $ ls dist
  # $ # upload the wheel to a public location, e.g. https://wheels.vllm.ai/flashinfer/524304395bd1d8cd7d07db083859523fcaa246a4/flashinfer_python-0.2.1.post1+cu124torch2.5-cp38-abi3-linux_x86_64.whl
  
  RUN --mount=type=cache,target=/root/.cache/uv \
  . /etc/environment && \
  if [ "$TARGETPLATFORM" != "linux/arm64" ]; then \
      uv pip install --system https://github.com/flashinfer-ai/flashinfer/releases/download/v0.2.1.post2/flashinfer_python-0.2.1.post2+cu124torch2.6-cp38-abi3-linux_x86_64.whl ; \
  fi
  COPY examples examples
  
  # Although we build Flashinfer with AOT mode, there's still
  # some issues w.r.t. JIT compilation. Therefore we need to
  # install build dependencies for JIT compilation.
  # TODO: Remove this once FlashInfer AOT wheel is fixed
  COPY requirements/build.txt requirements/build.txt
  RUN --mount=type=cache,target=/root/.cache/uv \
      uv pip install --system -r requirements/build.txt
  
  #################### vLLM installation IMAGE ####################
  
  #################### TEST IMAGE ####################
  # image to run unit testing suite
  # note that this uses vllm installed by `pip`
  FROM vllm-base AS test
  
  ADD . /vllm-workspace/
  
  # This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
  # Reference: https://github.com/astral-sh/uv/pull/1694
  ENV UV_HTTP_TIMEOUT=500
  
  # install development dependencies (for testing)
  RUN --mount=type=cache,target=/root/.cache/uv \
      uv pip install --system -r requirements/dev.txt
  
  # install development dependencies (for testing)
  RUN --mount=type=cache,target=/root/.cache/uv \
      uv pip install --system -e tests/vllm_test_utils
  
  # enable fast downloads from hf (for testing)
  RUN --mount=type=cache,target=/root/.cache/uv \
      uv pip install --system hf_transfer
  ENV HF_HUB_ENABLE_HF_TRANSFER 1
  
  # Copy in the v1 package for testing (it isn't distributed yet)
  COPY vllm/v1 /usr/local/lib/python3.12/dist-packages/vllm/v1
  
  # doc requires source code
  # we hide them inside `test_docs/` , so that this source code
  # will not be imported by other tests
  RUN mkdir test_docs
  RUN mv docs test_docs/
  RUN mv vllm test_docs/
  #################### TEST IMAGE ####################
  
  #################### OPENAI API SERVER ####################
  # base openai image with additional requirements, for any subsequent openai-style images
  FROM vllm-base AS vllm-openai-base
  
  # This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
  # Reference: https://github.com/astral-sh/uv/pull/1694
  ENV UV_HTTP_TIMEOUT=500
  
  # install additional dependencies for openai api server
  RUN --mount=type=cache,target=/root/.cache/uv \
      if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \
          uv pip install --system accelerate hf_transfer 'modelscope!=1.15.0' 'bitsandbytes>=0.42.0' 'timm==0.9.10' boto3 runai-model-streamer runai-model-streamer[s3]; \
      else \
          uv pip install --system accelerate hf_transfer 'modelscope!=1.15.0' 'bitsandbytes>=0.45.3' 'timm==0.9.10' boto3 runai-model-streamer runai-model-streamer[s3]; \
      fi
  
  ENV VLLM_USAGE_SOURCE production-docker-image
  
  # define sagemaker first, so it is not default from `docker build`
  FROM vllm-openai-base AS vllm-sagemaker
  
  COPY examples/online_serving/sagemaker-entrypoint.sh .
  RUN chmod +x sagemaker-entrypoint.sh
  ENTRYPOINT ["./sagemaker-entrypoint.sh"]
  
  FROM vllm-openai-base AS vllm-openai
  
  ENTRYPOINT ["python3", "-m", "vllm.entrypoints.openai.api_server"]
  #################### OPENAI API SERVER ####################
  
Dockerfile.rocm: # default base image
  ARG REMOTE_VLLM="0"
  ARG USE_CYTHON="0"
  ARG BUILD_RPD="1"
  ARG COMMON_WORKDIR=/app
  ARG BASE_IMAGE=rocm/vllm-dev:base
  
  FROM ${BASE_IMAGE} AS base
  
  ARG ARG_PYTORCH_ROCM_ARCH
  ENV PYTORCH_ROCM_ARCH=${ARG_PYTORCH_ROCM_ARCH:-${PYTORCH_ROCM_ARCH}}
  
  # Install some basic utilities
  RUN apt-get update -q -y && apt-get install -q -y \
      sqlite3 libsqlite3-dev libfmt-dev libmsgpack-dev libsuitesparse-dev \
      apt-transport-https ca-certificates wget curl
  # Remove sccache    
  RUN python3 -m pip install --upgrade pip && pip install setuptools_scm
  RUN apt-get purge -y sccache; python3 -m pip uninstall -y sccache; rm -f "$(which sccache)"
  ARG COMMON_WORKDIR
  WORKDIR ${COMMON_WORKDIR}
  
  
  # -----------------------
  # vLLM fetch stages
  FROM base AS fetch_vllm_0
  ONBUILD COPY ./ vllm/
  FROM base AS fetch_vllm_1
  ARG VLLM_REPO="https://github.com/vllm-project/vllm.git"
  ARG VLLM_BRANCH="main"
  ONBUILD RUN git clone ${VLLM_REPO} \
  	    && cd vllm \
  	    && git checkout ${VLLM_BRANCH}
  FROM fetch_vllm_${REMOTE_VLLM} AS fetch_vllm
  
  # -----------------------
  # vLLM build stages
  FROM fetch_vllm AS build_vllm
  ARG USE_CYTHON
  # Build vLLM
  RUN cd vllm \
      && python3 -m pip install -r requirements/rocm.txt \
      && python3 setup.py clean --all  \
      && if [ ${USE_CYTHON} -eq "1" ]; then python3 tests/build_cython.py build_ext --inplace; fi \
      && python3 setup.py bdist_wheel --dist-dir=dist
  FROM scratch AS export_vllm
  ARG COMMON_WORKDIR
  COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/dist/*.whl /
  COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/requirements /requirements
  COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/benchmarks /benchmarks
  COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/tests /tests
  COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/examples /examples
  COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/.buildkite /.buildkite
  
  # -----------------------
  # Test vLLM image
  FROM base AS test
  
  RUN python3 -m pip install --upgrade pip && rm -rf /var/lib/apt/lists/*
  
  # Install vLLM
  RUN --mount=type=bind,from=export_vllm,src=/,target=/install \
      cd /install \
      && pip install -U -r requirements/rocm.txt \
      && pip install -U -r requirements/rocm-test.txt \
      && pip uninstall -y vllm \
      && pip install *.whl
  
  WORKDIR /vllm-workspace
  ARG COMMON_WORKDIR
  COPY --from=build_vllm ${COMMON_WORKDIR}/vllm /vllm-workspace
  
  # install development dependencies (for testing)
  RUN cd /vllm-workspace \
      && rm -rf vllm \
      && python3 -m pip install -e tests/vllm_test_utils \
      && python3 -m pip install lm-eval[api]==0.4.4 \
      && python3 -m pip install pytest-shard
  
  # -----------------------
  # Final vLLM image
  FROM base AS final
  
  RUN python3 -m pip install --upgrade pip && rm -rf /var/lib/apt/lists/*
  # Error related to odd state for numpy 1.20.3 where there is no METADATA etc, but an extra LICENSES_bundled.txt.
  # Manually remove it so that later steps of numpy upgrade can continue
  RUN case "$(which python3)" in \
          *"/opt/conda/envs/py_3.9"*) \
              rm -rf /opt/conda/envs/py_3.9/lib/python3.9/site-packages/numpy-1.20.3.dist-info/;; \
          *) ;; esac
  
  RUN python3 -m pip install --upgrade huggingface-hub[cli]
  ARG BUILD_RPD
  RUN if [ ${BUILD_RPD} -eq "1" ]; then \
      git clone -b nvtx_enabled https://github.com/ROCm/rocmProfileData.git \
      && cd rocmProfileData/rpd_tracer \
      && pip install -r requirements.txt && cd ../ \
      && make && make install \
      && cd hipMarker && python3 setup.py install ; fi
  
  # Install vLLM
  RUN --mount=type=bind,from=export_vllm,src=/,target=/install \
      cd /install \
      && pip install -U -r requirements/rocm.txt \
      && pip uninstall -y vllm \
      && pip install *.whl
  
  ARG COMMON_WORKDIR
  
  # Copy over the benchmark scripts as well
  COPY --from=export_vllm /benchmarks ${COMMON_WORKDIR}/vllm/benchmarks
  COPY --from=export_vllm /examples ${COMMON_WORKDIR}/vllm/examples
  
  ENV RAY_EXPERIMENTAL_NOSET_ROCR_VISIBLE_DEVICES=1
  ENV TOKENIZERS_PARALLELISM=false
  
  # Performance environment variable.
  ENV HIP_FORCE_DEV_KERNARG=1
  
  CMD ["/bin/bash"]
  
  
Dockerfile.cpu: # This vLLM Dockerfile is used to construct image that can build and run vLLM on x86 CPU platform.
  #
  # Build targets:
  #   vllm-openai (default): used for serving deployment
  #   vllm-test: used for CI tests
  #   vllm-dev: used for development
  #
  # Build arguments:
  #   PYTHON_VERSION=3.12 (default)|3.11|3.10|3.9
  #   VLLM_CPU_DISABLE_AVX512=false (default)|true
  #
  
  ######################### BASE IMAGE #########################
  FROM ubuntu:22.04 AS base
  
  WORKDIR /workspace/
  
  ARG PYTHON_VERSION=3.12
  ARG PIP_EXTRA_INDEX_URL="https://download.pytorch.org/whl/cpu"
  
  ENV LD_PRELOAD=""
  
  # Install minimal dependencies and uv
  RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
      --mount=type=cache,target=/var/lib/apt,sharing=locked \
      apt-get update -y \
      && apt-get install -y --no-install-recommends ccache git curl wget ca-certificates \
          gcc-12 g++-12 libtcmalloc-minimal4 libnuma-dev ffmpeg libsm6 libxext6 libgl1 \
      && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 10 --slave /usr/bin/g++ g++ /usr/bin/g++-12 \
      && curl -LsSf https://astral.sh/uv/install.sh | sh
  
  ENV CCACHE_DIR=/root/.cache/ccache
  ENV CMAKE_CXX_COMPILER_LAUNCHER=ccache
  
  ENV PATH="/root/.local/bin:$PATH"
  ENV VIRTUAL_ENV="/opt/venv"
  ENV UV_PYTHON_INSTALL_DIR=/opt/uv/python
  RUN uv venv --python ${PYTHON_VERSION} --seed ${VIRTUAL_ENV}
  ENV PATH="$VIRTUAL_ENV/bin:$PATH"
  
  ENV UV_HTTP_TIMEOUT=500
  
  # Install Python dependencies 
  ENV PIP_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL}
  ENV UV_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL}
  ENV UV_INDEX_STRATEGY="unsafe-best-match"
  ENV UV_LINK_MODE="copy"
  RUN --mount=type=cache,target=/root/.cache/uv \
      --mount=type=bind,src=requirements/common.txt,target=requirements/common.txt \
      --mount=type=bind,src=requirements/cpu.txt,target=requirements/cpu.txt \
      uv pip install --upgrade pip && \
      uv pip install -r requirements/cpu.txt
  
  RUN --mount=type=cache,target=/root/.cache/uv \
      uv pip install intel-openmp==2024.2.1 intel_extension_for_pytorch==2.6.0
  
  ENV LD_PRELOAD="/usr/lib/x86_64-linux-gnu/libtcmalloc_minimal.so.4:/opt/venv/lib/libiomp5.so:$LD_PRELOAD"
  
  RUN echo 'ulimit -c 0' >> ~/.bashrc
  
  ######################### BUILD IMAGE #########################
  FROM base AS vllm-build
  
  ARG GIT_REPO_CHECK=0
  # Support for building with non-AVX512 vLLM: docker build --build-arg VLLM_CPU_DISABLE_AVX512="true" ...
  ARG VLLM_CPU_DISABLE_AVX512
  ENV VLLM_CPU_DISABLE_AVX512=${VLLM_CPU_DISABLE_AVX512}
  
  WORKDIR /workspace/vllm
  
  RUN --mount=type=cache,target=/root/.cache/uv \
      --mount=type=bind,src=requirements/build.txt,target=requirements/build.txt \
      uv pip install -r requirements/build.txt
  
  COPY . .
  RUN --mount=type=bind,source=.git,target=.git \
      if [ "$GIT_REPO_CHECK" != 0 ]; then bash tools/check_repo.sh ; fi
  
  RUN --mount=type=cache,target=/root/.cache/uv \
      --mount=type=cache,target=/root/.cache/ccache \
      --mount=type=bind,source=.git,target=.git \
      VLLM_TARGET_DEVICE=cpu python3 setup.py bdist_wheel 
  
  ######################### DEV IMAGE #########################
  FROM vllm-build AS vllm-dev
  
  WORKDIR /workspace/vllm
  
  RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
      --mount=type=cache,target=/var/lib/apt,sharing=locked \
      apt-get install -y --no-install-recommends vim numactl
  
  # install development dependencies (for testing)
  RUN --mount=type=cache,target=/root/.cache/uv \
      uv pip install -e tests/vllm_test_utils 
  
  RUN --mount=type=cache,target=/root/.cache/uv \
      --mount=type=cache,target=/root/.cache/ccache \
      --mount=type=bind,source=.git,target=.git \
      VLLM_TARGET_DEVICE=cpu python3 setup.py develop 
  
  RUN --mount=type=cache,target=/root/.cache/uv \
      uv pip install -r requirements/dev.txt && \
      pre-commit install --hook-type pre-commit --hook-type commit-msg
  
  ENTRYPOINT ["bash"]
  
  ######################### TEST IMAGE #########################
  FROM base AS vllm-test
  
  WORKDIR /workspace/
  
  RUN --mount=type=cache,target=/root/.cache/uv \
      --mount=type=bind,src=requirements/test.txt,target=requirements/test.txt \
      uv pip install -r requirements/test.txt
  
  RUN --mount=type=cache,target=/root/.cache/uv \
      --mount=type=bind,from=vllm-build,src=/workspace/vllm/dist,target=dist \
      uv pip install dist/*.whl
  
  ADD ./tests/ ./tests/
  ADD ./examples/ ./examples/
  ADD ./benchmarks/ ./benchmarks/
  
  # install development dependencies (for testing)
  RUN --mount=type=cache,target=/root/.cache/uv \
      uv pip install -e tests/vllm_test_utils 
  
  ENTRYPOINT ["bash"]
  
  ######################### RELEASE IMAGE #########################
  FROM base AS vllm-openai
  
  WORKDIR /workspace/
  
  RUN --mount=type=cache,target=/root/.cache/uv \
      --mount=type=cache,target=/root/.cache/ccache \
      --mount=type=bind,from=vllm-build,src=/workspace/vllm/dist,target=dist \
      uv pip install dist/*.whl
  
  ENTRYPOINT ["python3", "-m", "vllm.entrypoints.openai.api_server"]
  
Dockerfile.neuron: # default base image
  # https://gallery.ecr.aws/neuron/pytorch-inference-neuronx
  ARG BASE_IMAGE="public.ecr.aws/neuron/pytorch-inference-neuronx:2.5.1-neuronx-py310-sdk2.22.0-ubuntu22.04"
  
  FROM $BASE_IMAGE
  
  RUN echo "Base image is $BASE_IMAGE"
  
  # Install some basic utilities
  RUN apt-get update && \
      apt-get install -y \
          git \
          python3 \
          python3-pip \
          ffmpeg libsm6 libxext6 libgl1
  
  ### Mount Point ###
  # When launching the container, mount the code directory to /workspace
  ARG APP_MOUNT=/workspace
  VOLUME [ ${APP_MOUNT} ]
  WORKDIR ${APP_MOUNT}/vllm
  
  RUN python3 -m pip install --upgrade pip
  RUN python3 -m pip install --no-cache-dir fastapi ninja tokenizers pandas tenacity
  RUN python3 -m pip install sentencepiece transformers==4.48.0 -U
  RUN python3 -m pip install neuronx-cc==2.17.194.0 --extra-index-url=https://pip.repos.neuron.amazonaws.com -U
  RUN python3 -m pip install pytest
  
  # uninstall transformers-neuronx package explicitly to avoid version conflict
  RUN python3 -m pip uninstall -y transformers-neuronx
  
  COPY . .
  ARG GIT_REPO_CHECK=0
  RUN --mount=type=bind,source=.git,target=.git \
      if [ "$GIT_REPO_CHECK" != 0 ]; then bash tools/check_repo.sh ; fi
  
  RUN python3 -m pip install -U \
          'cmake>=3.26' ninja packaging 'setuptools-scm>=8' wheel jinja2 \
          -r requirements/neuron.txt
  
  ENV VLLM_TARGET_DEVICE neuron
  RUN --mount=type=bind,source=.git,target=.git \
      pip install --no-build-isolation -v -e .
  
  # install development dependencies (for testing)
  RUN python3 -m pip install -e tests/vllm_test_utils
  
  # install transformers-neuronx package as an optional dependencies (for V0)
  # FIXME: `--no-deps` argument is temporarily added to resolve transformers package version conflict
  RUN python3 -m pip install transformers-neuronx==0.13.* --extra-index-url=https://pip.repos.neuron.amazonaws.com -U --no-deps
  
  # overwrite entrypoint to run bash script
  RUN echo "import subprocess; import sys; subprocess.check_call(sys.argv[1:])" > /usr/local/bin/dockerd-entrypoint.py
  
  CMD ["/bin/bash"]
  
Dockerfile.tpu: ARG NIGHTLY_DATE="20250124"
  ARG BASE_IMAGE="us-central1-docker.pkg.dev/tpu-pytorch-releases/docker/xla:nightly_3.10_tpuvm_$NIGHTLY_DATE"
  
  FROM $BASE_IMAGE
  WORKDIR /workspace/vllm
  
  # Install some basic utilities
  RUN apt-get update && apt-get install -y \
      git \
      ffmpeg libsm6 libxext6 libgl1
  
  # Build vLLM.
  COPY . .
  ARG GIT_REPO_CHECK=0
  RUN --mount=type=bind,source=.git,target=.git \
      if [ "$GIT_REPO_CHECK" != 0 ]; then bash tools/check_repo.sh; fi
  
  # Remove existing versions of dependencies
  RUN pip uninstall -y torch torch_xla torchvision
  
  ENV VLLM_TARGET_DEVICE="tpu"
  RUN --mount=type=cache,target=/root/.cache/pip \
      --mount=type=bind,source=.git,target=.git \
      python3 -m pip install \
          -r requirements/tpu.txt
  RUN python3 setup.py develop
  
  # install development dependencies (for testing)
  RUN python3 -m pip install -e tests/vllm_test_utils
  
  CMD ["/bin/bash"]
  
Dockerfile.ppc64le: ARG BASE_UBI_IMAGE_TAG=9.5-1741850109
  
  ###############################################################
  # base stage with basic dependencies
  ###############################################################
  
  FROM registry.access.redhat.com/ubi9/ubi-minimal:${BASE_UBI_IMAGE_TAG} AS base-builder
  
  ARG PYTHON_VERSION=3.12
  ARG OPENBLAS_VERSION=0.3.29
  
  # Set Environment Variables for venv, cargo & openblas
  ENV VIRTUAL_ENV=/opt/vllm
  ENV PATH=${VIRTUAL_ENV}/bin:/root/.cargo/bin:$PATH
  ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig/
  ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib64:/usr/local/lib:/usr/lib64:/usr/lib
  ENV UV_LINK_MODE=copy
  
  # install gcc-13, python, rust, openblas
  # Note: A symlink for libatomic.so is created for gcc-13 (linker fails to find libatomic otherwise - reqd. for sentencepiece)
  # Note: A dummy file 'control' is created in /tmp/ to artificially create dependencies between stages when building stages in parallel
  #       when `--jobs=<N>` is passed with podman build command
  RUN microdnf install -y openssl-devel dnf \
      && dnf install -y https://mirror.stream.centos.org/9-stream/BaseOS/`arch`/os/Packages/centos-gpg-keys-9.0-24.el9.noarch.rpm \
          https://mirror.stream.centos.org/9-stream/BaseOS/`arch`/os/Packages/centos-stream-repos-9.0-24.el9.noarch.rpm \
          https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm \
      && dnf config-manager --add-repo https://mirror.stream.centos.org/9-stream/BaseOS/`arch`/os \
      && dnf config-manager --add-repo https://mirror.stream.centos.org/9-stream/AppStream/`arch`/os \
      && dnf config-manager --set-enabled crb \
      && dnf install -y \
         git tar gcc-toolset-13 automake libtool numactl-devel lapack-devel \
         pkgconfig xsimd zeromq-devel kmod findutils protobuf* \
         libtiff-devel libjpeg-devel openjpeg2-devel zlib-devel \
         freetype-devel lcms2-devel libwebp-devel tcl-devel tk-devel \
         harfbuzz-devel fribidi-devel libraqm-devel libimagequant-devel libxcb-devel \
         python${PYTHON_VERSION}-devel python${PYTHON_VERSION}-pip \
      && dnf clean all \
      && ln -sf /usr/lib64/libatomic.so.1 /usr/lib64/libatomic.so \
      && python${PYTHON_VERSION} -m venv ${VIRTUAL_ENV} \
      && python -m pip install -U pip uv \
      && uv pip install wheel build "setuptools<70" setuptools_scm setuptools_rust meson-python 'cmake<4' ninja cython scikit_build_core scikit_build \
      && curl -sL https://ftp2.osuosl.org/pub/ppc64el/openblas/latest/Openblas_${OPENBLAS_VERSION}_ppc64le.tar.gz | tar xvf - -C /usr/local \
      && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y \
      && cd /tmp && touch control
  
  ###############################################################
  # Stage to build torch family
  ###############################################################
  
  FROM base-builder AS torch-builder
  
  ARG MAX_JOBS
  ARG TORCH_VERSION=2.6.0
  ARG _GLIBCXX_USE_CXX11_ABI=1
  RUN --mount=type=cache,target=/root/.cache/uv \
      source /opt/rh/gcc-toolset-13/enable &&  \
      git clone --recursive https://github.com/pytorch/pytorch.git -b v${TORCH_VERSION} && \
      cd pytorch && \
      uv pip install -r requirements.txt && \
      python setup.py develop && \
      rm -f dist/torch*+git*whl && \
      MAX_JOBS=${MAX_JOBS:-$(nproc)} \
      PYTORCH_BUILD_VERSION=${TORCH_VERSION} PYTORCH_BUILD_NUMBER=1 uv build --wheel --out-dir /torchwheels/
  
  ARG TORCHVISION_VERSION=0.21.0
  ARG TORCHVISION_USE_NVJPEG=0
  ARG TORCHVISION_USE_FFMPEG=0
  RUN --mount=type=cache,target=/root/.cache/uv \
      source /opt/rh/gcc-toolset-13/enable && \
      git clone --recursive https://github.com/pytorch/vision.git -b v${TORCHVISION_VERSION} && \
      cd vision && \
      MAX_JOBS=${MAX_JOBS:-$(nproc)} \
      BUILD_VERSION=${TORCHVISION_VERSION} \
      uv build --wheel --out-dir /torchwheels/ --no-build-isolation
  
  ARG TORCHAUDIO_VERSION=2.6.0
  ARG BUILD_SOX=1
  ARG BUILD_KALDI=1
  ARG BUILD_RNNT=1
  ARG USE_FFMPEG=0
  ARG USE_ROCM=0
  ARG USE_CUDA=0
  ARG TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_FFMPEG=1
  RUN --mount=type=cache,target=/root/.cache/uv \
      source /opt/rh/gcc-toolset-13/enable && \
      git clone --recursive https://github.com/pytorch/audio.git -b v${TORCHAUDIO_VERSION} && \
      cd audio && \
      MAX_JOBS=${MAX_JOBS:-$(nproc)} \
      BUILD_VERSION=${TORCHAUDIO_VERSION} \
      uv build --wheel --out-dir /torchwheels/ --no-build-isolation
  
  ###############################################################
  # Stage to build pyarrow
  ###############################################################
  
  FROM base-builder AS arrow-builder
  
  ARG MAX_JOBS
  ARG PYARROW_PARALLEL
  ARG PYARROW_VERSION=19.0.1
  RUN --mount=type=cache,target=/root/.cache/uv \
      source /opt/rh/gcc-toolset-13/enable && \
      git clone --recursive https://github.com/apache/arrow.git -b apache-arrow-${PYARROW_VERSION} && \
      cd arrow/cpp && \
      mkdir build && cd build && \
      cmake -DCMAKE_BUILD_TYPE=release \
          -DCMAKE_INSTALL_PREFIX=/usr/local \
          -DARROW_PYTHON=ON \
          -DARROW_BUILD_TESTS=OFF \
          -DARROW_JEMALLOC=ON \
          -DARROW_BUILD_STATIC="OFF" \
          -DARROW_PARQUET=ON \
          .. && \
      make install -j ${MAX_JOBS:-$(nproc)} && \
      cd ../../python/ && \
      uv pip install -v -r requirements-wheel-build.txt && \
      PYARROW_PARALLEL=${PYARROW_PARALLEL:-$(nproc)} \
      python setup.py build_ext \
      --build-type=release --bundle-arrow-cpp \
      bdist_wheel --dist-dir /arrowwheels/
  
  ###############################################################
  # Stage to build opencv
  ###############################################################
  
  FROM base-builder AS cv-builder
  
  ARG MAX_JOBS
  ARG OPENCV_VERSION=84
  ARG ENABLE_HEADLESS=1
  RUN --mount=type=cache,target=/root/.cache/uv \
      source /opt/rh/gcc-toolset-13/enable && \
      git clone --recursive https://github.com/opencv/opencv-python.git -b ${OPENCV_VERSION} && \
      cd opencv-python && \
      sed -i 's/"setuptools==59.2.0",/"setuptools<70.0",/g' pyproject.toml && \
      python -m build --wheel --installer=uv --outdir /opencvwheels/
  
  ###############################################################
  # Stage to build vllm - this stage builds and installs
  # vllm, tensorizer and vllm-tgis-adapter and builds uv cache
  # for transitive dependencies - eg. grpcio
  ###############################################################
  
  FROM base-builder AS vllmcache-builder
  
  COPY --from=torch-builder /tmp/control /dev/null
  COPY --from=arrow-builder /tmp/control /dev/null
  COPY --from=cv-builder /tmp/control /dev/null
  
  ARG VLLM_TARGET_DEVICE=cpu
  
  # this step installs vllm and populates uv cache
  # with all the transitive dependencies
  RUN --mount=type=cache,target=/root/.cache/uv \
      --mount=type=bind,from=torch-builder,source=/torchwheels/,target=/torchwheels/,ro \
      --mount=type=bind,from=arrow-builder,source=/arrowwheels/,target=/arrowwheels/,ro \
      --mount=type=bind,from=cv-builder,source=/opencvwheels/,target=/opencvwheels/,ro \
      --mount=type=bind,src=.,dst=/src/,rw \
      source /opt/rh/gcc-toolset-13/enable && \
      uv pip install /opencvwheels/*.whl /arrowwheels/*.whl /torchwheels/*.whl && \
      sed -i -e 's/.*torch.*//g' /src/pyproject.toml /src/requirements/*.txt && \
      uv pip install pandas pythran pybind11 && \
      # sentencepiece.pc is in some pkgconfig inside uv cache
      export PKG_CONFIG_PATH=$(find / -type d -name "pkgconfig" 2>/dev/null | tr '\n' ':') && \
      uv pip install -r /src/requirements/common.txt -r /src/requirements/cpu.txt -r /src/requirements/build.txt --no-build-isolation && \
      cd /src/ && \
      uv build --wheel --out-dir /vllmwheel/ --no-build-isolation && \
      uv pip install /vllmwheel/*.whl
  
  
  ###############################################################
  # Stage to build numactl
  ###############################################################
  
  FROM base-builder AS numa-builder
  
  # Note: Building numactl with gcc-11. Compiling with gcc-13 in this builder stage will
  # trigger recompilation with gcc-11 (and require libtool) in the final stage where we do not have gcc-13
  ARG MAX_JOBS
  ARG NUMACTL_VERSION=2.0.19
  RUN git clone --recursive https://github.com/numactl/numactl.git -b v${NUMACTL_VERSION} \
      && cd numactl \
      && autoreconf -i && ./configure \
      && make -j ${MAX_JOBS:-$(nproc)}
  
  ###############################################################
  # Stage to build lapack
  ###############################################################
  
  FROM base-builder AS lapack-builder
  
  ARG MAX_JOBS
  ARG LAPACK_VERSION=3.12.1
  RUN git clone --recursive https://github.com/Reference-LAPACK/lapack.git -b v${LAPACK_VERSION} \
      && cd lapack && source /opt/rh/gcc-toolset-13/enable \
      && cmake -B build -S . \
      && cmake --build build -j ${MAX_JOBS:-$(nproc)}
  
  
  ###############################################################
  #                   FINAL VLLM IMAGE STAGE                    #
  ###############################################################
  
  FROM registry.access.redhat.com/ubi9/ubi-minimal:${BASE_UBI_IMAGE_TAG} AS vllm-openai
  
  ARG PYTHON_VERSION=3.12
  ARG OPENBLAS_VERSION=0.3.29
  
  # Set Environment Variables for venv & openblas
  ENV VIRTUAL_ENV=/opt/vllm
  ENV PATH=${VIRTUAL_ENV}/bin:$PATH
  ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig/
  ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib64:/usr/local/lib:/usr/lib64:/usr/lib
  ENV UV_LINK_MODE=copy
  
  # create artificial dependencies between stages for independent stages to build in parallel
  COPY --from=torch-builder /tmp/control /dev/null
  COPY --from=arrow-builder /tmp/control /dev/null
  COPY --from=cv-builder /tmp/control /dev/null
  COPY --from=vllmcache-builder /tmp/control /dev/null
  COPY --from=numa-builder /tmp/control /dev/null
  COPY --from=lapack-builder /tmp/control /dev/null
  
  # install gcc-11, python, openblas, numactl, lapack
  RUN --mount=type=cache,target=/root/.cache/uv \
      --mount=type=bind,from=numa-builder,source=/numactl/,target=/numactl/,rw \
      --mount=type=bind,from=lapack-builder,source=/lapack/,target=/lapack/,rw \
      rpm -ivh https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm && \
      microdnf install --nodocs -y \
      tar findutils openssl \
      pkgconfig xsimd g++ gcc-fortran libsndfile \
      libtiff libjpeg openjpeg2 zlib zeromq \
      freetype lcms2 libwebp tcl tk utf8proc \
      harfbuzz fribidi libraqm libimagequant libxcb \
      python${PYTHON_VERSION}-devel python${PYTHON_VERSION}-pip \
      && microdnf clean all \
      && python${PYTHON_VERSION} -m venv ${VIRTUAL_ENV} \
      && python -m pip install -U pip uv --no-cache \
      && curl -sL https://ftp2.osuosl.org/pub/ppc64el/openblas/latest/Openblas_${OPENBLAS_VERSION}_ppc64le.tar.gz | tar xvf - -C /usr/local \
      && make -C /numactl install \
      && uv pip install 'cmake<4' \
      && cmake --install /lapack/build \
      && uv pip uninstall cmake
  
  # consume previously built wheels (including vllm)
  RUN --mount=type=cache,target=/root/.cache/uv \
      --mount=type=bind,from=torch-builder,source=/torchwheels/,target=/torchwheels/,ro \
      --mount=type=bind,from=arrow-builder,source=/arrowwheels/,target=/arrowwheels/,ro \
      --mount=type=bind,from=cv-builder,source=/opencvwheels/,target=/opencvwheels/,ro \
      --mount=type=bind,from=vllmcache-builder,source=/vllmwheel/,target=/vllmwheel/,ro \
      HOME=/root uv pip install /opencvwheels/*.whl /arrowwheels/*.whl /torchwheels/*.whl /vllmwheel/*.whl
  
  COPY ./ /workspace/vllm
  WORKDIR /workspace/vllm
  ARG GIT_REPO_CHECK=0
  RUN --mount=type=bind,source=.git,target=.git \
      if [ "$GIT_REPO_CHECK" != 0 ]; then bash tools/check_repo.sh; fi
  
  # install development dependencies (for testing)
  RUN --mount=type=cache,target=/root/.cache/uv \
      uv pip install -e tests/vllm_test_utils
  
  WORKDIR /workspace/
  
  RUN ln -s /workspace/vllm/tests && ln -s /workspace/vllm/examples && ln -s /workspace/vllm/benchmarks
  
  ENTRYPOINT ["python", "-m", "vllm.entrypoints.openai.api_server"]
  
Dockerfile.xpu: # oneapi 2025.0.2 docker base image use rolling 2448 package. https://dgpu-docs.intel.com/releases/packages.html?release=Rolling+2448.13&os=Ubuntu+22.04, and we don't need install driver manually.
  FROM intel/deep-learning-essentials:2025.0.2-0-devel-ubuntu22.04 AS vllm-base
  
  RUN rm /etc/apt/sources.list.d/intel-graphics.list
  
  RUN apt-get update -y && \
      apt-get install -y --no-install-recommends --fix-missing \
      curl \
      ffmpeg \
      git \
      libsndfile1 \
      libsm6 \
      libxext6 \
      libgl1 \
      lsb-release \
      numactl \
      python3 \
      python3-dev \
      python3-pip \
      wget
  
  WORKDIR /workspace/vllm
  COPY requirements/xpu.txt /workspace/vllm/requirements/xpu.txt
  COPY requirements/common.txt /workspace/vllm/requirements/common.txt
  
  RUN --mount=type=cache,target=/root/.cache/pip \
      pip install --no-cache-dir \
      -r requirements/xpu.txt
  
  ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/local/lib/"
  
  COPY . .
  ARG GIT_REPO_CHECK=0
  RUN --mount=type=bind,source=.git,target=.git \
      if [ "$GIT_REPO_CHECK" != 0 ]; then bash tools/check_repo.sh; fi
  
  ENV VLLM_TARGET_DEVICE=xpu
  
  RUN --mount=type=cache,target=/root/.cache/pip \
      --mount=type=bind,source=.git,target=.git \
      python3 setup.py install
  
  # Please refer xpu doc, we need manually install intel-extension-for-pytorch 2.6.10+xpu due to there are some conflict dependencies with torch 2.6.0+xpu
  # FIXME: This will be fix in ipex 2.7. just leave this here for awareness.
  RUN --mount=type=cache,target=/root/.cache/pip \
      pip install intel-extension-for-pytorch==2.6.10+xpu \
      --extra-index-url=https://pytorch-extension.intel.com/release-whl/stable/xpu/us/
  
  CMD ["/bin/bash"]
  
  FROM vllm-base AS vllm-openai
  
  # install additional dependencies for openai api server
  RUN --mount=type=cache,target=/root/.cache/pip \
      pip install accelerate hf_transfer 'modelscope!=1.15.0'
  
  ENV VLLM_USAGE_SOURCE production-docker-image \
      TRITON_XPU_PROFILE 1
  # install development dependencies (for testing)
  RUN python3 -m pip install -e tests/vllm_test_utils
  ENTRYPOINT ["python3", "-m", "vllm.entrypoints.openai.api_server"]
  
Dockerfile.hpu: FROM vault.habana.ai/gaudi-docker/1.20.1/ubuntu22.04/habanalabs/pytorch-installer-2.6.0:latest
  
  COPY ./ /workspace/vllm
  
  WORKDIR /workspace/vllm
  
  RUN pip install -v -r requirements/hpu.txt
  
  ENV no_proxy=localhost,127.0.0.1
  ENV PT_HPU_ENABLE_LAZY_COLLECTIVES=true
  
  RUN VLLM_TARGET_DEVICE=hpu python3 setup.py install
  
  # install development dependencies (for testing)
  RUN python3 -m pip install -e tests/vllm_test_utils
  
  WORKDIR /workspace/
  
  RUN ln -s /workspace/vllm/tests && ln -s /workspace/vllm/examples && ln -s /workspace/vllm/benchmarks
  
  ENTRYPOINT ["python3", "-m", "vllm.entrypoints.openai.api_server"]
  
Dockerfile.arm: # This vLLM Dockerfile is used to construct an image that can build and run vLLM on ARM CPU platform.
  
  FROM ubuntu:22.04 AS cpu-test-arm
  
  ENV CCACHE_DIR=/root/.cache/ccache
  
  ENV CMAKE_CXX_COMPILER_LAUNCHER=ccache
  
  RUN --mount=type=cache,target=/var/cache/apt \
      apt-get update -y \
      && apt-get install -y curl ccache git wget vim numactl gcc-12 g++-12 python3 python3-pip libtcmalloc-minimal4 libnuma-dev \
      && apt-get install -y ffmpeg libsm6 libxext6 libgl1 \
      && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 10 --slave /usr/bin/g++ g++ /usr/bin/g++-12
  
  # tcmalloc provides better memory allocation efficiency, e.g., holding memory in caches to speed up access of commonly-used objects.
  RUN --mount=type=cache,target=/root/.cache/pip \
      pip install py-cpuinfo  # Use this to gather CPU info and optimize based on ARM Neoverse cores
  
  # Set LD_PRELOAD for tcmalloc on ARM
  ENV LD_PRELOAD="/usr/lib/aarch64-linux-gnu/libtcmalloc_minimal.so.4"
  
  RUN echo 'ulimit -c 0' >> ~/.bashrc
  
  WORKDIR /workspace
  
  ARG PIP_EXTRA_INDEX_URL="https://download.pytorch.org/whl/cpu"
  ENV PIP_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL}
  RUN --mount=type=cache,target=/root/.cache/pip \
      --mount=type=bind,src=requirements/build.txt,target=requirements/build.txt \
      pip install --upgrade pip && \
      pip install -r requirements/build.txt
  
  FROM cpu-test-arm AS build
  
  WORKDIR /workspace/vllm
  
  RUN --mount=type=cache,target=/root/.cache/pip \
      --mount=type=bind,src=requirements/common.txt,target=requirements/common.txt \
      --mount=type=bind,src=requirements/cpu.txt,target=requirements/cpu.txt \
      pip install -v -r requirements/cpu.txt
  
  COPY . .
  ARG GIT_REPO_CHECK=0
  RUN --mount=type=bind,source=.git,target=.git \
      if [ "$GIT_REPO_CHECK" != 0 ]; then bash tools/check_repo.sh ; fi
  
  # Disabling AVX512 specific optimizations for ARM
  ARG VLLM_CPU_DISABLE_AVX512="true"
  ENV VLLM_CPU_DISABLE_AVX512=${VLLM_CPU_DISABLE_AVX512}
  
  RUN --mount=type=cache,target=/root/.cache/pip \
      --mount=type=cache,target=/root/.cache/ccache \
      --mount=type=bind,source=.git,target=.git \
      VLLM_TARGET_DEVICE=cpu python3 setup.py bdist_wheel && \
      pip install dist/*.whl && \
      rm -rf dist
  
  WORKDIR /workspace/
  
  RUN ln -s /workspace/vllm/tests && ln -s /workspace/vllm/examples && ln -s /workspace/vllm/benchmarks
  
  ENTRYPOINT ["python3", "-m", "vllm.entrypoints.openai.api_server"]
Dockerfile.rocm_base: ARG BASE_IMAGE=rocm/dev-ubuntu-22.04:6.3.1-complete
  ARG HIPBLASLT_BRANCH="db8e93b4"
  ARG HIPBLAS_COMMON_BRANCH="7c1566b"
  ARG LEGACY_HIPBLASLT_OPTION=
  ARG RCCL_BRANCH="648a58d"
  ARG RCCL_REPO="https://github.com/ROCm/rccl"
  ARG TRITON_BRANCH="e5be006"
  ARG TRITON_REPO="https://github.com/triton-lang/triton.git"
  ARG PYTORCH_BRANCH="295f2ed4"
  ARG PYTORCH_VISION_BRANCH="v0.21.0"
  ARG PYTORCH_REPO="https://github.com/pytorch/pytorch.git"
  ARG PYTORCH_VISION_REPO="https://github.com/pytorch/vision.git"
  ARG FA_BRANCH="1a7f4dfa"
  ARG FA_REPO="https://github.com/Dao-AILab/flash-attention.git"
  ARG AITER_BRANCH="8970b25b"
  ARG AITER_REPO="https://github.com/ROCm/aiter.git"
  
  FROM ${BASE_IMAGE} AS base
  
  ENV PATH=/opt/rocm/llvm/bin:$PATH
  ENV ROCM_PATH=/opt/rocm
  ENV LD_LIBRARY_PATH=/opt/rocm/lib:/usr/local/lib:
  ARG PYTORCH_ROCM_ARCH=gfx90a;gfx942;gfx1100;gfx1101;gfx1200;gfx1201
  ENV PYTORCH_ROCM_ARCH=${PYTORCH_ROCM_ARCH}
  
  ARG PYTHON_VERSION=3.12
  
  RUN mkdir -p /app
  WORKDIR /app
  ENV DEBIAN_FRONTEND=noninteractive
  
  # Install Python and other dependencies
  RUN apt-get update -y \
      && apt-get install -y software-properties-common git curl sudo vim less libgfortran5 \
      && add-apt-repository ppa:deadsnakes/ppa \
      && apt-get update -y \
      && apt-get install -y python${PYTHON_VERSION} python${PYTHON_VERSION}-dev python${PYTHON_VERSION}-venv \
         python${PYTHON_VERSION}-lib2to3 python-is-python3  \
      && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1 \
      && update-alternatives --set python3 /usr/bin/python${PYTHON_VERSION} \
      && ln -sf /usr/bin/python${PYTHON_VERSION}-config /usr/bin/python3-config \
      && curl -sS https://bootstrap.pypa.io/get-pip.py | python${PYTHON_VERSION} \
      && python3 --version && python3 -m pip --version
  
  RUN pip install -U packaging 'cmake<4' ninja wheel setuptools pybind11 Cython
  
  FROM base AS build_hipblaslt
  ARG HIPBLASLT_BRANCH
  ARG HIPBLAS_COMMON_BRANCH
  # Set to "--legacy_hipblas_direct" for ROCm<=6.2
  ARG LEGACY_HIPBLASLT_OPTION
  RUN git clone https://github.com/ROCm/hipBLAS-common.git
  RUN cd hipBLAS-common \
      && git checkout ${HIPBLAS_COMMON_BRANCH} \
      && mkdir build \
      && cd build \
      && cmake .. \
      && make package \
      && dpkg -i ./*.deb
  RUN git clone https://github.com/ROCm/hipBLASLt
  RUN cd hipBLASLt \
      && git checkout ${HIPBLASLT_BRANCH} \
      && apt-get install -y llvm-dev \
      && ./install.sh -dc --architecture ${PYTORCH_ROCM_ARCH} ${LEGACY_HIPBLASLT_OPTION} \
      && cd build/release \
      && make package
  RUN mkdir -p /app/install && cp /app/hipBLASLt/build/release/*.deb /app/hipBLAS-common/build/*.deb /app/install
  
  FROM base AS build_rccl
  ARG RCCL_BRANCH
  ARG RCCL_REPO
  RUN git clone ${RCCL_REPO}
  RUN cd rccl \
      && git checkout ${RCCL_BRANCH} \
      && ./install.sh -p --amdgpu_targets ${PYTORCH_ROCM_ARCH}
  RUN mkdir -p /app/install && cp /app/rccl/build/release/*.deb /app/install
  
  FROM base AS build_triton
  ARG TRITON_BRANCH
  ARG TRITON_REPO
  RUN git clone ${TRITON_REPO}
  RUN cd triton \
      && git checkout ${TRITON_BRANCH} \
      && cd python \
      && python3 setup.py bdist_wheel --dist-dir=dist
  RUN mkdir -p /app/install && cp /app/triton/python/dist/*.whl /app/install
  
  FROM base AS build_amdsmi
  RUN cd /opt/rocm/share/amd_smi \
      && pip wheel . --wheel-dir=dist
  RUN mkdir -p /app/install && cp /opt/rocm/share/amd_smi/dist/*.whl /app/install
  
  FROM base AS build_pytorch
  ARG PYTORCH_BRANCH
  ARG PYTORCH_VISION_BRANCH
  ARG PYTORCH_REPO
  ARG PYTORCH_VISION_REPO
  ARG FA_BRANCH
  ARG FA_REPO
  RUN git clone ${PYTORCH_REPO} pytorch
  RUN cd pytorch && git checkout ${PYTORCH_BRANCH} && \
      pip install -r requirements.txt && git submodule update --init --recursive \
      && python3 tools/amd_build/build_amd.py \
      && CMAKE_PREFIX_PATH=$(python3 -c 'import sys; print(sys.prefix)') python3 setup.py bdist_wheel --dist-dir=dist \
      && pip install dist/*.whl
  RUN git clone ${PYTORCH_VISION_REPO} vision
  RUN cd vision && git checkout ${PYTORCH_VISION_BRANCH} \
      && python3 setup.py bdist_wheel --dist-dir=dist \
      && pip install dist/*.whl
  RUN git clone ${FA_REPO}
  RUN cd flash-attention \
      && git checkout ${FA_BRANCH} \
      && git submodule update --init \
      && GPU_ARCHS=$(echo ${PYTORCH_ROCM_ARCH} | sed -e 's/;gfx1[0-9]\{3\}//g') python3 setup.py bdist_wheel --dist-dir=dist
  RUN mkdir -p /app/install && cp /app/pytorch/dist/*.whl /app/install \
      && cp /app/vision/dist/*.whl /app/install \
      && cp /app/flash-attention/dist/*.whl /app/install
  
  FROM base AS build_aiter
  ARG AITER_BRANCH
  ARG AITER_REPO
  RUN --mount=type=bind,from=build_pytorch,src=/app/install/,target=/install \
      pip install /install/*.whl
  RUN git clone --recursive ${AITER_REPO}
  RUN cd aiter \
      && git checkout ${AITER_BRANCH} \
      && git submodule update --init --recursive \
      && pip install -r requirements.txt
  RUN pip install pyyaml && cd aiter && PREBUILD_KERNELS=1 GPU_ARCHS=gfx942 python3 setup.py bdist_wheel --dist-dir=dist && ls /app/aiter/dist/*.whl
  RUN mkdir -p /app/install && cp /app/aiter/dist/*.whl /app/install
  
  FROM base AS final
  RUN --mount=type=bind,from=build_hipblaslt,src=/app/install/,target=/install \
      dpkg -i /install/*deb \
      && sed -i 's/, hipblaslt-dev \(.*\), hipcub-dev/, hipcub-dev/g' /var/lib/dpkg/status \
      && sed -i 's/, hipblaslt \(.*\), hipfft/, hipfft/g' /var/lib/dpkg/status
  RUN --mount=type=bind,from=build_rccl,src=/app/install/,target=/install \
      dpkg -i /install/*deb \
      && sed -i 's/, rccl-dev \(.*\), rocalution/, rocalution/g' /var/lib/dpkg/status \
      && sed -i 's/, rccl \(.*\), rocalution/, rocalution/g' /var/lib/dpkg/status
  RUN --mount=type=bind,from=build_triton,src=/app/install/,target=/install \
      pip install /install/*.whl
  RUN --mount=type=bind,from=build_amdsmi,src=/app/install/,target=/install \
      pip install /install/*.whl
  RUN --mount=type=bind,from=build_pytorch,src=/app/install/,target=/install \
      pip install /install/*.whl
  RUN --mount=type=bind,from=build_aiter,src=/app/install/,target=/install \
      pip install /install/*.whl
  
  ARG BASE_IMAGE
  ARG HIPBLAS_COMMON_BRANCH
  ARG HIPBLASLT_BRANCH
  ARG LEGACY_HIPBLASLT_OPTION
  ARG RCCL_BRANCH
  ARG RCCL_REPO
  ARG TRITON_BRANCH
  ARG TRITON_REPO
  ARG PYTORCH_BRANCH
  ARG PYTORCH_VISION_BRANCH
  ARG PYTORCH_REPO
  ARG PYTORCH_VISION_REPO
  ARG FA_BRANCH
  ARG FA_REPO
  ARG AITER_BRANCH
  ARG AITER_REPO
  RUN echo "BASE_IMAGE: ${BASE_IMAGE}" > /app/versions.txt \
      && echo "HIPBLAS_COMMON_BRANCH: ${HIPBLAS_COMMON_BRANCH}" >> /app/versions.txt \
      && echo "HIPBLASLT_BRANCH: ${HIPBLASLT_BRANCH}" >> /app/versions.txt \
      && echo "LEGACY_HIPBLASLT_OPTION: ${LEGACY_HIPBLASLT_OPTION}" >> /app/versions.txt \
      && echo "RCCL_BRANCH: ${RCCL_BRANCH}" >> /app/versions.txt \
      && echo "RCCL_REPO: ${RCCL_REPO}" >> /app/versions.txt \
      && echo "TRITON_BRANCH: ${TRITON_BRANCH}" >> /app/versions.txt \
      && echo "TRITON_REPO: ${TRITON_REPO}" >> /app/versions.txt \
      && echo "PYTORCH_BRANCH: ${PYTORCH_BRANCH}" >> /app/versions.txt \
      && echo "PYTORCH_VISION_BRANCH: ${PYTORCH_VISION_BRANCH}" >> /app/versions.txt \
      && echo "PYTORCH_REPO: ${PYTORCH_REPO}" >> /app/versions.txt \
      && echo "PYTORCH_VISION_REPO: ${PYTORCH_VISION_REPO}" >> /app/versions.txt \
      && echo "FA_BRANCH: ${FA_BRANCH}" >> /app/versions.txt \
      && echo "AITER_BRANCH: ${AITER_BRANCH}" >> /app/versions.txt \
      && echo "AITER_REPO: ${AITER_REPO}" >> /app/versions.txt
  
Dockerfile.s390x: # Base UBI image for s390x architecture
  ARG BASE_UBI_IMAGE_TAG=9.5-1736404155
  ARG PYTHON_VERSION=3.12
  FROM registry.access.redhat.com/ubi9/ubi-minimal:${BASE_UBI_IMAGE_TAG} AS base
  
  # Install basic dependencies
  ARG PYTHON_VERSION
  ENV PYTHON_VERSION=${PYTHON_VERSION}
  
  WORKDIR /workspace
  
  ENV LANG=C.UTF-8 \
      LC_ALL=C.UTF-8
  
  # Install development utilities
  RUN microdnf install -y \
      which procps findutils tar vim git gcc gcc-gfortran g++ make patch zlib-devel \
      libjpeg-turbo-devel libtiff-devel libpng-devel libwebp-devel freetype-devel harfbuzz-devel \
      openssl-devel openblas openblas-devel autoconf automake libtool cmake && \
      microdnf clean all
  
  # Python Installation
  FROM base AS python-install
  ARG PYTHON_VERSION
  
  ENV VIRTUAL_ENV=/opt/vllm
  ENV PATH="$VIRTUAL_ENV/bin:$PATH"
  ENV PYTHON_VERSION=${PYTHON_VERSION}
  RUN microdnf install -y \
      python${PYTHON_VERSION}-devel python${PYTHON_VERSION}-pip python${PYTHON_VERSION}-wheel  && \
      python${PYTHON_VERSION} -m venv $VIRTUAL_ENV && pip install --no-cache -U pip wheel uv && microdnf clean all
  
  FROM python-install AS pyarrow
  
  # Build Apache Arrow
  WORKDIR /tmp
  RUN --mount=type=cache,target=/root/.cache/uv \
      git clone https://github.com/apache/arrow.git && \
      cd arrow/cpp && \
      mkdir release && cd release && \
      cmake -DCMAKE_BUILD_TYPE=Release \
            -DCMAKE_INSTALL_PREFIX=/usr/local \
            -DARROW_PYTHON=ON \
            -DARROW_PARQUET=ON \
            -DARROW_ORC=ON \
            -DARROW_FILESYSTEM=ON \
            -DARROW_WITH_LZ4=ON \
            -DARROW_WITH_ZSTD=ON \
            -DARROW_WITH_SNAPPY=ON \
            -DARROW_JSON=ON \
            -DARROW_CSV=ON \
            -DARROW_DATASET=ON \
            -DPROTOBUF_PROTOC_EXECUTABLE=/usr/bin/protoc \
            -DARROW_DEPENDENCY_SOURCE=BUNDLED \
            .. && \
      make -j$(nproc) && \
      make install && \
      cd ../../python && \
      export PYARROW_PARALLEL=4 && \
      export ARROW_BUILD_TYPE=release && \
      uv pip install -r requirements/build.txt && \
      python setup.py build_ext --build-type=$ARROW_BUILD_TYPE --bundle-arrow-cpp bdist_wheel
  
  FROM python-install AS numa-build
  # Install numactl (needed for numa.h dependency)
  WORKDIR /tmp
  RUN curl -LO https://github.com/numactl/numactl/archive/refs/tags/v2.0.16.tar.gz && \
      tar -xvzf v2.0.16.tar.gz && \
      cd numactl-2.0.16 && \
      ./autogen.sh && \
      ./configure && \
      make
  
  # Set include path
  ENV C_INCLUDE_PATH="/usr/local/include:$C_INCLUDE_PATH"
  
  FROM python-install AS rust
  ENV CARGO_HOME=/root/.cargo
  ENV RUSTUP_HOME=/root/.rustup
  ENV PATH="$CARGO_HOME/bin:$RUSTUP_HOME/bin:$PATH"
  
  RUN curl https://sh.rustup.rs -sSf | sh -s -- -y && \
      . "$CARGO_HOME/env" && \
      rustup default stable && \
      rustup show
  
  FROM python-install AS torch-vision
  # Install torchvision
  ARG TORCH_VERSION=2.7.0.dev20250304
  ARG TORCH_VISION_VERSION=v0.20.1
  WORKDIR /tmp
  RUN --mount=type=cache,target=/root/.cache/uv \
      git clone https://github.com/pytorch/vision.git && \
      cd vision && \
      git checkout $TORCH_VISION_VERSION && \
      uv pip install -v torch==${TORCH_VERSION} --extra-index-url https://download.pytorch.org/whl/nightly/cpu && \
      python setup.py bdist_wheel
  
  # Final build stage
  FROM python-install AS vllm-cpu
  ARG PYTHON_VERSION
  
  # Set correct library path for torch and numactl
  ENV LD_LIBRARY_PATH="/opt/vllm/lib64/python${PYTHON_VERSION}/site-packages/torch/lib:/usr/local/lib:$LD_LIBRARY_PATH"
  ENV C_INCLUDE_PATH="/usr/local/include:$C_INCLUDE_PATH"
  ENV UV_LINK_MODE=copy
  ENV CARGO_HOME=/root/.cargo
  ENV RUSTUP_HOME=/root/.rustup
  ENV PATH="$CARGO_HOME/bin:$RUSTUP_HOME/bin:$PATH"
  
  COPY . /workspace/vllm
  WORKDIR /workspace/vllm
  
  RUN --mount=type=bind,from=numa-build,src=/tmp/numactl-2.0.16,target=/numactl \
      make -C /numactl install
  
  # Install dependencies, including PyTorch and Apache Arrow
  RUN --mount=type=cache,target=/root/.cache/uv \
      --mount=type=bind,from=rust,source=/root/.cargo,target=/root/.cargo,rw \
      --mount=type=bind,from=rust,source=/root/.rustup,target=/root/.rustup,rw \
      --mount=type=bind,from=pyarrow,source=/tmp/arrow/python/dist,target=/tmp/arrow-wheels \
      --mount=type=bind,from=torch-vision,source=/tmp/vision/dist,target=/tmp/vision-wheels/ \
       sed -i '/^torch/d' requirements/build.txt && \
       ARROW_WHL_FILE=$(ls /tmp/arrow-wheels/pyarrow-*.whl | head -n 1) && \
       VISION_WHL_FILE=$(ls /tmp/vision-wheels/*.whl | head -n 1) && \
      uv pip install -v \    
          $ARROW_WHL_FILE  \
          $VISION_WHL_FILE \
          --extra-index-url https://download.pytorch.org/whl/nightly/cpu \
          --index-strategy unsafe-best-match \
          -r requirements/build.txt \
          -r requirements/cpu.txt 
  
  # Build and install vllm
  RUN --mount=type=cache,target=/root/.cache/uv \
      VLLM_TARGET_DEVICE=cpu python setup.py bdist_wheel && \
      uv pip install "$(echo dist/*.whl)[tensorizer]"
  
  # setup non-root user for vllm
  RUN umask 002 && \
      useradd --uid 2000 --gid 0 vllm && \
      mkdir -p /home/vllm && \
      chmod g+rwx /home/vllm
  
  COPY LICENSE /licenses/vllm.md
  COPY examples/*.jinja /app/data/template/
  
  USER 2000
  WORKDIR /home/vllm
  
  # Set the default entrypoint
  ENTRYPOINT ["python", "-m", "vllm.entrypoints.openai.api_server"]