Dockerfile: # The vLLM Dockerfile is used to construct vLLM image that can be directly used
  # to run the OpenAI compatible server.
  
  # Please update any changes made here to
  # docs/source/dev/dockerfile/dockerfile.rst and
  # docs/source/assets/dev/dockerfile-stages-dependency.png
  
  #################### BASE BUILD IMAGE ####################
  # prepare basic build environment
  FROM nvidia/cuda:12.4.1-devel-ubuntu22.04 AS dev
  
  RUN apt-get update -y \
      && apt-get install -y python3-pip git curl sudo
  
  # Workaround for https://github.com/openai/triton/issues/2507 and
  # https://github.com/pytorch/pytorch/issues/107960 -- hopefully
  # this won't be needed for future versions of this docker image
  # or future versions of triton.
  RUN ldconfig /usr/local/cuda-12.4/compat/
  
  WORKDIR /workspace
  
  # install build and runtime dependencies
  COPY requirements-common.txt requirements-common.txt
  COPY requirements-cuda.txt requirements-cuda.txt
  RUN --mount=type=cache,target=/root/.cache/pip \
      pip install -r requirements-cuda.txt
  
  # install development dependencies
  COPY requirements-dev.txt requirements-dev.txt
  RUN --mount=type=cache,target=/root/.cache/pip \
      pip install -r requirements-dev.txt
  
  # cuda arch list used by torch
  # can be useful for both `dev` and `test`
  # explicitly set the list to avoid issues with torch 2.2
  # see https://github.com/pytorch/pytorch/pull/123243
  ARG torch_cuda_arch_list='7.0 7.5 8.0 8.6 8.9 9.0+PTX'
  ENV TORCH_CUDA_ARCH_LIST=${torch_cuda_arch_list}
  #################### BASE BUILD IMAGE ####################
  
  
  #################### WHEEL BUILD IMAGE ####################
  FROM dev AS build
  
  # install build dependencies
  COPY requirements-build.txt requirements-build.txt
  RUN --mount=type=cache,target=/root/.cache/pip \
      pip install -r requirements-build.txt
  
  # install compiler cache to speed up compilation leveraging local or remote caching
  RUN apt-get update -y && apt-get install -y ccache
  
  # files and directories related to build wheels
  COPY csrc csrc
  COPY setup.py setup.py
  COPY cmake cmake
  COPY CMakeLists.txt CMakeLists.txt
  COPY requirements-common.txt requirements-common.txt
  COPY requirements-cuda.txt requirements-cuda.txt
  COPY pyproject.toml pyproject.toml
  COPY vllm vllm
  
  # max jobs used by Ninja to build extensions
  ARG max_jobs=2
  ENV MAX_JOBS=${max_jobs}
  # number of threads used by nvcc
  ARG nvcc_threads=8
  ENV NVCC_THREADS=$nvcc_threads
  # make sure punica kernels are built (for LoRA)
  ENV VLLM_INSTALL_PUNICA_KERNELS=1
  
  ARG USE_SCCACHE
  # if USE_SCCACHE is set, use sccache to speed up compilation
  RUN --mount=type=cache,target=/root/.cache/pip \
      if [ "$USE_SCCACHE" = "1" ]; then \
          echo "Installing sccache..." \
          && curl -L -o sccache.tar.gz https://github.com/mozilla/sccache/releases/download/v0.8.1/sccache-v0.8.1-x86_64-unknown-linux-musl.tar.gz \
          && tar -xzf sccache.tar.gz \
          && sudo mv sccache-v0.8.1-x86_64-unknown-linux-musl/sccache /usr/bin/sccache \
          && rm -rf sccache.tar.gz sccache-v0.8.1-x86_64-unknown-linux-musl \
          && export SCCACHE_BUCKET=vllm-build-sccache \
          && export SCCACHE_REGION=us-west-2 \
          && sccache --show-stats \
          && python3 setup.py bdist_wheel --dist-dir=dist \
          && sccache --show-stats; \
      fi
  
  ENV CCACHE_DIR=/root/.cache/ccache
  RUN --mount=type=cache,target=/root/.cache/ccache \
      --mount=type=cache,target=/root/.cache/pip \
      if [ "$USE_SCCACHE" != "1" ]; then \
          python3 setup.py bdist_wheel --dist-dir=dist; \
      fi
  
  # check the size of the wheel, we cannot upload wheels larger than 100MB
  COPY .buildkite/check-wheel-size.py check-wheel-size.py
  RUN python3 check-wheel-size.py dist
  
  #################### EXTENSION Build IMAGE ####################
  
  #################### vLLM installation IMAGE ####################
  # image with vLLM installed
  FROM nvidia/cuda:12.4.1-base-ubuntu22.04 AS vllm-base
  WORKDIR /vllm-workspace
  
  RUN apt-get update -y \
      && apt-get install -y python3-pip git vim
  
  # Workaround for https://github.com/openai/triton/issues/2507 and
  # https://github.com/pytorch/pytorch/issues/107960 -- hopefully
  # this won't be needed for future versions of this docker image
  # or future versions of triton.
  RUN ldconfig /usr/local/cuda-12.4/compat/
  
  # install vllm wheel first, so that torch etc will be installed
  RUN --mount=type=bind,from=build,src=/workspace/dist,target=/vllm-workspace/dist \
      --mount=type=cache,target=/root/.cache/pip \
      pip install dist/*.whl --verbose
  #################### vLLM installation IMAGE ####################
  
  
  #################### TEST IMAGE ####################
  # image to run unit testing suite
  # note that this uses vllm installed by `pip`
  FROM vllm-base AS test
  
  ADD . /vllm-workspace/
  
  # install development dependencies (for testing)
  RUN --mount=type=cache,target=/root/.cache/pip \
      pip install -r requirements-dev.txt
  
  # doc requires source code
  # we hide them inside `test_docs/` , so that this source code
  # will not be imported by other tests
  RUN mkdir test_docs
  RUN mv docs test_docs/
  RUN mv vllm test_docs/
  
  #################### TEST IMAGE ####################
  
  #################### OPENAI API SERVER ####################
  # openai api server alternative
  FROM vllm-base AS vllm-openai
  
  # install additional dependencies for openai api server
  RUN --mount=type=cache,target=/root/.cache/pip \
      pip install accelerate hf_transfer modelscope
  
  ENV VLLM_USAGE_SOURCE production-docker-image
  
  ENTRYPOINT ["python3", "-m", "vllm.entrypoints.openai.api_server"]
  #################### OPENAI API SERVER ####################
  
Dockerfile.rocm: # default base image
  ARG BASE_IMAGE="rocm/pytorch:rocm6.0_ubuntu20.04_py3.9_pytorch_2.1.1"
  
  FROM $BASE_IMAGE
  
  ARG BASE_IMAGE="rocm/pytorch:rocm6.0_ubuntu20.04_py3.9_pytorch_2.1.1"
  
  RUN echo "Base image is $BASE_IMAGE"
  
  # BASE_IMAGE for ROCm_5.7: "rocm/pytorch:rocm5.7_ubuntu22.04_py3.10_pytorch_2.0.1"
  # BASE_IMAGE for ROCm_6.0: "rocm/pytorch:rocm6.0_ubuntu20.04_py3.9_pytorch_2.1.1"
  
  
  ARG FA_GFX_ARCHS="gfx90a;gfx942"
  RUN echo "FA_GFX_ARCHS is $FA_GFX_ARCHS"
  
  ARG FA_BRANCH="ae7928c"
  RUN echo "FA_BRANCH is $FA_BRANCH"
  
  # whether to build flash-attention
  # if 0, will not build flash attention
  # this is useful for gfx target where flash-attention is not supported
  # In that case, we need to use the python reference attention implementation in vllm
  ARG BUILD_FA="1"
  
  # whether to build triton on rocm
  ARG BUILD_TRITON="1"
  
  # Install some basic utilities
  RUN apt-get update && apt-get install python3 python3-pip -y
  
  # Install some basic utilities
  RUN apt-get update && apt-get install -y \
      curl \
      ca-certificates \
      sudo \
      git \
      bzip2 \
      libx11-6 \
      build-essential \
      wget \
      unzip \
      nvidia-cuda-toolkit \
      tmux \
   && rm -rf /var/lib/apt/lists/*
  
  ### Mount Point ###
  # When launching the container, mount the code directory to /app
  ARG APP_MOUNT=/vllm-workspace
  VOLUME [ ${APP_MOUNT} ]
  WORKDIR ${APP_MOUNT}
  
  RUN python3 -m pip install --upgrade pip
  RUN python3 -m pip install --no-cache-dir fastapi ninja tokenizers pandas
  
  ENV LLVM_SYMBOLIZER_PATH=/opt/rocm/llvm/bin/llvm-symbolizer
  ENV PATH=$PATH:/opt/rocm/bin:/libtorch/bin:
  ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/rocm/lib/:/libtorch/lib:
  ENV CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH:/libtorch/include:/libtorch/include/torch/csrc/api/include/:/opt/rocm/include/:
  
  # Install ROCm flash-attention
  RUN if [ "$BUILD_FA" = "1" ]; then \
      mkdir libs \
      && cd libs \
      && git clone https://github.com/ROCm/flash-attention.git \
      && cd flash-attention \
      && git checkout ${FA_BRANCH} \
      && git submodule update --init \
      && export GPU_ARCHS=${FA_GFX_ARCHS} \
      && if [ "$BASE_IMAGE" = "rocm/pytorch:rocm5.7_ubuntu22.04_py3.10_pytorch_2.0.1" ]; then \
          patch /opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/utils/hipify/hipify_python.py hipify_patch.patch; fi \
      && python3 setup.py install \
      && cd ..; \
      fi
  
  # Error related to odd state for numpy 1.20.3 where there is no METADATA etc, but an extra LICENSES_bundled.txt.
  # Manually removed it so that later steps of numpy upgrade can continue
  RUN if [ "$BASE_IMAGE" = "rocm/pytorch:rocm6.0_ubuntu20.04_py3.9_pytorch_2.1.1" ]; then \
      rm -rf /opt/conda/envs/py_3.9/lib/python3.9/site-packages/numpy-1.20.3.dist-info/; fi
  
  # build triton
  RUN if [ "$BUILD_TRITON" = "1" ]; then \
      mkdir -p libs \
      && cd libs \
      && pip uninstall -y triton \
      && git clone https://github.com/ROCm/triton.git \
      && cd triton/python \
      && pip3 install . \
      && cd ../..; \
      fi
  
  WORKDIR /vllm-workspace
  COPY . .
  
  #RUN python3 -m pip install pynvml # to be removed eventually
  RUN python3 -m pip install --upgrade pip numba
  
  # make sure punica kernels are built (for LoRA)
  ENV VLLM_INSTALL_PUNICA_KERNELS=1
  # Workaround for ray >= 2.10.0
  ENV RAY_EXPERIMENTAL_NOSET_ROCR_VISIBLE_DEVICES=1
  
  ENV VLLM_NCCL_SO_PATH=/opt/rocm/lib/librccl.so
  
  RUN --mount=type=cache,target=/root/.cache/pip \
      pip install -U -r requirements-rocm.txt \
      && patch /opt/rocm/include/hip/amd_detail/amd_hip_bf16.h ./rocm_patch/rocm_bf16.patch \
      && python3 setup.py install \
      && cp build/lib.linux-x86_64-cpython-39/vllm/_C.abi3.so vllm/ \
      && cp build/lib.linux-x86_64-cpython-39/vllm/_punica_C.abi3.so vllm/ \
      && cp build/lib.linux-x86_64-cpython-39/vllm/_moe_C.abi3.so vllm/ \
      && cd ..
  
  
  CMD ["/bin/bash"]
  
Dockerfile.cpu: # This vLLM Dockerfile is used to construct image that can build and run vLLM on x86 CPU platform.
  
  FROM ubuntu:22.04 AS cpu-test-1
  
  RUN apt-get update  -y \
      && apt-get install -y git wget vim numactl gcc-12 g++-12 python3 python3-pip libtcmalloc-minimal4 \
      && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 10 --slave /usr/bin/g++ g++ /usr/bin/g++-12
  
  RUN echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libtcmalloc_minimal.so.4:$LD_PRELOAD' >> ~/.bashrc
  
  RUN pip install https://intel-extension-for-pytorch.s3.amazonaws.com/ipex_dev/cpu/intel_extension_for_pytorch-2.3.100%2Bgit0eb3473-cp310-cp310-linux_x86_64.whl
  
  RUN pip install --upgrade pip \
      && pip install wheel packaging ninja "setuptools>=49.4.0" numpy
  
  FROM cpu-test-1 AS build
  
  COPY ./ /workspace/vllm
  
  WORKDIR /workspace/vllm
  
  RUN pip install -v -r requirements-cpu.txt --extra-index-url https://download.pytorch.org/whl/cpu
  
  RUN VLLM_TARGET_DEVICE=cpu python3 setup.py install
  
  WORKDIR /workspace/
  
  RUN ln -s /workspace/vllm/tests && ln -s /workspace/vllm/examples && ln -s /workspace/vllm/benchmarks
  
  CMD ["/bin/bash"]
  
Dockerfile.neuron: # default base image
  ARG BASE_IMAGE="763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-inference-neuronx:2.1.1-neuronx-py310-sdk2.17.0-ubuntu20.04"
  
  FROM $BASE_IMAGE
  
  RUN echo "Base image is $BASE_IMAGE"
  
  # Install some basic utilities
  RUN apt-get update && apt-get install python3 python3-pip -y
  
  ### Mount Point ###
  # When launching the container, mount the code directory to /app
  ARG APP_MOUNT=/app
  VOLUME [ ${APP_MOUNT} ]
  WORKDIR ${APP_MOUNT}
  
  RUN python3 -m pip install --upgrade pip
  RUN python3 -m pip install --no-cache-dir fastapi ninja tokenizers pandas
  RUN python3 -m pip install sentencepiece transformers==4.36.2 -U
  RUN python3 -m pip install transformers-neuronx --extra-index-url=https://pip.repos.neuron.amazonaws.com -U
  RUN python3 -m pip install --pre neuronx-cc==2.12.* --extra-index-url=https://pip.repos.neuron.amazonaws.com -U
  
  COPY ./vllm /app/vllm/vllm
  COPY ./setup.py /app/vllm/setup.py
  COPY ./requirements-common.txt /app/vllm/requirements-common.txt
  COPY ./requirements-neuron.txt /app/vllm/requirements-neuron.txt
  
  RUN cd /app/vllm \
      && python3 -m pip install -U -r requirements-neuron.txt
  
  ENV VLLM_TARGET_DEVICE neuron
  RUN cd /app/vllm \
      && pip install -e . \
      && cd ..
  
  CMD ["/bin/bash"]
  
Dockerfile.tpu: ARG NIGHTLY_DATE="20240601"
  ARG BASE_IMAGE="us-central1-docker.pkg.dev/tpu-pytorch-releases/docker/xla:nightly_3.10_tpuvm_$NIGHTLY_DATE"
  
  FROM $BASE_IMAGE
  
  WORKDIR /workspace
  COPY . /workspace/vllm
  
  ENV VLLM_TARGET_DEVICE="tpu"
  # Install aiohttp separately to avoid build errors.
  RUN pip install aiohttp
  # Install the TPU and Pallas dependencies.
  RUN pip install torch_xla[tpu] -f https://storage.googleapis.com/libtpu-releases/index.html
  RUN pip install torch_xla[pallas] -f https://storage.googleapis.com/jax-releases/jax_nightly_releases.html -f https://storage.googleapis.com/jax-releases/jaxlib_nightly_releases.html
  
  # Build vLLM.
  RUN cd /workspace/vllm && python setup.py develop
  
  CMD ["/bin/bash"]
  