LegacyCudaDockerfile

FROM nvidia/cuda:12.1.1-devel-ubuntu22.04

# Install dependencies
RUN apt-get update && apt-get install -y \
    python3-pip \
    python3-dev \
    cmake \
    git \
    build-essential \
    && apt-get clean \
    && rm -rf /var/lib/apt/lists/*

WORKDIR /code

ENV GGML_CUDA=on
# Pascal lacks Tensor Cores, so we compile for the base SIMT cores.
ENV GGML_CUDA_ARCH="61"
ENV CUDA_DOCKER_ARCH=61

RUN CMAKE_ARGS="-DGGML_CUDA=on -DGGML_CUDA_ARCH=${GGML_CUDA_ARCH}" \
    FORCE_CMAKE=1 \
    CMAKE_BUILD_PARALLEL_LEVEL=2 \
    pip install llama-cpp-python --no-cache-dir --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cu121

# Install Python dependencies
RUN pip install "fastapi[standard]" "uvicorn[standard]" httpx

COPY ./app /code

EXPOSE 8080

CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8080"]