poc move to uv

Author: JacobCoffee

Dockerfile

@@ -43,6 +43,9 @@ RUN NODE_ENV=production npm run build
 # We'll build a light-weight layer along the way with just docs stuff
 FROM python:${PYTHON_IMAGE_VERSION} AS docs
 
+# Pull the uv binary in so we can use it as a faster pip / pip-compile.
+COPY --from=ghcr.io/astral-sh/uv:0.11.7 /uv /uvx /usr/local/bin/
+
 # By default, Docker has special steps to avoid keeping APT caches in the layers, which
 # is good, but in our case, we're going to mount a special cache volume (kept between
 # builds), so we WANT the cache to persist.
@@ -68,36 +71,33 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
     && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 # We create an /opt directory with a virtual environment in it to store our
-# application in.
+# application in. uv picks this up via VIRTUAL_ENV below.
 RUN set -x \
     && python3 -m venv /opt/warehouse
 
-# Now that we've created our virtual environment, we'll go ahead and update
-# our $PATH to refer to it first.
+# Point uv (and shells) at the venv we just created.
+# Hynek-recommended uv tuning (see https://hynek.me/articles/docker-uv/):
+#   UV_LINK_MODE=copy        — silence hardlink-across-FS warnings (cache mount)
+#   UV_COMPILE_BYTECODE=1    — pay the compile cost once, at build time
+#   UV_PYTHON_DOWNLOADS=never — never auto-fetch a Python; use the system one
+#   UV_PYTHON                — pin uv to the venv's interpreter
+ENV VIRTUAL_ENV="/opt/warehouse" \
+    UV_LINK_MODE=copy \
+    UV_COMPILE_BYTECODE=1 \
+    UV_PYTHON_DOWNLOADS=never \
+    UV_PYTHON="/opt/warehouse/bin/python"
 ENV PATH="/opt/warehouse/bin:${PATH}"
 
-# Next, we want to update pip inside of this virtual
-# environment to ensure that we have the latest version.
-RUN pip --no-cache-dir --disable-pip-version-check install --upgrade pip
-
-# We copy this into the docker container prior to copying in the rest of our
-# application so that we can skip installing requirements if the only thing
-# that has changed is the Warehouse code itself.
-COPY requirements /tmp/requirements
-
-# Install the Python level Warehouse requirements, this is done after copying
-# the requirements but prior to copying Warehouse itself into the container so
-# that code changes don't require triggering an entire install of all of
-# Warehouse's dependencies.
-RUN --mount=type=cache,target=/root/.cache/pip \
+# Install the Python level Warehouse requirements. requirements/ is bind-mounted
+# rather than COPY'd to keep it out of the layer.
+RUN --mount=type=cache,target=/root/.cache/uv \
+    --mount=type=bind,source=requirements,target=/tmp/requirements \
     set -x \
-    && pip --disable-pip-version-check \
-            install --no-deps --only-binary :all: \
+    && uv pip install --no-deps --only-binary :all: \
             -r /tmp/requirements/docs-dev.txt \
             -r /tmp/requirements/docs-user.txt \
             -r /tmp/requirements/docs-blog.txt \
-    && pip check \
-    && find /opt/warehouse -name '*.pyc' -delete
+    && uv pip check
 
 WORKDIR /opt/warehouse/src/
 
@@ -117,6 +117,9 @@ USER docs
 # image that it gets deployed into.
 FROM python:${PYTHON_IMAGE_VERSION} AS build
 
+# Pull the uv binary in so we can use it as a faster pip / pip-compile.
+COPY --from=ghcr.io/astral-sh/uv:0.11.7 /uv /uvx /usr/local/bin/
+
 # Define whether we're building a production or a development image. This will
 # generally be used to control whether or not we install our development and
 # test dependencies.
@@ -132,47 +135,42 @@ ARG CI=no
 ARG IPYTHON=no
 
 # We create an /opt directory with a virtual environment in it to store our
-# application in.
+# application in. uv picks this up via VIRTUAL_ENV below.
 RUN set -x \
     && python3 -m venv /opt/warehouse
 
-# Now that we've created our virtual environment, we'll go ahead and update
-# our $PATH to refer to it first.
+# Point uv (and shells) at the venv we just created.
+# See the docs stage above for the rationale behind each UV_* variable.
+ENV VIRTUAL_ENV="/opt/warehouse" \
+    UV_LINK_MODE=copy \
+    UV_COMPILE_BYTECODE=1 \
+    UV_PYTHON_DOWNLOADS=never \
+    UV_PYTHON="/opt/warehouse/bin/python"
 ENV PATH="/opt/warehouse/bin:${PATH}"
 
-# Next, we want to update pip inside of this virtual
-# environment to ensure that we have the latest version.
-RUN pip --no-cache-dir --disable-pip-version-check install --upgrade pip
-
-# We copy this into the docker container prior to copying in the rest of our
-# application so that we can skip installing requirements if the only thing
-# that has changed is the Warehouse code itself.
-COPY requirements /tmp/requirements
-
 # Install our development dependencies if we're building a development install
-# otherwise this will do nothing.
-RUN --mount=type=cache,target=/root/.cache/pip \
+# otherwise this will do nothing. requirements/ is bind-mounted to stay out of
+# the layer.
+RUN --mount=type=cache,target=/root/.cache/uv \
+    --mount=type=bind,source=requirements,target=/tmp/requirements \
     set -x \
-    && if [ "$DEVEL" = "yes" ]; then pip --disable-pip-version-check install -r /tmp/requirements/dev.txt; fi
+    && if [ "$DEVEL" = "yes" ]; then uv pip install -r /tmp/requirements/dev.txt; fi
 
-RUN --mount=type=cache,target=/root/.cache/pip \
+RUN --mount=type=cache,target=/root/.cache/uv \
+    --mount=type=bind,source=requirements,target=/tmp/requirements \
     set -x \
-    && if [ "$DEVEL" = "yes" ] && [ "$IPYTHON" = "yes" ]; then pip --disable-pip-version-check install -r /tmp/requirements/ipython.txt; fi
+    && if [ "$DEVEL" = "yes" ] && [ "$IPYTHON" = "yes" ]; then uv pip install -r /tmp/requirements/ipython.txt; fi
 
-# Install the Python level Warehouse requirements, this is done after copying
-# the requirements but prior to copying Warehouse itself into the container so
-# that code changes don't require triggering an entire install of all of
-# Warehouse's dependencies.
-RUN --mount=type=cache,target=/root/.cache/pip \
+# Install the Python level Warehouse requirements.
+RUN --mount=type=cache,target=/root/.cache/uv \
+    --mount=type=bind,source=requirements,target=/tmp/requirements \
     set -x \
-    && pip --disable-pip-version-check \
-            install --no-deps --only-binary :all: \
+    && uv pip install --no-deps --only-binary :all: \
                     -r /tmp/requirements/deploy.txt \
                     -r /tmp/requirements/main.txt \
                     $(if [ "$DEVEL" = "yes" ]; then echo '-r /tmp/requirements/tests.txt -r /tmp/requirements/lint.txt'; fi) \
                     $(if [ "$CI" = "yes" ]; then echo '-r /tmp/requirements/docs-dev.txt -r /tmp/requirements/docs-user.txt -r /tmp/requirements/docs-blog.txt'; fi ) \
-    && pip check \
-    && find /opt/warehouse -name '*.pyc' -delete
+    && uv pip check
 
 
 

Makefile

@@ -124,7 +124,7 @@ translations: .state/docker-build-base
 	docker compose run --rm base bin/translations
 
 requirements/%.txt: requirements/%.in
-	docker compose run --rm base pip-compile --generate-hashes --output-file=$@ $<
+	docker compose run --rm base uv pip compile --generate-hashes --output-file=$@ $<
 
 resetdb: .state/docker-build-base
 	docker compose pause web worker

bin/deps

@@ -3,14 +3,14 @@ set -ex
 
 export TMPDIR=$(mktemp -d)
 cp requirements/*.txt $TMPDIR/
-pip-compile -o $TMPDIR/deploy.txt requirements/deploy.in > /dev/null
-pip-compile -o $TMPDIR/main.txt requirements/main.in > /dev/null
-pip-compile -o $TMPDIR/lint.txt requirements/lint.in > /dev/null
-pip-compile -o $TMPDIR/tests.txt requirements/tests.in > /dev/null
-pip-compile -o $TMPDIR/dev.txt requirements/dev.in > /dev/null
-pip-compile -o $TMPDIR/docs-dev.txt requirements/docs-dev.in > /dev/null
-pip-compile -o $TMPDIR/docs-user.txt requirements/docs-user.in > /dev/null
-pip-compile -o $TMPDIR/docs-blog.txt requirements/docs-blog.in > /dev/null
+uv pip compile --quiet -o $TMPDIR/deploy.txt requirements/deploy.in
+uv pip compile --quiet -o $TMPDIR/main.txt requirements/main.in
+uv pip compile --quiet -o $TMPDIR/lint.txt requirements/lint.in
+uv pip compile --quiet -o $TMPDIR/tests.txt requirements/tests.in
+uv pip compile --quiet -o $TMPDIR/dev.txt requirements/dev.in
+uv pip compile --quiet -o $TMPDIR/docs-dev.txt requirements/docs-dev.in
+uv pip compile --quiet -o $TMPDIR/docs-user.txt requirements/docs-user.in
+uv pip compile --quiet -o $TMPDIR/docs-blog.txt requirements/docs-blog.in
 python bin/depchecker.py $TMPDIR/deploy.txt requirements/deploy.txt
 python bin/depchecker.py $TMPDIR/main.txt requirements/main.txt
 python bin/depchecker.py $TMPDIR/lint.txt requirements/lint.txt
@@ -20,4 +20,4 @@ python bin/depchecker.py $TMPDIR/docs-dev.txt requirements/docs-dev.txt
 python bin/depchecker.py $TMPDIR/docs-user.txt requirements/docs-user.txt
 python bin/depchecker.py $TMPDIR/docs-blog.txt requirements/docs-blog.txt
 rm -r $TMPDIR
-pip check
+uv pip check

bin/deps-upgrade

@@ -16,7 +16,7 @@ extract_argument() {
   echo "${2:-${1#*=}}"
 }
 
-pip_compile_args=""
+uv_compile_args=""
 
 handle_options() {
   while [ $# -gt 0 ]; do
@@ -26,7 +26,7 @@ handle_options() {
         exit 0
         ;;
       -a | --all)
-        pip_compile_args="--upgrade"
+        uv_compile_args="--upgrade"
         ;;
       -p | --package*)
         if ! has_argument $@; then
@@ -35,7 +35,7 @@ handle_options() {
           exit 1
         fi
 
-        pip_compile_args="--upgrade-package $(extract_argument $@)"
+        uv_compile_args="--upgrade-package $(extract_argument $@)"
 
         shift
         ;;
@@ -53,10 +53,10 @@ handle_options() {
 handle_options "$@"
 
 set -ex
-pip-compile --allow-unsafe --generate-hashes $pip_compile_args --output-file=requirements/deploy.txt requirements/deploy.in
-pip-compile --allow-unsafe --generate-hashes $pip_compile_args --output-file=requirements/main.txt requirements/main.in
-pip-compile --allow-unsafe --generate-hashes $pip_compile_args --output-file=requirements/lint.txt requirements/lint.in
-pip-compile --allow-unsafe --generate-hashes $pip_compile_args --output-file=requirements/tests.txt requirements/tests.in
-pip-compile --allow-unsafe --generate-hashes $pip_compile_args --output-file=requirements/docs-dev.txt requirements/docs-dev.in
-pip-compile --allow-unsafe --generate-hashes $pip_compile_args --output-file=requirements/docs-user.txt requirements/docs-user.in
-pip-compile --allow-unsafe --generate-hashes $pip_compile_args --output-file=requirements/docs-blog.txt requirements/docs-blog.in
+uv pip compile --generate-hashes $uv_compile_args --output-file=requirements/deploy.txt requirements/deploy.in
+uv pip compile --generate-hashes $uv_compile_args --output-file=requirements/main.txt requirements/main.in
+uv pip compile --generate-hashes $uv_compile_args --output-file=requirements/lint.txt requirements/lint.in
+uv pip compile --generate-hashes $uv_compile_args --output-file=requirements/tests.txt requirements/tests.in
+uv pip compile --generate-hashes $uv_compile_args --output-file=requirements/docs-dev.txt requirements/docs-dev.in
+uv pip compile --generate-hashes $uv_compile_args --output-file=requirements/docs-user.txt requirements/docs-user.in
+uv pip compile --generate-hashes $uv_compile_args --output-file=requirements/docs-blog.txt requirements/docs-blog.in