(POC) Investigate uv

.github/workflows/ci.yml

@@ -54,7 +54,7 @@ jobs:
           - name: Developer Documentation
             command: bin/dev-docs
           - name: Dependencies
-            command: bin/deps
+            command: uv lock --check
           - name: Licenses
             command: bin/licenses
           - name: Translations
@@ -96,7 +96,7 @@ jobs:
         with:
           path: |
               dev/.mypy_cache
-          key: ${{ runner.os }}-mypy-${{ env.pythonLocation }}-${{ hashFiles('requirements.txt', 'requirements/*.txt') }}
+          key: ${{ runner.os }}-mypy-${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml', 'uv.lock') }}
       - name: Run ${{ matrix.name }}
         run: ${{ matrix.command }}
 

Dockerfile

@@ -43,6 +43,9 @@ RUN NODE_ENV=production npm run build
 # We'll build a light-weight layer along the way with just docs stuff
 FROM python:${PYTHON_IMAGE_VERSION} AS docs
 
+# Pull the uv binary in so we can use it as a faster pip / pip-compile.
+COPY --from=ghcr.io/astral-sh/uv:0.11.7 /uv /uvx /usr/local/bin/
+
 # By default, Docker has special steps to avoid keeping APT caches in the layers, which
 # is good, but in our case, we're going to mount a special cache volume (kept between
 # builds), so we WANT the cache to persist.
@@ -68,36 +71,33 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
     && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 # We create an /opt directory with a virtual environment in it to store our
-# application in.
+# application in. uv picks this up via VIRTUAL_ENV below.
 RUN set -x \
     && python3 -m venv /opt/warehouse
 
-# Now that we've created our virtual environment, we'll go ahead and update
-# our $PATH to refer to it first.
+# Point uv (and shells) at the venv we just created.
+# Hynek-recommended uv tuning (see https://hynek.me/articles/docker-uv/):
+#   UV_LINK_MODE=copy        — silence hardlink-across-FS warnings (cache mount)
+#   UV_COMPILE_BYTECODE=1    — pay the compile cost once, at build time
+#   UV_PYTHON_DOWNLOADS=never — never auto-fetch a Python; use the system one
+#   UV_PYTHON                — pin uv to the venv's interpreter
+ENV VIRTUAL_ENV="/opt/warehouse" \
+    UV_PROJECT_ENVIRONMENT="/opt/warehouse" \
+    UV_LINK_MODE=copy \
+    UV_COMPILE_BYTECODE=1 \
+    UV_PYTHON_DOWNLOADS=never \
+    UV_PYTHON="/opt/warehouse/bin/python"
 ENV PATH="/opt/warehouse/bin:${PATH}"
 
-# Next, we want to update pip inside of this virtual
-# environment to ensure that we have the latest version.
-RUN pip --no-cache-dir --disable-pip-version-check install --upgrade pip
-
-# We copy this into the docker container prior to copying in the rest of our
-# application so that we can skip installing requirements if the only thing
-# that has changed is the Warehouse code itself.
-COPY requirements /tmp/requirements
-
-# Install the Python level Warehouse requirements, this is done after copying
-# the requirements but prior to copying Warehouse itself into the container so
-# that code changes don't require triggering an entire install of all of
-# Warehouse's dependencies.
-RUN --mount=type=cache,target=/root/.cache/pip \
+# Install Python deps via uv sync from pyproject.toml + uv.lock.
+# Bind-mount the lock + manifest so they don't enter the layer.
+RUN --mount=type=cache,target=/root/.cache/uv \
+    --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
+    --mount=type=bind,source=uv.lock,target=uv.lock \
     set -x \
-    && pip --disable-pip-version-check \
-            install --no-deps --only-binary :all: \
-            -r /tmp/requirements/docs-dev.txt \
-            -r /tmp/requirements/docs-user.txt \
-            -r /tmp/requirements/docs-blog.txt \
-    && pip check \
-    && find /opt/warehouse -name '*.pyc' -delete
+    && uv sync --frozen --no-default-groups \
+            --group docs-dev --group docs-user --group docs-blog \
+    && uv pip check
 
 WORKDIR /opt/warehouse/src/
 
@@ -117,6 +117,9 @@ USER docs
 # image that it gets deployed into.
 FROM python:${PYTHON_IMAGE_VERSION} AS build
 
+# Pull the uv binary in so we can use it as a faster pip / pip-compile.
+COPY --from=ghcr.io/astral-sh/uv:0.11.7 /uv /uvx /usr/local/bin/
+
 # Define whether we're building a production or a development image. This will
 # generally be used to control whether or not we install our development and
 # test dependencies.
@@ -132,47 +135,33 @@ ARG CI=no
 ARG IPYTHON=no
 
 # We create an /opt directory with a virtual environment in it to store our
-# application in.
+# application in. uv picks this up via VIRTUAL_ENV below.
 RUN set -x \
     && python3 -m venv /opt/warehouse
 
-# Now that we've created our virtual environment, we'll go ahead and update
-# our $PATH to refer to it first.
+# Point uv (and shells) at the venv we just created.
+# See the docs stage above for the rationale behind each UV_* variable.
+ENV VIRTUAL_ENV="/opt/warehouse" \
+    UV_PROJECT_ENVIRONMENT="/opt/warehouse" \
+    UV_LINK_MODE=copy \
+    UV_COMPILE_BYTECODE=1 \
+    UV_PYTHON_DOWNLOADS=never \
+    UV_PYTHON="/opt/warehouse/bin/python"
 ENV PATH="/opt/warehouse/bin:${PATH}"
 
-# Next, we want to update pip inside of this virtual
-# environment to ensure that we have the latest version.
-RUN pip --no-cache-dir --disable-pip-version-check install --upgrade pip
-
-# We copy this into the docker container prior to copying in the rest of our
-# application so that we can skip installing requirements if the only thing
-# that has changed is the Warehouse code itself.
-COPY requirements /tmp/requirements
-
-# Install our development dependencies if we're building a development install
-# otherwise this will do nothing.
-RUN --mount=type=cache,target=/root/.cache/pip \
+# Install Python deps via uv sync from pyproject.toml + uv.lock.
+# [project].dependencies (was main.in) is always installed; additional groups
+# are conditionally pulled in based on build args.
+RUN --mount=type=cache,target=/root/.cache/uv \
+    --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
+    --mount=type=bind,source=uv.lock,target=uv.lock \
     set -x \
-    && if [ "$DEVEL" = "yes" ]; then pip --disable-pip-version-check install -r /tmp/requirements/dev.txt; fi
-
-RUN --mount=type=cache,target=/root/.cache/pip \
-    set -x \
-    && if [ "$DEVEL" = "yes" ] && [ "$IPYTHON" = "yes" ]; then pip --disable-pip-version-check install -r /tmp/requirements/ipython.txt; fi
-
-# Install the Python level Warehouse requirements, this is done after copying
-# the requirements but prior to copying Warehouse itself into the container so
-# that code changes don't require triggering an entire install of all of
-# Warehouse's dependencies.
-RUN --mount=type=cache,target=/root/.cache/pip \
-    set -x \
-    && pip --disable-pip-version-check \
-            install --no-deps --only-binary :all: \
-                    -r /tmp/requirements/deploy.txt \
-                    -r /tmp/requirements/main.txt \
-                    $(if [ "$DEVEL" = "yes" ]; then echo '-r /tmp/requirements/tests.txt -r /tmp/requirements/lint.txt'; fi) \
-                    $(if [ "$CI" = "yes" ]; then echo '-r /tmp/requirements/docs-dev.txt -r /tmp/requirements/docs-user.txt -r /tmp/requirements/docs-blog.txt'; fi ) \
-    && pip check \
-    && find /opt/warehouse -name '*.pyc' -delete
+    && uv sync --frozen --no-default-groups \
+            --group deploy \
+            $(if [ "$DEVEL" = "yes" ]; then echo '--group dev --group tests --group lint'; fi) \
+            $(if [ "$DEVEL" = "yes" ] && [ "$IPYTHON" = "yes" ]; then echo '--group ipython'; fi) \
+            $(if [ "$CI" = "yes" ]; then echo '--group docs-dev --group docs-user --group docs-blog'; fi) \
+    && uv pip check
 
 
 
@@ -181,6 +170,10 @@ RUN --mount=type=cache,target=/root/.cache/pip \
 # pull in the static files that were built above.
 FROM python:${PYTHON_IMAGE_VERSION}
 
+# Pull the uv binary in so it is available at runtime. Used by bin/lint
+# (uv lock --check), make deps_upgrade_*, and any in-container shell work.
+COPY --from=ghcr.io/astral-sh/uv:0.11.7 /uv /uvx /usr/local/bin/
+
 # Setup some basic environment variables that are ~never going to change.
 ENV PYTHONUNBUFFERED 1
 ENV PYTHONPATH /opt/warehouse/src/

Makefile

@@ -38,7 +38,7 @@ default:
 	@echo
 	@exit 1
 
-.state/docker-build-base: Dockerfile package.json package-lock.json requirements/main.txt requirements/deploy.txt requirements/lint.txt requirements/tests.txt requirements/dev.txt
+.state/docker-build-base: Dockerfile package.json package-lock.json pyproject.toml uv.lock
 	# Build our base container for this project.
 	docker compose build --build-arg IPYTHON=$(IPYTHON) --force-rm base
 
@@ -54,7 +54,7 @@ default:
 	mkdir -p .state
 	touch .state/docker-build-static
 
-.state/docker-build-docs: Dockerfile requirements/docs-dev.txt requirements/docs-blog.txt requirements/docs-user.txt
+.state/docker-build-docs: Dockerfile pyproject.toml uv.lock
 	# Build the worker container for this project
 	docker compose build --build-arg  USER_ID=$(shell id -u)  --build-arg GROUP_ID=$(shell id -g) --force-rm dev-docs
 
@@ -112,20 +112,17 @@ licenses: .state/docker-build-base
 	docker compose run --rm base bin/licenses
 
 deps: .state/docker-build-base
-	docker compose run --rm base bin/deps
+	docker compose run --rm base uv lock --check
 
 deps_upgrade_all: .state/docker-build-base
-	docker compose run --rm base bin/deps-upgrade -a
+	docker compose run --rm base uv lock --upgrade
 
 deps_upgrade_project: .state/docker-build-base
-	docker compose run --rm base bin/deps-upgrade -p $(P)
+	docker compose run --rm base uv lock --upgrade-package $(P)
 
 translations: .state/docker-build-base
 	docker compose run --rm base bin/translations
 
-requirements/%.txt: requirements/%.in
-	docker compose run --rm base pip-compile --generate-hashes --output-file=$@ $<
-
 resetdb: .state/docker-build-base
 	docker compose pause web worker
 	docker compose up -d db

bin/lint

@@ -1,6 +1,8 @@
 #!/bin/bash
 set -ex
 
+# Fail if pyproject.toml has changed without regenerating uv.lock.
+uv lock --check
 
 # When on GitHub Actions, configure ruff to use the github output style
 if [ -n "$GITHUB_ACTIONS" ]; then

bin/rtd-docs

@@ -4,35 +4,36 @@ set -e
 # Print all the following commands
 set -x
 
+# Install uv into the RTD-provided Python so we can use it below.
+pip install uv
+asdf reshim
+
 if [ "${READTHEDOCS_PROJECT}" = "warehouse" ]; then
-  if [ "$READTHEDOCS_VERSION_TYPE" = "external" ] && git diff --quiet origin/main -- docs/dev requirements/docs-dev.txt .readthedocs.yaml docs/mkdocs-dev-docs.yml; then
+  if [ "$READTHEDOCS_VERSION_TYPE" = "external" ] && git diff --quiet origin/main -- docs/dev pyproject.toml uv.lock .readthedocs.yaml docs/mkdocs-dev-docs.yml; then
     echo "Nothing relevant changed, skipping build..."
     exit 183;
   fi
-  pip install -r requirements/docs-dev.txt
-  asdf reshim
-  mkdocs build -f docs/mkdocs-dev-docs.yml
+  uv sync --frozen --no-default-groups --group docs-dev
+  uv run --no-sync mkdocs build -f docs/mkdocs-dev-docs.yml
   mkdir _readthedocs && mv docs/dev-site _readthedocs/html
 fi
 
 if [ "${READTHEDOCS_PROJECT}" = "docspypiorg" ]; then
-  if [ "$READTHEDOCS_VERSION_TYPE" = "external" ] && git diff --quiet origin/main -- docs/user requirements/docs-user.txt .readthedocs.yaml docs/mkdocs-user-docs.yml; then
+  if [ "$READTHEDOCS_VERSION_TYPE" = "external" ] && git diff --quiet origin/main -- docs/user pyproject.toml uv.lock .readthedocs.yaml docs/mkdocs-user-docs.yml; then
     echo "Nothing relevant changed, skipping build..."
     exit 183;
   fi
-  pip install -r requirements/docs-user.txt
-  asdf reshim
-  mkdocs build -f docs/mkdocs-user-docs.yml
+  uv sync --frozen --no-default-groups --group docs-user
+  uv run --no-sync mkdocs build -f docs/mkdocs-user-docs.yml
   mkdir _readthedocs && mv docs/user-site _readthedocs/html
 fi
 
 if [ "${READTHEDOCS_PROJECT}" = "blogpypiorg" ] || [ "${READTHEDOCS_PROJECT}" = "psf-warehouse-private" ]; then
-  if [ "$READTHEDOCS_VERSION_TYPE" = "external" ] && git diff --quiet origin/main -- docs/blog requirements/docs-blog.txt .readthedocs.yaml docs/mkdocs-blog.yml; then
+  if [ "$READTHEDOCS_VERSION_TYPE" = "external" ] && git diff --quiet origin/main -- docs/blog pyproject.toml uv.lock .readthedocs.yaml docs/mkdocs-blog.yml; then
     echo "Nothing relevant changed, skipping build..."
     exit 183;
   fi
-  pip install -r requirements/docs-blog.txt
-  asdf reshim
-  mkdocs build -f docs/mkdocs-blog.yml
+  uv sync --frozen --no-default-groups --group docs-blog
+  uv run --no-sync mkdocs build -f docs/mkdocs-blog.yml
   mkdir _readthedocs && mv docs/blog-site _readthedocs/html
 fi