discuss: implement dependency cooldowns in more areas

.github/dependabot.yml

@@ -9,5 +9,6 @@ updates:
       actions:
         patterns:
           - "*"
+    # NOTE: keep this in sync with bin/_cooldown.py
     cooldown:
       default-days: 7

.github/workflows/update-dependencies.yml

@@ -11,6 +11,11 @@ on:
       - 'docs/data/projects.yml'
       - 'noxfile.py'
   workflow_dispatch:
+    inputs:
+      ignore-cooldown:
+        description: 'Ignore the cooldown window and pick up the very latest releases'
+        type: boolean
+        default: false
   schedule:
     - cron: '0 6 * * 1'  # "At 06:00 on Monday."
 
@@ -45,8 +50,12 @@ jobs:
 
     - name: "Run update: dependencies"
       run: uvx nox --force-color -s update_constraints
+      env:
+        CIBW_IGNORE_COOLDOWN: ${{ github.event.inputs.ignore-cooldown || 'false' }}
     - name: "Run update: python configs"
       run: uvx nox --force-color -s update_pins
+      env:
+        CIBW_IGNORE_COOLDOWN: ${{ github.event.inputs.ignore-cooldown || 'false' }}
     - name: "Run update: docs user projects"
       run: uvx nox --force-color -s update_proj -- --auth=${{ secrets.GITHUB_TOKEN }}
 

.pre-commit-config.yaml

@@ -29,7 +29,7 @@ repos:
   - id: mypy
     name: mypy 3.11 on cibuildwheel/
     args: ["--python-version=3.11"]
-    exclude: ^cibuildwheel/resources/android/_cross_venv.py$  # Requires Python 3.13 or later
+    exclude: (^cibuildwheel/resources/android/_cross_venv.py$|^bin/_cooldown\.py$)  # _cross_venv.py requires Python 3.13 or later and _cooldown.py is a support module for bin/ scripts
     additional_dependencies: &mypy-dependencies
       - bracex
       - build
@@ -54,6 +54,7 @@ repos:
   - id: mypy
     name: mypy 3.14
     args: ["--python-version=3.14"]
+    exclude: ^bin/_cooldown\.py$  # support module, not a top-level file
     additional_dependencies: *mypy-dependencies
 
 - repo: https://github.com/shellcheck-py/shellcheck-py

bin/_cooldown.py

@@ -0,0 +1,10 @@
+import os
+
+# Number of days a release must age before the update scripts will pick it up.
+# This is intentionally different from Dependabot because we want to have these
+# updates coming faster to align with the latest releases.
+COOLDOWN_DAYS = 2
+
+# Set CIBW_IGNORE_COOLDOWN to a truthy value to bypass the cooldown and always pick
+# up the very latest releases regardless of age.
+IGNORE_COOLDOWN = os.environ.get("CIBW_IGNORE_COOLDOWN", "").lower() in ("1", "true")

bin/update_nodejs.py

@@ -6,21 +6,27 @@
 #   "packaging",
 #   "requests",
 #   "rich",
+#   "cibuildwheel",
 # ]
+#
+# [tool.uv.sources]
+# cibuildwheel = { path = ".." }
 # ///
 
 
 import dataclasses
 import difflib
 import logging
 import tomllib
+from datetime import UTC, date, datetime, timedelta
 from pathlib import Path
 from typing import Final
 
 import click
 import packaging.specifiers
 import requests
 import rich
+from _cooldown import COOLDOWN_DAYS, IGNORE_COOLDOWN
 from packaging.version import InvalidVersion, Version
 from rich.logging import RichHandler
 from rich.syntax import Syntax
@@ -47,6 +53,11 @@ def parse_nodejs_index() -> list[VersionTuple]:
     response = requests.get(NODEJS_INDEX)
     response.raise_for_status()
     versions_info = response.json()
+    cutoff_date: date = (
+        date.max
+        if IGNORE_COOLDOWN
+        else (datetime.now(tz=UTC) - timedelta(days=COOLDOWN_DAYS)).date()
+    )
     for version_info in versions_info:
         version_string = version_info.get("version", "???")
         if not version_info.get("lts", False):
@@ -57,6 +68,9 @@ def parse_nodejs_index() -> list[VersionTuple]:
                 "Ignoring release %r which does not include a linux-x64 binary", version_string
             )
             continue
+        if date.fromisoformat(version_info["date"]) > cutoff_date:
+            log.info("Ignoring release %r within cooldown period", version_string)
+            continue
         try:
             version = Version(version_string)
             if version.is_devrelease:

bin/update_python_build_standalone.py

@@ -10,6 +10,9 @@
 # ///
 
 import json
+from datetime import UTC, datetime, timedelta
+
+from _cooldown import COOLDOWN_DAYS, IGNORE_COOLDOWN
 
 from cibuildwheel.extra import github_api_request
 from cibuildwheel.util.python_build_standalone import (
@@ -29,6 +32,14 @@ def main() -> None:
     latest_release = github_api_request("repos/astral-sh/python-build-standalone/releases/latest")
     latest_tag = latest_release["tag_name"]
 
+    published_at = datetime.fromisoformat(latest_release["published_at"])
+    if not IGNORE_COOLDOWN and datetime.now(tz=UTC) - published_at < timedelta(days=COOLDOWN_DAYS):
+        print(
+            f"Skipping update: latest release {latest_tag!r} was published "
+            f"less than {COOLDOWN_DAYS} days ago."
+        )
+        return
+
     # Get the list of assets for the latest release
     github_assets = github_api_request(
         f"repos/astral-sh/python-build-standalone/releases/tags/{latest_tag}"

bin/update_pythons.py

@@ -20,13 +20,16 @@
 import re
 import tomllib
 from collections.abc import Mapping, MutableMapping
+from datetime import UTC, date, datetime, timedelta
+from email.utils import parsedate_to_datetime
 from pathlib import Path
 from typing import Any, Final, Literal, TypedDict
 from xml.etree import ElementTree as ET
 
 import click
 import requests
 import rich
+from _cooldown import COOLDOWN_DAYS, IGNORE_COOLDOWN
 from packaging.specifiers import Specifier
 from packaging.version import Version
 from rich.logging import RichHandler
@@ -65,14 +68,14 @@ class ConfigPyodide(Config):
 
 
 class WindowsVersions:
-    def __init__(self, arch_str: ArchStr, free_threaded: bool) -> None:
+    def __init__(self, arch_str: ArchStr, free_threaded: bool, cutoff_date: date) -> None:
         response = requests.get("https://api.nuget.org/v3/index.json")
         response.raise_for_status()
         api_info = response.json()
 
-        for resource in api_info["resources"]:
-            if resource["@type"] == "PackageBaseAddress/3.0.0":
-                endpoint = resource["@id"]
+        reg_endpoint = next(
+            r["@id"] for r in api_info["resources"] if r["@type"] == "RegistrationsBaseUrl/3.6.0"
+        )
 
         ARCH_DICT = {"32": "win32", "64": "win_amd64", "ARM64": "win_arm64"}
         PACKAGE_DICT = {"32": "pythonx86", "64": "python", "ARM64": "pythonarm64"}
@@ -85,11 +88,30 @@ def __init__(self, arch_str: ArchStr, free_threaded: bool) -> None:
         if free_threaded:
             package = f"{package}-freethreaded"
 
-        response = requests.get(f"{endpoint}{package}/index.json")
-        response.raise_for_status()
-        cp_info = response.json()
-
-        self.version_dict = {Version(v): v for v in cp_info["versions"]}
+        # NuGet serves registration responses gzip-compressed; requests decompresses
+        # automatically via Accept-Encoding negotiation
+        reg_response = requests.get(f"{reg_endpoint}{package}/index.json")
+        reg_response.raise_for_status()
+        reg_data = reg_response.json()
+
+        # NuGet uses 1900-01-01 as a sentinel for packages whose publish date was
+        # not recorded. The NuGet SDK looks at this as null and treats it as
+        # "allow" (and dependabot-core does the same):
+        # https://github.com/dependabot/dependabot-core/blob/b0090acfa61b7541c040e302c760a84c702217a3/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/Cooldown.cs#L70-L75
+        NUGET_DATE_SENTINEL = date(1900, 1, 1)
+
+        self.version_dict: dict[Version, str] = {}
+        for page_meta in reg_data["items"]:
+            # Registration pages may be inlined in the index response or kept external
+            # See: https://github.com/microsoft/NativeAOTDependencyHelper/blob/9ad3f7be5b919f6166d60a228691e1c802797909/NativeAOTDependencyHelper.Core/Checks/NuGetRecentlyUpdatedCheck.cs#L36-L45
+            # pythonarm64 and all freethreaded packages have fully-inlined pages and need no extra fetches.
+            items = page_meta.get("items") or requests.get(page_meta["@id"]).json()["items"]
+            for item in items:
+                entry = item["catalogEntry"]
+                published = datetime.fromisoformat(entry["published"]).date()
+                if published != NUGET_DATE_SENTINEL and published > cutoff_date:
+                    continue
+                self.version_dict[Version(entry["version"])] = entry["version"]
 
     def update_version_windows(self, spec: Specifier) -> Config | None:
         # Specifier.filter selects all non pre-releases that match the spec,
@@ -113,7 +135,7 @@ def update_version_windows(self, spec: Specifier) -> Config | None:
 
 
 class GraalPyVersions:
-    def __init__(self) -> None:
+    def __init__(self, cutoff_date: date) -> None:
         response = requests.get("https://api.github.com/repos/oracle/graalpython/releases")
         response.raise_for_status()
 
@@ -128,7 +150,13 @@ def __init__(self) -> None:
             if m:
                 release["python_version"] = Version(m.group(1))
 
-        self.releases = [r for r in releases if "graalpy_version" in r and "python_version" in r]
+        self.releases = [
+            r
+            for r in releases
+            if "graalpy_version" in r
+            and "python_version" in r
+            and datetime.fromisoformat(r["published_at"]).date() <= cutoff_date
+        ]
 
     def update_version(self, identifier: str, spec: Specifier) -> ConfigUrl | None:
         if "x86_64" in identifier or "amd64" in identifier:
@@ -185,7 +213,7 @@ def update_version(self, identifier: str, spec: Specifier) -> ConfigUrl | None:
 
 
 class PyPyVersions:
-    def __init__(self, arch_str: ArchStr):
+    def __init__(self, arch_str: ArchStr, cutoff_date: date):
         response = requests.get("https://downloads.python.org/pypy/versions.json")
         response.raise_for_status()
 
@@ -197,7 +225,9 @@ def __init__(self, arch_str: ArchStr):
         self.releases = [
             r
             for r in releases
-            if not r["pypy_version"].is_prerelease and not r["pypy_version"].is_devrelease
+            if not r["pypy_version"].is_prerelease
+            and not r["pypy_version"].is_devrelease
+            and date.fromisoformat(r["date"]) <= cutoff_date
         ]
         self.arch = arch_str
 
@@ -263,7 +293,7 @@ def update_version_macos(self, spec: Specifier) -> ConfigUrl:
 
 
 class CPythonVersions:
-    def __init__(self) -> None:
+    def __init__(self, cutoff_date: date) -> None:
         response = requests.get(
             "https://www.python.org/api/v2/downloads/release/?is_published=true"
         )
@@ -277,6 +307,10 @@ def __init__(self) -> None:
             if not release["slug"].startswith("python"):
                 continue
 
+            release_date_str = release.get("release_date")
+            if release_date_str and datetime.fromisoformat(release_date_str).date() > cutoff_date:
+                continue
+
             # Removing the prefix
             version = Version(release["name"].removeprefix("Python "))
             self.versions_dict[version] = release["resource_uri"]
@@ -318,7 +352,7 @@ def update_version_android(self, identifier: str, spec: Specifier) -> ConfigUrl
 class MavenVersions:
     MAVEN_URL = "https://repo.maven.apache.org/maven2/com/chaquo/python/python"
 
-    def __init__(self) -> None:
+    def __init__(self, cutoff_date: date) -> None:
         response = requests.get(f"{self.MAVEN_URL}/maven-metadata.xml")
         response.raise_for_status()
         root = ET.fromstring(response.text)
@@ -329,24 +363,34 @@ def __init__(self) -> None:
             assert isinstance(version_str, str), version_str
             self.versions.append(Version(version_str))
 
+        self.cutoff_date = cutoff_date
+
     def update_version_android(self, identifier: str, spec: Specifier) -> ConfigUrl | None:
         sorted_versions = sorted(spec.filter(self.versions), reverse=True)
 
-        # Return a config using the highest version for the given specifier.
-        if sorted_versions:
-            max_version = sorted_versions[0]
+        # maven-metadata.xml only carries a package-level timestamp, not per-version
+        # dates, so we check the Last-Modified header on each candidate's POM file
+        for max_version in sorted_versions:
             triplet = android_triplet(identifier)
+            pom_url = f"{self.MAVEN_URL}/{max_version}/python-{max_version}.pom"
+            head_response = requests.head(pom_url)
+            head_response.raise_for_status()
+            last_modified_str = head_response.headers.get("Last-Modified")
+            if last_modified_str:
+                published = parsedate_to_datetime(last_modified_str).date()
+                if published > self.cutoff_date:
+                    log.info("Skipping %s: published %s is within cooldown", max_version, published)
+                    continue
             return ConfigUrl(
                 identifier=identifier,
                 version=f"{max_version.major}.{max_version.minor}",
                 url=f"{self.MAVEN_URL}/{max_version}/python-{max_version}-{triplet}.tar.gz",
             )
-        else:
-            return None
+        return None
 
 
 class CPythonIOSVersions:
-    def __init__(self) -> None:
+    def __init__(self, cutoff_date: date) -> None:
         response = requests.get(
             "https://api.github.com/repos/beeware/Python-Apple-support/releases",
             headers={
@@ -361,6 +405,9 @@ def __init__(self) -> None:
 
         # Each release has a name like "3.13-b4"
         for release in releases_info:
+            if datetime.fromisoformat(release["published_at"]).date() > cutoff_date:
+                continue
+
             py_version, build = release["name"].split("-")
             version = Version(py_version)
             self.versions_dict.setdefault(version, {})
@@ -426,22 +473,28 @@ def update_version_pyodide(
 
 class AllVersions:
     def __init__(self) -> None:
-        self.windows_32 = WindowsVersions("32", False)
-        self.windows_t_32 = WindowsVersions("32", True)
-        self.windows_64 = WindowsVersions("64", False)
-        self.windows_t_64 = WindowsVersions("64", True)
-        self.windows_arm64 = WindowsVersions("ARM64", False)
-        self.windows_t_arm64 = WindowsVersions("ARM64", True)
-        self.windows_pypy_64 = PyPyVersions("64")
-
-        self.cpython = CPythonVersions()
-        self.macos_pypy = PyPyVersions("64")
-        self.macos_pypy_arm64 = PyPyVersions("ARM64")
-
-        self.maven = MavenVersions()
-        self.ios_cpython = CPythonIOSVersions()
-
-        self.graalpy = GraalPyVersions()
+        cutoff_date: date = (
+            date.max
+            if IGNORE_COOLDOWN
+            else (datetime.now(tz=UTC) - timedelta(days=COOLDOWN_DAYS)).date()
+        )
+
+        self.windows_32 = WindowsVersions("32", False, cutoff_date)
+        self.windows_t_32 = WindowsVersions("32", True, cutoff_date)
+        self.windows_64 = WindowsVersions("64", False, cutoff_date)
+        self.windows_t_64 = WindowsVersions("64", True, cutoff_date)
+        self.windows_arm64 = WindowsVersions("ARM64", False, cutoff_date)
+        self.windows_t_arm64 = WindowsVersions("ARM64", True, cutoff_date)
+        self.windows_pypy_64 = PyPyVersions("64", cutoff_date)
+
+        self.cpython = CPythonVersions(cutoff_date)
+        self.macos_pypy = PyPyVersions("64", cutoff_date)
+        self.macos_pypy_arm64 = PyPyVersions("ARM64", cutoff_date)
+
+        self.maven = MavenVersions(cutoff_date)
+        self.ios_cpython = CPythonIOSVersions(cutoff_date)
+
+        self.graalpy = GraalPyVersions(cutoff_date)
 
         self.pyodide = PyodideVersions()