Fix: DeprecationWarning for datetime methods in Python 3.12 (#5953)

.mypy.ini

@@ -96,8 +96,5 @@ follow_untyped_imports = True
 ; https://github.com/python/typeshed/issues/13439
 follow_untyped_imports = True
 
-[mypy-aws_requests_auth.boto_utils]
-follow_untyped_imports = True
-
 [mypy-googleapiclient.*]
 follow_untyped_imports = True

requirements.all.txt

@@ -1,21 +1,20 @@
 atomicwrites==1.4.1
 attrs==26.1.0
-aws-requests-auth==0.4.3
 blessed==1.39.0
 boto3==1.42.97
 boto3-stubs-lite==1.42.97
 botocore==1.42.97
 botocore-stubs==1.42.41
 certifi==2026.4.22
 cffi==2.0.0
-chalice==1.32.0+20
+chalice==1.32.0+21
 charset-normalizer==3.4.7
 chevron==0.14.0
 click==8.3.3
 coverage==7.13.5
 cryptography==48.0.0
 docker==7.1.0
-editor==1.7.0
+editor==1.8.0
 et_xmlfile==2.0.0
 events==0.5
 fastavro==1.12.2
@@ -27,22 +26,22 @@ gitpython==3.1.48
 google-api-core==2.30.3
 google-api-python-client==2.194.0
 google-auth==2.49.2
-google-auth-httplib2==0.3.1
+google-auth-httplib2==0.4.0
 google-cloud-bigquery==3.41.0
 google-cloud-bigquery-reservation==1.23.0
-google-cloud-core==2.5.1
+google-cloud-core==2.6.0
 google-cloud-storage==3.10.1
 google-crc32c==1.8.0
-google-resumable-media==2.8.2
-googleapis-common-protos==1.74.0
+google-resumable-media==2.9.0
+googleapis-common-protos==1.75.0
 greenlet==3.5.0
 grpc-google-iam-v1==0.14.4
 grpcio==1.80.0
 grpcio-status==1.80.0
 http-message-signatures==2.0.1
 http_sfv==0.9.9
 httplib2==0.31.2
-idna==3.13
+idna==3.15
 inquirer==3.4.1
 isort==8.0.1
 jinja2==3.1.6
@@ -52,7 +51,7 @@ jsonschema==4.26.0
 jsonschema-path==0.3.4
 jsonschema-specifications==2025.9.1
 lazy-object-proxy==1.12.0
-librt==0.10.0
+librt==0.11.0
 markupsafe==3.0.3
 mccabe==0.7.0
 more-itertools==11.0.2
@@ -88,7 +87,7 @@ pathable==0.4.4
 pathspec==1.1.1
 pip==26.0.1
 posix_ipc==1.3.2
-proto-plus==1.27.2
+proto-plus==1.28.0
 protobuf==6.33.6
 py-partiql-parser==0.6.3
 pyasn1==0.6.3
@@ -134,6 +133,6 @@ werkzeug==3.1.8
 wheel==0.46.3
 www-authenticate==0.9.2
 xmltodict==1.0.4
-xmod==1.9.0
+xmod==1.10.0
 zope.event==6.2
 zope.interface==8.4

requirements.dev.trans.txt

@@ -1,18 +1,18 @@
 blessed==1.39.0
 botocore-stubs==1.42.41
 click==8.3.3
-editor==1.7.0
+editor==1.8.0
 et_xmlfile==2.0.0
 gitdb==4.0.12
-google-auth-httplib2==0.3.1
+google-auth-httplib2==0.4.0
 greenlet==3.5.0
 grpc-google-iam-v1==0.14.4
 httplib2==0.31.2
 inquirer==3.4.1
 jinja2==3.1.6
 jsonschema-path==0.3.4
 lazy-object-proxy==1.12.0
-librt==0.10.0
+librt==0.11.0
 mccabe==0.7.0
 mypy-boto3-apigateway==1.42.68
 mypy-boto3-cloudwatch==1.42.95
@@ -52,6 +52,6 @@ uritemplate==4.2.0
 wcwidth==0.7.0
 www-authenticate==0.9.2
 xmltodict==1.0.4
-xmod==1.9.0
+xmod==1.10.0
 zope.event==6.2
 zope.interface==8.4

requirements.dev.txt

@@ -1,8 +1,7 @@
---find-links https://github.com/DataBiosphere/azul-chalice/releases/expanded_assets/v1.32.0+20  # match version of chalice requirement below
 --find-links https://github.com/DataBiosphere/azul-requirements-parser/releases/expanded_assets/v0.13.0+6  # match version of requirements-parser requirement below
 atomicwrites==1.4.1
 boto3-stubs-lite[apigateway,cloudwatch,dynamodb,ec2,ecr,iam,kms,lambda,opensearch,s3,secretsmanager,securityhub,sns,sqs,ssm,stepfunctions,sts]==1.42.97  # match this with the version of the `boto3` runtime dendency
-chalice==1.32.0+20  # match version with find-links entry above
+git+https://github.com/DataBiosphere/azul-chalice@hotfix/1.32.0+21/azul-updates#egg=chalice  # match version with find-links entry above
 coverage==7.13.5
 docker==7.1.0
 flake8==7.3.0

requirements.trans.txt

@@ -3,18 +3,18 @@ cffi==2.0.0
 charset-normalizer==3.4.7
 cryptography==48.0.0
 events==0.5
-google-cloud-core==2.5.1
+google-cloud-core==2.6.0
 google-crc32c==1.8.0
-google-resumable-media==2.8.2
-googleapis-common-protos==1.74.0
+google-resumable-media==2.9.0
+googleapis-common-protos==1.75.0
 grpcio==1.80.0
 grpcio-status==1.80.0
-idna==3.13
+idna==3.15
 jsonschema-specifications==2025.9.1
 markupsafe==3.0.3
 orderedmultidict==1.0.2
 packaging==26.2
-proto-plus==1.27.2
+proto-plus==1.28.0
 protobuf==6.33.6
 pyasn1==0.6.3
 pyasn1_modules==0.4.2

requirements.txt

@@ -1,6 +1,5 @@
 --find-links  https://github.com/DataBiosphere/azul-resumablehash/releases/expanded_assets/1.5  # match version of resumablehash requirement below
 attrs==26.1.0
-aws-requests-auth==0.4.3
 boto3==1.42.97  # Match version of the `boto3-stubs` dev dependency
 botocore==1.42.97
 chevron==0.14.0  # Match with types-chevron in requirements.dev.txt

src/azul/opensearch.py

@@ -5,33 +5,24 @@
 from typing import (
     Any,
     Mapping,
-    cast,
 )
 from urllib.parse import (
     urlencode,
 )
 
-from aws_requests_auth.boto_utils import (
-    BotoAWSRequestsAuth,
-)
 from opensearchpy import (
     Connection,
     OpenSearch,
+    Urllib3AWSV4SignerAuth,
     Urllib3HttpConnection,
 )
-import requests
-import requests.auth
-import urllib3
 
 from azul import (
     config,
 )
 from azul.deployment import (
     aws,
 )
-from azul.http import (
-    HttpClient,
-)
 from azul.lib import (
     lru_cache,
 )
@@ -43,19 +34,6 @@
 log = logging.getLogger(__name__)
 
 
-class CachedBotoAWSRequestsAuth(BotoAWSRequestsAuth):
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        # We use the botocore session from Boto3 since it is pre-configured by
-        # envhook.py to use cached credentials for the AssumeRoleProvider. This
-        # avoids repeated entry of MFA tokens when running this code locally.
-        # noinspection PyProtectedMember
-        credentials = aws.boto3_session.get_credentials()
-        assert credentials is not None, R'Need credentials'
-        self._refreshable_credentials = credentials
-
-
 class AzulConnection(Connection):
     """
     Improves the request logging by the OpenSearch client library with
@@ -138,78 +116,15 @@ def _log_response(self,
         opensearch_log.log(log_level, http_body_log_message('response', response))
 
 
-class AWSAuthHttpClient(HttpClient):
-    """
-    Decorates a urllib3 HTTPConnectionPool instance so that requests are
-    signed with AWS's Signature Version 4 flavor of HMAC.
-    """
-
-    def __init__(self,
-                 pool: urllib3.HTTPConnectionPool,
-                 http_auth: BotoAWSRequestsAuth):
-        super().__init__()
-        self._inner = pool
-        self._http_auth = http_auth
-
-    def urlopen(self,  # type: ignore[override]
-                method: str,
-                url: str,
-                body: bytes | None = None,
-                headers: Mapping[str, str] | None = None,
-                **kwargs
-                ) -> urllib3.BaseHTTPResponse:
-        # self._http_auth is an instance of BotoAWSRequestsAuth, a subclass of
-        # AuthBase from the Requests library. To use that instance with urllib3
-        # directly, we need to prepare a Requests request object, sign it with
-        # self._http_auth and pass the resulting signature header to urllib3's
-        # urlopen() method.
-        request = requests.models.PreparedRequest()
-        request.method = method
-        # Because urllib3 connection pools are host-specific, URLs passed to a
-        # connection pool's urlencode() must be relative and path-absolute. And
-        # while PreparedRequest.prepare() requires an absolute URL, we can sneak
-        # a relative one in by setting the attribute directly. This neatly
-        # avoids having to compose an absolute URL and the URL-encoding
-        # ambiguities that entails. The OpenSearch client, for example,
-        # encodes colons in absolute paths even though the leading slash in such
-        # a path makes that unnecessary. These ambiguities could lead to an
-        # invalid signature. The AWS signature algorithm only looks at path and
-        # query of URLs.
-        assert url.startswith('/'), url
-        request.url = url
-        request.headers = headers
-        request.body = body
-        request = self._http_auth(request)
-        # Note that the various urlopen() implementations in urllib3 declare the
-        # `body` argument with a default value, making it a keyword argument,
-        # the ES client passes it as a positional. If this were ever to change,
-        # this method would get a duplicate of the `body` argument as part of
-        # `kwargs`, resulting in a TypeError.
-        return self._inner.urlopen(method, url, body, headers=request.headers, **kwargs)
-
-    def close(self):
-        self._inner.close()
-
-
 class AzulUrllib3HttpConnection(AzulConnection, Urllib3HttpConnection):
+    """
+    Combines AzulConnection's improved request logging with Urllib3HttpConnection's
+    HTTP transport and native SigV4 signing via the ``http_auth`` callable interface.
 
-    def __init__(self,
-                 *args,
-                 http_auth: BotoAWSRequestsAuth | None = None,
-                 **kwargs
-                 ) -> None:
-        super().__init__(*args, **kwargs)
-        if http_auth is not None:
-            # We can't extend the pool class because we don't control the
-            # instantiation. We therefore have to decorate the pool instance.
-            # Looking at the source of Urllib3HttpConnection we notice that only
-            # the methods `urlopen()` and `close()` are called. This means that
-            # the decorating class doesn't need to implement (or extend) a full
-            # HTTPConnectionPool, only the much slimmer RequestMethods.
-            client = AWSAuthHttpClient(self.pool, http_auth)
-            # We still need the cast because the stub declares `self.pool` to be
-            # an instance of HTTPConnectionPool.
-            self.pool = cast(urllib3.HTTPConnectionPool, client)
+    Since Urllib3HttpConnection already supports callable http_auth natively,
+    AzulUrllib3HttpConnection.__init__ is no longer needed.
+    """
+    pass
 
 
 class OpenSearchClientFactory:
@@ -234,9 +149,13 @@ def _create_client(cls, host, port, timeout):
                              timeout=timeout,
                              max_retries=0)
         if host.endswith('.amazonaws.com'):
-            aws_auth = CachedBotoAWSRequestsAuth(aws_host=host,
-                                                 aws_region=aws.region_name,
-                                                 aws_service='es')
+            # We use the botocore session from Boto3 since it is pre-configured
+            # by envhook.py to use cached credentials for the AssumeRoleProvider.
+            # This avoids repeated entry of MFA tokens when running this code
+            # locally.
+            refreshable_credentials = aws.boto3_session.get_credentials()
+            assert refreshable_credentials is not None, R'Need credentials'
+            aws_auth = Urllib3AWSV4SignerAuth(refreshable_credentials, aws.region_name)
             return OpenSearch(http_auth=aws_auth,
                               use_ssl=True,
                               verify_certs=True,

test/azul_test_case.py

@@ -148,10 +148,6 @@ class RE(str):
                     'and will be removed in .*. Instead use .*'
                 ),
 
-                # FIXME: DeprecationWarning for datetime methods in Python 3.12
-                #        https://github.com/DataBiosphere/azul/issues/5953
-                'datetime.datetime.utcnow() is deprecated',
-
                 # FIXME: DeprecationWarning for patch_source_cache
                 #        https://github.com/DataBiosphere/azul/issues/7838
                 'Instead of decorating your test case, or its test methods in '