diff --git a/.mypy.ini b/.mypy.ini index 91dc65a0ec..e605509f77 100644 --- a/.mypy.ini +++ b/.mypy.ini @@ -96,8 +96,5 @@ follow_untyped_imports = True ; https://github.com/python/typeshed/issues/13439 follow_untyped_imports = True -[mypy-aws_requests_auth.boto_utils] -follow_untyped_imports = True - [mypy-googleapiclient.*] follow_untyped_imports = True diff --git a/bin/wheels/runtime/aws_requests_auth-0.4.3-py2.py3-none-any.whl b/bin/wheels/runtime/aws_requests_auth-0.4.3-py2.py3-none-any.whl deleted file mode 100644 index a7c9e7c61f..0000000000 Binary files a/bin/wheels/runtime/aws_requests_auth-0.4.3-py2.py3-none-any.whl and /dev/null differ diff --git a/bin/wheels/runtime/google_cloud_core-2.5.1-py3-none-any.whl b/bin/wheels/runtime/google_cloud_core-2.6.0-py3-none-any.whl similarity index 78% rename from bin/wheels/runtime/google_cloud_core-2.5.1-py3-none-any.whl rename to bin/wheels/runtime/google_cloud_core-2.6.0-py3-none-any.whl index aa826623d4..109d5634f9 100644 Binary files a/bin/wheels/runtime/google_cloud_core-2.5.1-py3-none-any.whl and b/bin/wheels/runtime/google_cloud_core-2.6.0-py3-none-any.whl differ diff --git a/bin/wheels/runtime/google_resumable_media-2.8.2-py3-none-any.whl b/bin/wheels/runtime/google_resumable_media-2.9.0-py3-none-any.whl similarity index 93% rename from bin/wheels/runtime/google_resumable_media-2.8.2-py3-none-any.whl rename to bin/wheels/runtime/google_resumable_media-2.9.0-py3-none-any.whl index 2d130e6eb5..dcb8be42bb 100644 Binary files a/bin/wheels/runtime/google_resumable_media-2.8.2-py3-none-any.whl and b/bin/wheels/runtime/google_resumable_media-2.9.0-py3-none-any.whl differ diff --git a/bin/wheels/runtime/googleapis_common_protos-1.74.0-py3-none-any.whl b/bin/wheels/runtime/googleapis_common_protos-1.75.0-py3-none-any.whl similarity index 79% rename from bin/wheels/runtime/googleapis_common_protos-1.74.0-py3-none-any.whl rename to bin/wheels/runtime/googleapis_common_protos-1.75.0-py3-none-any.whl index 0e40212c5d..780ae72c0b 100644 Binary files a/bin/wheels/runtime/googleapis_common_protos-1.74.0-py3-none-any.whl and b/bin/wheels/runtime/googleapis_common_protos-1.75.0-py3-none-any.whl differ diff --git a/bin/wheels/runtime/idna-3.13-py3-none-any.whl b/bin/wheels/runtime/idna-3.13-py3-none-any.whl deleted file mode 100644 index 82401620eb..0000000000 Binary files a/bin/wheels/runtime/idna-3.13-py3-none-any.whl and /dev/null differ diff --git a/bin/wheels/runtime/idna-3.15-py3-none-any.whl b/bin/wheels/runtime/idna-3.15-py3-none-any.whl new file mode 100644 index 0000000000..cffdab65bb Binary files /dev/null and b/bin/wheels/runtime/idna-3.15-py3-none-any.whl differ diff --git a/bin/wheels/runtime/proto_plus-1.27.2-py3-none-any.whl b/bin/wheels/runtime/proto_plus-1.28.0-py3-none-any.whl similarity index 75% rename from bin/wheels/runtime/proto_plus-1.27.2-py3-none-any.whl rename to bin/wheels/runtime/proto_plus-1.28.0-py3-none-any.whl index ef7b39e052..b4fadaeaff 100644 Binary files a/bin/wheels/runtime/proto_plus-1.27.2-py3-none-any.whl and b/bin/wheels/runtime/proto_plus-1.28.0-py3-none-any.whl differ diff --git a/requirements.all.txt b/requirements.all.txt index 8fa062bd56..33d2ec447f 100644 --- a/requirements.all.txt +++ b/requirements.all.txt @@ -1,6 +1,5 @@ atomicwrites==1.4.1 attrs==26.1.0 -aws-requests-auth==0.4.3 blessed==1.39.0 boto3==1.42.97 boto3-stubs-lite==1.42.97 @@ -8,14 +7,14 @@ botocore==1.42.97 botocore-stubs==1.42.41 certifi==2026.4.22 cffi==2.0.0 -chalice==1.32.0+20 +chalice==1.32.0+21 charset-normalizer==3.4.7 chevron==0.14.0 click==8.3.3 coverage==7.13.5 cryptography==48.0.0 docker==7.1.0 -editor==1.7.0 +editor==1.8.0 et_xmlfile==2.0.0 events==0.5 fastavro==1.12.2 @@ -27,14 +26,14 @@ gitpython==3.1.48 google-api-core==2.30.3 google-api-python-client==2.194.0 google-auth==2.49.2 -google-auth-httplib2==0.3.1 +google-auth-httplib2==0.4.0 google-cloud-bigquery==3.41.0 google-cloud-bigquery-reservation==1.23.0 -google-cloud-core==2.5.1 +google-cloud-core==2.6.0 google-cloud-storage==3.10.1 google-crc32c==1.8.0 -google-resumable-media==2.8.2 -googleapis-common-protos==1.74.0 +google-resumable-media==2.9.0 +googleapis-common-protos==1.75.0 greenlet==3.5.0 grpc-google-iam-v1==0.14.4 grpcio==1.80.0 @@ -42,7 +41,7 @@ grpcio-status==1.80.0 http-message-signatures==2.0.1 http_sfv==0.9.9 httplib2==0.31.2 -idna==3.13 +idna==3.15 inquirer==3.4.1 isort==8.0.1 jinja2==3.1.6 @@ -52,7 +51,7 @@ jsonschema==4.26.0 jsonschema-path==0.3.4 jsonschema-specifications==2025.9.1 lazy-object-proxy==1.12.0 -librt==0.10.0 +librt==0.11.0 markupsafe==3.0.3 mccabe==0.7.0 more-itertools==11.0.2 @@ -88,7 +87,7 @@ pathable==0.4.4 pathspec==1.1.1 pip==26.0.1 posix_ipc==1.3.2 -proto-plus==1.27.2 +proto-plus==1.28.0 protobuf==6.33.6 py-partiql-parser==0.6.3 pyasn1==0.6.3 @@ -134,6 +133,6 @@ werkzeug==3.1.8 wheel==0.46.3 www-authenticate==0.9.2 xmltodict==1.0.4 -xmod==1.9.0 +xmod==1.10.0 zope.event==6.2 zope.interface==8.4 diff --git a/requirements.dev.trans.txt b/requirements.dev.trans.txt index 6ad7e933f2..a97a63fc77 100644 --- a/requirements.dev.trans.txt +++ b/requirements.dev.trans.txt @@ -1,10 +1,10 @@ blessed==1.39.0 botocore-stubs==1.42.41 click==8.3.3 -editor==1.7.0 +editor==1.8.0 et_xmlfile==2.0.0 gitdb==4.0.12 -google-auth-httplib2==0.3.1 +google-auth-httplib2==0.4.0 greenlet==3.5.0 grpc-google-iam-v1==0.14.4 httplib2==0.31.2 @@ -12,7 +12,7 @@ inquirer==3.4.1 jinja2==3.1.6 jsonschema-path==0.3.4 lazy-object-proxy==1.12.0 -librt==0.10.0 +librt==0.11.0 mccabe==0.7.0 mypy-boto3-apigateway==1.42.68 mypy-boto3-cloudwatch==1.42.95 @@ -52,6 +52,6 @@ uritemplate==4.2.0 wcwidth==0.7.0 www-authenticate==0.9.2 xmltodict==1.0.4 -xmod==1.9.0 +xmod==1.10.0 zope.event==6.2 zope.interface==8.4 diff --git a/requirements.dev.txt b/requirements.dev.txt index 3eb8fedaab..0f703eb34b 100644 --- a/requirements.dev.txt +++ b/requirements.dev.txt @@ -1,8 +1,7 @@ ---find-links https://github.com/DataBiosphere/azul-chalice/releases/expanded_assets/v1.32.0+20 # match version of chalice requirement below --find-links https://github.com/DataBiosphere/azul-requirements-parser/releases/expanded_assets/v0.13.0+6 # match version of requirements-parser requirement below atomicwrites==1.4.1 boto3-stubs-lite[apigateway,cloudwatch,dynamodb,ec2,ecr,iam,kms,lambda,opensearch,s3,secretsmanager,securityhub,sns,sqs,ssm,stepfunctions,sts]==1.42.97 # match this with the version of the `boto3` runtime dendency -chalice==1.32.0+20 # match version with find-links entry above +git+https://github.com/DataBiosphere/azul-chalice@hotfix/1.32.0+21/azul-updates#egg=chalice # match version with find-links entry above coverage==7.13.5 docker==7.1.0 flake8==7.3.0 diff --git a/requirements.trans.txt b/requirements.trans.txt index 9e5979b273..8e4d2e7024 100644 --- a/requirements.trans.txt +++ b/requirements.trans.txt @@ -3,18 +3,18 @@ cffi==2.0.0 charset-normalizer==3.4.7 cryptography==48.0.0 events==0.5 -google-cloud-core==2.5.1 +google-cloud-core==2.6.0 google-crc32c==1.8.0 -google-resumable-media==2.8.2 -googleapis-common-protos==1.74.0 +google-resumable-media==2.9.0 +googleapis-common-protos==1.75.0 grpcio==1.80.0 grpcio-status==1.80.0 -idna==3.13 +idna==3.15 jsonschema-specifications==2025.9.1 markupsafe==3.0.3 orderedmultidict==1.0.2 packaging==26.2 -proto-plus==1.27.2 +proto-plus==1.28.0 protobuf==6.33.6 pyasn1==0.6.3 pyasn1_modules==0.4.2 diff --git a/requirements.txt b/requirements.txt index 261fc454b9..b83d9587a3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,5 @@ --find-links https://github.com/DataBiosphere/azul-resumablehash/releases/expanded_assets/1.5 # match version of resumablehash requirement below attrs==26.1.0 -aws-requests-auth==0.4.3 boto3==1.42.97 # Match version of the `boto3-stubs` dev dependency botocore==1.42.97 chevron==0.14.0 # Match with types-chevron in requirements.dev.txt diff --git a/src/azul/opensearch.py b/src/azul/opensearch.py index aaf2db2d22..27b7ae8618 100644 --- a/src/azul/opensearch.py +++ b/src/azul/opensearch.py @@ -5,23 +5,17 @@ from typing import ( Any, Mapping, - cast, ) from urllib.parse import ( urlencode, ) -from aws_requests_auth.boto_utils import ( - BotoAWSRequestsAuth, -) from opensearchpy import ( Connection, OpenSearch, + Urllib3AWSV4SignerAuth, Urllib3HttpConnection, ) -import requests -import requests.auth -import urllib3 from azul import ( config, @@ -29,9 +23,6 @@ from azul.deployment import ( aws, ) -from azul.http import ( - HttpClient, -) from azul.lib import ( lru_cache, ) @@ -43,19 +34,6 @@ log = logging.getLogger(__name__) -class CachedBotoAWSRequestsAuth(BotoAWSRequestsAuth): - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - # We use the botocore session from Boto3 since it is pre-configured by - # envhook.py to use cached credentials for the AssumeRoleProvider. This - # avoids repeated entry of MFA tokens when running this code locally. - # noinspection PyProtectedMember - credentials = aws.boto3_session.get_credentials() - assert credentials is not None, R'Need credentials' - self._refreshable_credentials = credentials - - class AzulConnection(Connection): """ Improves the request logging by the OpenSearch client library with @@ -138,78 +116,15 @@ def _log_response(self, opensearch_log.log(log_level, http_body_log_message('response', response)) -class AWSAuthHttpClient(HttpClient): - """ - Decorates a urllib3 HTTPConnectionPool instance so that requests are - signed with AWS's Signature Version 4 flavor of HMAC. - """ - - def __init__(self, - pool: urllib3.HTTPConnectionPool, - http_auth: BotoAWSRequestsAuth): - super().__init__() - self._inner = pool - self._http_auth = http_auth - - def urlopen(self, # type: ignore[override] - method: str, - url: str, - body: bytes | None = None, - headers: Mapping[str, str] | None = None, - **kwargs - ) -> urllib3.BaseHTTPResponse: - # self._http_auth is an instance of BotoAWSRequestsAuth, a subclass of - # AuthBase from the Requests library. To use that instance with urllib3 - # directly, we need to prepare a Requests request object, sign it with - # self._http_auth and pass the resulting signature header to urllib3's - # urlopen() method. - request = requests.models.PreparedRequest() - request.method = method - # Because urllib3 connection pools are host-specific, URLs passed to a - # connection pool's urlencode() must be relative and path-absolute. And - # while PreparedRequest.prepare() requires an absolute URL, we can sneak - # a relative one in by setting the attribute directly. This neatly - # avoids having to compose an absolute URL and the URL-encoding - # ambiguities that entails. The OpenSearch client, for example, - # encodes colons in absolute paths even though the leading slash in such - # a path makes that unnecessary. These ambiguities could lead to an - # invalid signature. The AWS signature algorithm only looks at path and - # query of URLs. - assert url.startswith('/'), url - request.url = url - request.headers = headers - request.body = body - request = self._http_auth(request) - # Note that the various urlopen() implementations in urllib3 declare the - # `body` argument with a default value, making it a keyword argument, - # the ES client passes it as a positional. If this were ever to change, - # this method would get a duplicate of the `body` argument as part of - # `kwargs`, resulting in a TypeError. - return self._inner.urlopen(method, url, body, headers=request.headers, **kwargs) - - def close(self): - self._inner.close() - - class AzulUrllib3HttpConnection(AzulConnection, Urllib3HttpConnection): + """ + Combines AzulConnection's improved request logging with Urllib3HttpConnection's + HTTP transport and native SigV4 signing via the ``http_auth`` callable interface. - def __init__(self, - *args, - http_auth: BotoAWSRequestsAuth | None = None, - **kwargs - ) -> None: - super().__init__(*args, **kwargs) - if http_auth is not None: - # We can't extend the pool class because we don't control the - # instantiation. We therefore have to decorate the pool instance. - # Looking at the source of Urllib3HttpConnection we notice that only - # the methods `urlopen()` and `close()` are called. This means that - # the decorating class doesn't need to implement (or extend) a full - # HTTPConnectionPool, only the much slimmer RequestMethods. - client = AWSAuthHttpClient(self.pool, http_auth) - # We still need the cast because the stub declares `self.pool` to be - # an instance of HTTPConnectionPool. - self.pool = cast(urllib3.HTTPConnectionPool, client) + Since Urllib3HttpConnection already supports callable http_auth natively, + AzulUrllib3HttpConnection.__init__ is no longer needed. + """ + pass class OpenSearchClientFactory: @@ -234,9 +149,13 @@ def _create_client(cls, host, port, timeout): timeout=timeout, max_retries=0) if host.endswith('.amazonaws.com'): - aws_auth = CachedBotoAWSRequestsAuth(aws_host=host, - aws_region=aws.region_name, - aws_service='es') + # We use the botocore session from Boto3 since it is pre-configured + # by envhook.py to use cached credentials for the AssumeRoleProvider. + # This avoids repeated entry of MFA tokens when running this code + # locally. + refreshable_credentials = aws.boto3_session.get_credentials() + assert refreshable_credentials is not None, R'Need credentials' + aws_auth = Urllib3AWSV4SignerAuth(refreshable_credentials, aws.region_name) return OpenSearch(http_auth=aws_auth, use_ssl=True, verify_certs=True, diff --git a/test/azul_test_case.py b/test/azul_test_case.py index 20d482a261..cd875d2dd2 100644 --- a/test/azul_test_case.py +++ b/test/azul_test_case.py @@ -148,10 +148,6 @@ class RE(str): 'and will be removed in .*. Instead use .*' ), - # FIXME: DeprecationWarning for datetime methods in Python 3.12 - # https://github.com/DataBiosphere/azul/issues/5953 - 'datetime.datetime.utcnow() is deprecated', - # FIXME: DeprecationWarning for patch_source_cache # https://github.com/DataBiosphere/azul/issues/7838 'Instead of decorating your test case, or its test methods in '