diff --git a/.env.default b/.env.default index 2492d6cc..93698c16 100644 --- a/.env.default +++ b/.env.default @@ -20,6 +20,7 @@ SECRET_KEY=dummy HEALTHCHECK_ENDPOINT=healthcheck ALLOWED_HOSTS=* MANAGED_BUCKET_COLLECTION_PATTERNS=ch.meteoschweiz.ogd-,ch.bgdi-test. +MANAGED_BUCKET_COLLECTION_PATTERNS_BLACKLIST=ch.meteoschweiz.ogd-precipitation # these are just here for completeness AWS_ROLE_ARN=some-arn diff --git a/README.md b/README.md index 5123c6f8..341377ac 100644 --- a/README.md +++ b/README.md @@ -548,6 +548,7 @@ The service is configured by Environment Variable: | AWS_S3_CUSTOM_DOMAIN | `None` | | | AWS_PRESIGNED_URL_EXPIRES | 3600 | AWS presigned url for asset upload expire time in seconds | | MANAGED_BUCKET_COLLECTION_PATTERNS | - | A list of prefix patterns for collections that go to the managed bucket | +| MANAGED_BUCKET_COLLECTION_PATTERNS_BLACKLIST | - | A list of prefix patterns for collection that explicitly should not go to the managed bucket | | EXTERNAL_URL_REACHABLE_TIMEOUT | `5` | How long the external asset URL validator should try to connect to given asset in seconds | #### **Development settings (only for local environment and DEV staging)** diff --git a/app/config/settings_dev.py b/app/config/settings_dev.py index 650f8437..59cd1adc 100644 --- a/app/config/settings_dev.py +++ b/app/config/settings_dev.py @@ -52,11 +52,16 @@ SHELL_PLUS_POST_IMPORTS = ['from tests.data_factory import Factory'] -# Regex patterns of collections that should go to the managed bucket +# Patterns prefixes of collections that should go to the managed bucket MANAGED_BUCKET_COLLECTION_PATTERNS = env.list( 'MANAGED_BUCKET_COLLECTION_PATTERNS', default=["ch.meteoschweiz.ogd-"] ) +# Patterns prefixes of collections that should should *not* go to the managed bucket +MANAGED_BUCKET_COLLECTION_PATTERNS_BLACKLIST = env.list( + 'MANAGED_BUCKET_COLLECTION_PATTERNS_BLACKLIST', default=["ch.meteoschweiz.ogd-precipitation"] +) + # Since it's impossible to recreate the service-account situation with minio # we inject some configuration in here to access the second bucket # in the same way as first bucket, via access/secrets diff --git a/app/config/settings_prod.py b/app/config/settings_prod.py index 2b958b7e..8e37a19c 100644 --- a/app/config/settings_prod.py +++ b/app/config/settings_prod.py @@ -358,9 +358,14 @@ def get_logging_config(): ) # if None, the host is taken from the request url STAC_BROWSER_BASE_PATH = env('STAC_BROWSER_BASE_PATH', default='browser/index.html') -# Regex patterns of collections that should go to the managed bucket +# Pattern prefixes of collections that should go to the managed bucket MANAGED_BUCKET_COLLECTION_PATTERNS = env.list('MANAGED_BUCKET_COLLECTION_PATTERNS', default=[]) +# Pattern prefixes of collections that should should *not* go to the managed bucket +MANAGED_BUCKET_COLLECTION_PATTERNS_BLACKLIST = env.list( + 'MANAGED_BUCKET_COLLECTION_PATTERNS_BLACKLIST', default=[""] +) + # the duration in seconds that the validator should try and reach the external URL EXTERNAL_URL_REACHABLE_TIMEOUT = env.int('EXTERNAL_URL_REACHABLE_TIMEOUT', default=5) diff --git a/app/stac_api/utils.py b/app/stac_api/utils.py index 48e58020..726edeea 100644 --- a/app/stac_api/utils.py +++ b/app/stac_api/utils.py @@ -602,10 +602,15 @@ def select_s3_bucket(collection_name) -> AVAILABLE_S3_BUCKETS: Select the correct s3 bucket based on matching patterns with the collection name """ - patterns = settings.MANAGED_BUCKET_COLLECTION_PATTERNS - - for pattern in patterns: - if collection_name.startswith(pattern): + whitelist_patterns = settings.MANAGED_BUCKET_COLLECTION_PATTERNS + blacklist_patterns = settings.MANAGED_BUCKET_COLLECTION_PATTERNS_BLACKLIST + + for whitelist_pattern in whitelist_patterns: + if collection_name.startswith(whitelist_pattern): + # if a pattern is found, let's also check it against the blacklist + for blacklist_pattern in blacklist_patterns: + if collection_name.startswith(blacklist_pattern): + return AVAILABLE_S3_BUCKETS.legacy return AVAILABLE_S3_BUCKETS.managed return AVAILABLE_S3_BUCKETS.legacy diff --git a/app/tests/base_test_admin_page.py b/app/tests/base_test_admin_page.py index e4f2321e..83ce1c3c 100644 --- a/app/tests/base_test_admin_page.py +++ b/app/tests/base_test_admin_page.py @@ -1,4 +1,5 @@ import logging +import re import time from django.contrib.auth import get_user_model @@ -18,6 +19,22 @@ logger = logging.getLogger(__name__) +def normalize_wkt(wkt_string): + """Normalize WKT string by rounding floating point numbers to 10 decimal places. + + This handles floating point precision differences that occur during + database storage/retrieval. + """ + wkt = re.sub(r'(\d+\.\d+)', lambda m: str(round(float(m.group(1)), 10)), wkt_string) + # Canonicalize spacing so POLYGON ((.. becomes POLYGON((.. and commas have single spaces + wkt = re.sub(r'([A-Z]+)\s+\(', r'\1(', wkt) + wkt = re.sub(r'\(\s+', '(', wkt) + wkt = re.sub(r'\s+\)', ')', wkt) + wkt = re.sub(r',\s*', ', ', wkt) + wkt = re.sub(r'\s+', ' ', wkt) + return wkt.strip() + + class AdminBaseTestCase(TestCase): def setUp(self): @@ -175,7 +192,17 @@ def _create_item(self, collection, with_link=False, extra=None, data=None): elif key.startswith('links-'): continue else: - self.assertEqual(getattr(item, key), value, msg=f"Item field {key} value missmatch") + if key == 'geometry': + geom_obj = getattr(item, key) + actual_wkt = normalize_wkt(str(geom_obj)) + expected_wkt = normalize_wkt(value) + self.assertEqual( + actual_wkt, expected_wkt, msg=f"Item field {key} value missmatch" + ) + else: + self.assertEqual( + getattr(item, key), value, msg=f"Item field {key} value missmatch" + ) return item, data, link diff --git a/app/tests/test_multiple_bucket_setup.py b/app/tests/test_multiple_bucket_setup.py index 2e168b31..b4d40ff5 100644 --- a/app/tests/test_multiple_bucket_setup.py +++ b/app/tests/test_multiple_bucket_setup.py @@ -178,7 +178,7 @@ def test_managed_bucket_patterns(self, collection_name): default environment explicitly here. This might appear a bit artificial, but otherwise we have no way to machine-test the functioning of getting the values from the env - list, use them as regex, and match the collection name. + list and match the collection name. """ env = environ.Env() env.read_env("../.local.default") @@ -188,3 +188,22 @@ def test_managed_bucket_patterns(self, collection_name): bucket_name = select_s3_bucket(collection_name) self.assertEqual(bucket_name, AVAILABLE_S3_BUCKETS.managed) + + @parameterized.expand([ + 'ch.meteoschweiz.ogd-precipitation', + ]) + def test_managed_bucket_patterns_blacklist(self, collection_name): + """Test if the patterns in the environment work correctly. We take the + default environment explicitly here. + This might appear a bit artificial, but otherwise we have no way + to machine-test the functioning of getting the values from the env + list and match the collection name. + """ + env = environ.Env() + env.read_env("../.local.default") + + patterns = env.list('MANAGED_BUCKET_COLLECTION_PATTERNS_BLACKLIST') + with self.settings(MANAGED_BUCKET_COLLECTION_PATTERNS=patterns): + bucket_name = select_s3_bucket(collection_name) + + self.assertEqual(bucket_name, AVAILABLE_S3_BUCKETS.legacy) diff --git a/docker-compose.yml b/docker-compose.yml index b3693e2c..028b02fc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,11 +1,11 @@ services: db: - image: kartoza/postgis:16 + image: kartoza/postgis:18-3.6 environment: - POSTGRES_DB=${DB_NAME:-service_stac_local} - POSTGRES_USER=postgres - POSTGRES_PASSWORD=postgres - - POSTGRES_MULTIPLE_EXTENSIONS=postgis,postgis_topology + - POSTGRES_MULTIPLE_EXTENSIONS=postgis - EXTRA_CONF=log_min_messages = ${DB_LOG_LEVEL:-FATAL} user: ${UID} ports: