Merge pull request #627 from geoadmin/develop

New Release v2.5.0 - #minor

Author: schtibe

.env.default

@@ -20,6 +20,7 @@ SECRET_KEY=dummy
 HEALTHCHECK_ENDPOINT=healthcheck
 ALLOWED_HOSTS=*
 MANAGED_BUCKET_COLLECTION_PATTERNS=ch.meteoschweiz.ogd-,ch.bgdi-test.
+MANAGED_BUCKET_COLLECTION_PATTERNS_BLACKLIST=ch.meteoschweiz.ogd-precipitation
 
 # these are just here for completeness
 AWS_ROLE_ARN=some-arn

README.md

@@ -548,6 +548,7 @@ The service is configured by Environment Variable:
 | AWS_S3_CUSTOM_DOMAIN | `None` | |
 | AWS_PRESIGNED_URL_EXPIRES | 3600 | AWS presigned url for asset upload expire time in seconds |
 | MANAGED_BUCKET_COLLECTION_PATTERNS | - | A list of prefix patterns for collections that go to the managed bucket |
+| MANAGED_BUCKET_COLLECTION_PATTERNS_BLACKLIST | - | A list of prefix patterns for collection that explicitly should not go to the managed bucket |
 | EXTERNAL_URL_REACHABLE_TIMEOUT | `5` | How long the external asset URL validator should try to connect to given asset in seconds |
 
 #### **Development settings (only for local environment and DEV staging)**

app/config/settings_dev.py

@@ -52,11 +52,16 @@
 
 SHELL_PLUS_POST_IMPORTS = ['from tests.data_factory import Factory']
 
-# Regex patterns of collections that should go to the managed bucket
+# Patterns prefixes of collections that should go to the managed bucket
 MANAGED_BUCKET_COLLECTION_PATTERNS = env.list(
     'MANAGED_BUCKET_COLLECTION_PATTERNS', default=["ch.meteoschweiz.ogd-"]
 )
 
+# Patterns prefixes of collections that should should *not* go to the managed bucket
+MANAGED_BUCKET_COLLECTION_PATTERNS_BLACKLIST = env.list(
+    'MANAGED_BUCKET_COLLECTION_PATTERNS_BLACKLIST', default=["ch.meteoschweiz.ogd-precipitation"]
+)
+
 # Since it's impossible to recreate the service-account situation with minio
 # we inject some configuration in here to access the second bucket
 # in the same way as first bucket, via access/secrets

app/config/settings_prod.py

@@ -358,9 +358,14 @@ def get_logging_config():
 )  # if None, the host is taken from the request url
 STAC_BROWSER_BASE_PATH = env('STAC_BROWSER_BASE_PATH', default='browser/index.html')
 
-# Regex patterns of collections that should go to the managed bucket
+# Pattern prefixes of collections that should go to the managed bucket
 MANAGED_BUCKET_COLLECTION_PATTERNS = env.list('MANAGED_BUCKET_COLLECTION_PATTERNS', default=[])
 
+# Pattern prefixes of collections that should should *not* go to the managed bucket
+MANAGED_BUCKET_COLLECTION_PATTERNS_BLACKLIST = env.list(
+    'MANAGED_BUCKET_COLLECTION_PATTERNS_BLACKLIST', default=[""]
+)
+
 # the duration in seconds that the validator should try and reach the external URL
 EXTERNAL_URL_REACHABLE_TIMEOUT = env.int('EXTERNAL_URL_REACHABLE_TIMEOUT', default=5)
 

app/stac_api/utils.py

@@ -602,10 +602,15 @@ def select_s3_bucket(collection_name) -> AVAILABLE_S3_BUCKETS:
     Select the correct s3 bucket based on matching patterns with the collection
     name
     """
-    patterns = settings.MANAGED_BUCKET_COLLECTION_PATTERNS
-
-    for pattern in patterns:
-        if collection_name.startswith(pattern):
+    whitelist_patterns = settings.MANAGED_BUCKET_COLLECTION_PATTERNS
+    blacklist_patterns = settings.MANAGED_BUCKET_COLLECTION_PATTERNS_BLACKLIST
+
+    for whitelist_pattern in whitelist_patterns:
+        if collection_name.startswith(whitelist_pattern):
+            # if a pattern is found, let's also check it against the blacklist
+            for blacklist_pattern in blacklist_patterns:
+                if collection_name.startswith(blacklist_pattern):
+                    return AVAILABLE_S3_BUCKETS.legacy
             return AVAILABLE_S3_BUCKETS.managed
 
     return AVAILABLE_S3_BUCKETS.legacy

app/tests/base_test_admin_page.py

@@ -1,4 +1,5 @@
 import logging
+import re
 import time
 
 from django.contrib.auth import get_user_model
@@ -18,6 +19,22 @@
 logger = logging.getLogger(__name__)
 
 
+def normalize_wkt(wkt_string):
+    """Normalize WKT string by rounding floating point numbers to 10 decimal places.
+
+    This handles floating point precision differences that occur during
+    database storage/retrieval.
+    """
+    wkt = re.sub(r'(\d+\.\d+)', lambda m: str(round(float(m.group(1)), 10)), wkt_string)
+    # Canonicalize spacing so POLYGON ((.. becomes POLYGON((.. and commas have single spaces
+    wkt = re.sub(r'([A-Z]+)\s+\(', r'\1(', wkt)
+    wkt = re.sub(r'\(\s+', '(', wkt)
+    wkt = re.sub(r'\s+\)', ')', wkt)
+    wkt = re.sub(r',\s*', ', ', wkt)
+    wkt = re.sub(r'\s+', ' ', wkt)
+    return wkt.strip()
+
+
 class AdminBaseTestCase(TestCase):
 
     def setUp(self):
@@ -175,7 +192,17 @@ def _create_item(self, collection, with_link=False, extra=None, data=None):
             elif key.startswith('links-'):
                 continue
             else:
-                self.assertEqual(getattr(item, key), value, msg=f"Item field {key} value missmatch")
+                if key == 'geometry':
+                    geom_obj = getattr(item, key)
+                    actual_wkt = normalize_wkt(str(geom_obj))
+                    expected_wkt = normalize_wkt(value)
+                    self.assertEqual(
+                        actual_wkt, expected_wkt, msg=f"Item field {key} value missmatch"
+                    )
+                else:
+                    self.assertEqual(
+                        getattr(item, key), value, msg=f"Item field {key} value missmatch"
+                    )
 
         return item, data, link
 

app/tests/test_multiple_bucket_setup.py

@@ -178,7 +178,7 @@ def test_managed_bucket_patterns(self, collection_name):
         default environment explicitly here.
         This might appear a bit artificial, but otherwise we have no way
         to machine-test the functioning of getting the values from the env
-        list, use them as regex, and match the collection name.
+        list and match the collection name.
         """
         env = environ.Env()
         env.read_env("../.local.default")
@@ -188,3 +188,22 @@ def test_managed_bucket_patterns(self, collection_name):
             bucket_name = select_s3_bucket(collection_name)
 
         self.assertEqual(bucket_name, AVAILABLE_S3_BUCKETS.managed)
+
+    @parameterized.expand([
+        'ch.meteoschweiz.ogd-precipitation',
+    ])
+    def test_managed_bucket_patterns_blacklist(self, collection_name):
+        """Test if the patterns in the environment work correctly. We take the
+        default environment explicitly here.
+        This might appear a bit artificial, but otherwise we have no way
+        to machine-test the functioning of getting the values from the env
+        list and match the collection name.
+        """
+        env = environ.Env()
+        env.read_env("../.local.default")
+
+        patterns = env.list('MANAGED_BUCKET_COLLECTION_PATTERNS_BLACKLIST')
+        with self.settings(MANAGED_BUCKET_COLLECTION_PATTERNS=patterns):
+            bucket_name = select_s3_bucket(collection_name)
+
+        self.assertEqual(bucket_name, AVAILABLE_S3_BUCKETS.legacy)

docker-compose.yml

@@ -1,11 +1,11 @@
 services:
   db:
-    image: kartoza/postgis:16
+    image: kartoza/postgis:18-3.6
     environment:
       - POSTGRES_DB=${DB_NAME:-service_stac_local}
       - POSTGRES_USER=postgres
       - POSTGRES_PASSWORD=postgres
-      - POSTGRES_MULTIPLE_EXTENSIONS=postgis,postgis_topology
+      - POSTGRES_MULTIPLE_EXTENSIONS=postgis
       - EXTRA_CONF=log_min_messages = ${DB_LOG_LEVEL:-FATAL}
     user: ${UID}
     ports: