PB-2131 Add possibility to blacklist patterns

Author: schtibe

.env.default

@@ -20,6 +20,7 @@ SECRET_KEY=dummy
 HEALTHCHECK_ENDPOINT=healthcheck
 ALLOWED_HOSTS=*
 MANAGED_BUCKET_COLLECTION_PATTERNS=ch.meteoschweiz.ogd-,ch.bgdi-test.
+MANAGED_BUCKET_COLLECTION_PATTERNS_BLACKLIST=ch.meteoschweiz.ogd-precipitation
 
 # these are just here for completeness
 AWS_ROLE_ARN=some-arn

README.md

@@ -548,6 +548,7 @@ The service is configured by Environment Variable:
 | AWS_S3_CUSTOM_DOMAIN | `None` | |
 | AWS_PRESIGNED_URL_EXPIRES | 3600 | AWS presigned url for asset upload expire time in seconds |
 | MANAGED_BUCKET_COLLECTION_PATTERNS | - | A list of prefix patterns for collections that go to the managed bucket |
+| MANAGED_BUCKET_COLLECTION_PATTERNS_BLACKLIST | - | A list of prefix patterns for collection that explicitly should not go to the managed bucket |
 | EXTERNAL_URL_REACHABLE_TIMEOUT | `5` | How long the external asset URL validator should try to connect to given asset in seconds |
 
 #### **Development settings (only for local environment and DEV staging)**

app/config/settings_dev.py

@@ -52,11 +52,16 @@
 
 SHELL_PLUS_POST_IMPORTS = ['from tests.data_factory import Factory']
 
-# Regex patterns of collections that should go to the managed bucket
+# Patterns prefixes of collections that should go to the managed bucket
 MANAGED_BUCKET_COLLECTION_PATTERNS = env.list(
     'MANAGED_BUCKET_COLLECTION_PATTERNS', default=["ch.meteoschweiz.ogd-"]
 )
 
+# Patterns prefixes of collections that should should *not* go to the managed bucket
+MANAGED_BUCKET_COLLECTION_PATTERNS_BLACKLIST = env.list(
+    'MANAGED_BUCKET_COLLECTION_PATTERNS_BLACKLIST', default=["ch.meteoschweiz.ogd-precipitation"]
+)
+
 # Since it's impossible to recreate the service-account situation with minio
 # we inject some configuration in here to access the second bucket
 # in the same way as first bucket, via access/secrets

app/config/settings_prod.py

@@ -358,9 +358,14 @@ def get_logging_config():
 )  # if None, the host is taken from the request url
 STAC_BROWSER_BASE_PATH = env('STAC_BROWSER_BASE_PATH', default='browser/index.html')
 
-# Regex patterns of collections that should go to the managed bucket
+# Pattern prefixes of collections that should go to the managed bucket
 MANAGED_BUCKET_COLLECTION_PATTERNS = env.list('MANAGED_BUCKET_COLLECTION_PATTERNS', default=[])
 
+# Pattern prefixes of collections that should should *not* go to the managed bucket
+MANAGED_BUCKET_COLLECTION_PATTERNS_BLACKLIST = env.list(
+    'MANAGED_BUCKET_COLLECTION_PATTERNS_BLACKLIST', default=[""]
+)
+
 # the duration in seconds that the validator should try and reach the external URL
 EXTERNAL_URL_REACHABLE_TIMEOUT = env.int('EXTERNAL_URL_REACHABLE_TIMEOUT', default=5)
 

app/stac_api/utils.py

@@ -602,10 +602,15 @@ def select_s3_bucket(collection_name) -> AVAILABLE_S3_BUCKETS:
     Select the correct s3 bucket based on matching patterns with the collection
     name
     """
-    patterns = settings.MANAGED_BUCKET_COLLECTION_PATTERNS
-
-    for pattern in patterns:
-        if collection_name.startswith(pattern):
+    whitelist_patterns = settings.MANAGED_BUCKET_COLLECTION_PATTERNS
+    blacklist_patterns = settings.MANAGED_BUCKET_COLLECTION_PATTERNS_BLACKLIST
+
+    for whitelist_pattern in whitelist_patterns:
+        if collection_name.startswith(whitelist_pattern):
+            # if a pattern is found, let's also check it against the blacklist
+            for blacklist_pattern in blacklist_patterns:
+                if collection_name.startswith(blacklist_pattern):
+                    return AVAILABLE_S3_BUCKETS.legacy
             return AVAILABLE_S3_BUCKETS.managed
 
     return AVAILABLE_S3_BUCKETS.legacy

app/tests/test_multiple_bucket_setup.py

@@ -178,7 +178,7 @@ def test_managed_bucket_patterns(self, collection_name):
         default environment explicitly here.
         This might appear a bit artificial, but otherwise we have no way
         to machine-test the functioning of getting the values from the env
-        list, use them as regex, and match the collection name.
+        list and match the collection name.
         """
         env = environ.Env()
         env.read_env("../.local.default")
@@ -188,3 +188,22 @@ def test_managed_bucket_patterns(self, collection_name):
             bucket_name = select_s3_bucket(collection_name)
 
         self.assertEqual(bucket_name, AVAILABLE_S3_BUCKETS.managed)
+
+    @parameterized.expand([
+        'ch.meteoschweiz.ogd-precipitation',
+    ])
+    def test_managed_bucket_patterns_blacklist(self, collection_name):
+        """Test if the patterns in the environment work correctly. We take the
+        default environment explicitly here.
+        This might appear a bit artificial, but otherwise we have no way
+        to machine-test the functioning of getting the values from the env
+        list and match the collection name.
+        """
+        env = environ.Env()
+        env.read_env("../.local.default")
+
+        patterns = env.list('MANAGED_BUCKET_COLLECTION_PATTERNS_BLACKLIST')
+        with self.settings(MANAGED_BUCKET_COLLECTION_PATTERNS=patterns):
+            bucket_name = select_s3_bucket(collection_name)
+
+        self.assertEqual(bucket_name, AVAILABLE_S3_BUCKETS.legacy)