diff --git a/app/stac_api/migrations/0071_remove_asset_update_asset_item_file_size_trigger_and_more.py b/app/stac_api/migrations/0071_remove_asset_update_asset_item_file_size_trigger_and_more.py new file mode 100644 index 00000000..b5149ae6 --- /dev/null +++ b/app/stac_api/migrations/0071_remove_asset_update_asset_item_file_size_trigger_and_more.py @@ -0,0 +1,38 @@ +# Generated by Django 5.2.11 on 2026-06-03 06:08 + +import pgtrigger.compiler +import pgtrigger.migrations + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('stac_api', '0070_alter_asset_media_type_and_more'), + ] + + operations = [ + pgtrigger.migrations.RemoveTrigger( + model_name='asset', + name='update_asset_item_file_size_trigger', + ), + pgtrigger.migrations.AddTrigger( + model_name='asset', + trigger=pgtrigger.compiler.Trigger( + name='update_asset_item_file_size_trigger', + sql=pgtrigger.compiler.UpsertTriggerSql( + condition='WHEN (OLD."file_size" IS DISTINCT FROM (NEW."file_size"))', + declare= + 'DECLARE asset_instance stac_api_asset%ROWTYPE; updated_total_data_size NUMERIC;', + func= + "\n asset_instance = COALESCE(NEW, OLD);\n\n -- Update related item file_size variables\n UPDATE stac_api_item SET\n total_data_size = total_data_size + COALESCE(NEW.file_size, 0) - COALESCE(OLD.file_size, 0)\n WHERE id = asset_instance.item_id\n RETURNING total_data_size INTO updated_total_data_size;\n\n IF updated_total_data_size < 0\n THEN RAISE WARNING 'item.id=% total_data_size has negative value %',\n asset_instance.item_id, updated_total_data_size;\n END IF;\n\n RAISE INFO 'item.id=% total_data_size updated, due to asset.name=% updates.',\n asset_instance.item_id, asset_instance.name;\n\n RETURN asset_instance;\n ", + hash='2e0243f743d6b4ba265e71d9a738d73551d84431', + operation='UPDATE', + pgid='pgtrigger_update_asset_item_file_size_trigger_316e8', + table='stac_api_asset', + when='AFTER' + ) + ), + ), + ] diff --git a/app/stac_api/migrations/0072_remove_collectionasset_upd_dec_col_asset_proj_epsg_trigger_and_more.py b/app/stac_api/migrations/0072_remove_collectionasset_upd_dec_col_asset_proj_epsg_trigger_and_more.py new file mode 100644 index 00000000..05aa71c4 --- /dev/null +++ b/app/stac_api/migrations/0072_remove_collectionasset_upd_dec_col_asset_proj_epsg_trigger_and_more.py @@ -0,0 +1,60 @@ +# Generated by Django 5.2.11 on 2026-06-03 06:12 + +import pgtrigger.compiler +import pgtrigger.migrations + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('stac_api', '0071_remove_asset_update_asset_item_file_size_trigger_and_more'), + ] + + operations = [ + pgtrigger.migrations.RemoveTrigger( + model_name='collectionasset', + name='upd_dec_col_asset_proj_epsg_trigger', + ), + pgtrigger.migrations.RemoveTrigger( + model_name='collectionasset', + name='upd_inc_col_asset_proj_epsg_trigger', + ), + pgtrigger.migrations.AddTrigger( + model_name='collectionasset', + trigger=pgtrigger.compiler.Trigger( + name='upd_dec_col_asset_proj_epsg_trigger', + sql=pgtrigger.compiler.UpsertTriggerSql( + condition='WHEN (OLD."proj_epsg" IS DISTINCT FROM (NEW."proj_epsg"))', + declare= + 'DECLARE asset_instance stac_api_collectionasset%ROWTYPE; related_collection_id INT;', + func= + "\n asset_instance = OLD;\n\n related_collection_id = asset_instance.collection_id;\n\n -- Remove entry when count will reach 0\n DELETE FROM stac_api_projepsgcount\n WHERE collection_id = related_collection_id\n AND value = asset_instance.proj_epsg\n AND count = 1;\n\n IF NOT FOUND THEN\n UPDATE stac_api_projepsgcount\n SET count = count-1\n WHERE collection_id = related_collection_id\n AND value = asset_instance.proj_epsg;\n\n RAISE INFO\n 'stac_api_projepsgcount (collection_id, value) (% %) count updated, due to asset.name=% update.',\n related_collection_id, asset_instance.proj_epsg, asset_instance.name;\n\n RETURN asset_instance;\n END IF;\n\n RAISE INFO\n 'stac_api_projepsgcount (collection_id, value) (% %) deleted, due to asset.name=% update.',\n related_collection_id, asset_instance.proj_epsg, asset_instance.name;\n\n RETURN asset_instance;\n ", + hash='56e4e1fae63cd83c85410bb5e49f1672f644c015', + operation='UPDATE', + pgid='pgtrigger_upd_dec_col_asset_proj_epsg_trigger_07a9d', + table='stac_api_collectionasset', + when='AFTER' + ) + ), + ), + pgtrigger.migrations.AddTrigger( + model_name='collectionasset', + trigger=pgtrigger.compiler.Trigger( + name='upd_inc_col_asset_proj_epsg_trigger', + sql=pgtrigger.compiler.UpsertTriggerSql( + condition='WHEN (OLD."proj_epsg" IS DISTINCT FROM (NEW."proj_epsg"))', + declare= + 'DECLARE asset_instance stac_api_collectionasset%ROWTYPE; related_collection_id INT;', + func= + "\n asset_instance = NEW;\n\n related_collection_id = asset_instance.collection_id;\n\n INSERT INTO stac_api_projepsgcount (collection_id, value, count)\n VALUES (related_collection_id, asset_instance.proj_epsg, 1)\n ON CONFLICT (collection_id, value)\n DO UPDATE SET count = stac_api_projepsgcount.count+1;\n\n RAISE INFO\n 'projepsgcount (collection_id, value) (% %) count updated, due to asset.name=% update.',\n related_collection_id, asset_instance.proj_epsg, asset_instance.name;\n\n RETURN asset_instance;\n ", + hash='0626a8387cba7cffb40dd81594e3d5c5c16e9ea4', + operation='UPDATE', + pgid='pgtrigger_upd_inc_col_asset_proj_epsg_trigger_d717d', + table='stac_api_collectionasset', + when='AFTER' + ) + ), + ), + ] diff --git a/app/stac_api/pgtriggers.py b/app/stac_api/pgtriggers.py index 7caf29fb..acecb391 100644 --- a/app/stac_api/pgtriggers.py +++ b/app/stac_api/pgtriggers.py @@ -31,7 +31,7 @@ def auto_variables_triggers(name, *fields): ] -def child_triggers(parent_name, child_name): +def child_triggers(parent_name, child_name, *fields): '''Triggers used by various tables to update the `updated` and `etag` fields of the parent table when a child gets inserted, updated or deleted. @@ -63,7 +63,7 @@ def child_triggers(parent_name, child_name): name=f"update_{parent_name}_child_trigger", operation=pgtrigger.Update, when=pgtrigger.After, - condition=pgtrigger.Condition('OLD.* IS DISTINCT FROM NEW.*'), + condition=pgtrigger.AnyChange(*fields), func=child_update_func.format( parent_name=parent_name, child_obj="NEW", child_name=child_name ) @@ -225,7 +225,7 @@ class ItemFileSizeTrigger(pgtrigger.Trigger): ItemFileSizeTrigger( name='update_asset_item_file_size_trigger', operation=pgtrigger.Update, - condition=pgtrigger.Condition('OLD.file_size IS DISTINCT FROM NEW.file_size'), + condition=pgtrigger.AnyChange('file_size'), ) ] @@ -327,7 +327,7 @@ class IncreaseCounterTrigger(pgtrigger.Trigger): DecreaseCounterTrigger( name='upd_dec_col_asset_proj_epsg_trigger', operation=pgtrigger.Update, - condition=pgtrigger.Condition('''OLD.proj_epsg IS DISTINCT FROM NEW.proj_epsg''') + condition=pgtrigger.AnyChange('proj_epsg') ), DecreaseCounterTrigger( name='del_col_asset_proj_epsg_trigger', @@ -336,7 +336,7 @@ class IncreaseCounterTrigger(pgtrigger.Trigger): IncreaseCounterTrigger( name='upd_inc_col_asset_proj_epsg_trigger', operation=pgtrigger.Update, - condition=pgtrigger.Condition('''OLD.proj_epsg IS DISTINCT FROM NEW.proj_epsg''') + condition=pgtrigger.AnyChange('proj_epsg') ), IncreaseCounterTrigger( name='add_col_asset_proj_epsg_trigger', @@ -349,7 +349,7 @@ class IncreaseCounterTrigger(pgtrigger.Trigger): CollectionFileSizeTrigger( name='update_col_asset_col_file_size_trigger', operation=pgtrigger.Update, - condition=pgtrigger.Condition('OLD.* IS DISTINCT FROM NEW.*'), + condition=pgtrigger.AnyChange(), ) ] @@ -480,7 +480,7 @@ def generates_asset_upload_triggers(): pgtrigger.Trigger( name="update_asset_upload_trigger", operation=pgtrigger.Update, - condition=pgtrigger.Condition('OLD.* IS DISTINCT FROM NEW.*'), + condition=pgtrigger.AnyChange(), when=pgtrigger.Before, func=etag_func ), diff --git a/app/tests/tests_10/test_pgtriggers.py b/app/tests/tests_10/test_pgtriggers.py index 88073267..f7b4d3bc 100644 --- a/app/tests/tests_10/test_pgtriggers.py +++ b/app/tests/tests_10/test_pgtriggers.py @@ -1,5 +1,9 @@ import logging +from parameterized import parameterized + +from stac_api.models.item import AssetUpload + from tests.tests_10.base_test import StacBaseTransactionTestCase from tests.tests_10.data_factory import Factory from tests.tests_10.sample_data.asset_samples import FILE_CONTENT_1 @@ -73,17 +77,66 @@ class PgTriggersUpdated(MockS3PerTestMixin, StacBaseTransactionTestCase): def setUp(self): super().setUp() self.factory = Factory() - self.collection = self.factory.create_collection_sample().model - self.item = self.factory.create_item_sample(collection=self.collection).model + self.collection = self.factory.create_collection_sample(db_create=True).model + self.collection_asset = self.factory.create_collection_asset_sample( + collection=self.collection, db_create=True + ).model + self.item = self.factory.create_item_sample( + collection=self.collection, db_create=True + ).model self.asset = self.factory.create_asset_sample( self.item, sample='asset-1', db_create=True ).model - def test_checksum_update_changes_timestamps(self): - prev_mtime = self.asset.updated + @parameterized.expand([ + ('asset', 'asset', 'checksum_multihash'), + ('asset', 'item', 'checksum_multihash'), + ('item', 'collection', 'name'), + ('collection_asset', 'collection', 'checksum_multihash'), + ]) + def test_timestamp_updated(self, source_name, destination_name, field_name): + destination = getattr(self, destination_name) + source = getattr(self, source_name) + + prev_etag = destination.etag + prev_mtime = destination.updated + setattr(source, field_name, 'new value') + source.save() + destination.refresh_from_db() + + self.assertGreater(destination.updated, prev_mtime) + self.assertNotEqual(destination.etag, prev_etag) + - self.asset.checksum_multihash = 'new hash' - self.asset.save() - self.asset.refresh_from_db() +class PgTriggerAssetUploads(MockS3PerTestMixin, StacBaseTransactionTestCase): - self.assertGreater(self.asset.updated, prev_mtime) + def setUp(self): + super().setUp() + self.factory = Factory() + self.collection = self.factory.create_collection_sample(db_create=True).model + self.collection_asset = self.factory.create_collection_asset_sample( + collection=self.collection, db_create=True + ).model + self.item = self.factory.create_item_sample( + collection=self.collection, db_create=True + ).model + self.asset = self.factory.create_asset_sample( + self.item, sample='asset-1', db_create=True + ).model + self.asset_upload = AssetUpload( + asset=self.asset, + upload_id=42, + checksum_multihash=b'some hash', + md5_parts=['totally a valid md5'], + number_parts=1, + ) + self.asset_upload.full_clean() + self.asset_upload.save() + self.asset_upload.refresh_from_db() + + def test_etag_updated(self): + prev_etag = self.asset_upload.etag + self.asset_upload.status = AssetUpload.Status.ABORTED + self.asset_upload.save() + self.asset_upload.refresh_from_db() + self.assertNotEqual(self.asset_upload.etag, prev_etag)