diff --git a/README.md b/README.md index 29a56171..5123c6f8 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ - [Table of Content](#table-of-content) - [Summary of the project](#summary-of-the-project) - [SPEC](#spec) +- [Logging Standard Django Management Commands](#logging-standard-django-management-commands) - [Local development](#local-development) - [Dependencies](#dependencies) - [Python3.12](#python312) @@ -53,6 +54,15 @@ See [SPEC](./spec/README.md) +## Logging Standard Django Management Commands + +This project uses a modified `manage.py` that supports redirecting the output of the standard +Django management commands to the logger. For this, simply add `--redirect-std-to-logger`, e.g.: + +```bash +app/manage.py migrate --redirect-std-to-logger +``` + ## Local development ### Dependencies diff --git a/app/helpers/logging.py b/app/helpers/logging.py new file mode 100644 index 00000000..9e25829c --- /dev/null +++ b/app/helpers/logging.py @@ -0,0 +1,65 @@ +from contextlib import contextmanager +from contextlib import redirect_stderr +from contextlib import redirect_stdout +from io import StringIO +from logging import ERROR +from logging import INFO +from logging import getLogger +from logging.config import dictConfig +from time import time +from typing import Generator + +from django.conf import settings + + +class TimestampedStringIO(StringIO): + """ A StringIO-like in-memory text buffer that logs each write and stores a timestamp for when + the content was appended. + + """ + + def __init__(self, level: int) -> None: + super().__init__() + self.level = level + self.messages: list[tuple[float, int, str]] = [] + + def write(self, s: str) -> int: + message = s.strip() + if message: + self.messages.append((time(), self.level, message)) + return len(s) + + +@contextmanager +def redirect_std_to_logger(logger_name: str, + stderr_level: int = ERROR, + stdout_level: int = INFO) -> Generator[None, None, None]: + """ A context manager that redirects sys.stdout and sys.stderr to the logger using the given + levels. + + Use it like this: + + import sys + from utils.logging import redirect_std_to_logger + + with redirect_std_to_logger('my_module'): + sys.out('This gets logged with level INFO') + sys.err('This gets logged with level ERROR') + + """ + + stderr = TimestampedStringIO(stderr_level) + stdout = TimestampedStringIO(stdout_level) + exception: Exception | None = None + with redirect_stderr(stderr), redirect_stdout(stdout): + try: + yield + except Exception as e: # pylint: disable=broad-exception-caught + exception = e + + logger = getLogger(logger_name) + dictConfig(settings.LOGGING) + for _, level, message in sorted(stderr.messages + stdout.messages): + logger.log(level, message) + if exception: + logger.exception(exception) diff --git a/app/manage.py b/app/manage.py index 51df0478..ba440d5c 100755 --- a/app/manage.py +++ b/app/manage.py @@ -5,6 +5,7 @@ from pathlib import Path from dotenv import load_dotenv +from helpers.logging import redirect_std_to_logger def main(): @@ -41,4 +42,9 @@ def main(): if __name__ == '__main__': - main() + if '--redirect-std-to-logger' in sys.argv: + sys.argv.remove('--redirect-std-to-logger') + with redirect_std_to_logger(__name__): + main() + else: + main() diff --git a/app/stac_api/management/commands/calculate_extent.py b/app/stac_api/management/commands/calculate_extent.py index b0d257dc..433d9124 100644 --- a/app/stac_api/management/commands/calculate_extent.py +++ b/app/stac_api/management/commands/calculate_extent.py @@ -4,7 +4,6 @@ from django.db import connection from stac_api.models.collection import Collection -from stac_api.utils import CommandHandler from stac_api.utils import CustomBaseCommand @@ -15,18 +14,30 @@ def boolean_input(question, default=None): return len(result) > 0 and result[0].lower() == "y" -class Handler(CommandHandler): +class Command(CustomBaseCommand): + help = """Calculate the collection spacial and temporal extent for all collections that have + 'extent_out_of_sync' set to true. After update, 'extent_out_of_sync' will be set to False. + This command is thought to be scheduled as cron job. + """ - def run(self): + def add_arguments(self, parser: CommandParser) -> None: + super().add_arguments(parser) + parser.add_argument( + '-a', '--all', action='store_true', help='Update extent for all collections' + ) + parser.add_argument( + '-f', '--force', action='store_true', help='Run all without confirmation' + ) + + def handle(self, *args, **options): self.print_success('running command to update collection extents') - # print(self.options) qry = Collection.objects.filter(extent_out_of_sync=True) - if self.options['all']: + if options['all']: qry = Collection.objects collections = qry.values_list('id', flat=True) # Prompt user to confirm update of all collections if force was not provided. - if self.options['all'] and not self.options['force']: + if options['all'] and not options['force']: cont = boolean_input( f"You are about to update {len(collections)} collections!\n" + "Are you sure you want to continue? (y/n): ", @@ -78,22 +89,3 @@ def run(self): f"successfully updated extent of {len(collections)} collections", extra={"duration": time.monotonic() - start} ) - - -class Command(CustomBaseCommand): - help = """Calculate the collection spacial and temporal extent for all collections that have - 'extent_out_of_sync' set to true. After update, 'extent_out_of_sync' will be set to False. - This command is thought to be scheduled as cron job. - """ - - def add_arguments(self, parser: CommandParser) -> None: - super().add_arguments(parser) - parser.add_argument( - '-a', '--all', action='store_true', help='Update extent for all collections' - ) - parser.add_argument( - '-f', '--force', action='store_true', help='Run all without confirmation' - ) - - def handle(self, *args, **options): - Handler(self, options).run() diff --git a/app/stac_api/management/commands/dummy_asset.py b/app/stac_api/management/commands/dummy_asset.py index b5e65891..92f67ea3 100644 --- a/app/stac_api/management/commands/dummy_asset.py +++ b/app/stac_api/management/commands/dummy_asset.py @@ -1,23 +1,65 @@ import hashlib -import logging import random import uuid from io import BytesIO from django.conf import settings -from django.core.management.base import BaseCommand -from stac_api.utils import CommandHandler +from stac_api.utils import CustomBaseCommand from stac_api.utils import get_s3_resource from stac_api.utils import get_sha256_multihash from stac_api.validators import MEDIA_TYPES -logger = logging.getLogger(__name__) - PREFIX = 'dummy-obj-' -class DummyAssetHandler(CommandHandler): +class Command(CustomBaseCommand): + help = f"""Upload dummy asset file on S3 for testing. + + The command upload dummy asset file with random data on S3 for testing. + By default only one file is uploaded to + /{PREFIX}collection-1/{PREFIX}item-1/{PREFIX}asset-1.txt + + Optionally you can create more than one asset on several items and collections. + If more than one asset is uploaded, then its extension is chosen randomly. + + The asset uploaded is then printed to the console with its checksum:multishash. + """ + + def add_arguments(self, parser): + super().add_arguments(parser) + parser.add_argument( + 'action', + type=str, + choices=['upload', 'clean'], + default='upload', + help='Define the action to be performed, either "upload" (default) to create and ' + 'upload dummy asset file or "clean" to delete them', + ) + + parser.add_argument( + '--collections', + type=int, + default=1, + help="Number of collections to create (default 1)" + ) + + parser.add_argument( + '--items', + type=int, + default=1, + help="Number of items per collection to create (default 1)" + ) + + parser.add_argument( + '--assets', type=int, default=1, help="Number of assets per item to create (default 1)" + ) + + def handle(self, *args, **options): + if options['action'] == 'clean': + self.clean() + elif options['action'] == 'upload': + self.upload() def clean(self): self.print_warning("Deleting all assets with prefix %s on S3...", PREFIX) @@ -64,52 +106,3 @@ def upload(self): self.print('%s,%s', file, get_sha256_multihash(content), level=2) self.print('-' * 100, level=2) self.print_success('Done') - - -class Command(BaseCommand): - help = f"""Upload dummy asset file on S3 for testing. - - The command upload dummy asset file with random data on S3 for testing. - By default only one file is uploaded to - /{PREFIX}collection-1/{PREFIX}item-1/{PREFIX}asset-1.txt - - Optionally you can create more than one asset on several items and collections. - If more than one asset is uploaded, then its extension is chosen randomly. - - The asset uploaded is then printed to the console with its checksum:multishash. - """ - - def add_arguments(self, parser): - parser.add_argument( - 'action', - type=str, - choices=['upload', 'clean'], - default='upload', - help='Define the action to be performed, either "upload" (default) to create and ' - 'upload dummy asset file or "clean" to delete them', - ) - - parser.add_argument( - '--collections', - type=int, - default=1, - help="Number of collections to create (default 1)" - ) - - parser.add_argument( - '--items', - type=int, - default=1, - help="Number of items per collection to create (default 1)" - ) - - parser.add_argument( - '--assets', type=int, default=1, help="Number of assets per item to create (default 1)" - ) - - def handle(self, *args, **options): - handler = DummyAssetHandler(self, options) - if options['action'] == 'clean': - handler.clean() - elif options['action'] == 'upload': - handler.upload() diff --git a/app/stac_api/management/commands/dummy_asset_upload.py b/app/stac_api/management/commands/dummy_asset_upload.py index 3d1effc9..e17d73a2 100644 --- a/app/stac_api/management/commands/dummy_asset_upload.py +++ b/app/stac_api/management/commands/dummy_asset_upload.py @@ -1,38 +1,75 @@ -import logging import os from django.conf import settings -from django.core.management.base import BaseCommand from stac_api.models.general import BaseAssetUpload from stac_api.models.item import Asset from stac_api.models.item import AssetUpload from stac_api.s3_multipart_upload import MultipartUpload from stac_api.utils import AVAILABLE_S3_BUCKETS -from stac_api.utils import CommandHandler +from stac_api.utils import CustomBaseCommand from stac_api.utils import get_asset_path from stac_api.utils import get_sha256_multihash -logger = logging.getLogger(__name__) +class Command(CustomBaseCommand): + help = """Start dummy Multipart upload for asset file on S3 for testing. + """ + + def add_arguments(self, parser): + super().add_arguments(parser) + + subparsers = parser.add_subparsers( + dest='action', + required=True, + help='Define the action to be performed, either "start" (default) to initate ' + ' and upload, "complete" to end it or "abort" to cancel it' + ) + + # create the parser for the "start" command + parser_start = subparsers.add_parser('start', help='start help') + parser_start.add_argument( + '--asset-id', + type=str, + required=True, + help="The asset-id for which data should be uploaded" + ) + parser_start.set_defaults() + + # create the parser for the "complete" command + parser_complete = subparsers.add_parser('complete', help='complete help') + parser_complete.add_argument( + '--upload-id', type=str, required=True, help="The upload-id to complete" + ) -class DummyAssetUploadHandler(CommandHandler): + # create the parser for the "abort" command + parser_abort = subparsers.add_parser('abort', help='abort help') + parser_abort.add_argument( + '--upload-id', type=str, required=True, help="The upload-id to abort" + ) - def __init__(self, *args, **kwargs): + def handle(self, *args, **options): + if options['action'] == 'start': + self.start() + elif options['action'] == 'list': + self.list() + elif options['action'] == 'complete': + self.complete() + elif options['action'] == 'abort': + self.abort() + def start(self): # Note: the command is currently just used to be able to manipulate # AssetUpload objects, not to actually upload content to a bucket, so # the bucket configuration here is just to be able to create a MultipartUpload # object. - s3_bucket = kwargs.pop('s3_bucket', AVAILABLE_S3_BUCKETS.legacy) - super().__init__(*args, **kwargs) + s3_bucket = AVAILABLE_S3_BUCKETS.legacy self.print_success( f"connect MultipartUploader to s3 bucket " f"{settings.AWS_SETTINGS[s3_bucket.name]['S3_BUCKET_NAME']}" ) - self.uploader = MultipartUpload(s3_bucket) + uploader = MultipartUpload(s3_bucket) - def start(self): self.print_success(f"Starting upload for {self.options['asset_id']}") asset = Asset.objects.filter(name=self.options['asset_id']).first() if not asset: @@ -45,7 +82,7 @@ def start(self): file_like = os.urandom(size) checksum_multihash = get_sha256_multihash(file_like) - upload_id = self.uploader.create_multipart_upload( + upload_id = uploader.create_multipart_upload( key=key, asset=asset, checksum_multihash=checksum_multihash, @@ -59,7 +96,7 @@ def start(self): def list(self): for upload in AssetUpload.objects.filter(status=BaseAssetUpload.Status.IN_PROGRESS): - print(f"> {upload.upload_id} (asset: {upload.asset.name})") + self.print(f"> {upload.upload_id} (asset: {upload.asset.name})") def complete(self): try: @@ -76,50 +113,3 @@ def abort(self): upload.save() except AssetUpload.DoesNotExist: self.print_error(f"upload_id {self.options['upload_id']} doesn't exist") - - -class Command(BaseCommand): - help = """Start dummy Multipart upload for asset file on S3 for testing. - """ - - def add_arguments(self, parser): - - subparsers = parser.add_subparsers( - dest='action', - required=True, - help='Define the action to be performed, either "start" (default) to initate ' - ' and upload, "complete" to end it or "abort" to cancel it' - ) - - # create the parser for the "start" command - parser_start = subparsers.add_parser('start', help='start help') - parser_start.add_argument( - '--asset-id', - type=str, - required=True, - help="The asset-id for which data should be uploaded" - ) - parser_start.set_defaults() - - # create the parser for the "complete" command - parser_complete = subparsers.add_parser('complete', help='complete help') - parser_complete.add_argument( - '--upload-id', type=str, required=True, help="The upload-id to complete" - ) - - # create the parser for the "abort" command - parser_abort = subparsers.add_parser('abort', help='abort help') - parser_abort.add_argument( - '--upload-id', type=str, required=True, help="The upload-id to abort" - ) - - def handle(self, *args, **options): - handler = DummyAssetUploadHandler(self, options) - if options['action'] == 'start': - handler.start() - elif options['action'] == 'list': - handler.list() - elif options['action'] == 'complete': - handler.complete() - elif options['action'] == 'abort': - handler.abort() diff --git a/app/stac_api/management/commands/dummy_data.py b/app/stac_api/management/commands/dummy_data.py index a69702db..d6fa24d4 100644 --- a/app/stac_api/management/commands/dummy_data.py +++ b/app/stac_api/management/commands/dummy_data.py @@ -1,5 +1,4 @@ import datetime -import logging import random import string import time @@ -11,16 +10,13 @@ from django.contrib.gis.geos import Polygon from django.core.files.uploadedfile import SimpleUploadedFile -from django.core.management.base import BaseCommand from stac_api.models.collection import Collection from stac_api.models.item import Asset from stac_api.models.item import Item -from stac_api.utils import CommandHandler +from stac_api.utils import CustomBaseCommand from stac_api.validators import MEDIA_TYPES -logger = logging.getLogger(__name__) - # Min/Max extent (roughly) of CH in LV95 XMIN = 2570000 XMAX = 2746000 @@ -41,7 +37,67 @@ def random_datetime(start, end): ) -class DummyDataHandler(CommandHandler): +class Command(CustomBaseCommand): + help = """Manage dummy data for performance testing. + + The command populates the database by default with + 30 collections, 300 items per collection and 2 assets per item. + Number of collections, items and assets can be changed. + + The generated data is randomized where necessary, i.e. the field + that are also likely to be queried. + """ + + def add_arguments(self, parser): + super().add_arguments(parser) + parser.add_argument( + 'action', + type=str, + choices=['populate', 'clean'], + default='populate', + help='Define the action to be performed, either "populate" (default) to create ' + 'dummy data or "clean" to delete it', + ) + + parser.add_argument( + '--collections', + type=str, + default='30', + help="Number of collections to create (default 30), or alternatively a comma separated " + f"list of collection names to create (a common prefix '{NAME_PREFIX}' is added to " + "these names)" + ) + + parser.add_argument( + '--items', + type=int, + default=300, + help="Number of items per collection to create (default 300)" + ) + + parser.add_argument( + '--assets', type=int, default=2, help="Number of assets per item to create (default 2)" + ) + + parser.add_argument( + '--parallel-collections', + type=int, + default=1, + help="Number of collection created in parallel (default 1)" + ) + + parser.add_argument( + '--parallel-items', + type=int, + default=5, + help="Number of items created in parallel (default 5)" + ) + + def handle(self, *args, **options): + if options['action'] == 'clean': + self.clean() + elif options['action'] == 'populate': + self.populate() def clean(self): self.print_warning('Deleting all collections starting with "%s"...', NAME_PREFIX) @@ -250,67 +306,3 @@ def create_asset(self, item, asset_id): } ) self.print('Asset %s/%s/%s created', item.collection.name, item.name, asset_id, level=3) - - -class Command(BaseCommand): - help = """Manage dummy data for performance testing. - - The command populates the database by default with - 30 collections, 300 items per collection and 2 assets per item. - Number of collections, items and assets can be changed. - - The generated data is randomized where necessary, i.e. the field - that are also likely to be queried. - """ - - def add_arguments(self, parser): - parser.add_argument( - 'action', - type=str, - choices=['populate', 'clean'], - default='populate', - help='Define the action to be performed, either "populate" (default) to create ' - 'dummy data or "clean" to delete it', - ) - - parser.add_argument( - '--collections', - type=str, - default='30', - help="Number of collections to create (default 30), or alternatively a comma separated " - f"list of collection names to create (a common prefix '{NAME_PREFIX}' is added to " - "these names)" - ) - - parser.add_argument( - '--items', - type=int, - default=300, - help="Number of items per collection to create (default 300)" - ) - - parser.add_argument( - '--assets', type=int, default=2, help="Number of assets per item to create (default 2)" - ) - - parser.add_argument( - '--parallel-collections', - type=int, - default=1, - help="Number of collection created in parallel (default 1)" - ) - - parser.add_argument( - '--parallel-items', - type=int, - default=5, - help="Number of items created in parallel (default 5)" - ) - - def handle(self, *args, **options): - handler = DummyDataHandler(self, options) - - if options['action'] == 'clean': - handler.clean() - elif options['action'] == 'populate': - handler.populate() diff --git a/app/stac_api/management/commands/list_asset_uploads.py b/app/stac_api/management/commands/list_asset_uploads.py index 24bb58ce..49a1c015 100644 --- a/app/stac_api/management/commands/list_asset_uploads.py +++ b/app/stac_api/management/commands/list_asset_uploads.py @@ -1,53 +1,93 @@ import json -import logging -from django.core.management.base import BaseCommand from django.core.serializers.json import DjangoJSONEncoder from stac_api.models.item import AssetUpload from stac_api.s3_multipart_upload import MultipartUpload from stac_api.serializers.upload import AssetUploadSerializer -from stac_api.utils import CommandHandler +from stac_api.utils import CustomBaseCommand from stac_api.utils import get_asset_path -logger = logging.getLogger(__name__) +class Command(CustomBaseCommand): + help = """List all asset uploads object (DB and/or S3) + + This checks for all asset uploads object in DB (by default only returning the `in-progress` + status objects) as well as the open S3 multipart uploads (S3 has only `in-progress` uploads, + once the upload is completed it is automatically deleted). This command is in addition to the + .../assets//uploads which only list the uploads of one asset, while the command list + all uploads for all assets. + + WARNINGS: + - Although pagination is implemented, if there is more uploads than the limit, the sync + algorithm will not work because it only search for common upload on the page context and + uploads are not sorted. + - The S3 minio server for local development doesn't supports the list_multipart_uploads + methods, therefore the output will only contains the DB entries. + """ + + def add_arguments(self, parser): + self.prog = parser.prog # pylint: disable=attribute-defined-outside-init + super().add_arguments(parser) + + parser.add_argument( + '--status', + type=str, + default=AssetUpload.Status.IN_PROGRESS, + help=f"Filter by status (default '{AssetUpload.Status.IN_PROGRESS}')" + ) -class ListAssetUploadsHandler(CommandHandler): + default_limit = 50 + parser.add_argument( + '--limit', + type=int, + default=default_limit, + help=f"Limit the output (default {default_limit})" + ) - def __init__(self, command, options): - super().__init__(command, options) - self.s3 = MultipartUpload() + parser.add_argument( + '--start', type=int, default=0, help="Start the list at the given index (default 0)" + ) - def list_asset_uploads(self): + parser.add_argument('--db-only', type=bool, default=False, help="List only DB objects") + + parser.add_argument('--s3-only', type=bool, default=False, help="List only S3 objects") + + parser.add_argument( + '--s3-key-start', type=str, default=None, help='Next S3 key for pagination' + ) + parser.add_argument( + '--s3-upload-id-start', type=str, default=None, help='Next S3 upload ID for pagination' + ) + + def handle(self, *args, **options): # pylint: disable=too-many-locals + s3 = MultipartUpload() uploads = [] only_s3_uploads = [] only_db_uploads = [] s3_has_next = False db_has_next = False - limit = self.options['limit'] - start = self.options['start'] - s3_key_start = self.options['s3_key_start'] - s3_upload_id_start = self.options['s3_upload_id_start'] + limit = options['limit'] + start = options['start'] + s3_key_start = options['s3_key_start'] + s3_upload_id_start = options['s3_upload_id_start'] db_uploads_qs = None s3_next_key = None s3_next_upload_id = None s3_uploads = [] - if not self.options['db_only']: + if not options['db_only']: # get all s3 multipart uploads ( s3_uploads, s3_has_next, s3_next_key, s3_next_upload_id, - ) = self.s3.list_multipart_uploads( - limit=limit, key=s3_key_start, start=s3_upload_id_start - ) + ) = s3.list_multipart_uploads(limit=limit, key=s3_key_start, start=s3_upload_id_start) - if not self.options['s3_only']: - queryset = AssetUpload.objects.filter_by_status(self.options['status']) + if not options['s3_only']: + queryset = AssetUpload.objects.filter_by_status(options['status']) count = queryset.count() if count > limit: queryset = queryset[start:start + limit] @@ -55,7 +95,7 @@ def list_asset_uploads(self): if start + limit < count: db_has_next = True - if not self.options['db_only'] and not self.options['s3_only']: + if not options['db_only'] and not options['s3_only']: def are_uploads_equal(s3_upload, db_upload): if ( @@ -93,12 +133,12 @@ def are_uploads_equal(s3_upload, db_upload): ) if db_upload is None: only_s3_uploads.append(s3_upload) - elif self.options['db_only']: + elif options['db_only']: only_db_uploads = AssetUploadSerializer(instance=list(db_uploads_qs), many=True).data - elif self.options['s3_only']: + elif options['s3_only']: only_s3_uploads = s3_uploads - print( + self.print( json.dumps( { 'uploads': uploads, @@ -106,7 +146,7 @@ def are_uploads_equal(s3_upload, db_upload): 's3_uploads': only_s3_uploads, 'next': ' '.join([ - f'./{self.command.prog}', + f'./{self.prog}', f'--limit={limit}', f'--start={start}' if db_has_next else '', f'--s3-key-start={s3_next_key}' if s3_has_next else '', @@ -117,57 +157,3 @@ def are_uploads_equal(s3_upload, db_upload): cls=DjangoJSONEncoder, ) ) - - -class Command(BaseCommand): - help = """List all asset uploads object (DB and/or S3) - - This checks for all asset uploads object in DB (by default only returning the `in-progress` - status objects) as well as the open S3 multipart uploads (S3 has only `in-progress` uploads, - once the upload is completed it is automatically deleted). This command is in addition to the - .../assets//uploads which only list the uploads of one asset, while the command list - all uploads for all assets. - - WARNINGS: - - Although pagination is implemented, if there is more uploads than the limit, the sync - algorithm will not work because it only search for common upload on the page context and - uploads are not sorted. - - The S3 minio server for local development doesn't supports the list_multipart_uploads - methods, therefore the output will only contains the DB entries. - """ - - def add_arguments(self, parser): - self.prog = parser.prog # pylint: disable=attribute-defined-outside-init - - parser.add_argument( - '--status', - type=str, - default=AssetUpload.Status.IN_PROGRESS, - help=f"Filter by status (default '{AssetUpload.Status.IN_PROGRESS}')" - ) - - default_limit = 50 - parser.add_argument( - '--limit', - type=int, - default=default_limit, - help=f"Limit the output (default {default_limit})" - ) - - parser.add_argument( - '--start', type=int, default=0, help="Start the list at the given index (default 0)" - ) - - parser.add_argument('--db-only', type=bool, default=False, help="List only DB objects") - - parser.add_argument('--s3-only', type=bool, default=False, help="List only S3 objects") - - parser.add_argument( - '--s3-key-start', type=str, default=None, help='Next S3 key for pagination' - ) - parser.add_argument( - '--s3-upload-id-start', type=str, default=None, help='Next S3 upload ID for pagination' - ) - - def handle(self, *args, **options): - ListAssetUploadsHandler(self, options).list_asset_uploads() diff --git a/app/stac_api/management/commands/manage_superuser.py b/app/stac_api/management/commands/manage_superuser.py index 63d67ee2..56dfffb2 100644 --- a/app/stac_api/management/commands/manage_superuser.py +++ b/app/stac_api/management/commands/manage_superuser.py @@ -4,13 +4,12 @@ from django.contrib.auth import get_user_model -from stac_api.utils import CommandHandler from stac_api.utils import CustomBaseCommand env = environ.Env() -class Handler(CommandHandler): +class Command(CustomBaseCommand): """Create or update superuser from information from the environment This command is used to make sure that the superuser is created and @@ -18,7 +17,9 @@ class Handler(CommandHandler): This will help with the password rotation. """ - def run(self) -> None: + help = "Superuser management (creating or updating)" + + def handle(self, *args: Any, **options: Any) -> None: User = get_user_model() # pylint: disable=invalid-name username = env.str('DJANGO_SUPERUSER_USERNAME', default='').strip() email = env.str('DJANGO_SUPERUSER_EMAIL', default='').strip() @@ -42,10 +43,3 @@ def run(self) -> None: admin.save() self.print_success('%s the superuser %s', operation, username) - - -class Command(CustomBaseCommand): - help = "Superuser management (creating or updating)" - - def handle(self, *args: Any, **options: Any) -> None: - Handler(self, options).run() diff --git a/app/stac_api/management/commands/populate_testdb.py b/app/stac_api/management/commands/populate_testdb.py index 92762a1c..a4d135d2 100644 --- a/app/stac_api/management/commands/populate_testdb.py +++ b/app/stac_api/management/commands/populate_testdb.py @@ -1,31 +1,15 @@ -import logging import os from django.conf import settings -from django.core.management.base import BaseCommand from stac_api.sample_data import importer -from stac_api.utils import CommandHandler +from stac_api.utils import CustomBaseCommand # path definition relative to the directory that contains manage.py DATADIR = settings.BASE_DIR / 'app/stac_api/sample_data/' -logger = logging.getLogger(__name__) -class Handler(CommandHandler): - - def populate(self): - # loop over the collection directories inside sample_data - for collection_dir in os.scandir(DATADIR): - if collection_dir.is_dir() and not collection_dir.name.startswith('_'): - self.print('Import collection %s', collection_dir.name, level=1) - importer.import_collection(collection_dir) - else: - self.print('Ignore file %s', collection_dir.name, level=2) - self.print_success('Done') - - -class Command(BaseCommand): +class Command(CustomBaseCommand): help = """Populates the local test database with sample data The sample data has to be located in stac_api/management/sample_data and @@ -38,4 +22,11 @@ class Command(BaseCommand): """ def handle(self, *args, **options): - Handler(self, options).populate() + # loop over the collection directories inside sample_data + for collection_dir in os.scandir(DATADIR): + if collection_dir.is_dir() and not collection_dir.name.startswith('_'): + self.print('Import collection %s', collection_dir.name, level=1) + importer.import_collection(collection_dir) + else: + self.print('Ignore file %s', collection_dir.name, level=2) + self.print_success('Done') diff --git a/app/stac_api/management/commands/profile_cursor_paginator.py b/app/stac_api/management/commands/profile_cursor_paginator.py index 18c6174c..d4d98bc5 100644 --- a/app/stac_api/management/commands/profile_cursor_paginator.py +++ b/app/stac_api/management/commands/profile_cursor_paginator.py @@ -1,26 +1,42 @@ import cProfile -import logging import os import pstats from django.conf import settings -from django.core.management.base import BaseCommand from rest_framework.pagination import CursorPagination from rest_framework.request import Request from rest_framework.test import APIRequestFactory from stac_api.models.item import Item -from stac_api.utils import CommandHandler - -logger = logging.getLogger(__name__) +from stac_api.utils import CustomBaseCommand STAC_BASE_V = f'{settings.STAC_BASE}/v1' -class Handler(CommandHandler): +class Command(CustomBaseCommand): + help = """Paginator paginate_queryset() profiling command + + Profiling of the method paginator.paginate_queryset(qs, request) + + See https://docs.python.org/3.7/library/profile.html + """ + + def add_arguments(self, parser): + super().add_arguments(parser) + parser.add_argument( + '--collection', + type=str, + default='perftest-collection-0', + help="Collection ID to use for the queryset profiling" + ) + parser.add_argument('--limit', type=int, default=100, help="Limit to use for the queryset") + parser.add_argument('--sort', type=str, default='tottime', help="Profiling output sorting") + parser.add_argument( + '--lines', type=str, default=50, help="Profiling output numbers of line to show" + ) - def profiling(self): + def handle(self, *args, **options): # pylint: disable=import-outside-toplevel,possibly-unused-variable collection_id = self.options["collection"] qs = Item.objects.filter(collection__name=collection_id).prefetch_related('assets', 'links') @@ -42,28 +58,3 @@ def profiling(self): stats.sort_stats(self.options['sort']).print_stats() self.print_success('Done') - - -class Command(BaseCommand): - help = """Paginator paginate_queryset() profiling command - - Profiling of the method paginator.paginate_queryset(qs, request) - - See https://docs.python.org/3.7/library/profile.html - """ - - def add_arguments(self, parser): - parser.add_argument( - '--collection', - type=str, - default='perftest-collection-0', - help="Collection ID to use for the queryset profiling" - ) - parser.add_argument('--limit', type=int, default=100, help="Limit to use for the queryset") - parser.add_argument('--sort', type=str, default='tottime', help="Profiling output sorting") - parser.add_argument( - '--lines', type=str, default=50, help="Profiling output numbers of line to show" - ) - - def handle(self, *args, **options): - Handler(self, options).profiling() diff --git a/app/stac_api/management/commands/profile_item_serializer.py b/app/stac_api/management/commands/profile_item_serializer.py index 8439598d..35206711 100644 --- a/app/stac_api/management/commands/profile_item_serializer.py +++ b/app/stac_api/management/commands/profile_item_serializer.py @@ -1,24 +1,37 @@ import cProfile -import logging import os import pstats from django.conf import settings -from django.core.management.base import BaseCommand from rest_framework.test import APIRequestFactory from stac_api.models.item import Item -from stac_api.utils import CommandHandler - -logger = logging.getLogger(__name__) +from stac_api.utils import CustomBaseCommand STAC_BASE_V = f'{settings.STAC_BASE}/v1' -class Handler(CommandHandler): +class Command(CustomBaseCommand): + help = """ItemSerializer profiling command + + Profiling of the serialization of many items. + + See https://docs.python.org/3.7/library/profile.html + """ - def profiling(self): + def add_arguments(self, parser): + super().add_arguments(parser) + parser.add_argument( + '--collection', + type=str, + default='perftest-collection-0', + help="Collection ID to use for the ItemSerializer profiling" + ) + parser.add_argument('--limit', type=int, default=100, help="Limit to use for the query") + parser.add_argument('--sort', type=str, default='tottime', help="Profiling output sorting") + + def handle(self, *args, **options): # pylint: disable=import-outside-toplevel,possibly-unused-variable from stac_api.serializers.item import ItemSerializer collection_id = self.options["collection"] @@ -38,25 +51,3 @@ def profiling(self): stats.sort_stats(self.options['sort']).print_stats() self.print_success('Done') - - -class Command(BaseCommand): - help = """ItemSerializer profiling command - - Profiling of the serialization of many items. - - See https://docs.python.org/3.7/library/profile.html - """ - - def add_arguments(self, parser): - parser.add_argument( - '--collection', - type=str, - default='perftest-collection-0', - help="Collection ID to use for the ItemSerializer profiling" - ) - parser.add_argument('--limit', type=int, default=100, help="Limit to use for the query") - parser.add_argument('--sort', type=str, default='tottime', help="Profiling output sorting") - - def handle(self, *args, **options): - Handler(self, options).profiling() diff --git a/app/stac_api/management/commands/profile_serializer_vs_no_drf.py b/app/stac_api/management/commands/profile_serializer_vs_no_drf.py index 28ed2a2d..ce8ef947 100644 --- a/app/stac_api/management/commands/profile_serializer_vs_no_drf.py +++ b/app/stac_api/management/commands/profile_serializer_vs_no_drf.py @@ -1,23 +1,36 @@ import json -import logging from timeit import timeit from django.conf import settings -from django.core.management.base import BaseCommand from rest_framework.test import APIRequestFactory from stac_api.models.item import Item -from stac_api.utils import CommandHandler - -logger = logging.getLogger(__name__) +from stac_api.utils import CustomBaseCommand STAC_BASE_V = f'{settings.STAC_BASE}/v1' -class Handler(CommandHandler): +class Command(CustomBaseCommand): + help = """ItemSerializer vs simple serializer profiling command + + Profiling of the serialization of many items using DRF vs using a simple function. + + See https://docs.python.org/3.7/library/profile.html + """ - def profiling(self): + def add_arguments(self, parser): + super().add_arguments(parser) + parser.add_argument( + '--collection', + type=str, + default='perftest-collection-0', + help="Collection ID to use for the ItemSerializer profiling" + ) + parser.add_argument('--limit', type=int, default=100, help="Limit to use for the query") + parser.add_argument('--repeat', type=int, default=100, help="Repeat the measurement") + + def handle(self, *args, **options): # pylint: disable=import-outside-toplevel,possibly-unused-variable self.print('Starting profiling') @@ -83,25 +96,3 @@ def serialize(qs): self.print_success('DRF time: %fms', serializer_time / self.options['repeat'] * 1000) self.print_success('NO DRF time: %fms', no_drf_time / self.options['repeat'] * 1000) - - -class Command(BaseCommand): - help = """ItemSerializer vs simple serializer profiling command - - Profiling of the serialization of many items using DRF vs using a simple function. - - See https://docs.python.org/3.7/library/profile.html - """ - - def add_arguments(self, parser): - parser.add_argument( - '--collection', - type=str, - default='perftest-collection-0', - help="Collection ID to use for the ItemSerializer profiling" - ) - parser.add_argument('--limit', type=int, default=100, help="Limit to use for the query") - parser.add_argument('--repeat', type=int, default=100, help="Repeat the measurement") - - def handle(self, *args, **options): - Handler(self, options).profiling() diff --git a/app/stac_api/management/commands/remove_expired_items.py b/app/stac_api/management/commands/remove_expired_items.py index d7d195f6..40f2366c 100644 --- a/app/stac_api/management/commands/remove_expired_items.py +++ b/app/stac_api/management/commands/remove_expired_items.py @@ -9,7 +9,6 @@ from stac_api.models.item import Asset from stac_api.models.item import AssetUpload from stac_api.models.item import Item -from stac_api.utils import CommandHandler from stac_api.utils import CustomBaseCommand @@ -20,7 +19,69 @@ def __str__(self): f" {self.args[1]} > {self.args[0]}.") -class Handler(CommandHandler): +class Command(CustomBaseCommand): + help = """Remove items and their assets that have expired more than + DELETE_EXPIRED_ITEMS_OLDER_THAN_HOURS hours ago. + This command is thought to be scheduled as cron job. + """ + + def _validate_int(self, candidate, min_value=None, max_value=None): + value = int(candidate) + if min_value is not None and value < min_value: + raise ValueError(f"{value} is less than {min_value}") + if max_value is not None and value > max_value: + raise ValueError(f"{value} is greater than {max_value}") + return value + + def add_arguments(self, parser: CommandParser) -> None: + super().add_arguments(parser) + parser.register('type', 'positive_int', functools.partial(self._validate_int, min_value=0)) + parser.register( + 'type', + 'percentage_int', + functools.partial(self._validate_int, min_value=0, max_value=100) + ) + parser.add_argument( + '--dry-run', + action='store_true', + help='Simulate deleting items, without actually deleting them' + ) + default_batch_size = settings.DELETE_EXPIRED_ITEMS_BATCH_SIZE + parser.add_argument( + '--batch-size', + type='positive_int', + default=default_batch_size, + help=f"How many rows to delete at a time ({default_batch_size})" + ) + default_min_age = settings.DELETE_EXPIRED_ITEMS_OLDER_THAN_HOURS + parser.add_argument( + '--min-age-hours', + type='positive_int', + default=default_min_age, + help=f"Minimum hours the item must have been expired for (default {default_min_age})" + ) + default_max_deletions = settings.DELETE_EXPIRED_ITEMS_MAX + parser.add_argument( + '--max-deletions', + type='positive_int', + default=default_max_deletions, + help=( + f"Maximum number of items to delete. If that number of items" + f" have expired, this programm will fail. Default value:" + f" {default_max_deletions}." + ) + ) + default_max_deletions_percentage = settings.DELETE_EXPIRED_ITEMS_MAX_PERCENTAGE + parser.add_argument( + '--max-deletions-percentage', + type='percentage_int', + default=default_max_deletions_percentage, + help=( + f"Maximum percentage of items to delete. If that percentage of" + f" items are expired, this program will fail." + f" Default value: {default_max_deletions_percentage}." + ) + ) def delete_by_batch(self, queryset, object_type, batch_size): # When many rows are involved, looping over each one is very slow. @@ -78,7 +139,7 @@ def _raise_if_too_many_deletions(self, max_deletions, max_deletions_pct, items_c self.print_error("%s", str(exception)) raise exception - def run(self): + def handle(self, *args, **options): self.print_success('running command to remove expired items') batch_size = self.options['batch_size'] min_age_hours = self.options['min_age_hours'] @@ -116,71 +177,3 @@ def run(self): self.print_success(f'[dry run] would have removed {items_count} expired items') else: self.print_success(f'successfully removed {items_count} expired items') - - -class Command(CustomBaseCommand): - help = """Remove items and their assets that have expired more than - DELETE_EXPIRED_ITEMS_OLDER_THAN_HOURS hours ago. - This command is thought to be scheduled as cron job. - """ - - def _validate_int(self, candidate, min_value=None, max_value=None): - value = int(candidate) - if min_value is not None and value < min_value: - raise ValueError(f"{value} is less than {min_value}") - if max_value is not None and value > max_value: - raise ValueError(f"{value} is greater than {max_value}") - return value - - def add_arguments(self, parser: CommandParser) -> None: - super().add_arguments(parser) - parser.register('type', 'positive_int', functools.partial(self._validate_int, min_value=0)) - parser.register( - 'type', - 'percentage_int', - functools.partial(self._validate_int, min_value=0, max_value=100) - ) - parser.add_argument( - '--dry-run', - action='store_true', - help='Simulate deleting items, without actually deleting them' - ) - default_batch_size = settings.DELETE_EXPIRED_ITEMS_BATCH_SIZE - parser.add_argument( - '--batch-size', - type='positive_int', - default=default_batch_size, - help=f"How many rows to delete at a time ({default_batch_size})" - ) - default_min_age = settings.DELETE_EXPIRED_ITEMS_OLDER_THAN_HOURS - parser.add_argument( - '--min-age-hours', - type='positive_int', - default=default_min_age, - help=f"Minimum hours the item must have been expired for (default {default_min_age})" - ) - default_max_deletions = settings.DELETE_EXPIRED_ITEMS_MAX - parser.add_argument( - '--max-deletions', - type='positive_int', - default=default_max_deletions, - help=( - f"Maximum number of items to delete. If that number of items" - f" have expired, this programm will fail. Default value:" - f" {default_max_deletions}." - ) - ) - default_max_deletions_percentage = settings.DELETE_EXPIRED_ITEMS_MAX_PERCENTAGE - parser.add_argument( - '--max-deletions-percentage', - type='percentage_int', - default=default_max_deletions_percentage, - help=( - f"Maximum percentage of items to delete. If that percentage of" - f" items are expired, this program will fail." - f" Default value: {default_max_deletions_percentage}." - ) - ) - - def handle(self, *args, **options): - Handler(self, options).run() diff --git a/app/stac_api/management/commands/reset_counter_tables.py b/app/stac_api/management/commands/reset_counter_tables.py index b3e57ecd..e2febe5d 100644 --- a/app/stac_api/management/commands/reset_counter_tables.py +++ b/app/stac_api/management/commands/reset_counter_tables.py @@ -1,14 +1,19 @@ import time -from django.core.management.base import BaseCommand from django.db import connection -from stac_api.utils import CommandHandler +from stac_api.utils import CustomBaseCommand -class Handler(CommandHandler): +class Command(CustomBaseCommand): + help = """Reset the summary counter tables. - def run(self): + Truncates all the summary counter tables and repopulates with current data to make sure they are + in sync with the values in the asset table. Unless the triggers are disabled or values in the + counter tables are changed manually, this should not be required. + """ + + def handle(self, *args, **options): self.print_success('running query to update counter tables...') start = time.monotonic() @@ -57,15 +62,3 @@ def run(self): self.print_success( f"successfully updated counter tables in {(time.monotonic()-start):.3f}s" ) - - -class Command(BaseCommand): - help = """Reset the summary counter tables. - - Truncates all the summary counter tables and repopulates with current data to make sure they are - in sync with the values in the asset table. Unless the triggers are disabled or values in the - counter tables are changed manually, this should not be required. - """ - - def handle(self, *args, **options): - Handler(self, options).run() diff --git a/app/stac_api/management/commands/update_asset_file_size.py b/app/stac_api/management/commands/update_asset_file_size.py index 2631211d..1cc3a4f3 100644 --- a/app/stac_api/management/commands/update_asset_file_size.py +++ b/app/stac_api/management/commands/update_asset_file_size.py @@ -1,23 +1,32 @@ import logging -from django.core.management.base import BaseCommand from django.core.management.base import CommandParser from stac_api.models.collection import CollectionAsset from stac_api.models.item import Asset -from stac_api.utils import CommandHandler - -logger = logging.getLogger(__name__) +from stac_api.utils import CustomBaseCommand # increase the log level so boto3 doesn't spam the output logging.getLogger('boto3').setLevel(logging.WARNING) logging.getLogger('botocore').setLevel(logging.WARNING) -class Handler(CommandHandler): +class Command(CustomBaseCommand): + help = """Requests the file size of every asset / collection asset from the s3 bucket and + updates the value in the database""" + + def add_arguments(self, parser: CommandParser) -> None: + super().add_arguments(parser) + parser.add_argument( + '-c', + '--count', + help="The amount of assets to process at once", + required=True, + type=int + ) - def update(self): - self.print_success('Running command to update file size') + def handle(self, *args, **options): + self.print('Running command to update file size') asset_limit = self.options['count'] @@ -25,7 +34,7 @@ def update(self): total_asset_count = asset_qs.count() assets = asset_qs.all()[:asset_limit] - self.print_success(f'Update file size for {len(assets)} assets out of {total_asset_count}') + self.print(f'Update file size for {len(assets)} assets out of {total_asset_count}') for asset in assets: try: @@ -45,14 +54,13 @@ def update(self): asset.file_size = None asset.save() print("_", end="", flush=True) - logger.error('file %s could not be found', asset.file) - print() + self.print_error('file %s could not be found', asset.file) collection_asset_qs = CollectionAsset.objects.filter(file_size=0) total_asset_count = collection_asset_qs.count() collection_assets = collection_asset_qs.all()[:asset_limit] - self.print_success( + self.print( f"Update file size for {len(collection_assets)} collection assets out of " f"{total_asset_count}" ) @@ -73,26 +81,6 @@ def update(self): # bucket. collection_asset.file_size = None collection_asset.save() - print("_", end="", flush=True) - logger.error('file %s could not be found', collection_asset.file) + self.print_error('file %s could not be found', collection_asset.file) - print() self.print_success('Update completed') - - -class Command(BaseCommand): - help = """Requests the file size of every asset / collection asset from the s3 bucket and - updates the value in the database""" - - def add_arguments(self, parser: CommandParser) -> None: - super().add_arguments(parser) - parser.add_argument( - '-c', - '--count', - help="The amount of assets to process at once", - required=True, - type=int - ) - - def handle(self, *args, **options): - Handler(self, options).update() diff --git a/app/stac_api/management/commands/write_perftests.py b/app/stac_api/management/commands/write_perftests.py index 130d87f1..800f830a 100644 --- a/app/stac_api/management/commands/write_perftests.py +++ b/app/stac_api/management/commands/write_perftests.py @@ -1,17 +1,13 @@ -import logging import time from statistics import mean import requests from django.contrib.gis.geos import GEOSGeometry -from django.core.management.base import BaseCommand -from stac_api.utils import CommandHandler +from stac_api.utils import CustomBaseCommand from stac_api.validators import get_media_type -logger = logging.getLogger(__name__) - GEOMETRIES = { 'switzerland-west': GEOSGeometry( @@ -52,7 +48,64 @@ } -class Handler(CommandHandler): +class Command(CustomBaseCommand): + help = """Run write performance tests + + The following three steps are consecutively runned: + 1. create n (items | assets) + 2. update n (items | assets) + 3. delete n (items | assets) + """ + + def add_arguments(self, parser): + super().add_arguments(parser) + + parser.add_argument( + 'object_type', + type=str, + choices=['items', 'assets'], + help='Define which object type to create/update/deletes', + ) + + parser.add_argument( + '--clean', + action='store_true', + help='Clean all object created by the scripts. Usefull if the script failed.' + ) + + parser.add_argument( + '-n', type=int, default=50, help="Number of object to create/update/delete" + ) + + parser.add_argument( + '--url', type=str, default='http://localhost:8000', help="Url to run the test against" + ) + + parser.add_argument('--key', type=str, help='Token used for authentication') + parser.add_argument('--auth', type=str, help='Basic authentication in form user:pass') + + parser.add_argument( + '--collection', + type=str, + default='perftest-collection-1', + help="Collection on which to run the tests." + ) + + parser.add_argument( + '--item', + type=str, + default='perftest-item-1', + help="Item on which to run the tests, only valid for 'assets' object_type." + ) + + def handle(self, *args, **options): + try: + if options['clean']: + self.clean() + else: + self.start() + except RuntimeError: + pass def clean(self): # pylint: disable=missing-timeout @@ -124,13 +177,13 @@ def start(self): self.print_success('Done') def get_auth(self): - if 'auth' in self.options: + if self.options.get('auth'): return (*self.options['auth'].split(':', maxsplit=1),) return None def get_headers(self): headers = {} - if 'key' in self.options: + if self.options.get('key'): headers['Authorization'] = f'Token {self.options["key"]}' return headers @@ -229,61 +282,3 @@ def get_item_properties(self, i): }, ] return properties[i % len(properties)] - - -class Command(BaseCommand): - help = """Run write performance tests - - The following three steps are consecutively runned: - 1. create n (items | assets) - 2. update n (items | assets) - 3. delete n (items | assets) - """ - - def add_arguments(self, parser): - parser.add_argument( - 'object_type', - type=str, - choices=['items', 'assets'], - help='Define which object type to create/update/deletes', - ) - - parser.add_argument( - '--clean', - action='store_true', - help='Clean all object created by the scripts. Usefull if the script failed.' - ) - - parser.add_argument( - '-n', type=int, default=50, help="Number of object to create/update/delete" - ) - - parser.add_argument( - '--url', type=str, default='http://localhost:8000', help="Url to run the test against" - ) - - parser.add_argument('--key', type=str, help='Token used for authentication') - parser.add_argument('--auth', type=str, help='Basic authentication in form user:pass') - - parser.add_argument( - '--collection', - type=str, - default='perftest-collection-1', - help="Collection on which to run the tests." - ) - - parser.add_argument( - '--item', - type=str, - default='perftest-item-1', - help="Item on which to run the tests, only valid for 'assets' object_type." - ) - - def handle(self, *args, **options): - try: - if options['clean']: - Handler(self, options).clean() - else: - Handler(self, options).start() - except RuntimeError: - pass diff --git a/app/stac_api/utils.py b/app/stac_api/utils.py index de125695..48e58020 100644 --- a/app/stac_api/utils.py +++ b/app/stac_api/utils.py @@ -1,5 +1,4 @@ import hashlib -import inspect import json import logging import os @@ -10,6 +9,8 @@ from decimal import InvalidOperation from enum import Enum from io import StringIO +from typing import Any +from typing import TextIO from urllib import parse import boto3 @@ -21,6 +22,7 @@ from django.contrib.gis.geos import Polygon from django.core.management import call_command from django.core.management.base import BaseCommand +from django.core.management.base import CommandParser from django.urls import reverse from stac_api.exceptions import NotImplementedException @@ -370,40 +372,72 @@ def remove_query_params(url, keys): return parse.urlunsplit((scheme, netloc, path, query, fragment)) +# This class is also used in service-control. Ensure that any changes made here are reflected there +# as well. class CustomBaseCommand(BaseCommand): + """ + A custom Django management command that adds proper support for logging. + + Example how to subclass: + + class MyCommand(CustomBaseCommand): + + def add_arguments(self, parser: CommandParser) -> None: + super().add_arguments(parser) + parser.add_argument('--flag', action='store_true') + + def handle(self, *args: Any, **options: dict['str', Any]) -> None: + if options['flag']: # or self.options['flag'] + self.print('flag was set') + self.print_success('done') - def handle(self, *args, **options): + """ + + def __init__( + self, + stdout: TextIO | None = None, + stderr: TextIO | None = None, + no_color: bool = False, + force_color: bool = False + ): + super().__init__(stdout, stderr, no_color, force_color) + self.logger = logging.getLogger(self.__module__) + self.options: dict['str', Any] = {} + + def add_arguments(self, parser: CommandParser) -> None: """ - The actual logic of the command. Subclasses must implement - this method. + Entry point for add custom arguments. Options will also be available as self.options during + handle. + + Subclasses may want to extend this method. """ - raise NotImplementedError("subclasses of CustomBaseCommand must provide a handle() method") - def add_arguments(self, parser): parser.add_argument('--logger', action='store_true', help='use logger configuration') + def handle(self, *args: Any, **options: dict['str', Any]) -> None: + """ + The actual logic of the command. -class CommandHandler(): - '''Base class for management command handler + Subclasses must implement this method. + """ - This class add proper support for printing to the console for management command - ''' + raise NotImplementedError("subclasses of CustomBaseCommand must provide a handle() method") + + def execute(self, *args: Any, **options: dict['str', Any]) -> None: + """ Try to execute the command and log any exceptions if the logger is configured. """ - def __init__(self, command, options): - frm = inspect.stack()[1] - mod = inspect.getmodule(frm[0]) - self.logger = logging.getLogger(mod.__name__) self.options = options - self.verbosity = options['verbosity'] - self.use_logger = options.get('logger') - self.stdout = command.stdout - self.stderr = command.stderr - self.style = command.style - self.command = command - - def print(self, message, *args, level=2, **kwargs): - if self.verbosity >= level: - if self.use_logger: + if self.options['logger']: + try: + super().execute(*args, **options) + except Exception as e: # pylint: disable=broad-exception-caught + self.print_error(e, exc_info=True) + else: + super().execute(*args, **options) + + def print(self, message: str, *args: Any, level: int = 2, **kwargs: Any) -> None: + if self.options['verbosity'] >= level: + if self.options['logger']: self.logger.info(message, *args, **kwargs) else: if len(kwargs) > 0: @@ -412,10 +446,10 @@ def print(self, message, *args, level=2, **kwargs): ) self.stdout.write(message % (args)) - def print_warning(self, message, *args, level=1, **kwargs): - if self.verbosity >= level: - if self.use_logger: - self.logger.warning(self.style.WARNING(message % (args)), **kwargs) + def print_warning(self, message: str, *args: Any, level: int = 1, **kwargs: Any) -> None: + if self.options['verbosity'] >= level: + if self.options['logger']: + self.logger.warning(message, *args, **kwargs) else: if len(kwargs) > 0: message = message + " " + ", ".join( @@ -423,10 +457,10 @@ def print_warning(self, message, *args, level=1, **kwargs): ) self.stdout.write(self.style.WARNING(message % (args))) - def print_success(self, message, *args, level=1, **kwargs): - if self.verbosity >= level: - if self.use_logger: - self.logger.info(self.style.SUCCESS(message % (args)), **kwargs) + def print_success(self, message: str, *args: Any, level: int = 1, **kwargs: Any) -> None: + if self.options['verbosity'] >= level: + if self.options['logger']: + self.logger.info(message, *args, **kwargs) else: if len(kwargs) > 0: message = message + " " + ", ".join( @@ -434,10 +468,11 @@ def print_success(self, message, *args, level=1, **kwargs): ) self.stdout.write(self.style.SUCCESS(message % (args))) - def print_error(self, message, *args, **kwargs): - if self.use_logger: - self.logger.error(self.style.ERROR(message % (args)), **kwargs) + def print_error(self, message: str | Exception, *args: Any, **kwargs: Any) -> None: + if self.options['logger']: + self.logger.error(message, *args, **kwargs) else: + message = str(message) if len(kwargs) > 0: message = message + "\n" + ", ".join( f"{key}={value}" for key, value in kwargs.items() diff --git a/app/tests/test_helpers.py b/app/tests/test_helpers.py new file mode 100644 index 00000000..72b574f8 --- /dev/null +++ b/app/tests/test_helpers.py @@ -0,0 +1,78 @@ +import sys +from logging import DEBUG +from logging import ERROR +from logging import FATAL +from logging import INFO +from unittest.mock import call +from unittest.mock import patch + +from helpers.logging import TimestampedStringIO +from helpers.logging import redirect_std_to_logger + +from django.test import TestCase + + +class LoggingHelperTests(TestCase): + + def test_timestamped_string_io(self): + out = TimestampedStringIO(level=1) + + with patch('helpers.logging.time', return_value=100): + self.assertEqual(out.write('test'), 4) + self.assertEqual(out.messages, [(100, 1, 'test')]) + + def test_redirect_std_to_logger(self): + with patch('helpers.logging.getLogger') as logger: + with redirect_std_to_logger('test'): + sys.stdout.write('stdout 1') + sys.stderr.write('stderr 1') + sys.stderr.write('stderr 2\n') + sys.stdout.write(' stdout 2') + + self.assertEqual( + logger.mock_calls, + [ + call('test'), + call().log(INFO, 'stdout 1'), + call().log(ERROR, 'stderr 1'), + call().log(ERROR, 'stderr 2'), + call().log(INFO, 'stdout 2'), + ] + ) + + def test_redirect_std_to_logger_custom_level(self): + with patch('helpers.logging.getLogger') as logger: + with redirect_std_to_logger('test', stderr_level=FATAL, stdout_level=DEBUG): + sys.stdout.write('stdout 1') + sys.stderr.write('stderr 1') + sys.stderr.write('stderr 2\n') + sys.stdout.write(' stdout 2') + + self.assertEqual( + logger.mock_calls, + [ + call('test'), + call().log(DEBUG, 'stdout 1'), + call().log(FATAL, 'stderr 1'), + call().log(FATAL, 'stderr 2'), + call().log(DEBUG, 'stdout 2'), + ] + ) + + def test_redirect_std_to_logger_exception(self): + exception = RuntimeError('abort') + with patch('helpers.logging.getLogger') as logger: + with redirect_std_to_logger('test'): + sys.stdout.write('stdout 1') + sys.stderr.write(' stderr 1\n') + raise exception + + self.assertEqual( + logger.mock_calls, + [ + call('test'), + call().log(INFO, 'stdout 1'), + call().log(ERROR, 'stderr 1'), + call().exception(exception), + ] + ) diff --git a/publiccode.yml b/publiccode.yml index 117607f5..ce7970dd 100644 --- a/publiccode.yml +++ b/publiccode.yml @@ -1,4 +1,4 @@ -publiccodeYmlVersion: 0.4.0 +publiccodeYmlVersion: 0.5.0 name: STAC API applicationSuite: geo.admin.ch url: https://github.com/geoadmin/service-stac.git @@ -12,6 +12,9 @@ usedBy: - Federal Office of Meteorology and Climatology MeteoSwiss developmentStatus: stable softwareType: standalone/web +organisation: + uri: https://ld.admin.ch/office/IV.1.5a + name: Federal Office of Topography description: en: localisedName: STAC API @@ -23,8 +26,6 @@ description: implementation conforms to the core STAC API specification. The underlying data model follows the STAC schema and is extended by the forecast extension to support weather forecast data. - documentation: https://www.geo.admin.ch/en/rest-interface-stac-api - apiDocumentation: https://data.geo.admin.ch/api/stac/static/spec/v1/api.html features: - List geodata organized into collections, items and assets (STAC compatible) - Search items by field value or by spatial criteria, e.g., a bounding box