diff --git a/nautobot_golden_config/datasources.py b/nautobot_golden_config/datasources.py index b48358cfe..4ba98a6be 100644 --- a/nautobot_golden_config/datasources.py +++ b/nautobot_golden_config/datasources.py @@ -14,6 +14,7 @@ ComplianceRule, ConfigRemove, ConfigReplace, + GoldenConfigSetting, RemediationSetting, ) from nautobot_golden_config.utilities.constant import ENABLE_BACKUP, ENABLE_COMPLIANCE, ENABLE_INTENDED @@ -232,42 +233,45 @@ def update_git_gc_properties(golden_config_path, job_result, gc_config_item): # datasource_contents = [] -if ENABLE_INTENDED or ENABLE_COMPLIANCE: - datasource_contents.append( - ( - "extras.gitrepository", - DatasourceContent( - name="intended configs", - content_identifier="nautobot_golden_config.intendedconfigs", - icon="mdi-file-document-outline", - callback=refresh_git_intended, - ), - ) +# if GoldenConfigSetting.objects.filter(enable_intended=True).exists() or GoldenConfigSetting.objects.filter(enable_compliance=True).exists(): +# if ENABLE_INTENDED or ENABLE_COMPLIANCE: +datasource_contents.append( + ( + "extras.gitrepository", + DatasourceContent( + name="intended configs", + content_identifier="nautobot_golden_config.intendedconfigs", + icon="mdi-file-document-outline", + callback=refresh_git_intended, + ), ) -if ENABLE_INTENDED: - datasource_contents.append( - ( - "extras.gitrepository", - DatasourceContent( - name="jinja templates", - content_identifier="nautobot_golden_config.jinjatemplate", - icon="mdi-text-box-check-outline", - callback=refresh_git_jinja, - ), - ) +) +# if GoldenConfigSetting.objects.filter(enable_intended=True).exists(): +# if ENABLE_INTENDED: +datasource_contents.append( + ( + "extras.gitrepository", + DatasourceContent( + name="jinja templates", + content_identifier="nautobot_golden_config.jinjatemplate", + icon="mdi-text-box-check-outline", + callback=refresh_git_jinja, + ), ) -if ENABLE_BACKUP or ENABLE_COMPLIANCE: - datasource_contents.append( - ( - "extras.gitrepository", - DatasourceContent( - name="backup configs", - content_identifier="nautobot_golden_config.backupconfigs", - icon="mdi-file-code", - callback=refresh_git_backup, - ), - ) +) +# if GoldenConfigSetting.objects.filter(enable_backup=True).exists() or GoldenConfigSetting.objects.filter(enable_compliance=True).exists(): +# if ENABLE_BACKUP or ENABLE_COMPLIANCE: +datasource_contents.append( + ( + "extras.gitrepository", + DatasourceContent( + name="backup configs", + content_identifier="nautobot_golden_config.backupconfigs", + icon="mdi-file-code", + callback=refresh_git_backup, + ), ) +) datasource_contents.append( ( diff --git a/nautobot_golden_config/details.py b/nautobot_golden_config/details.py index 22052bc85..ed8ff7226 100644 --- a/nautobot_golden_config/details.py +++ b/nautobot_golden_config/details.py @@ -89,7 +89,7 @@ def hyperlinked_field_with_icon(url, title, icon_class="mdi mdi-text-box-check-o label="General Settings", section=ui.SectionChoices.LEFT_HALF, weight=100, - fields=("weight", "description"), + fields=("weight", "description", "enable_backup", "enable_intended", "enable_compliance", "enable_plan", "enable_deploy"), ), ui.KeyValueTablePanel( section=ui.SectionChoices.LEFT_HALF, diff --git a/nautobot_golden_config/jobs.py b/nautobot_golden_config/jobs.py index 7a1482f32..8b05b4806 100644 --- a/nautobot_golden_config/jobs.py +++ b/nautobot_golden_config/jobs.py @@ -20,7 +20,7 @@ StringVar, TextVar, ) -from nautobot.extras.models import DynamicGroup, Role, Status, Tag +from nautobot.extras.models import Role, Status, Tag from nautobot.tenancy.models import Tenant, TenantGroup from nautobot_plugin_nornir.plugins.inventory.nautobot_orm import NautobotORMInventory from nornir.core.plugins.inventory import InventoryPluginRegister @@ -28,22 +28,26 @@ from nautobot_golden_config.choices import ConfigPlanTypeChoice from nautobot_golden_config.exceptions import BackupFailure, ComplianceFailure, IntendedGenerationFailure -from nautobot_golden_config.models import ComplianceFeature, ConfigPlan, GoldenConfig +from nautobot_golden_config.models import ComplianceFeature, ConfigPlan, GoldenConfig, GoldenConfigSetting from nautobot_golden_config.nornir_plays.config_backup import config_backup from nautobot_golden_config.nornir_plays.config_compliance import config_compliance from nautobot_golden_config.nornir_plays.config_deployment import config_deployment from nautobot_golden_config.nornir_plays.config_intended import config_intended -from nautobot_golden_config.utilities import constant from nautobot_golden_config.utilities.config_plan import ( config_plan_default_status, generate_config_set_from_compliance_feature, generate_config_set_from_manual, ) +from nautobot_golden_config.utilities.constant import JOB_FUNCTION_MAP from nautobot_golden_config.utilities.git import GitRepo from nautobot_golden_config.utilities.helper import ( get_device_to_settings_map, + # get_golden_config_settings, + get_inscope_settings_from_device_qs, get_job_filter, update_dynamic_groups_cache, + # verify_config_plan_eligibility, + # verify_deployment_eligibility, ) InventoryPluginRegister.register("nautobot-inventory", NautobotORMInventory) @@ -54,34 +58,53 @@ def get_repo_types_for_job(job_name): """Logic to determine which repo_types are needed based on job + plugin settings.""" repo_types = [] - if constant.ENABLE_BACKUP and job_name == "nautobot_golden_config.jobs.BackupJob": - repo_types.extend(["backup_repository"]) - if constant.ENABLE_INTENDED and job_name == "nautobot_golden_config.jobs.IntendedJob": + if job_name == "backup": + repo_types.append("backup_repository") + if job_name == "intended": repo_types.extend(["jinja_repository", "intended_repository"]) - if constant.ENABLE_COMPLIANCE and job_name == "nautobot_golden_config.jobs.ComplianceJob": + if job_name == "compliance": repo_types.extend(["intended_repository", "backup_repository"]) - if "All" in job_name: + if "all" in job_name.lower(): repo_types.extend(["backup_repository", "jinja_repository", "intended_repository"]) return repo_types -def get_refreshed_repos(job_obj, repo_types, data=None): - """Small wrapper to pull latest branch, and return a GitRepo app specific object.""" - dynamic_groups = DynamicGroup.objects.exclude(golden_config_setting__isnull=True) - repository_records = set() - for group in dynamic_groups: - # Make sure the data(device qs) device exist in the dg first. - if data.filter(group.generate_query()).exists(): - for repo_type in repo_types: - repo = getattr(group.golden_config_setting, repo_type, None) - if repo: - repository_records.add(repo) - - repositories = {} +# def get_refreshed_repos(job_obj, repository_records, gc_setting): +# """Small wrapper to pull latest branch, and return a GitRepo app specific object.""" +# repositories = {} +# for repository_record in repository_records: +# ensure_git_repository(repository_record, job_obj.logger) +# # TODO: Should this not point to non-nautobot.core import +# # We should ask in nautobot core for the `from_url` constructor to be it's own function +# git_info = get_repo_from_url_to_path_and_from_branch(repository_record) +# git_repo = GitRepo( +# repository_record.filesystem_path, +# git_info.from_url, +# clone_initially=False, +# base_url=repository_record.remote_url, +# nautobot_repo_obj=repository_record, +# ) +# commit = False + +# if ( +# gc_setting.intended_enabled +# and "nautobot_golden_config.intendedconfigs" in git_repo.nautobot_repo_obj.provided_contents +# ): +# commit = True +# if ( +# gc_setting.backup_enabled +# and "nautobot_golden_config.backupconfigs" in git_repo.nautobot_repo_obj.provided_contents +# ): +# commit = True + +# repositories[str(git_repo.nautobot_repo_obj.id)] = {"repo_obj": git_repo, "to_commit": commit} +# return repositories + + +def get_refreshed_reposv2(repository_records): + """Small wrapper to pull latest branch, and return a list of GitRepo app specific objects.""" + gitrepo_obj = [] for repository_record in repository_records: - ensure_git_repository(repository_record, job_obj.logger) - # TODO: Should this not point to non-nautobot.core import - # We should ask in nautobot core for the `from_url` constructor to be it's own function git_info = get_repo_from_url_to_path_and_from_branch(repository_record) git_repo = GitRepo( repository_record.filesystem_path, @@ -90,47 +113,66 @@ def get_refreshed_repos(job_obj, repo_types, data=None): base_url=repository_record.remote_url, nautobot_repo_obj=repository_record, ) - commit = False - - if ( - constant.ENABLE_INTENDED - and "nautobot_golden_config.intendedconfigs" in git_repo.nautobot_repo_obj.provided_contents - ): - commit = True - if ( - constant.ENABLE_BACKUP - and "nautobot_golden_config.backupconfigs" in git_repo.nautobot_repo_obj.provided_contents - ): - commit = True - repositories[str(git_repo.nautobot_repo_obj.id)] = {"repo_obj": git_repo, "to_commit": commit} - return repositories - - -def gc_repo_prep(job, data): - """Prepare Golden Config git repos for work. - - Args: - job (Job): Nautobot Job object with logger and other vars. - data (dict): Data being passed from Job. - - Returns: - List[GitRepo]: List of GitRepos to be used with Job(s). - """ - job.logger.debug("Compiling device data for GC job.", extra={"grouping": "Get Job Filter"}) - job.qs = get_job_filter(data) - job.logger.debug(f"In scope device count for this job: {job.qs.count()}", extra={"grouping": "Get Job Filter"}) - job.logger.debug("Mapping device(s) to GC Settings.", extra={"grouping": "Device to Settings Map"}) - job.device_to_settings_map = get_device_to_settings_map(queryset=job.qs) - gitrepo_types = list(set(get_repo_types_for_job(job.class_path))) - job.logger.debug( - f"Repository types to sync: {', '.join(sorted(gitrepo_types))}", - extra={"grouping": "GC Repo Syncs"}, - ) - current_repos = get_refreshed_repos(job_obj=job, repo_types=gitrepo_types, data=job.qs) - return current_repos - - -def gc_repo_push(job, current_repos, commit_message=""): + gitrepo_obj.append(git_repo) + return gitrepo_obj + + +# def gc_repo_prep(job, inscope_gc_settings): +# """Prepare Golden Config git repos for work. + +# Args: +# job (Job): Nautobot Job object with logger and other vars. +# data (dict): Data being passed from Job. + +# Returns: +# List[GitRepo]: List of GitRepos to be used with Job(s). +# """ +# gitrepo_types = list(set(get_repo_types_for_job(job.class_path))) +# if inscope_gc_settings: +# for gcs in inscope_gc_settings: +# repos = GoldenConfigSetting.objects.get_repos_for_setting(setting=gcs, repo_types=gitrepo_types) +# job.logger.debug( +# f"Repositories to sync for GC Setting {gcs.name}: {', '.join(sorted([repo.name for repo in repos]))}", +# extra={"grouping": "GC Repo Syncs"}, +# ) +# current_repos = get_refreshed_repos(job_obj=job, repository_records=repos, gc_setting=gcs) +# return current_repos +# return [] + + +# def gc_repo_push(job, current_repos, commit_message=""): +# """Push any work from worker to git repos in Job. + +# Args: +# job (Job): Nautobot Job with logger and other attributes. +# current_repos (List[GitRepo]): List of GitRepos to be used with Job(s). +# """ +# now = make_aware(datetime.now()) +# job.logger.debug( +# f"Finished the {job.Meta.name} job execution.", +# extra={"grouping": "GC After Run"}, +# ) +# if current_repos: +# for _, repo in current_repos.items(): +# if repo["to_commit"]: +# job.logger.debug( +# f"Pushing {job.Meta.name} results to repo {repo['repo_obj'].base_url}.", +# extra={"grouping": "GC Repo Commit and Push"}, +# ) +# if not commit_message: +# commit_message = f"{job.Meta.name.upper()} JOB {now}" +# repo["repo_obj"].commit_with_added(commit_message) +# repo["repo_obj"].push() +# job.logger.info( +# f'{repo["repo_obj"].nautobot_repo_obj.name}: the new Git repository hash is "{repo["repo_obj"].head}"', +# extra={ +# "grouping": "GC Repo Commit and Push", +# "object": repo["repo_obj"].nautobot_repo_obj, +# }, +# ) + + +def gc_repo_pushv2(job, current_repos, commit_message=""): """Push any work from worker to git repos in Job. Args: @@ -143,31 +185,55 @@ def gc_repo_push(job, current_repos, commit_message=""): extra={"grouping": "GC After Run"}, ) if current_repos: - for _, repo in current_repos.items(): - if repo["to_commit"]: - job.logger.debug( - f"Pushing {job.Meta.name} results to repo {repo['repo_obj'].base_url}.", - extra={"grouping": "GC Repo Commit and Push"}, - ) - if not commit_message: - commit_message = f"{job.Meta.name.upper()} JOB {now}" - repo["repo_obj"].commit_with_added(commit_message) - repo["repo_obj"].push() - job.logger.info( - f'{repo["repo_obj"].nautobot_repo_obj.name}: the new Git repository hash is "{repo["repo_obj"].head}"', - extra={ - "grouping": "GC Repo Commit and Push", - "object": repo["repo_obj"].nautobot_repo_obj, - }, - ) + for repo in current_repos: + job.logger.debug( + f"Pushing {job.Meta.name} results to repo {repo.base_url}.", + extra={"grouping": "GC Repo Commit and Push"}, + ) + if not commit_message: + commit_message = f"{job.Meta.name.upper()} JOB {now}" + repo.commit_with_added(commit_message) + repo.push() + job.logger.info( + f'{repo.nautobot_repo_obj.name}: the new Git repository hash is "{repo.head}"', + extra={ + "grouping": "GC Repo Commit and Push", + "object": repo.nautobot_repo_obj, + }, + ) -def gc_repos(func): +# def gc_repos(func): +# """Decorator used for handle repo syncing, commiting, and pushing.""" + +# def gc_repo_wrapper(self, *args, **kwargs): +# """Decorator used for handle repo syncing, commiting, and pushing.""" +# self.qs = get_job_filter(data=kwargs) +# # self.gc_advanced_filter = GCSettingsDeviceFilterSet(self.qs) +# self.gc_advanced_filter = get_device_to_settings_map(self.qs, self.name) +# active_settings = set(list(self.gc_advanced_filter[JOB_FUNCTION_MAP[self.name]][True].values())) +# current_repos = gc_repo_prep(job=self, inscope_gc_settings=active_settings) +# # This is where the specific jobs run method runs via this decorator. +# try: +# func(self, *args, **kwargs) +# except Exception as error: # pylint: disable=broad-exception-caught +# error_msg = f"`E3001:` General Exception handler, original error message ```{error}```" +# # Raise error only if the job kwarg (checkbox) is selected to do so on the job execution form. +# if kwargs.get("fail_job_on_task_failure"): +# raise NornirNautobotException(error_msg) from error +# finally: +# gc_repo_push(job=self, current_repos=current_repos, commit_message=kwargs.get("commit_message", "")) + +# return gc_repo_wrapper + + +def gc_job_helper(func): """Decorator used for handle repo syncing, commiting, and pushing.""" - def gc_repo_wrapper(self, *args, **kwargs): - """Decorator used for handle repo syncing, commiting, and pushing.""" - current_repos = gc_repo_prep(job=self, data=kwargs) + def gc_job_wrapper(self, *args, **kwargs): + """Decorator used for GC job setup, repo syncing, commiting, and pushing.""" + # self.gc_job_setup(data=kwargs, all_job=False) + self.gc_job_setup(data=kwargs) # This is where the specific jobs run method runs via this decorator. try: func(self, *args, **kwargs) @@ -177,9 +243,13 @@ def gc_repo_wrapper(self, *args, **kwargs): if kwargs.get("fail_job_on_task_failure"): raise NornirNautobotException(error_msg) from error finally: - gc_repo_push(job=self, current_repos=current_repos, commit_message=kwargs.get("commit_message")) + gc_repo_pushv2( + job=self, + current_repos=get_refreshed_reposv2(self.repos_to_push), + commit_message=kwargs.get("commit_message", ""), + ) - return gc_repo_wrapper + return gc_job_wrapper class FormEntry: # pylint disable=too-few-public-method @@ -223,12 +293,69 @@ class GoldenConfigJobMixin(Job): # pylint: disable=abstract-method def __init__(self, *args, **kwargs): """Initialize the job.""" super().__init__(*args, **kwargs) - self.qs = None - self.device_to_settings_map = {} + self.qs = Device.objects.none() + self.task_qs = Device.objects.none() + self.gc_advanced_settings_filter = {} + self.job_function = "" + self.repos_to_push = [] + + def gc_job_setup(self, data): + """Handles the setup for the Golden Config job.""" + self.job_function = JOB_FUNCTION_MAP[self.name] + self.qs = get_job_filter(data=data) + self.gc_advanced_settings_filter = get_device_to_settings_map(self.qs, self.job_function) + if self.job_function.lower() == "all": + # If the job is "all", we need to set the job_function to each individual job. + # If the job is one of the all jobs, we need to loop through each job and run the setup for each. + return + enabled_qs, disabled_qs = self._get_filtered_queryset(self.job_function) + self._log_out_of_scope_devices(disabled_qs) + if enabled_qs.count() == 0: + self.logger.warning( + f"E3039: No devices found with Golden Config settings enabled for the {self.job_function} job." + ) + return + self._get_repos_to_sync(enabled_qs) + + def _get_repos_to_sync(self, enabled_qs): + inscope_gcs = get_inscope_settings_from_device_qs(enabled_qs) + repos_to_sync, self.repos_to_push = GoldenConfigSetting.objects.get_repos_for_settings( + inscope_gcs, get_repo_types_for_job(self.job_function) + ) + if repos_to_sync: + for repository_record in repos_to_sync: + ensure_git_repository(repository_record, self.logger) + + def _log_out_of_scope_devices(self, disabled_devices_qs): + """Log devices that are out of scope for the job.""" + if disabled_devices_qs.count() > 0: + for device in disabled_devices_qs: + self.logger.warning( + f"E3038: Device {device.name} does not have the required settings to run the job. Skipping device.", + extra={"object": device}, + ) + + def _get_filtered_queryset(self, job_function): + """Helper for gc_advanced_settings_filter to get filtered queryset.""" + enabled_devs = list(self.gc_advanced_settings_filter[job_function][True].keys()) + disabled_devs = list(self.gc_advanced_settings_filter[job_function][False].keys()) + enabled_qs = self.qs.filter(pk__in=enabled_devs) + disabled_qs = self.qs.filter(pk__in=disabled_devs) + + self.logger.debug( + f"Device(s) with settings enabled for {job_function} job: {enabled_qs.count()}", + extra={"grouping": "Get Filtered Queryset"}, + ) + self.logger.debug( + f"Device(s) with settings disabled for {job_function} job: {disabled_qs.count()}", + extra={"grouping": "Get Filtered Queryset"}, + ) + self.task_qs = enabled_qs + return enabled_qs, disabled_qs class ComplianceJob(GoldenConfigJobMixin, FormEntry): - """Job to to run the compliance engine.""" + """Job to run the compliance engine.""" class Meta: """Meta object boilerplate for compliance.""" @@ -237,14 +364,18 @@ class Meta: description = "Run configuration compliance on your network infrastructure." has_sensitive_variables = False - @gc_repos + @gc_job_helper def run(self, *args, **data): # pylint: disable=unused-argument """Run config compliance report script.""" - self.logger.warning("Starting config compliance nornir play.") - if not constant.ENABLE_COMPLIANCE: - self.logger.critical("Compliance is disabled in application settings.") - raise ValueError("Compliance is disabled in application settings.") - config_compliance(self) + if self.task_qs.count() == 0: + return + try: + self.logger.debug("Starting config compliance nornir play.") + config_compliance(self) + except NornirNautobotException as error: + error_msg = str(error) + self.logger.error(error_msg) + raise NornirNautobotException(error_msg) from error class IntendedJob(GoldenConfigJobMixin, FormEntry): @@ -257,14 +388,18 @@ class Meta: description = "Generate the configuration for your intended state." has_sensitive_variables = False - @gc_repos + @gc_job_helper def run(self, *args, **data): # pylint: disable=unused-argument """Run config generation script.""" - self.logger.debug("Building device settings mapping and running intended config nornir play.") - if not constant.ENABLE_INTENDED: - self.logger.critical("Intended Generation is disabled in application settings.") - raise ValueError("Intended Generation is disabled in application settings.") - config_intended(self) + if self.task_qs.count() == 0: + return + try: + self.logger.debug("Building device settings mapping and running intended config nornir play.") + config_intended(self) + except NornirNautobotException as error: + error_msg = str(error) + self.logger.error(error_msg) + raise NornirNautobotException(error_msg) from error class BackupJob(GoldenConfigJobMixin, FormEntry): @@ -277,14 +412,18 @@ class Meta: description = "Backup the configurations of your network devices." has_sensitive_variables = False - @gc_repos + @gc_job_helper def run(self, *args, **data): # pylint: disable=unused-argument """Run config backup process.""" - self.logger.debug("Starting config backup nornir play.") - if not constant.ENABLE_BACKUP: - self.logger.critical("Backups are disabled in application settings.") - raise ValueError("Backups are disabled in application settings.") - config_backup(self) + if self.task_qs.count() == 0: + return + try: + self.logger.debug("Starting config backup nornir play.") + config_backup(self) + except NornirNautobotException as error: + error_msg = str(error) + self.logger.error(error_msg) + raise NornirNautobotException(error_msg) from error class AllGoldenConfig(GoldenConfigJobMixin): @@ -300,43 +439,68 @@ class Meta: description = "Process to run all Golden Configuration jobs configured." has_sensitive_variables = False + @gc_job_helper def run(self, *args, **data): # pylint: disable=unused-argument, too-many-branches """Run all jobs on a single device.""" - current_repos = gc_repo_prep(job=self, data=data) failed_jobs = [] - error_msg, jobs_list = "", "All" - for enabled, play in [ - (constant.ENABLE_INTENDED, config_intended), - (constant.ENABLE_BACKUP, config_backup), - (constant.ENABLE_COMPLIANCE, config_compliance), - ]: - try: - if enabled: - play(self) - except BackupFailure: - self.logger.error("Backup failure occurred!") - failed_jobs.append("Backup") - except IntendedGenerationFailure: - self.logger.error("Intended failure occurred!") - failed_jobs.append("Intended") - except ComplianceFailure: - self.logger.error("Compliance failure occurred!") - failed_jobs.append("Compliance") - except Exception as error: # pylint: disable=broad-exception-caught - error_msg = f"`E3001:` General Exception handler, original error message ```{error}```" - gc_repo_push(job=self, current_repos=current_repos, commit_message=data.get("commit_message")) - if len(failed_jobs) > 1: - jobs_list = ", ".join(failed_jobs) - elif len(failed_jobs) == 1: - jobs_list = failed_jobs[0] - failure_msg = f"`E3030:` Failure during {jobs_list} Job(s)." - if len(failed_jobs) > 0: - self.logger.error(failure_msg) - if (len(failed_jobs) > 0 or error_msg) and data["fail_job_on_task_failure"]: - if not error_msg: - error_msg = failure_msg - # Raise error only if the job kwarg (checkbox) is selected to do so on the job execution form. - raise NornirNautobotException(error_msg) + # error_msg, jobs_list = "", "All" + # self.gc_job_setup(data) + # gc_setting = GoldenConfigSetting.objects.get_for_device(data["device"]) + # repos_to_sync, self.repos_to_push = GoldenConfigSetting.objects.get_repos_for_settings( + # gc_setting, + # get_repo_types_for_job(self.job_function), + # ) + # if repos_to_sync: + # for repository_record in repos_to_sync: + # ensure_git_repository(repository_record, self.logger) + try: + for nornir_play in [config_intended, config_backup, config_compliance]: + # "backup", "intended", "compliance" + self.task_qs, disabled_qs = self._get_filtered_queryset(nornir_play.__name__.split("_")[1]) + self._get_repos_to_sync(self.task_qs) + self._log_out_of_scope_devices(disabled_qs) + try: + if self.task_qs.count() == 0: + self.logger.warning( + f"E3039: No devices found with Golden Config settings enabled for the {nornir_play.__name__.split('_')[1]} job." + ) + continue + nornir_play(self) + except BackupFailure: + self.logger.error("Backup failure occurred!") + failed_jobs.append("Backup") + except IntendedGenerationFailure: + self.logger.error("Intended failure occurred!") + failed_jobs.append("Intended") + except ComplianceFailure: + self.logger.error("Compliance failure occurred!") + failed_jobs.append("Compliance") + except Exception as error: # pylint: disable=broad-exception-caught + error_msg = f"`E3001:` General Exception handler, original error message ```{error}```" + failure_msg = f"`E3030:` Failure during {', '.join(failed_jobs)} Job(s)." + if len(failed_jobs) > 0: + self.logger.error(failure_msg) + except NornirNautobotException as error: + error_msg = str(error) + self.logger.error(error_msg) + raise NornirNautobotException(error_msg) from error + # gc_repo_pushv2( + # job=self, + # current_repos=get_refreshed_reposv2(self.repos_to_push), + # commit_message=data.get("commit_message", ""), + # ) + # if len(failed_jobs) > 1: + # jobs_list = ", ".join(failed_jobs) + # elif len(failed_jobs) == 1: + # jobs_list = failed_jobs[0] + # failure_msg = f"`E3030:` Failure during {', '.join(failed_jobs)} Job(s)." + # if len(failed_jobs) > 0: + # self.logger.error(failure_msg) + # if (len(failed_jobs) > 0 or error_msg) and data["fail_job_on_task_failure"]: + # if not error_msg: + # error_msg = failure_msg + # # Raise error only if the job kwarg (checkbox) is selected to do so on the job execution form. + # raise NornirNautobotException(error_msg) class AllDevicesGoldenConfig(GoldenConfigJobMixin, FormEntry): @@ -351,17 +515,21 @@ class Meta: def run(self, *args, **data): # pylint: disable=unused-argument, too-many-branches """Run all jobs on multiple devices.""" - current_repos = gc_repo_prep(job=self, data=data) + self.gc_job_setup(data) failed_jobs = [] error_msg, jobs_list = "", "All" - for enabled, play in [ - (constant.ENABLE_INTENDED, config_intended), - (constant.ENABLE_BACKUP, config_backup), - (constant.ENABLE_COMPLIANCE, config_compliance), - ]: + inscope_gcs = get_inscope_settings_from_device_qs(self.qs) + repos_to_sync, self.repos_to_push = GoldenConfigSetting.objects.get_repos_for_settings( + inscope_gcs, + get_repo_types_for_job(self.job_function), + ) + if repos_to_sync: + for repository_record in repos_to_sync: + ensure_git_repository(repository_record, self.logger) + for nornir_play in [config_intended, config_backup, config_compliance]: + self.task_qs, _ = self._get_filtered_queryset(nornir_play.__name__.split("_")[1]) try: - if enabled: - play(self) + nornir_play(self) except BackupFailure: self.logger.error("Backup failure occurred!") failed_jobs.append("Backup") @@ -373,7 +541,11 @@ def run(self, *args, **data): # pylint: disable=unused-argument, too-many-branc failed_jobs.append("Compliance") except Exception as error: # pylint: disable=broad-exception-caught error_msg = f"`E3001:` General Exception handler, original error message ```{error}```" - gc_repo_push(job=self, current_repos=current_repos, commit_message=data.get("commit_message")) + gc_repo_pushv2( + job=self, + current_repos=get_refreshed_reposv2(self.repos_to_push), + commit_message=data.get("commit_message", ""), + ) if len(failed_jobs) > 1: jobs_list = ", ".join(failed_jobs) elif len(failed_jobs) == 1: @@ -503,16 +675,22 @@ def _generate_config_plan_from_manual(self): def run(self, **data): """Run config plan generation process.""" - self.logger.debug("Updating Dynamic Group Cache.") - update_dynamic_groups_cache() self.logger.debug("Starting config plan generation job.") + settings = None + self._validate_inputs(data) try: self._device_qs = get_job_filter(data) + + # Verify plan eligibility for each device + # for device in self._device_qs: + # verify_config_plan_eligibility(self.logger, device, settings) + except NornirNautobotException as error: error_msg = str(error) self.logger.error(error_msg) raise NornirNautobotException(error_msg) from error + if self._plan_type in ["intended", "missing", "remediation"]: self.logger.debug("Starting config plan generation for compliance features.") self._generate_config_plan_from_feature() @@ -550,6 +728,12 @@ def run(self, **data): # pylint: disable=arguments-differ update_dynamic_groups_cache() self.logger.debug("Starting config plan deployment job.") self.data = data + settings = None + + # Verify deployment eligibility for each config plan + # for config_plan in self.data["config_plan"]: + # verify_deployment_eligibility(self.logger, config_plan, settings) + try: config_deployment(self) except Exception as error: # pylint: disable=broad-exception-caught diff --git a/nautobot_golden_config/migrations/0031_goldenconfigsetting_enable_backup_and_more.py b/nautobot_golden_config/migrations/0031_goldenconfigsetting_enable_backup_and_more.py new file mode 100644 index 000000000..17f3f60b7 --- /dev/null +++ b/nautobot_golden_config/migrations/0031_goldenconfigsetting_enable_backup_and_more.py @@ -0,0 +1,38 @@ +# Generated by Django 4.2.23 on 2025-08-30 20:13 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('nautobot_golden_config', '0030_alter_goldenconfig_device'), + ] + + operations = [ + migrations.AddField( + model_name='goldenconfigsetting', + name='enable_backup', + field=models.BooleanField(default=True), + ), + migrations.AddField( + model_name='goldenconfigsetting', + name='enable_compliance', + field=models.BooleanField(default=True), + ), + migrations.AddField( + model_name='goldenconfigsetting', + name='enable_deploy', + field=models.BooleanField(default=True), + ), + migrations.AddField( + model_name='goldenconfigsetting', + name='enable_intended', + field=models.BooleanField(default=True), + ), + migrations.AddField( + model_name='goldenconfigsetting', + name='enable_plan', + field=models.BooleanField(default=True), + ), + ] diff --git a/nautobot_golden_config/models.py b/nautobot_golden_config/models.py index 32eb30283..034e5cfd9 100644 --- a/nautobot_golden_config/models.py +++ b/nautobot_golden_config/models.py @@ -23,7 +23,14 @@ from xmldiff import actions, main from nautobot_golden_config.choices import ComplianceRuleConfigTypeChoice, ConfigPlanTypeChoice, RemediationTypeChoice -from nautobot_golden_config.utilities.constant import ENABLE_SOTAGG, PLUGIN_CFG +from nautobot_golden_config.utilities.constant import ( + ENABLE_BACKUP, + ENABLE_COMPLIANCE, + ENABLE_DEPLOY, + ENABLE_INTENDED, + ENABLE_PLAN, + PLUGIN_CFG, +) LOGGER = logging.getLogger(__name__) GRAPHQL_STR_START = "query ($device_id: ID!)" @@ -547,6 +554,29 @@ def get_for_device(self, device): return dynamic_group.order_by("-golden_config_setting__weight").first().golden_config_setting return None + def get_repos_for_settings(self, gcs_queryset, job_types): + """Return all enabled repos for all settings in a restricted queryset.""" + repos_to_sync, repos_to_push = [], [] + if isinstance(gcs_queryset, GoldenConfigSetting): + gcs_queryset = [gcs_queryset] + for setting in gcs_queryset: + for job_type in job_types: + if job_type == "backup_repository": + if setting.enable_backup and setting.backup_repository: + repos_to_sync.append(setting.backup_repository) + repos_to_push.append(setting.backup_repository) + if not setting.enable_backup and setting.backup_repository: + repos_to_sync.append(setting.backup_repository) + if job_type == "intended_repository": + if setting.enable_intended and setting.intended_repository: + repos_to_sync.append(setting.intended_repository) + repos_to_push.append(setting.intended_repository) + if setting.jinja_repository: + repos_to_sync.append(setting.jinja_repository) + if not setting.enable_intended and setting.intended_repository: + repos_to_sync.append(setting.intended_repository) + return list(set(repos_to_sync)), list(set(repos_to_push)) + @extras_features( "graphql", @@ -575,6 +605,11 @@ class GoldenConfigSetting(PrimaryModel): # pylint: disable=too-many-ancestors verbose_name="Backup Path in Jinja Template Form", help_text="The Jinja path representation of where the backup file will be found. The variable `obj` is available as the device instance object of a given device, as is the case for all Jinja templates. e.g. `{{obj.location.name|slugify}}/{{obj.name}}.cfg`", ) + enable_backup = models.BooleanField( + default=ENABLE_BACKUP, + verbose_name="Enable Backup", + help_text="Whether or not backups are performed by Golden Config. This can be disabled if backups are fetched from another process.", + ) intended_repository = models.ForeignKey( to="extras.GitRepository", on_delete=models.PROTECT, @@ -589,6 +624,11 @@ class GoldenConfigSetting(PrimaryModel): # pylint: disable=too-many-ancestors verbose_name="Intended Path in Jinja Template Form", help_text="The Jinja path representation of where the generated file will be placed. e.g. `{{obj.location.name|slugify}}/{{obj.name}}.cfg`", ) + enable_intended = models.BooleanField( + default=ENABLE_INTENDED, + verbose_name="Enable Intended", + help_text="Whether or not intended config tasks are performed by Golden Config. This can be disabled if intended configs are fetched from another process.", + ) jinja_repository = models.ForeignKey( to="extras.GitRepository", on_delete=models.PROTECT, @@ -621,6 +661,21 @@ class GoldenConfigSetting(PrimaryModel): # pylint: disable=too-many-ancestors on_delete=models.PROTECT, related_name="golden_config_setting", ) + enable_compliance = models.BooleanField( + default=ENABLE_COMPLIANCE, + verbose_name="Enable Compliance", + help_text="Whether or not compliance tasks are performed by Golden Config.", + ) + enable_plan = models.BooleanField( + default=ENABLE_PLAN, + verbose_name="Enable Config Plan", + help_text="Whether or not config plan tasks are performed by Golden Config.", + ) + enable_deploy = models.BooleanField( + default=ENABLE_DEPLOY, + verbose_name="Enable Deploy", + help_text="Whether or not deploy tasks are performed by Golden Config.", + ) is_dynamic_group_associable_model = False objects = GoldenConfigSettingManager() @@ -653,8 +708,12 @@ def clean(self): """Validate the scope and GraphQL query.""" super().clean() - if ENABLE_SOTAGG and not self.sot_agg_query: - raise ValidationError("A GraphQL query must be defined when `ENABLE_SOTAGG` is True") + if self.enable_intended and ( + not self.jinja_repository or not self.sot_agg_query or not self.jinja_path_template + ): + raise ValidationError( + "When Intended is enabled, you must be define a `Sot agg query`, `Jinja repository` and `Jinja Template Path`." + ) if self.sot_agg_query: LOGGER.debug("GraphQL - test query start with: `%s`", GRAPHQL_STR_START) diff --git a/nautobot_golden_config/navigation.py b/nautobot_golden_config/navigation.py index 0b3aa46d2..eebb07760 100644 --- a/nautobot_golden_config/navigation.py +++ b/nautobot_golden_config/navigation.py @@ -2,129 +2,91 @@ from nautobot.apps.ui import NavMenuAddButton, NavMenuGroup, NavMenuItem, NavMenuTab -from nautobot_golden_config.utilities.constant import ENABLE_BACKUP, ENABLE_COMPLIANCE, ENABLE_PLAN - items_operate = [ NavMenuItem( link="plugins:nautobot_golden_config:goldenconfig_list", name="Config Overview", permissions=["nautobot_golden_config.view_goldenconfig"], - ) -] - -items_setup = [] - -if ENABLE_COMPLIANCE: - items_operate.append( - NavMenuItem( - link="plugins:nautobot_golden_config:configcompliance_list", - name="Config Compliance", - permissions=["nautobot_golden_config.view_configcompliance"], - ) - ) - -if ENABLE_COMPLIANCE: - items_setup.append( - NavMenuItem( - link="plugins:nautobot_golden_config:compliancerule_list", - name="Compliance Rules", - permissions=["nautobot_golden_config.view_compliancerule"], - buttons=( - NavMenuAddButton( - link="plugins:nautobot_golden_config:compliancerule_add", - permissions=["nautobot_golden_config.add_compliancerule"], - ), + ), + NavMenuItem( + link="plugins:nautobot_golden_config:configcompliance_list", + name="Config Compliance", + permissions=["nautobot_golden_config.view_configcompliance"], + ), + NavMenuItem( + link="plugins:nautobot_golden_config:configcompliance_overview", + name="Compliance Report", + permissions=["nautobot_golden_config.view_configcompliance"], + ), + NavMenuItem( + link="plugins:nautobot_golden_config:configplan_list", + name="Config Plans", + permissions=["nautobot_golden_config.view_configplan"], + buttons=( + NavMenuAddButton( + link="plugins:nautobot_golden_config:configplan_add", + permissions=["nautobot_golden_config.add_configplan"], ), - ) + ), ) +] -if ENABLE_COMPLIANCE: - items_setup.append( - NavMenuItem( - link="plugins:nautobot_golden_config:compliancefeature_list", - name="Compliance Features", - permissions=["nautobot_golden_config.view_compliancefeature"], - buttons=( - NavMenuAddButton( - link="plugins:nautobot_golden_config:compliancefeature_add", - permissions=["nautobot_golden_config.add_compliancefeature"], - ), +items_setup = [ + NavMenuItem( + link="plugins:nautobot_golden_config:compliancerule_list", + name="Compliance Rules", + permissions=["nautobot_golden_config.view_compliancerule"], + buttons=( + NavMenuAddButton( + link="plugins:nautobot_golden_config:compliancerule_add", + permissions=["nautobot_golden_config.add_compliancerule"], ), - ) - ) - - -if ENABLE_COMPLIANCE: - items_operate.append( - NavMenuItem( - link="plugins:nautobot_golden_config:configcompliance_overview", - name="Compliance Report", - permissions=["nautobot_golden_config.view_configcompliance"], - ) - ) - -if ENABLE_PLAN: - items_operate.append( - NavMenuItem( - link="plugins:nautobot_golden_config:configplan_list", - name="Config Plans", - permissions=["nautobot_golden_config.view_configplan"], - buttons=( - NavMenuAddButton( - link="plugins:nautobot_golden_config:configplan_add", - permissions=["nautobot_golden_config.add_configplan"], - ), + ), + ), + NavMenuItem( + link="plugins:nautobot_golden_config:compliancefeature_list", + name="Compliance Features", + permissions=["nautobot_golden_config.view_compliancefeature"], + buttons=( + NavMenuAddButton( + link="plugins:nautobot_golden_config:compliancefeature_add", + permissions=["nautobot_golden_config.add_compliancefeature"], ), - ) - ) - -if ENABLE_BACKUP: - items_setup.append( - NavMenuItem( - link="plugins:nautobot_golden_config:configremove_list", - name="Config Removals", - permissions=["nautobot_golden_config.view_configremove"], - buttons=( - NavMenuAddButton( - link="plugins:nautobot_golden_config:configremove_add", - permissions=["nautobot_golden_config.add_configremove"], - ), + ), + ), + NavMenuItem( + link="plugins:nautobot_golden_config:configremove_list", + name="Config Removals", + permissions=["nautobot_golden_config.view_configremove"], + buttons=( + NavMenuAddButton( + link="plugins:nautobot_golden_config:configremove_add", + permissions=["nautobot_golden_config.add_configremove"], ), - ) - ) - -if ENABLE_BACKUP: - items_setup.append( - NavMenuItem( - link="plugins:nautobot_golden_config:configreplace_list", - name="Config Replacements", - permissions=["nautobot_golden_config.view_configreplace"], - buttons=( - NavMenuAddButton( - link="plugins:nautobot_golden_config:configreplace_add", - permissions=["nautobot_golden_config.add_configreplace"], - ), + ), + ), + NavMenuItem( + link="plugins:nautobot_golden_config:configreplace_list", + name="Config Replacements", + permissions=["nautobot_golden_config.view_configreplace"], + buttons=( + NavMenuAddButton( + link="plugins:nautobot_golden_config:configreplace_add", + permissions=["nautobot_golden_config.add_configreplace"], ), - ) - ) - - -if ENABLE_COMPLIANCE: - items_setup.append( - NavMenuItem( - link="plugins:nautobot_golden_config:remediationsetting_list", - name="Remediation Settings", - permissions=["nautobot_golden_config.view_remediationsetting"], - buttons=( - NavMenuAddButton( - link="plugins:nautobot_golden_config:remediationsetting_add", - permissions=["nautobot_golden_config.add_remediationsetting"], - ), + ), + ), + NavMenuItem( + link="plugins:nautobot_golden_config:remediationsetting_list", + name="Remediation Settings", + permissions=["nautobot_golden_config.view_remediationsetting"], + buttons=( + NavMenuAddButton( + link="plugins:nautobot_golden_config:remediationsetting_add", + permissions=["nautobot_golden_config.add_remediationsetting"], ), - ) - ) - -items_setup.append( + ), + ), NavMenuItem( link="plugins:nautobot_golden_config:goldenconfigsetting_list", name="Golden Config Settings", @@ -136,8 +98,7 @@ ), ), ), -) - +] menu_items = ( NavMenuTab( diff --git a/nautobot_golden_config/nornir_plays/config_backup.py b/nautobot_golden_config/nornir_plays/config_backup.py index 01390428f..6415faa06 100644 --- a/nautobot_golden_config/nornir_plays/config_backup.py +++ b/nautobot_golden_config/nornir_plays/config_backup.py @@ -3,6 +3,7 @@ # pylint: disable=relative-beyond-top-level import logging import os +import traceback from datetime import datetime from django.utils.timezone import make_aware @@ -21,7 +22,6 @@ from nautobot_golden_config.utilities.helper import ( dispatch_params, render_jinja_template, - verify_settings, ) from nautobot_golden_config.utilities.logger import NornirLogger @@ -101,9 +101,6 @@ def config_backup(job): now = make_aware(datetime.now()) logger = NornirLogger(job.job_result, job.logger.getEffectiveLevel()) - for settings in set(job.device_to_settings_map.values()): - verify_settings(logger, settings, ["backup_path_template"]) - # Build a dictionary, with keys of platform.network_driver, and the regex line in it for the netutils func. remove_regex_dict = {} for regex in ConfigRemove.objects.all(): @@ -126,7 +123,8 @@ def config_backup(job): "options": { "credentials_class": NORNIR_SETTINGS.get("credentials"), "params": NORNIR_SETTINGS.get("inventory_params"), - "queryset": job.qs, + "queryset": job.task_qs, + # "queryset": job.settings_filters["backup"][True].keys(), "defaults": {"now": now}, }, }, @@ -138,15 +136,16 @@ def config_backup(job): task=run_backup, name="BACKUP CONFIG", logger=logger, - device_to_settings_map=job.device_to_settings_map, + device_to_settings_map=job.gc_advanced_settings_filter["backup"][True], remove_regex_dict=remove_regex_dict, replace_regex_dict=replace_regex_dict, ) logger.debug("Completed configuration from devices.") except NornirNautobotException as err: - logger.error( - f"`E3027:` NornirNautobotException raised during backup tasks. Original exception message: ```{err}```" - ) + if job.job_result.task_kwargs["debug"]: + logger.error( + f"`E3027:` NornirNautobotException raised during backup tasks. Original exception message: ```{traceback.format_exc()}```" + ) # re-raise Exception if it's raised from nornir-nautobot or nautobot-app-nornir if str(err).startswith("`E2") or str(err).startswith("`E1"): raise NornirNautobotException(err) from err diff --git a/nautobot_golden_config/nornir_plays/config_compliance.py b/nautobot_golden_config/nornir_plays/config_compliance.py index 5a2531efd..0d62b2d8f 100644 --- a/nautobot_golden_config/nornir_plays/config_compliance.py +++ b/nautobot_golden_config/nornir_plays/config_compliance.py @@ -27,7 +27,6 @@ get_xml_config, get_xml_subtree_with_full_path, render_jinja_template, - verify_settings, ) from nautobot_golden_config.utilities.logger import NornirLogger @@ -209,11 +208,7 @@ def config_compliance(job): # pylint: disable=unused-argument """ now = make_aware(datetime.now()) logger = NornirLogger(job.job_result, job.logger.getEffectiveLevel()) - rules = get_rules() - - for settings in set(job.device_to_settings_map.values()): - verify_settings(logger, settings, ["backup_path_template", "intended_path_template"]) try: with InitNornir( runner=NORNIR_SETTINGS.get("runner"), @@ -223,7 +218,7 @@ def config_compliance(job): # pylint: disable=unused-argument "options": { "credentials_class": NORNIR_SETTINGS.get("credentials"), "params": NORNIR_SETTINGS.get("inventory_params"), - "queryset": job.qs, + "queryset": job.task_qs, "defaults": {"now": now}, }, }, @@ -235,7 +230,7 @@ def config_compliance(job): # pylint: disable=unused-argument task=run_compliance, name="RENDER COMPLIANCE TASK GROUP", logger=logger, - device_to_settings_map=job.device_to_settings_map, + device_to_settings_map=job.gc_advanced_settings_filter["compliance"][True], rules=rules, ) except NornirNautobotException as err: diff --git a/nautobot_golden_config/nornir_plays/config_intended.py b/nautobot_golden_config/nornir_plays/config_intended.py index dc97f3a35..4cd7ef900 100644 --- a/nautobot_golden_config/nornir_plays/config_intended.py +++ b/nautobot_golden_config/nornir_plays/config_intended.py @@ -23,7 +23,6 @@ dispatch_params, get_django_env, render_jinja_template, - verify_settings, ) from nautobot_golden_config.utilities.logger import NornirLogger @@ -107,10 +106,21 @@ def config_intended(job): """ now = make_aware(datetime.now()) logger = NornirLogger(job.job_result, job.logger.getEffectiveLevel()) - - for settings in set(job.device_to_settings_map.values()): - verify_settings(logger, settings, ["jinja_path_template", "intended_path_template", "sot_agg_query"]) - + # enabled_qs, disabled_qs = job.gc_advanced_filter.get_filtered_querysets("intended") + # device_filter = GCSettingsDeviceFilterSet(job.qs) + + # Verify intended feature is enabled and has required settings + # device_filter.verify_feature_enabled( + # logger, + # "intended", + # required_settings=["jinja_path_template", "intended_path_template", "sot_agg_query"], + # ) + # if job.job_result.task_kwargs["debug"]: + # for device in disabled_qs: + # logger.warning( + # f"E3038: Device {device.name} does not have the required settings to run the intended job. Skipping device.", + # extra={"object": device}, + # ) # Retrieve filters from the Django jinja template engine jinja_env = get_django_env() try: @@ -122,7 +132,7 @@ def config_intended(job): "options": { "credentials_class": NORNIR_SETTINGS.get("credentials"), "params": NORNIR_SETTINGS.get("inventory_params"), - "queryset": job.qs, + "queryset": job.task_qs, "defaults": {"now": now}, }, }, @@ -135,7 +145,7 @@ def config_intended(job): task=run_template, name="RENDER CONFIG", logger=logger, - device_to_settings_map=job.device_to_settings_map, + device_to_settings_map=job.gc_advanced_settings_filter["intended"][True], job_class_instance=job, jinja_env=jinja_env, ) diff --git a/nautobot_golden_config/tables.py b/nautobot_golden_config/tables.py index 0f9028f5a..0e266a6f5 100644 --- a/nautobot_golden_config/tables.py +++ b/nautobot_golden_config/tables.py @@ -10,30 +10,26 @@ from nautobot_golden_config.utilities.constant import CONFIG_FEATURES, ENABLE_BACKUP, ENABLE_COMPLIANCE, ENABLE_INTENDED ALL_ACTIONS = """ -{% if backup == True %} - {% if record.config_type == 'json' %} - +{% if record.config_type == 'json' %} + +{% else %} + {% if record.backup_config %} + + + {% else %} - {% if record.backup_config %} - - - - {% else %} - - {% endif %} + {% endif %} {% endif %} -{% if intended == True %} {% if record.config_type == 'json' %} - + +{% else %} + {% if record.intended_config %} + + + {% else %} - {% if record.intended_config %} - - - - {% else %} - - {% endif %} + {% endif %} {% endif %} {% if postprocessing == True %} @@ -45,14 +41,12 @@ {% endif %} {% endif %} -{% if compliance == True %} - {% if record.intended_config and record.backup_config %} - - - - {% else %} - - {% endif %} +{% if record.intended_config and record.backup_config %} + + + +{% else %} + {% endif %} {% if sotagg == True %} @@ -122,12 +116,12 @@ def actual_fields(): """Convienance function to conditionally toggle columns.""" active_fields = ["pk", "name"] - if ENABLE_BACKUP: - active_fields.append("backup_last_success_date") - if ENABLE_INTENDED: - active_fields.append("intended_last_success_date") - if ENABLE_COMPLIANCE: - active_fields.append("compliance_last_success_date") + # if ENABLE_BACKUP: + active_fields.append("backup_last_success_date") + # if ENABLE_INTENDED: + active_fields.append("intended_last_success_date") + # if ENABLE_COMPLIANCE: + active_fields.append("compliance_last_success_date") active_fields.append("actions") return tuple(active_fields) @@ -281,25 +275,33 @@ class GoldenConfigTable(BaseTable): verbose_name="Device", ) - if ENABLE_BACKUP: - backup_last_success_date = Column( - verbose_name="Backup Status", empty_values=(), order_by="backup_last_success_date" - ) - if ENABLE_INTENDED: - intended_last_success_date = Column( - verbose_name="Intended Status", - empty_values=(), - order_by="intended_last_success_date", - ) - if ENABLE_COMPLIANCE: - compliance_last_success_date = Column( - verbose_name="Compliance Status", - empty_values=(), - order_by="compliance_last_success_date", - ) + # if ENABLE_BACKUP: + backup_last_success_date = Column( + verbose_name="Backup Status", empty_values=(), order_by="backup_last_success_date" + ) + # if ENABLE_INTENDED: + intended_last_success_date = Column( + verbose_name="Intended Status", + empty_values=(), + order_by="intended_last_success_date", + ) + # if ENABLE_COMPLIANCE: + compliance_last_success_date = Column( + verbose_name="Compliance Status", + empty_values=(), + order_by="compliance_last_success_date", + ) + + config_features = { + "intended": True, # models.GoldenConfigSetting.objects.filter(enable_intended=True).exists() or True, + "compliance": True, #models.GoldenConfigSetting.objects.filter(enable_compliance=True).exists() or True, + "backup": True, #models.GoldenConfigSetting.objects.filter(enable_backup=True).exists() or True, + "sotagg": True, # Figure out if this is even needed + "postprocessing": True, # Figure out if this is even needed + } actions = TemplateColumn( - template_code=ALL_ACTIONS, verbose_name="Actions", extra_context=CONFIG_FEATURES, orderable=False + template_code=ALL_ACTIONS, verbose_name="Actions", extra_context=config_features, orderable=False ) def _render_last_success_date(self, record, column, value): @@ -435,35 +437,37 @@ class GoldenConfigSettingTable(BaseTable): pk = ToggleColumn() name = Column(order_by=("_name",), linkify=True) - jinja_repository = Column( - verbose_name="Jinja Repository", - empty_values=(), - ) - intended_repository = Column( - verbose_name="Intended Repository", - empty_values=(), - ) - backup_repository = Column( - verbose_name="Backup Repository", - empty_values=(), - ) + # kwargs = {"accessor": A("dynamic_group.pk"), "tab": "members", "verbose_name": "Dynamic Group"} + dynamic_group__members__count = LinkColumn(viewname="extras:dynamicgroup", kwargs={"pk": A("dynamic_group.pk")} ) + # jinja_repository = Column( + # verbose_name="Jinja Repository", + # empty_values=(), + # ) + # intended_repository = Column( + # verbose_name="Intended Repository", + # empty_values=(), + # ) + # backup_repository = Column( + # verbose_name="Backup Repository", + # empty_values=(), + # ) def _render_capability(self, record, column, record_attribute): # pylint: disable=unused-argument if getattr(record, record_attribute, None): return format_html('') return format_html('') - def render_backup_repository(self, record, column): - """Render backup repository boolean value.""" - return self._render_capability(record=record, column=column, record_attribute="backup_repository") + # def render_backup_repository(self, record, column): + # """Render backup repository boolean value.""" + # return self._render_capability(record=record, column=column, record_attribute="backup_repository") - def render_intended_repository(self, record, column): - """Render intended repository boolean value.""" - return self._render_capability(record=record, column=column, record_attribute="intended_repository") + # def render_intended_repository(self, record, column): + # """Render intended repository boolean value.""" + # return self._render_capability(record=record, column=column, record_attribute="intended_repository") - def render_jinja_repository(self, record, column): - """Render jinja repository boolean value.""" - return self._render_capability(record=record, column=column, record_attribute="jinja_repository") + # def render_jinja_repository(self, record, column): + # """Render jinja repository boolean value.""" + # return self._render_capability(record=record, column=column, record_attribute="jinja_repository") class Meta(BaseTable.Meta): """Meta attributes.""" @@ -473,10 +477,14 @@ class Meta(BaseTable.Meta): "pk", "name", "weight", - "description", - "backup_repository", - "intended_repository", - "jinja_repository", + # "description", + "enable_backup", + # "backup_repository", + "enable_intended", + # "intended_repository", + # "jinja_repository", + "enable_compliance", + "dynamic_group__members__count", ) diff --git a/nautobot_golden_config/template_content.py b/nautobot_golden_config/template_content.py index fb92b0fd8..58643ba9a 100644 --- a/nautobot_golden_config/template_content.py +++ b/nautobot_golden_config/template_content.py @@ -5,8 +5,8 @@ from django.urls import reverse from nautobot.extras.plugins import PluginTemplateExtension -from nautobot_golden_config.models import ConfigCompliance, GoldenConfig -from nautobot_golden_config.utilities.constant import CONFIG_FEATURES, ENABLE_COMPLIANCE +from nautobot_golden_config.models import ConfigCompliance, GoldenConfig, GoldenConfigSetting +# from nautobot_golden_config.utilities.constant import CONFIG_FEATURES, ENABLE_COMPLIANCE class ConfigComplianceDeviceCheck(PluginTemplateExtension): # pylint: disable=abstract-method @@ -100,13 +100,20 @@ def right_page(self): """Content to add to the configuration compliance.""" device = self.get_device() golden_config = GoldenConfig.objects.filter(device=device).first() + gc_setting = GoldenConfigSetting.objects.get_for_device(device) if not golden_config: return "" extra_context = { "device": self.get_device(), # device, "golden_config": golden_config, "template_type": "device-configs", - "config_features": CONFIG_FEATURES, + "config_features": { + "intended": gc_setting.enable_intended, + "compliance": gc_setting.enable_compliance, + "backup": gc_setting.enable_backup, + "sotagg": True, # Figure out if this is even needed + "postprocessing": True, # Figure out if this is even needed + }, } return self.render( "nautobot_golden_config/content_template.html", @@ -146,10 +153,10 @@ def right_page(self): extensions = [ConfigDeviceDetails] -if ENABLE_COMPLIANCE: - extensions.append(ConfigComplianceDeviceCheck) - extensions.append(ConfigComplianceLocationCheck) - extensions.append(ConfigComplianceTenantCheck) +# if GoldenConfigSetting.objects.filter(enable_compliance=True).exists(): +extensions.append(ConfigComplianceDeviceCheck) +extensions.append(ConfigComplianceLocationCheck) +extensions.append(ConfigComplianceTenantCheck) template_extensions = extensions diff --git a/nautobot_golden_config/templates/nautobot_golden_config/goldenconfigsetting_update.html b/nautobot_golden_config/templates/nautobot_golden_config/goldenconfigsetting_update.html index 417ad0909..014a271e8 100644 --- a/nautobot_golden_config/templates/nautobot_golden_config/goldenconfigsetting_update.html +++ b/nautobot_golden_config/templates/nautobot_golden_config/goldenconfigsetting_update.html @@ -10,11 +10,17 @@ {% render_field form.weight %} {% render_field form.description %} {% render_field form.dynamic_group %} + {% render_field form.enable_backup %} + {% render_field form.enable_intended %} + {% render_field form.enable_compliance %} + {% render_field form.enable_plan %} + {% render_field form.enable_deploy %}
Backup Configuration
+ {% render_field form.backup_repository %} {% render_field form.backup_path_template %} {% render_field form.backup_test_connectivity %} diff --git a/nautobot_golden_config/utilities/config_postprocessing.py b/nautobot_golden_config/utilities/config_postprocessing.py index cc8600735..0d49444dc 100644 --- a/nautobot_golden_config/utilities/config_postprocessing.py +++ b/nautobot_golden_config/utilities/config_postprocessing.py @@ -57,8 +57,8 @@ def get_secret_by_secret_group_name( def _get_device_agg_data(device, request): """Helper method to retrieve GraphQL data from a device.""" - settings = get_device_to_settings_map(Device.objects.filter(pk=device.pk))[device.id] - _, device_data = graph_ql_query(request, device, settings.sot_agg_query.query) + settings = get_device_to_settings_map(queryset=Device.objects.filter(pk=device.pk),job_name="intended") + _, device_data = graph_ql_query(request, device, settings["intended"][True][device.pk].sot_agg_query.query) return device_data diff --git a/nautobot_golden_config/utilities/constant.py b/nautobot_golden_config/utilities/constant.py index f1f5db283..371353e3f 100644 --- a/nautobot_golden_config/utilities/constant.py +++ b/nautobot_golden_config/utilities/constant.py @@ -27,3 +27,25 @@ raise ValueError("The `jinja_env` setting did not include the required key for `undefined`.") if isinstance(JINJA_ENV["undefined"], str): JINJA_ENV["undefined"] = import_string(JINJA_ENV["undefined"]) + +JOB_FUNCTION_MAP = { + "nautobot_golden_config.jobs.BackupJob": "backup", + "Backup Configurations": "backup", + "BackupJob": "backup", + "backup": "backup", + "nautobot_golden_config.jobs.IntendedJob": "intended", + "Generate Intended Configurations": "intended", + "intended": "intended", + "IntendedJob": "intended", + "nautobot_golden_config.jobs.ComplianceJob": "compliance", + "Perform Configuration Compliance": "compliance", + "ComplianceJob": "compliance", + "compliance": "compliance", + "Execute All Golden Configuration Jobs - Multiple Device": "all", + "nautobot_golden_config.jobs.AllDevicesGoldenConfig": "all", + "AllDevicesGoldenConfig": "all", + "Execute All Golden Configuration Jobs - Single Device": "all", + "nautobot_golden_config.jobs.AllGoldenConfig": "all", + "AllGoldenConfig": "all", + "all": "all", +} diff --git a/nautobot_golden_config/utilities/helper.py b/nautobot_golden_config/utilities/helper.py index 675081cf5..d2e538b26 100644 --- a/nautobot_golden_config/utilities/helper.py +++ b/nautobot_golden_config/utilities/helper.py @@ -173,7 +173,7 @@ def render_jinja_template(obj, logger, template): raise NornirNautobotException(error_msg) -def get_device_to_settings_map(queryset): +def get_device_to_settings_map(queryset, job_name): """Helper function to map heightest weighted GC settings to devices.""" update_dynamic_groups_cache() annotated_queryset = queryset.all().annotate( @@ -189,7 +189,16 @@ def get_device_to_settings_map(queryset): ) ) gcs = {gc.id: gc for gc in models.GoldenConfigSetting.objects.all()} - return {device.id: gcs[device.gc_settings] for device in annotated_queryset} + if job_name == "all": + job_name = ["backup", "intended", "compliance"] + else: + job_name = [job_name] + settings_filters2 = {setting: {True: {}, False: {}} for setting in job_name} + for device in annotated_queryset: + for setting in settings_filters2: + is_enabled = getattr(gcs[device.gc_settings], f"enable_{setting}", False) + settings_filters2[setting][is_enabled][device.id] = gcs[device.gc_settings] + return settings_filters2 def get_json_config(config): @@ -314,3 +323,12 @@ def get_error_message(error_code, **kwargs): except Exception: # pylint: disable=broad-except error_message = "Error Code was found, but failed to format message, unknown cause." return f"{error_code}: {error_message}" + +def get_inscope_settings_from_device_qs(queryset): + """Wrapper function to return a queryset of GoldenConfigSettings that are in scope for the provided queryset.""" + inscope_gcs = [] + for gc_setting in models.GoldenConfigSetting.objects.all(): + common_objects_queryset = queryset.intersection(gc_setting.dynamic_group.members) + if common_objects_queryset.count() > 0: + inscope_gcs.append(gc_setting) + return inscope_gcs diff --git a/nautobot_golden_config/views.py b/nautobot_golden_config/views.py index f055bdc3c..70cffba76 100644 --- a/nautobot_golden_config/views.py +++ b/nautobot_golden_config/views.py @@ -118,13 +118,16 @@ def get_extra_context(self, request, instance=None, **kwargs): context = super().get_extra_context(request, instance) if self.action == "retrieve": context["device_object"] = self._get_device_context(instance) - context["compliance"] = constant.ENABLE_COMPLIANCE - context["backup"] = constant.ENABLE_BACKUP - context["intended"] = constant.ENABLE_INTENDED + any_backup_enabled = models.GoldenConfigSetting.objects.filter(enable_backup=True).exists() + any_intended_enabled = models.GoldenConfigSetting.objects.filter(enable_intended=True).exists() + any_compliance_enabled = models.GoldenConfigSetting.objects.filter(enable_compliance=True).exists() + context["compliance"] = any_compliance_enabled + context["backup"] = any_backup_enabled + context["intended"] = any_intended_enabled jobs = [] - jobs.append(["BackupJob", constant.ENABLE_BACKUP]) - jobs.append(["IntendedJob", constant.ENABLE_INTENDED]) - jobs.append(["ComplianceJob", constant.ENABLE_COMPLIANCE]) + jobs.append(["BackupJob", any_backup_enabled]) + jobs.append(["IntendedJob", any_intended_enabled]) + jobs.append(["ComplianceJob", any_compliance_enabled]) add_message(jobs, request) return context @@ -283,10 +286,11 @@ def get_extra_context(self, request, instance=None, **kwargs): if self.action == "bulk_destroy": context["table"] = self.store_table - context["compliance"] = constant.ENABLE_COMPLIANCE - context["backup"] = constant.ENABLE_BACKUP - context["intended"] = constant.ENABLE_INTENDED - add_message([["ComplianceJob", constant.ENABLE_COMPLIANCE]], request) + any_compliance_enabled = models.GoldenConfigSetting.objects.filter(enable_compliance=True).exists() + context["compliance"] = any_compliance_enabled + context["backup"] = models.GoldenConfigSetting.objects.filter(enable_backup=True).exists() + context["intended"] = models.GoldenConfigSetting.objects.filter(enable_intended=True).exists() + add_message([["ComplianceJob", any_compliance_enabled]], request) return context def alter_queryset(self, request): @@ -407,7 +411,7 @@ def setup(self, request, *args, **kwargs): "device_visual": plot_visual(device_aggr), "feature_aggr": feature_aggr, "feature_visual": plot_visual(feature_aggr), - "compliance": constant.ENABLE_COMPLIANCE, + "compliance": models.GoldenConfigSetting.objects.filter(enable_compliance=True).exists(), } def extra_context(self): @@ -431,7 +435,7 @@ class ComplianceFeatureUIViewSet(views.NautobotUIViewSet): def get_extra_context(self, request, instance=None): """A ComplianceFeature helper function to warn if the Job is not enabled to run.""" - add_message([["ComplianceJob", constant.ENABLE_COMPLIANCE]], request) + add_message([["ComplianceJob", models.GoldenConfigSetting.objects.filter(enable_compliance=True).exists()]], request) return {} @@ -450,7 +454,7 @@ class ComplianceRuleUIViewSet(views.NautobotUIViewSet): def get_extra_context(self, request, instance=None): """A ComplianceRule helper function to warn if the Job is not enabled to run.""" - add_message([["ComplianceJob", constant.ENABLE_COMPLIANCE]], request) + add_message([["ComplianceJob", models.GoldenConfigSetting.objects.filter(enable_compliance=True).exists()]], request) return {} @@ -476,18 +480,18 @@ def get_extra_context(self, request, instance=None): context["dg_data"] = {"Dynamic Group": dg, "Filter Query Logic": dg.filter, "Scope of Devices": dg} jobs = [] - jobs.append(["BackupJob", constant.ENABLE_BACKUP]) - jobs.append(["IntendedJob", constant.ENABLE_INTENDED]) - jobs.append(["DeployConfigPlans", constant.ENABLE_DEPLOY]) - jobs.append(["ComplianceJob", constant.ENABLE_COMPLIANCE]) + jobs.append(["BackupJob", models.GoldenConfigSetting.objects.filter(enable_backup=True).exists()]) + jobs.append(["IntendedJob", models.GoldenConfigSetting.objects.filter(enable_intended=True).exists()]) + jobs.append(["DeployConfigPlans", models.GoldenConfigSetting.objects.filter(enable_deploy=True).exists()]) + jobs.append(["ComplianceJob", models.GoldenConfigSetting.objects.filter(enable_compliance=True).exists()]) jobs.append( [ "AllGoldenConfig", [ - constant.ENABLE_BACKUP, - constant.ENABLE_COMPLIANCE, - constant.ENABLE_DEPLOY, - constant.ENABLE_INTENDED, + models.GoldenConfigSetting.objects.filter(enable_backup=True).exists(), + models.GoldenConfigSetting.objects.filter(enable_compliance=True).exists(), + models.GoldenConfigSetting.objects.filter(enable_deploy=True).exists(), + models.GoldenConfigSetting.objects.filter(enable_intended=True).exists(), constant.ENABLE_SOTAGG, ], ] @@ -496,10 +500,10 @@ def get_extra_context(self, request, instance=None): [ "AllDevicesGoldenConfig", [ - constant.ENABLE_BACKUP, - constant.ENABLE_COMPLIANCE, - constant.ENABLE_DEPLOY, - constant.ENABLE_INTENDED, + models.GoldenConfigSetting.objects.filter(enable_backup=True).exists(), + models.GoldenConfigSetting.objects.filter(enable_compliance=True).exists(), + models.GoldenConfigSetting.objects.filter(enable_deploy=True).exists(), + models.GoldenConfigSetting.objects.filter(enable_intended=True).exists(), constant.ENABLE_SOTAGG, ], ] @@ -523,7 +527,7 @@ class ConfigRemoveUIViewSet(views.NautobotUIViewSet): def get_extra_context(self, request, instance=None): """A ConfigRemove helper function to warn if the Job is not enabled to run.""" - add_message([["BackupJob", constant.ENABLE_BACKUP]], request) + add_message([["BackupJob", models.GoldenConfigSetting.objects.filter(enable_backup=True).exists()]], request) return {} @@ -542,7 +546,7 @@ class ConfigReplaceUIViewSet(views.NautobotUIViewSet): def get_extra_context(self, request, instance=None): """A ConfigReplace helper function to warn if the Job is not enabled to run.""" - add_message([["BackupJob", constant.ENABLE_BACKUP]], request) + add_message([["BackupJob", models.GoldenConfigSetting.objects.filter(enable_backup=True).exists()]], request) return {} @@ -562,7 +566,7 @@ class RemediationSettingUIViewSet(views.NautobotUIViewSet): def get_extra_context(self, request, instance=None): """A RemediationSetting helper function to warn if the Job is not enabled to run.""" - add_message([["ComplianceJob", constant.ENABLE_COMPLIANCE]], request) + add_message([["ComplianceJob", models.GoldenConfigSetting.objects.filter(enable_compliance=True).exists()]], request) return {} @@ -591,9 +595,9 @@ def get_extra_context(self, request, instance=None): """A ConfigPlan helper function to warn if the Job is not enabled to run.""" context = super().get_extra_context(request, instance) jobs = [] - jobs.append(["GenerateConfigPlans", constant.ENABLE_PLAN]) - jobs.append(["DeployConfigPlans", constant.ENABLE_DEPLOY]) - jobs.append(["DeployConfigPlanJobButtonReceiver", constant.ENABLE_DEPLOY]) + jobs.append(["GenerateConfigPlans", models.GoldenConfigSetting.objects.filter(enable_plan=True).exists()]) + jobs.append(["DeployConfigPlans", models.GoldenConfigSetting.objects.filter(enable_deploy=True).exists()]) + jobs.append(["DeployConfigPlanJobButtonReceiver", models.GoldenConfigSetting.objects.filter(enable_deploy=True).exists()]) add_message(jobs, request) return context