Continued work on GC App Settings

.cookiecutter.json

@@ -21,16 +21,13 @@
         "_drift_manager": {
             "template": "https://github.com/nautobot/cookiecutter-nautobot-app.git",
             "template_dir": "nautobot-app",
-            "template_ref": "refs/tags/nautobot-app-v2.4.2",
+            "template_ref": "nautobot-app-v2.5.1",
             "cookie_dir": "",
             "branch_prefix": "drift-manager",
             "pull_request_strategy": "create",
-            "post_actions": [
-                "ruff",
-                "poetry"
-            ],
+            "post_actions": [],
             "draft": false,
-            "baked_commit_ref": "cc1a95931455ed52b0a30b22257e5c8fbdcd212d"
+            "baked_commit_ref": "bef141f8bdbc25beec0f7138010447b8e8e9fee4"
         }
     }
 }

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,6 +1,8 @@
 ---
 name: 🐛 Bug Report
 about: Report a reproducible bug in the current release of nautobot-golden-config
+labels:
+  - "type: bug"
 ---
 
 ### Environment

.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,7 +1,8 @@
 ---
 name: ✨ Feature Request
 about: Propose a new feature or enhancement
-
+labels:
+  - "type: feature"
 ---
 
 ### Environment

.github/pull_request_template.md

@@ -27,6 +27,7 @@
     Please feel free to update todos to keep track of your own notes for WIP PRs.
 -->
 - [ ] Explanation of Change(s)
+- [ ] Added change log fragment(s) (for more information see [the documentation](https://docs.nautobot.com/projects/core/en/stable/development/core/#creating-changelog-fragments))
 - [ ] Attached Screenshots, Payload Example
 - [ ] Unit, Integration Tests
 - [ ] Documentation Updates (when adding/changing features)

.github/workflows/ci.yml

@@ -82,12 +82,26 @@ jobs:
           poetry-version: "1.8.5"
       - name: "Linting: yamllint"
         run: "poetry run invoke yamllint"
+  markdownlint:
+    runs-on: "ubuntu-22.04"
+    env:
+      INVOKE_NAUTOBOT_GOLDEN_CONFIG_LOCAL: "True"
+    steps:
+      - name: "Check out repository code"
+        uses: "actions/checkout@v4"
+      - name: "Setup environment"
+        uses: "networktocode/gh-action-setup-poetry-environment@v6"
+        with:
+          poetry-version: "1.8.5"
+      - name: "Linting: markdownlint"
+        run: "poetry run invoke markdownlint"
   check-in-docker:
     needs:
       - "ruff-format"
       - "ruff-lint"
       - "poetry"
       - "yamllint"
+      - "markdownlint"
     runs-on: "ubuntu-22.04"
     strategy:
       fail-fast: true
@@ -110,9 +124,9 @@ jobs:
         run: "poetry run invoke lock --constrain-nautobot-ver --constrain-python-ver"
       - name: "Set up Docker Buildx"
         id: "buildx"
-        uses: "docker/setup-buildx-action@v3"
+        uses: "docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2"  # v3.10.0
       - name: "Build"
-        uses: "docker/build-push-action@v5"
+        uses: "docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25"  # v5.4.0
         with:
           builder: "${{ steps.buildx.outputs.name }}"
           context: "./"
@@ -126,8 +140,6 @@ jobs:
             NAUTOBOT_VER=${{ matrix.nautobot-version }}
             PYTHON_VER=${{ matrix.python-version }}
             CI=true
-      - name: "Copy credentials"
-        run: "cp development/creds.example.env development/creds.env"
       - name: "Linting: pylint"
         run: "poetry run invoke pylint"
       - name: "Checking: App Config"
@@ -140,7 +152,7 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        python-version: ["3.9", "3.12"]
+        python-version: ["3.9"]  # 3.12 stable is tested in unittest_report stage.
         db-backend: ["postgresql"]
         nautobot-version: ["stable"]
         include:
@@ -167,9 +179,60 @@ jobs:
         run: "poetry run invoke lock --constrain-nautobot-ver --constrain-python-ver"
       - name: "Set up Docker Buildx"
         id: "buildx"
-        uses: "docker/setup-buildx-action@v3"
+        uses: "docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2"  # v3.10.0
+      - name: "Build"
+        uses: "docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25"  # v5.4.0
+        with:
+          builder: "${{ steps.buildx.outputs.name }}"
+          context: "./"
+          push: false
+          load: true
+          tags: "${{ env.APP_NAME }}/nautobot:${{ matrix.nautobot-version }}-py${{ matrix.python-version }}"
+          file: "./development/Dockerfile"
+          cache-from: "type=gha,scope=${{ matrix.nautobot-version }}-py${{ matrix.python-version }}"
+          cache-to: "type=gha,scope=${{ matrix.nautobot-version }}-py${{ matrix.python-version }}"
+          build-args: |
+            NAUTOBOT_VER=${{ matrix.nautobot-version }}
+            PYTHON_VER=${{ matrix.python-version }}
+            CI=true
+      - name: "Use Mysql invoke settings when needed"
+        run: "cp invoke.mysql.yml invoke.yml"
+        if: "matrix.db-backend == 'mysql'"
+      - name: "Run Tests"
+        run: "poetry run invoke unittest"
+
+  unittest_report:
+    needs:
+      - "check-in-docker"
+    strategy:
+      fail-fast: true
+      matrix:
+        python-version: ["3.12"]
+        db-backend: ["postgresql"]
+        nautobot-version: ["stable"]
+    runs-on: "ubuntu-22.04"
+    permissions:
+      pull-requests: "write"
+      contents: "write"
+    env:
+      INVOKE_NAUTOBOT_GOLDEN_CONFIG_PYTHON_VER: "${{ matrix.python-version }}"
+      INVOKE_NAUTOBOT_GOLDEN_CONFIG_NAUTOBOT_VER: "${{ matrix.nautobot-version }}"
+    steps:
+      - name: "Check out repository code"
+        uses: "actions/checkout@v4"
+      - name: "Setup environment"
+        uses: "networktocode/gh-action-setup-poetry-environment@v6"
+        with:
+          poetry-version: "1.8.5"
+      - name: "Constrain Nautobot version and regenerate lock file"
+        env:
+          INVOKE_NAUTOBOT_GOLDEN_CONFIG_LOCAL: "true"
+        run: "poetry run invoke lock --constrain-nautobot-ver --constrain-python-ver"
+      - name: "Set up Docker Buildx"
+        id: "buildx"
+        uses: "docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2"  # v3.10.0
       - name: "Build"
-        uses: "docker/build-push-action@v5"
+        uses: "docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25"  # v5.4.0
         with:
           builder: "${{ steps.buildx.outputs.name }}"
           context: "./"
@@ -189,7 +252,28 @@ jobs:
         run: "cp invoke.mysql.yml invoke.yml"
         if: "matrix.db-backend == 'mysql'"
       - name: "Run Tests"
-        run: "poetry run invoke unittest"
+        run: "poetry run invoke unittest --coverage"
+      - name: "Generate Coverage Comment"
+        if: >
+          contains(fromJson('["develop","ltm-1.6"]'), github.base_ref) &&
+          (github.head_ref != 'main') && (!startsWith(github.head_ref, 'release'))
+        id: "coverage_comment"
+        uses: "py-cov-action/python-coverage-comment-action@d1ff8fbb5ff80feedb3faa0f6d7b424f417ad0e1"  # v3.30
+        with:
+          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+          MINIMUM_GREEN: 90
+          MINIMUM_ORANGE: 80
+          ANNOTATE_MISSING_LINES: true
+          ANNOTATION_TYPE: "warning"
+      - name: "Store Pull Request comment to be posted"
+        if: >
+          contains(fromJson('["develop","ltm-1.6"]'), github.base_ref) &&
+          (github.head_ref != 'main') && (!startsWith(github.head_ref, 'release'))
+        uses: "actions/upload-artifact@v4"
+        with:
+          name: "python-coverage-comment-action"
+          path: "python-coverage-comment-action.txt"
+
   changelog:
     if: >
       contains(fromJson('["develop","ltm-1.6"]'), github.base_ref) &&
@@ -211,41 +295,39 @@ jobs:
   publish_gh:
     needs:
       - "unittest"
+      - "unittest_report"
     name: "Publish to GitHub"
     runs-on: "ubuntu-22.04"
     if: "startsWith(github.ref, 'refs/tags/v')"
     env:
       INVOKE_NAUTOBOT_GOLDEN_CONFIG_LOCAL: "True"
+    permissions:
+      contents: "write"
     steps:
       - name: "Check out repository code"
         uses: "actions/checkout@v4"
-      - name: "Set up Python"
-        uses: "actions/setup-python@v5"
+      - name: "Setup environment"
+        uses: "networktocode/gh-action-setup-poetry-environment@v6"
         with:
+          poetry-version: "1.8.5"
           python-version: "3.12"
-      - name: "Install Python Packages"
-        run: "pip install poetry"
+          poetry-install-options: "--no-root"
       - name: "Set env"
         run: "echo RELEASE_VERSION=${GITHUB_REF:10} >> $GITHUB_ENV"
       - name: "Run Poetry Version"
         run: "poetry version $RELEASE_VERSION"
-      - name: "Install Dependencies (needed for mkdocs)"
-        run: "poetry install --no-root"
       - name: "Build Documentation"
         run: "poetry run invoke build-and-check-docs"
       - name: "Run Poetry Build"
         run: "poetry build"
       - name: "Upload binaries to release"
-        uses: "svenstaro/upload-release-action@v2"
-        with:
-          repo_token: "${{ secrets.GH_NAUTOBOT_BOT_TOKEN }}"
-          file: "dist/*"
-          tag: "${{ github.ref }}"
-          overwrite: true
-          file_glob: true
+        run: "gh release upload ${{ github.ref_name }} dist/*.{tar.gz,whl}"
+        env:
+          GH_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
   publish_pypi:
     needs:
       - "unittest"
+      - "unittest_report"
     name: "Push Package to PyPI"
     runs-on: "ubuntu-22.04"
     if: "startsWith(github.ref, 'refs/tags/v')"
@@ -254,24 +336,22 @@ jobs:
     steps:
       - name: "Check out repository code"
         uses: "actions/checkout@v4"
-      - name: "Set up Python"
-        uses: "actions/setup-python@v5"
+      - name: "Setup environment"
+        uses: "networktocode/gh-action-setup-poetry-environment@v6"
         with:
+          poetry-version: "1.8.5"
           python-version: "3.12"
-      - name: "Install Python Packages"
-        run: "pip install poetry"
+          poetry-install-options: "--no-root"
       - name: "Set env"
         run: "echo RELEASE_VERSION=${GITHUB_REF:10} >> $GITHUB_ENV"
       - name: "Run Poetry Version"
         run: "poetry version $RELEASE_VERSION"
-      - name: "Install Dependencies (needed for mkdocs)"
-        run: "poetry install --no-root"
       - name: "Build Documentation"
         run: "poetry run invoke build-and-check-docs"
       - name: "Run Poetry Build"
         run: "poetry build"
       - name: "Push to PyPI"
-        uses: "pypa/gh-action-pypi-publish@release/v1"
+        uses: "pypa/gh-action-pypi-publish@76f52bc884231f62b9a034ebfe128415bbaabdfc"  # v1.12.4
         with:
           user: "__token__"
           password: "${{ secrets.PYPI_API_TOKEN }}"
@@ -292,7 +372,7 @@ jobs:
         # ENVs cannot be used directly in job.if. This is a workaround to check
         # if SLACK_WEBHOOK_URL is present.
         if: "env.SLACK_WEBHOOK_URL != ''"
-        uses: "slackapi/slack-github-action@v1"
+        uses: "slackapi/slack-github-action@fcfb566f8b0aab22203f066d80ca1d7e4b5d05b3"  # v1.27.1
         with:
           payload: |
             {

.github/workflows/coverage.yml

@@ -0,0 +1,36 @@
+---
+name: "Post coverage comment"
+
+on: # yamllint disable-line rule:truthy rule:comments
+  workflow_run:
+    workflows: ["CI"]
+    types:
+      - "completed"
+
+jobs:
+  test:
+    name: "Post coverage comment to PR"
+    runs-on: "ubuntu-latest"
+    if: github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion == 'success' # yamllint disable-line rule:quoted-strings rule:comments
+    permissions:
+      # Gives the action the necessary permissions for publishing new
+      # comments in pull requests.
+      pull-requests: "write"
+      # Gives the action the necessary permissions for editing existing
+      # comments (to avoid publishing multiple comments in the same PR)
+      contents: "write" # yamllint disable-line rule:indentation rule:comments
+      # Gives the action the necessary permissions for looking up the
+      # workflow that launched this workflow, and download the related
+      # artifact that contains the comment to be published
+      actions: "read"
+    steps:
+      # DO NOT run actions/checkout here, for security reasons
+      # For details, refer to https://securitylab.github.com/research/github-actions-preventing-pwn-requests/
+      - name: "Post comment"
+        uses: "py-cov-action/python-coverage-comment-action@d1ff8fbb5ff80feedb3faa0f6d7b424f417ad0e13"  # v3.30
+        with:
+          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+          GITHUB_PR_RUN_ID: "${{ github.event.workflow_run.id }}"
+          # Update those if you changed the default values:
+          # COMMENT_ARTIFACT_NAME: python-coverage-comment-action
+          # COMMENT_FILENAME: python-coverage-comment-action.txt

.gitignore

@@ -2,6 +2,9 @@ nautobot_golden_config/transposer.py
 golden-config/
 nautobot.sql
 
+# Avoid accidental commit of analytics template override generated via ./docs/insert-analytics.sh
+docs/assets/overrides/main.html
+
 # Ansible Retry Files
 *.retry
 
@@ -60,6 +63,7 @@ coverage.xml
 *.py,cover
 .hypothesis/
 .pytest_cache/
+lcov.info
 
 # Translations
 *.mo

.readthedocs.yaml

@@ -11,6 +11,9 @@ build:
   os: "ubuntu-22.04"
   tools:
     python: "3.11"
+  jobs:
+    pre_build:
+      - "/bin/bash ./docs/insert-analytics.sh"
 
 mkdocs:
   configuration: "mkdocs.yml"

changes/+dynamic-form.changed

@@ -0,0 +1 @@
+Changed the Golden Config Setting form to use dynamic dropdowns for the related models.

changes/+nautobot-app-v2.5.0.housekeeping

@@ -0,0 +1 @@
+Rebaked from the cookie `nautobot-app-v2.5.0`.

changes/+nautobot-app-v2.5.1.housekeeping

@@ -0,0 +1 @@
+Rebaked from the cookie `nautobot-app-v2.5.1`.

changes/870.added

@@ -0,0 +1 @@
+Added the ability to sync remediation settings from a Git repository.

changes/898.added

@@ -0,0 +1 @@
+Added tests to ensure that mkdocs release notes are properly set.

changes/899.fixed

@@ -0,0 +1 @@
+Fixed template include errors during intended config rendering due to incorrect Jinja root path.

changes/902.changed

@@ -0,0 +1 @@
+Upgraded from hier_config v2.2.2 to v3.2.2, which is a breaking change from the hier_config side. The hier_config implementation was updated to reflect hier_config v3.

changes/921.added

@@ -0,0 +1 @@
+Add testing for the two issues with mkdocs versus markdown rendering and associated fixes.

changes/940.fixed

@@ -0,0 +1 @@
+Implement a more performant ORM/DB query in-place of the existing for loop logic for device_to_settings_map.

changes/951.fixed

@@ -0,0 +1 @@
+Fixed potential duplicates in data migration by adding a check to validate time uniqueness.

changes/955.added

@@ -0,0 +1 @@
+Added an option to fail the Config Plan Deployment Job if any tasks for any device fails.

changes/961.housekeeping

@@ -0,0 +1 @@
+Migrate Golden Config Setting, Config Plan, Golden Config models to UI Component Framework.

changes/969.documentation

@@ -0,0 +1 @@
+Added Analytics GTM template override only to the public ReadTheDocs build.

changes/970.housekeeping

@@ -0,0 +1 @@
+Regenerate lock file with poetry 1.8 version.