Add a +benchmark mode to act-wrapper.

act-wrapper

@@ -2,32 +2,45 @@
 
 set -e
 
+CHDIR_PATH="integration-tests"
+WORKFLOW_PATH="system-tests.yml"
+ACT_EXTRA_ARGS=
+
 function usage {
 	echo "The act wrapper script."
 	echo
 	echo "This script first builds the container image for running the tests"
 	echo "with a freshly compiled Cascade, before executing act with"
 	echo "the provided arguments."
 	echo
-	echo "If you want to skip the build step and run act with the latest"
-	echo "available version of the cascade test runner image you can use:"
+	echo "- If you want to skip the build step and run act with the latest"
+	echo "  available version of the cascade test runner image you can use:"
+	echo
+	echo "    '$0 +nobuild <act-args>'"
+	echo
+	echo "- If you want to build cascade within the container build, use:"
+	echo
+	echo "    '$0 +build-inside <act-args>'"
+	echo
+	echo "- If you want to run a benchmark test use:"
 	echo
-	echo "'$0 +nobuild <act-args>'"
+	echo "    '$0 +benchmark <ZONE NAME> <ZONE PATH> <POLICY PATH.toml> <TIMEOUT SECONDS>'"
 	echo
-	echo "If you want to build cascade within the container build, use:"
+	echo "  This will enable the +stats-graph option and invoke the"
+	echo "  integration-tests/benchmark-test.yml workflow using the supplied inputs"
+	echo "  with info level logging and a release mode build of Cascade."
 	echo
-	echo "'$0 +build-inside <act-args>'"
+	echo "- If you want to produce a report of approximate CPU and memory usage"
+	echo "  over time use:"
 	echo
-	echo "If you want to produce a report of approximate CPU and memory usage"
-	echo "over time use:"
+	echo "    '$0 +stats-report </path/to/write/report.csv>'"
 	echo
-	echo "'$0 +stats-report </path/to/write/report.csv>'"
+	echo "- To create a PNG graph of the usage use: (requires gnuplot)"
 	echo
-	echo "To create a PNG graph of the usage use: (requires gnuplot)"
-	echo "Note: If +stats-report is not used, a temporary file will be used"
-	echo "to capture resource usage in CSV format."
+	echo "    '$0 +stats-graph </path/to/write/report.png>'"
 	echo
-	echo "'$0 +stats-graph </path/to/write/report.png>'"
+	echo "  If +stats-report is not used, a temporary file will be used"
+	echo "  to capture resource usage in CSV format."
 	echo
 	echo "NOTE: '+XXX' arguments must be the first arguments to act-wrapper, and can be combined."
 	echo
@@ -84,6 +97,28 @@ while [[ "$1" =~ ^\+ ]]; do
 			shift
 			docker_stats_graph_path="$1"
 			;;
+		+benchmark)
+		  if [ $# -ne 5 ]; then
+		  	echo >&2 "ERROR: Incorrect number of arguments for +benchmark."
+		  	echo >&2
+		  	usage
+		  	exit 1
+		  fi
+		  shift; ZONE_NAME="$1"
+		  shift; ZONE_PATH="$1"
+		  shift; POLICY_PATH="$1"
+		  shift; TIMEOUT_SECS="$1"
+		  POLICY_NAME=$(basename -- "${POLICY_PATH}")
+      POLICY_NAME="${POLICY_NAME%.*}"
+		  NOW=$(date +"%Y%m%d_%H%M%S")
+		  WORKFLOW_PATH="benchmark-test.yml"
+      ACT_EXTRA_ARGS="--rm --input log-level=info --input zone-name=${ZONE_NAME} --input policy-name=${POLICY_NAME} --input timeout-seconds=${TIMEOUT_SECS}"
+		  docker_stats_graph=true
+		  docker_stats_graph_path="benchmark-${NOW}.png"
+		  mkdir "integration-tests/benchmark"
+      cp "${ZONE_PATH}" "integration-tests/benchmark/${ZONE_NAME}"
+      cp "${POLICY_PATH}" "integration-tests/benchmark/"
+		  ;;
 		*)
 			echo >&2 "ERROR: Unrecognized + argument: $1"
 			usage
@@ -165,7 +200,7 @@ fi
 # `docker cp` respect .gitignore or .dockerignore or have any other way I have
 # found to achieve the same result.
 echo "Running tests"
-act "${act_args[@]}" --log-prefix-job-id --pull=false -P "ubuntu-latest=nlnetlabs/cascade-tests-runner" -C integration-tests -W system-tests.yml "$@" && ACT_EXIT_CODE=$? || ACT_EXIT_CODE=$? 
+act "${act_args[@]}" --log-prefix-job-id --pull=false -P "ubuntu-latest=nlnetlabs/cascade-tests-runner" -C "${CHDIR_PATH}" -W "${WORKFLOW_PATH}" ${ACT_EXTRA_ARGS} "$@" && ACT_EXIT_CODE=$? || ACT_EXIT_CODE=$? 
 
 # Stop the docker stats collection process, if needed.
 if [[ "$docker_stats" == true ]]; then
@@ -196,7 +231,7 @@ if [[ "${docker_stats_graph}" == true && "${STATS_OKAY}" == true && "${ACT_EXIT_
 			# Generate the gnuplot control script.	
 			GNUPLOT_SCRIPT=$(cat <<EOF
 set locale "en_GB.UTF-8"
-set terminal png size 2048,1536
+set terminal png size 1024,768
 set datafile separator ','
 set title "Cascade ${GIT_VER}\n{/*0.75 $*}\n{/*0.75 ${CPU_INFO}"
 set timefmt "%s"

integration-tests/benchmark-test.yml

@@ -0,0 +1,95 @@
+name: System/Benchmark test
+
+defaults:
+  # see: https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-syntax#defaultsrunshell
+  run:
+    shell: bash --noprofile --norc -eo pipefail {0}
+
+on:
+  workflow_dispatch:
+    inputs:
+      log-level:
+        description: The level of logging that Cascade should output.
+        required: false
+        default: debug
+        type: choice
+        options:
+          - error
+          - warning
+          - info
+          - debug
+          - trace
+      zone-name:
+        description: The name of a zone file (and zone) in integration-tests/benchmark/ to load.
+        required: true
+        type: string
+      policy-name:
+        description: The name of a policy file minus .toml in integration-tests/benchmark/ to use.
+        required: true
+        type: string
+      timeout-seconds:
+        description: The maximum number of seconds to wait for loading and signing to complete.
+        required: false
+        type: number
+        default: 60
+
+jobs:
+  benchmark:
+    name: Benchmark
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: ./.github/actions/set-build-profile
+        with:
+          build-profile: release
+
+      - uses: ./.github/actions/setup-and-start-cascade
+        with:
+          log-level: ${{ inputs.log-level }}
+
+      - name: Add policy '${{ inputs.policy-name }}'
+        run: |
+          TEST_DIR="${PWD}/integration-tests/benchmark"
+          POLICY_DIR=$(integration-tests/scripts/get-default-path.sh policy-dir)
+          cp "${TEST_DIR}/${{ inputs.policy-name }}.toml" "${POLICY_DIR}/"
+          cascade policy reload
+
+      - name: Add zone '${{ inputs.zone-name }}'
+        run: |
+          TEST_DIR="${PWD}/integration-tests/benchmark"
+          cascade zone add --policy ${{ inputs.policy-name }} --source "${TEST_DIR}/${{ inputs.zone-name }}" ${{ inputs.zone-name }}
+
+      - name: Cascade should list the zone
+        run: |
+          timeout=10 # seconds
+          start=$(date +%s)
+          until cascade zone list | grep -q "${{ inputs.zone-name }}"; do
+            if (($(date +%s) > (start + timeout))); then
+              cascade zone list
+              echo "timeout: zone list output did not include the zone" >&2
+              exit 1
+            fi
+            sleep 1
+          done
+
+      - name: Check zone status
+        run: |
+          timeout=${{ inputs.timeout-seconds }}
+          start=$(date +%s)
+          until cascade zone status ${{ inputs.zone-name }} | grep -q "Published zone available"; do
+            if (($(date +%s) > (start + timeout))); then
+              cascade zone status ${{ inputs.zone-name }}
+              echo "timeout: zone status did not report published zone available" >&2
+              exit 1
+            fi
+            sleep 1
+            NOW="$(date)"
+            echo -n "${NOW}: "
+            cascade zone status ${{ inputs.zone-name }} 2>&1 | tail -n 1
+            tail -n 1 ${{ github.workspace }}/cascade-dir/cascade.log
+          done
+
+      - name: Print log files on any failure in this job
+        uses: ./.github/actions/print-logfiles
+        if: failure()