diff --git a/test/load/Readme.md b/test/load/Readme.md index 7dfb88af181..bd37cbe2d9d 100644 --- a/test/load/Readme.md +++ b/test/load/Readme.md @@ -13,6 +13,8 @@ Installation scripts and manuals are provided in [setup/Readme](./setup/Readme.m ## Usage +### Local Development + All test cases are organized in the `testing` folder. You can run the entire suite using: ```sh @@ -27,6 +29,18 @@ Alternatively you can run a subset of tests using standard `go test` syntax. E.g go test ./testing/... -run ^TestExample ``` +### Report Creation + +If you are running the tests to create reports, please use the `suite.sh` helper script. It automatically +ensures that all files get generated using unified timestamps in a `.loadtest-results` folder. + +```sh +./suite.sh TestExample +``` + +The script will show live output and is safe to run in jumphost environments where connectivity might +break. + ## Development The load-testing framework itself is organized in the `pkg` folder. You can run its unit diff --git a/test/load/go.mod b/test/load/go.mod index 8090da0ceea..b99828d280c 100644 --- a/test/load/go.mod +++ b/test/load/go.mod @@ -3,16 +3,19 @@ module github.com/kcp-dev/kcp/test/load go 1.26.0 require ( + github.com/kcp-dev/client-go v0.35.1 github.com/kcp-dev/logicalcluster/v3 v3.0.5 github.com/kcp-dev/sdk v0.31.0 github.com/montanaflynn/stats v0.7.1 github.com/stretchr/testify v1.11.1 + k8s.io/api v0.36.0 k8s.io/apimachinery v0.36.0 k8s.io/client-go v0.36.0 ) replace ( github.com/kcp-dev/apimachinery/v2 => ../../staging/src/github.com/kcp-dev/apimachinery + github.com/kcp-dev/client-go => ../../staging/src/github.com/kcp-dev/client-go github.com/kcp-dev/code-generator/v3 => ../../staging/src/github.com/kcp-dev/code-generator github.com/kcp-dev/sdk => ../../staging/src/github.com/kcp-dev/sdk ) @@ -56,12 +59,10 @@ require ( golang.org/x/term v0.42.0 // indirect golang.org/x/text v0.36.0 // indirect golang.org/x/time v0.15.0 // indirect - golang.org/x/tools v0.44.0 // indirect google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/api v0.36.0 // indirect k8s.io/apiextensions-apiserver v0.36.0 // indirect k8s.io/klog/v2 v2.140.0 // indirect k8s.io/kube-openapi v0.0.0-20260414162039-ec9c827d403f // indirect diff --git a/test/load/setup/manifests/dashboards/workspace-debug.json b/test/load/setup/manifests/dashboards/workspace-debug.json index d7f17e99d5c..09cdcb780ef 100644 --- a/test/load/setup/manifests/dashboards/workspace-debug.json +++ b/test/load/setup/manifests/dashboards/workspace-debug.json @@ -468,6 +468,420 @@ ], "title": "num logicalclusters (duplicate accounted)", "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 11, + "x": 0, + "y": 24 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(proxy_request_duration_seconds_count[1m])) by (code, method)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "fp req per second", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 11, + "x": 11, + "y": 24 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(proxy_request_duration_seconds_count[1m]))", + "legendFormat": "req", + "range": true, + "refId": "A" + } + ], + "title": "fp total req per second", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 11, + "x": 0, + "y": 36 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(proxy_request_duration_seconds_bucket[1m])) by (le, method, code))", + "legendFormat": "p99{code=\"{{code}}\", method=\"{{method}}\"}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "", + "legendFormat": "avg{code=\"{{code}}\", method=\"{{method}}\"}", + "range": true, + "refId": "B" + } + ], + "title": "p99", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 11, + "x": 11, + "y": 36 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(rate(proxy_request_duration_seconds_sum[1m])) by (method, code) / sum(rate(proxy_request_duration_seconds_count[1m])) by (method, code)", + "legendFormat": "avg{code=\"{{code}}\", method=\"{{method}}\"}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "", + "legendFormat": "avg{code=\"{{code}}\", method=\"{{method}}\"}", + "range": true, + "refId": "B" + } + ], + "title": "avg", + "type": "timeseries" } ], "preload": false, diff --git a/test/load/suite.sh b/test/load/suite.sh new file mode 100755 index 00000000000..664695576ae --- /dev/null +++ b/test/load/suite.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash + +# Copyright 2026 The kcp Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# suite is a simple helper script to execute load tests and capture their logs in a consistent and resilient way. +# It saves the logs to a file and outputs them on your screen. You can detach from the logs with Ctrl+C, while the test keeps running in the background. +# +# It can also be used in scripting/CI as well, as it exits once the go test has finished. +# Additionally it works with jumphosts in case the ssh connection terminates. + +set -euo pipefail + +# set to a fixed timezone so logfile names are consistent +export TZ="Europe/Berlin" +logdir=".loadtest-results" +mkdir -p "$logdir" + +# parse user input +testname=${1:-} +if [[ -z "$testname" ]]; then + echo "Usage: $0 " + exit 1 +fi + +# execute the test and capture logs +logfile="$logdir/${testname}_$(date '+%Y-%m-%d-%H:%M:%S').log" +echo "Executing test: $testname" +echo "Logs will be saved to: $logfile" + +go test -timeout 2h -test.fullpath=true -run "^${testname}$" -count=1 -v github.com/kcp-dev/kcp/test/load/testing &> "$logfile" & +test_pid=$! + +# on Ctrl+C: stop tailing but keep the test running +stop_tailing() { + echo "" + echo "Detached from output. Test (PID $test_pid) is still running. Kill it manually if required." + echo "Resume with: tail -f $logfile" + exit 0 +} +trap stop_tailing INT TERM + +# stream logs until the test process exits +tail -f --pid="$test_pid" "$logfile" + +wait "$test_pid" +exit_code=$? + +exit "$exit_code" diff --git a/test/load/testing/workspace.go b/test/load/testing/workspace.go new file mode 100644 index 00000000000..cf7192a735b --- /dev/null +++ b/test/load/testing/workspace.go @@ -0,0 +1,46 @@ +/* +Copyright 2026 The kcp Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testing + +import ( + "context" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + kcpclientset "github.com/kcp-dev/sdk/client/clientset/versioned/cluster" + + "github.com/kcp-dev/kcp/test/load/pkg/tree" +) + +// workspacesExist checks whether count workspaces already exist by +// verifying that the last workspace name is present. This is a cheap heuristic +// that avoids listing all workspaces. +func workspacesExist(wt tree.WorkspaceTree, client kcpclientset.ClusterInterface, count int) (bool, error) { + lastName := wt.WorkspaceName(count) + parentPath := wt.PathForSequenceNumber(wt.ParentSequenceNumber(count)) + _, err := client.Cluster(parentPath).TenancyV1alpha1().Workspaces().Get(context.Background(), lastName, metav1.GetOptions{}) + if err != nil { + // we also need IsForbidden here in case we are requesting a child ws whose + // parent does not exist either. + if apierrors.IsNotFound(err) || apierrors.IsForbidden(err) { + return false, nil + } + return false, err + } + return true, nil +} diff --git a/test/load/testing/workspace_simple_crud_test.go b/test/load/testing/workspace_simple_crud_test.go new file mode 100644 index 00000000000..0cb1a5b5b6f --- /dev/null +++ b/test/load/testing/workspace_simple_crud_test.go @@ -0,0 +1,161 @@ +/* +Copyright 2026 The kcp Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testing + +import ( + "context" + "fmt" + "os" + "testing" + "time" + + "github.com/stretchr/testify/require" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + kcpkubernetesclientset "github.com/kcp-dev/client-go/kubernetes" + kcpclientset "github.com/kcp-dev/sdk/client/clientset/versioned/cluster" + + "github.com/kcp-dev/kcp/test/load/pkg/framework" + "github.com/kcp-dev/kcp/test/load/pkg/measurement" + "github.com/kcp-dev/kcp/test/load/pkg/stats" + "github.com/kcp-dev/kcp/test/load/pkg/tree" + "github.com/kcp-dev/kcp/test/load/pkg/tuningset" +) + +const crudConfigMapQPS = 150 // equivalent to 600, see TODO comment below + +func TestWorkspaceSimpleCRUD(t *testing.T) { + cfg := framework.Require(t, framework.KCPFrontProxyKubeconfig) + + client, err := kcpclientset.NewForConfig(cfg.FrontProxyKubeconfig) + require.NoError(t, err) + + kubeClusterClient, err := kcpkubernetesclientset.NewForConfig(cfg.FrontProxyKubeconfig) + require.NoError(t, err) + + var sections []measurement.Section + + wt := defaultTree() + + // Ensure workspaces exist, creating them if necessary. + exist, err := workspacesExist(wt, client, workspaceCount) + require.NoError(t, err) + if exist { + t.Logf("workspaces already exist, skipping creation") + } else { + t.Logf("Creating required workspaces") + createSection := createWorkspaces(t, client, createWorkspaceQPS) + sections = append(sections, createSection) + } + + t.Logf("Running configmap CRUD operations") + crudSection := crudConfigMaps(t, wt, kubeClusterClient, crudConfigMapQPS) + sections = append(sections, crudSection) + + report := NewKCPReport(t, "Workspace Simple Configmap CRUD", cfg.FrontProxyKubeconfig) + report.Sections = sections + report.PrettyPrint(os.Stdout) + + for _, sec := range sections { + require.Empty(t, sec.Errors, "section %q encountered errors", sec.Title) + } +} + +// crudConfigMaps performs a Create/Update/Delete cycle for a ConfigMap in each +// of the workspaces. +func crudConfigMaps(t *testing.T, wt tree.WorkspaceTree, kubeClusterClient kcpkubernetesclientset.ClusterInterface, qps float64) measurement.Section { + t.Helper() + + section := measurement.Section{ + Title: "ConfigMap CRUD", + Parameters: []measurement.Parameter{ + {Key: "Workspaces", Value: fmt.Sprintf("%d", workspaceCount)}, + {Key: "QPS", Value: fmt.Sprintf("%f", qps*4)}, // TODO: for now multiply by 4 because we are doing 4 network calls per action, but this needs to be changed inside the framework next to be precise + }, + Sink: &measurement.Memory{ + Stats: []stats.NamedStat{stats.P99(), stats.Avg()}, + }, + } + + ts := tuningset.NewUniformQPS(qps, workspaceCount, 1) + section.Start() + action := func(seq int, s measurement.Sink) error { + cmClient := kubeClusterClient.Cluster(wt.PathForSequenceNumber(seq)).CoreV1().ConfigMaps("default") + + defer measurement.RecordElapsedDurationMS(time.Now(), s) + + ctx := context.Background() + cmName := fmt.Sprintf("loadtest-cm-%d", seq) + + // Create + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: cmName, + Namespace: "default", + }, + Data: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + } + + opStart := time.Now() + created, err := cmClient.Create(ctx, cm, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("failed to create configmap: %w", err) + } + s.Drop(measurement.Measurement{Name: "create_duration_ms", Value: time.Since(opStart).Seconds() * 1000}) + + // Get + opStart = time.Now() + _, err = cmClient.Get(ctx, cmName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get configmap: %w", err) + } + s.Drop(measurement.Measurement{Name: "get_duration_ms", Value: time.Since(opStart).Seconds() * 1000}) + + // Update + created.Data = map[string]string{ + "key1": "updated-value1", + "key2": "updated-value2", + "key3": "new-value3", + } + opStart = time.Now() + _, err = cmClient.Update(ctx, created, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("failed to update configmap: %w", err) + } + s.Drop(measurement.Measurement{Name: "update_duration_ms", Value: time.Since(opStart).Seconds() * 1000}) + + // Delete + opStart = time.Now() + err = cmClient.Delete(ctx, cmName, metav1.DeleteOptions{}) + if err != nil { + return fmt.Errorf("failed to delete configmap: %w", err) + } + s.Drop(measurement.Measurement{Name: "delete_duration_ms", Value: time.Since(opStart).Seconds() * 1000}) + + return nil + } + + section.Errors = framework.Execute(ts, action, section.Sink) + section.End() + + return section +}