fix: retry transient API errors in wait/poll loops instead of aborting

All four wait.Poll* call sites treated any non-NotFound error from Get
as a fatal poll error, causing PollUntilContext* to abort immediately
rather than retrying. This meant transient Kubernetes API errors (e.g.
Internal, ServerTimeout, network blips) would terminate the poll loop
well before the configured timeout was reached.

Change all four sites to treat non-NotFound errors as retryable by
returning (false, nil) instead of (false, err). If the error persists,
the poll will eventually time out as intended.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: porridge

internal/kubernetes/wait.go

@@ -21,8 +21,11 @@ func WaitForDelete(c *RetryClient, objs []runtime.Object) error {
 			actual := &unstructured.Unstructured{}
 			actual.SetGroupVersionKind(obj.GetObjectKind().GroupVersionKind())
 			err = c.Get(ctx, ObjectKey(obj), actual)
-			if err == nil || !errors.IsNotFound(err) {
-				return false, err
+			// Retry on transient API errors (return nil to keep polling) rather
+			// than aborting the entire wait, which would surface as a misleading
+			// "timed out" error well before the real deadline.
+			if !errors.IsNotFound(err) {
+				return false, nil
 			}
 		}
 
@@ -50,8 +53,9 @@ func WaitForSA(config *rest.Config, name, namespace string) error {
 		if errors.IsNotFound(err) {
 			return false, nil
 		}
+		// Retry on transient API errors rather than aborting the wait.
 		if err != nil {
-			return false, err
+			return false, nil
 		}
 		return true, nil
 	})

internal/kubernetes/wait_test.go

@@ -0,0 +1,82 @@
+package kubernetes
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
+)
+
+func testObj() runtime.Object {
+	obj := &unstructured.Unstructured{}
+	obj.SetGroupVersionKind(schema.GroupVersionKind{Group: "", Version: "v1", Kind: "ConfigMap"})
+	obj.SetName("cm")
+	obj.SetNamespace("default")
+	return obj
+}
+
+func TestWaitForDelete_AlreadyGone(t *testing.T) {
+	cl := fake.NewClientBuilder().Build()
+	rc := &RetryClient{Client: cl}
+
+	err := WaitForDelete(rc, []runtime.Object{testObj()})
+	require.NoError(t, err)
+}
+
+func TestWaitForDelete_TransientErrorThenGone(t *testing.T) {
+	callCount := 0
+	cl := fake.NewClientBuilder().WithInterceptorFuncs(interceptor.Funcs{
+		Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
+			callCount++
+			if callCount <= 3 {
+				return fmt.Errorf("transient API error")
+			}
+			return k8serrors.NewNotFound(schema.GroupResource{Resource: "configmaps"}, "cm")
+		},
+	}).Build()
+	rc := &RetryClient{Client: cl}
+
+	err := WaitForDelete(rc, []runtime.Object{testObj()})
+	require.NoError(t, err)
+	assert.Greater(t, callCount, 3)
+}
+
+func TestWaitForDelete_StillExistsThenGone(t *testing.T) {
+	callCount := 0
+	cl := fake.NewClientBuilder().WithInterceptorFuncs(interceptor.Funcs{
+		Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
+			callCount++
+			if callCount <= 2 {
+				return nil
+			}
+			return k8serrors.NewNotFound(schema.GroupResource{Resource: "configmaps"}, "cm")
+		},
+	}).Build()
+	rc := &RetryClient{Client: cl}
+
+	err := WaitForDelete(rc, []runtime.Object{testObj()})
+	require.NoError(t, err)
+	assert.Greater(t, callCount, 2)
+}
+
+func TestWaitForDelete_PersistentErrorTimesOut(t *testing.T) {
+	cl := fake.NewClientBuilder().WithInterceptorFuncs(interceptor.Funcs{
+		Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
+			return fmt.Errorf("persistent API error")
+		},
+	}).Build()
+	rc := &RetryClient{Client: cl}
+
+	err := WaitForDelete(rc, []runtime.Object{testObj()})
+	assert.Error(t, err)
+	assert.ErrorIs(t, err, context.DeadlineExceeded)
+}

internal/step/step.go

@@ -179,7 +179,7 @@ func (s *Step) DeleteExisting(namespace string) error {
 			}
 			if !k8serrors.IsNotFound(err) {
 				lastCheckMsg = fmt.Sprintf("checking existence of %v %s failed: %v", obj.GetObjectKind().GroupVersionKind(), obj.GetName(), err)
-				return false, err
+				return false, nil
 			}
 		}
 

internal/testcase/case.go

@@ -197,7 +197,8 @@ func (c *Case) deleteNamespace(cl clientWithKubeConfig) error {
 			return true, nil
 		}
 		if err != nil {
-			return false, fmt.Errorf("failed to check deletion of namespace %q: %w", c.ns.name, err)
+			cl.Logf("failed to check deletion of namespace %q (will retry): %v", c.ns.name, err)
+			return false, nil
 		}
 		return false, nil
 	})