debug

Signed-off-by: Vincent Hou <vhou@coreweave.com>

Author: houshengbo

test/e2e-common.sh

@@ -23,10 +23,10 @@ readonly PREVIOUS_SERVING_RELEASE_VERSION="1.21"
 readonly PREVIOUS_EVENTING_RELEASE_VERSION="1.21"
 # The target serving/eventing release to upgrade, installed by the operator. It can be a release available under
 # kodata or an incoming new release. This value should be in the semantic format of major.minor.
-readonly TARGET_RELEASE_VERSION="1.21"
+readonly TARGET_RELEASE_VERSION="1.22"
 # This is the branch name of knative repos, where we run the upgrade tests.
 # Default to empty for local runs (not in Prow).
-readonly KNATIVE_REPO_BRANCH="release-1.21"
+readonly KNATIVE_REPO_BRANCH="release-1.22"
 # Namespaces used for tests
 # This environment variable TEST_NAMESPACE defines the namespace to install Knative Serving.
 export TEST_NAMESPACE="${TEST_NAMESPACE:-knative-operator-testing}"

test/resources/knativeresources.go

@@ -92,6 +92,29 @@ func IsKnativeDeploymentReady(dpList *v1.DeploymentList, expectedDeployments []s
 		return false
 	}
 
+	// formatDeploymentStatus returns a compact, human-readable summary of a deployment's
+	// status conditions and replica counts to help diagnose why it is not ready.
+	formatDeploymentStatus := func(d *v1.Deployment) string {
+		s := d.Status
+		conds := make([]string, 0, len(s.Conditions))
+		for _, c := range s.Conditions {
+			conds = append(conds, fmt.Sprintf("%s=%s(reason=%s,msg=%s)", c.Type, c.Status, c.Reason, c.Message))
+		}
+		return fmt.Sprintf("replicas(desired=%d ready=%d available=%d updated=%d unavailable=%d) conditions=%v",
+			s.Replicas, s.ReadyReplicas, s.AvailableReplicas, s.UpdatedReplicas, s.UnavailableReplicas, conds)
+	}
+
+	// formatVersionLabel extracts the version-related label from a deployment for diagnostics.
+	formatVersionLabel := func(d *v1.Deployment) string {
+		labels := d.GetLabels()
+		for _, key := range []string{"serving.knative.dev/release", "eventing.knative.dev/release", "app.kubernetes.io/version", "networking.knative.dev/ingress-provider"} {
+			if val, ok := labels[key]; ok {
+				return fmt.Sprintf("%s=%s", key, val)
+			}
+		}
+		return "<no version label>"
+	}
+
 	isReady := func(d *v1.Deployment) bool {
 		for key, val := range d.GetObjectMeta().GetLabels() {
 			// Check if the version matches. As long as we find a value equals to the version, we can determine
@@ -139,14 +162,19 @@ func IsKnativeDeploymentReady(dpList *v1.DeploymentList, expectedDeployments []s
 		return false
 	}
 
+	logf("[deployment-check] version=%q existingVersion=%q expectedDeployments=%v",
+		version, existingVersion, expectedDeployments)
+
 	for _, name := range expectedDeployments {
 		dep := findDeployment(name, dpList.Items)
 		if dep == nil {
-			logf("The deployment %v is not found.", name)
+			logf("[deployment-check] deployment %q not found in namespace (found %d total deployments)",
+				name, len(dpList.Items))
 			return false, nil
 		}
 		if !isReady(dep) {
-			logf("The deployment %v is not ready.", dep.Name)
+			logf("[deployment-check] deployment %q not ready: versionLabel=%s status=%s",
+				dep.Name, formatVersionLabel(dep), formatDeploymentStatus(dep))
 			return false, nil
 		}
 	}

test/resources/verify.go

@@ -313,6 +313,68 @@ func AssertKnativeObsoleteResource(t *testing.T, clients *test.Clients, namespac
 func AssertKnativeDeploymentStatus(t *testing.T, clients *test.Clients, namespace string, version string, existingVersion string, expectedDeployments []string) {
 	if err := WaitForKnativeDeploymentState(clients, namespace, version, existingVersion, expectedDeployments, t.Logf,
 		IsKnativeDeploymentReady); err != nil {
+
+		t.Logf("=== DEPLOYMENT STATUS SNAPSHOT (namespace=%s, version=%q, existingVersion=%q) ===",
+			namespace, version, existingVersion)
+
+		// Dump all deployments in the namespace with full status.
+		dpList, listErr := clients.KubeClient.AppsV1().Deployments(namespace).List(context.TODO(), metav1.ListOptions{})
+		if listErr != nil {
+			t.Logf("  failed to list deployments: %v", listErr)
+		} else {
+			t.Logf("  total deployments found: %d", len(dpList.Items))
+			for _, d := range dpList.Items {
+				conds := make([]string, 0, len(d.Status.Conditions))
+				for _, c := range d.Status.Conditions {
+					conds = append(conds, fmt.Sprintf("%s=%s(reason=%s,msg=%s)", c.Type, c.Status, c.Reason, c.Message))
+				}
+				versionLabelVal := "<none>"
+				for _, key := range []string{"serving.knative.dev/release", "eventing.knative.dev/release", "app.kubernetes.io/version"} {
+					if val, ok := d.Labels[key]; ok {
+						versionLabelVal = fmt.Sprintf("%s=%s", key, val)
+						break
+					}
+				}
+				t.Logf("  deployment %q: versionLabel=%s replicas(desired=%d ready=%d available=%d updated=%d unavailable=%d) conditions=%v",
+					d.Name, versionLabelVal,
+					d.Status.Replicas, d.Status.ReadyReplicas, d.Status.AvailableReplicas,
+					d.Status.UpdatedReplicas, d.Status.UnavailableReplicas, conds)
+			}
+		}
+
+		// Dump pods in the namespace to expose CrashLoopBackOff / ImagePullBackOff / Pending.
+		podList, podErr := clients.KubeClient.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{})
+		if podErr != nil {
+			t.Logf("  failed to list pods: %v", podErr)
+		} else {
+			t.Logf("  total pods found: %d", len(podList.Items))
+			for _, p := range podList.Items {
+				containerStates := make([]string, 0, len(p.Status.ContainerStatuses))
+				for _, cs := range p.Status.ContainerStatuses {
+					state := "running"
+					if cs.State.Waiting != nil {
+						state = fmt.Sprintf("waiting(reason=%s,msg=%s)", cs.State.Waiting.Reason, cs.State.Waiting.Message)
+					} else if cs.State.Terminated != nil {
+						state = fmt.Sprintf("terminated(reason=%s,exitCode=%d)", cs.State.Terminated.Reason, cs.State.Terminated.ExitCode)
+					}
+					containerStates = append(containerStates, fmt.Sprintf("%s:%s(ready=%v,restarts=%d)", cs.Name, state, cs.Ready, cs.RestartCount))
+				}
+				t.Logf("  pod %q: phase=%s containers=%v", p.Name, p.Status.Phase, containerStates)
+			}
+		}
+
+		// Dump KnativeServing CR status conditions to show operator perspective.
+		ksList, ksErr := clients.KnativeServingAll().List(context.TODO(), metav1.ListOptions{})
+		if ksErr != nil {
+			t.Logf("  failed to list KnativeServings: %v", ksErr)
+		} else {
+			for _, ks := range ksList.Items {
+				conds := ks.Status.GetConditions()
+				t.Logf("  KnativeServing %s/%s: version=%q observedGen=%d conditions=%v",
+					ks.Namespace, ks.Name, ks.Status.Version, ks.Status.ObservedGeneration, conds)
+			}
+		}
+
 		t.Fatalf("Knative Serving deployments failed to meet the expected deployments: %v", err)
 	}
 }