OCPBUGS-59176: fix several failing tests in custom-dns jobs

Some e2e tests are failing with the job "gcp-custom-dns" for featuregate
"GCPClusterHostedDNSInstall" which is promoted to GA in 4.20. In the
"custom-dns" cluster OpenShift will start static CoreDNS pods to provide
DNS resolution for API, Internal API and Ingress services that are
essential for cluster creation. After cluster deployment is completed,
the customer will update their external DNS solution with the same
assigned LB IP addresses used for the configuration of the internal
CoreDNS instance.

The failing tests like http2 and grpc tests use dedicated
ingresscontrollers, and gateway also has separated LB and dnsrecord, so
the default wildcard created by the new static CoreDNS won't work for
those tests.

Make below changes to fix the failing tests:

- Update makeHTTPClient() DialContext to target LoadBalancer IP address
  directly when DNS is unmanaged, fixing both http2 and gatewayapi
  httproute tests.
- Update grpc Dial() to target LoadBalancer IP address directly when DNS
  is unmanaged, fixing the grpc test.
- Extract getLoadBalancerAddress() as a reusable helper from
  assertGatewayLoadbalancerReady() for use across http2, grpc and
  gatewayapi tests.
- Update gatewayapi assertDNSRecordStatus() to check the Published
  condition per-zone based on whether DNS is managed or unmanaged.
- Add isDNSManaged() helper that checks the DNSManaged condition on the
  default ingresscontroller, defaulting to managed=true when the
  condition is absent.
- Add getClusterBaseDomainName() and update http2/grpc/h2spec shard
  ingresscontroller to generate domain based on cluster baseDomain
  instead of ingress domain (e.g. "e2e-test-xxx.apps.baseDomain"),
  avoiding custom ingresscontroller DNS overlapping with default
  wildcard "*.apps.baseDomain".

Author: sadasu

test/extended/router/gatewayapi_upgrade.go

@@ -140,7 +140,7 @@ func (t *GatewayAPIUpgradeTest) Setup(ctx context.Context, f *e2e.Framework) {
 
 	if t.loadBalancerSupported && t.managedDNS {
 		g.By("Verifying HTTP connectivity before upgrade")
-		assertHttpRouteConnection(t.hostname)
+		assertHttpRouteConnection(t.oc, t.gatewayName+"-openshift-default", t.hostname)
 		e2e.Logf("HTTPRoute connectivity verified before upgrade")
 	}
 }
@@ -184,7 +184,7 @@ func (t *GatewayAPIUpgradeTest) Test(ctx context.Context, f *e2e.Framework, done
 
 	if t.loadBalancerSupported && t.managedDNS {
 		g.By("Verifying HTTP connectivity after upgrade")
-		assertHttpRouteConnection(t.hostname)
+		assertHttpRouteConnection(t.oc, t.gatewayName+"-openshift-default", t.hostname)
 	}
 
 	if migrationOccurred {

test/extended/router/gatewayapicontroller.go

@@ -2,7 +2,6 @@ package router
 
 import (
 	"context"
-	"crypto/tls"
 	"errors"
 	"fmt"
 	"net"
@@ -404,9 +403,7 @@ var _ = g.Describe("[sig-network-edge][OCPFeatureGate:GatewayAPIController][Feat
 		assertHttpRouteSuccessful(oc, gw, "test-httproute")
 
 		g.By("Validating the http connectivity to the backend application")
-		if loadBalancerSupported && managedDNS {
-			assertHttpRouteConnection(defaultRoutename)
-		}
+		assertHttpRouteConnection(oc, gw+"-openshift-default", defaultRoutename)
 	})
 
 	g.It("Ensure GIE is enabled after creating an inferencePool CRD", func() {
@@ -612,20 +609,14 @@ func getPlatformCapabilities(oc *exutil.CLI) (loadBalancerSupported bool, manage
 		loadBalancerSupported = false
 	}
 
-	managedDNS = isDNSManaged(oc)
+	var err error
+	managedDNS, err = isDNSManaged(oc, time.Minute)
+	o.Expect(err).NotTo(o.HaveOccurred(), "Failed to check if DNS is managed")
 
 	e2e.Logf("Platform: %s, LoadBalancer supported: %t, DNS managed: %t", infra.Status.PlatformStatus.Type, loadBalancerSupported, managedDNS)
 	return loadBalancerSupported, managedDNS
 }
 
-// isDNSManaged checks if the cluster has DNS zones configured (public or private).
-// On platforms like vSphere without external DNS, DNS records cannot be managed.
-func isDNSManaged(oc *exutil.CLI) bool {
-	dnsConfig, err := oc.AdminConfigClient().ConfigV1().DNSes().Get(context.Background(), "cluster", metav1.GetOptions{})
-	o.Expect(err).NotTo(o.HaveOccurred(), "Failed to get DNS config")
-	return dnsConfig.Spec.PrivateZone != nil || dnsConfig.Spec.PublicZone != nil
-}
-
 // isIPv6OrDualStack checks if the cluster is using IPv6 or dual-stack networking.
 // Returns true if any ServiceNetwork CIDR is IPv6 (indicates IPv6-only or dual-stack).
 func isIPv6OrDualStack(oc *exutil.CLI) (bool, error) {
@@ -753,18 +744,17 @@ func buildGateway(name, namespace, gcname, fromNs, domain string) *gatewayapiv1.
 	}
 }
 
-// assertGatewayLoadbalancerReady verifies that the given gateway has the service's load balancer address assigned.
-func assertGatewayLoadbalancerReady(oc *exutil.CLI, gwName, gwServiceName string) {
-	// check gateway LB service, note that External-IP might be hostname (AWS) or IP (Azure/GCP)
+// return LoadBalancer service address, note that External-IP might be hostname (AWS) or IP (Azure/GCP)
+func getLoadBalancerAddress(oc *exutil.CLI, serviceName string) string {
 	var lbAddress string
 	err := wait.PollUntilContextTimeout(context.Background(), 1*time.Second, loadBalancerReadyTimeout, false, func(context context.Context) (bool, error) {
-		lbService, err := oc.AdminKubeClient().CoreV1().Services(ingressNamespace).Get(context, gwServiceName, metav1.GetOptions{})
+		lbService, err := oc.AdminKubeClient().CoreV1().Services(ingressNamespace).Get(context, serviceName, metav1.GetOptions{})
 		if err != nil {
-			e2e.Logf("Failed to get service %q: %v, retrying...", gwServiceName, err)
+			e2e.Logf("Failed to get service %q: %v, retrying...", serviceName, err)
 			return false, nil
 		}
 		if len(lbService.Status.LoadBalancer.Ingress) == 0 {
-			e2e.Logf("Service %q has no load balancer; retrying...", gwServiceName)
+			e2e.Logf("Service %q has no load balancer; retrying...", serviceName)
 			return false, nil
 		}
 		if lbService.Status.LoadBalancer.Ingress[0].Hostname != "" {
@@ -773,11 +763,20 @@ func assertGatewayLoadbalancerReady(oc *exutil.CLI, gwName, gwServiceName string
 			lbAddress = lbService.Status.LoadBalancer.Ingress[0].IP
 		}
 		if lbAddress == "" {
-			e2e.Logf("No load balancer address for service %q, retrying", gwServiceName)
+			e2e.Logf("No load balancer address for service %q, retrying", serviceName)
 			return false, nil
 		}
-		e2e.Logf("Got load balancer address for service %q: %v", gwServiceName, lbAddress)
+		return true, nil
+	})
+	o.Expect(err).NotTo(o.HaveOccurred(), "Timed out to get load balancer address of service %q", serviceName)
+	e2e.Logf("Got load balancer address for service %q: %v", serviceName, lbAddress)
+	return lbAddress
+}
 
+// assertGatewayLoadbalancerReady verifies that the given gateway has the service's load balancer address assigned.
+func assertGatewayLoadbalancerReady(oc *exutil.CLI, gwName, gwServiceName string) {
+	lbAddress := getLoadBalancerAddress(oc, gwServiceName)
+	err := wait.PollUntilContextTimeout(context.Background(), 1*time.Second, loadBalancerReadyTimeout, false, func(context context.Context) (bool, error) {
 		gw, err := oc.AdminGatewayApiClient().GatewayV1().Gateways(ingressNamespace).Get(context, gwName, metav1.GetOptions{})
 		if err != nil {
 			e2e.Logf("Failed to get gateway %q: %v; retrying...", err, gwName)
@@ -795,10 +794,17 @@ func assertGatewayLoadbalancerReady(oc *exutil.CLI, gwName, gwServiceName string
 	o.Expect(err).NotTo(o.HaveOccurred(), "Timed out waiting for gateway %q to get load balancer address of service %q", gwName, gwServiceName)
 }
 
-// assertDNSRecordStatus polls until the DNSRecord's status in the default operand namespace is True.
+// assertDNSRecordStatus polls until the DNSRecord's status in the default operand namespace is ready.
+// When DNS is managed, it waits for all zones to have Published=True.
+// When DNS is unmanaged, it waits for all zones to have Published!=True (expected in custom-dns clusters).
 func assertDNSRecordStatus(oc *exutil.CLI, gatewayName string) {
-	// find the DNS Record and confirm its zone status is True
-	err := wait.PollUntilContextTimeout(context.Background(), 2*time.Second, 10*time.Minute, false, func(context context.Context) (bool, error) {
+	dnsManaged, err := isDNSManaged(oc, time.Minute)
+	if err != nil {
+		e2e.Failf("Failed to get default ingresscontroller DNSManaged status: %v", err)
+	}
+
+	// find the DNS Record and confirm its zone status
+	err = wait.PollUntilContextTimeout(context.Background(), 2*time.Second, 10*time.Minute, false, func(context context.Context) (bool, error) {
 		gatewayDNSRecord := &operatoringressv1.DNSRecord{}
 		gatewayDNSRecords, err := oc.AdminIngressClient().IngressV1().DNSRecords(ingressNamespace).List(context, metav1.ListOptions{})
 		if err != nil {
@@ -810,20 +816,43 @@ func assertDNSRecordStatus(oc *exutil.CLI, gatewayName string) {
 		for _, record := range gatewayDNSRecords.Items {
 			if record.Labels["gateway.networking.k8s.io/gateway-name"] == gatewayName {
 				gatewayDNSRecord = &record
+				e2e.Logf("Found the desired dnsrecord and spec is: %v", gatewayDNSRecord.Spec)
 				break
 			}
 		}
 
-		// checking the gateway DNS record status
+		if len(gatewayDNSRecord.Status.Zones) == 0 {
+			e2e.Logf("DNS record %q has no zones yet, retrying...", gatewayDNSRecord.Name)
+			return false, nil
+		}
+
+		// Check the Published condition for each zone
 		for _, zone := range gatewayDNSRecord.Status.Zones {
+			published := false
 			for _, condition := range zone.Conditions {
-				if condition.Type == "Published" && condition.Status == "True" {
-					return true, nil
+				if condition.Type != "Published" {
+					continue
+				}
+				e2e.Logf("The published status is %v for zone %v", condition.Status, zone.DNSZone)
+				if dnsManaged && condition.Status != "True" {
+					e2e.Logf("DNS record %q zone %v is not published yet, retrying...", gatewayDNSRecord.Name, zone.DNSZone)
+					return false, nil
+				}
+				if !dnsManaged && condition.Status == "True" {
+					e2e.Logf("DNS record %q zone %v is unexpectedly published for unmanaged DNS, retrying...", gatewayDNSRecord.Name, zone.DNSZone)
+					return false, nil
 				}
+				published = true
+				break
+			}
+			if !published {
+				e2e.Logf("DNS record %q zone %v has no Published condition, retrying...", gatewayDNSRecord.Name, zone.DNSZone)
+				return false, nil
 			}
 		}
-		e2e.Logf("DNS record %q is not ready, retrying...", gatewayDNSRecord.Name)
-		return false, nil
+
+		e2e.Logf("All zones are checked and DNS record %q is ready", gatewayDNSRecord.Name)
+		return true, nil
 	})
 	o.Expect(err).NotTo(o.HaveOccurred(), "Timed out waiting for gateway %q DNSRecord to become ready", gatewayName)
 }
@@ -1041,24 +1070,28 @@ func assertHttpRouteSuccessful(oc *exutil.CLI, gwName, name string) (*gatewayapi
 
 // assertHttpRouteConnection checks if the http route of the given name replies successfully,
 // and returns an error if not
-func assertHttpRouteConnection(hostname string) {
-	// Create the http client to check the response status code.
-	client := &http.Client{
-		Timeout: 10 * time.Second,
-		Transport: &http.Transport{
-			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
-		},
+func assertHttpRouteConnection(oc *exutil.CLI, gwServiceName, hostname string) {
+	isDNSManaged, err := isDNSManaged(oc, time.Minute)
+	if err != nil {
+		e2e.Failf("Failed to get default ingresscontroller DNSManaged status: %v", err)
+	}
+	lbAddress := ""
+	if isDNSManaged {
+		err := wait.PollUntilContextTimeout(context.Background(), 20*time.Second, dnsResolutionTimeout, false, func(context context.Context) (bool, error) {
+			_, err := net.LookupHost(hostname)
+			if err != nil {
+				e2e.Logf("[%v] Failed to resolve HTTP route's hostname %q: %v, retrying...", time.Now(), hostname, err)
+				return false, nil
+			}
+			return true, nil
+		})
+		o.Expect(err).NotTo(o.HaveOccurred(), "Timed out waiting for HTTP route's hostname %q to be resolved: %v", hostname, err)
+	} else {
+		lbAddress = getLoadBalancerAddress(oc, gwServiceName)
 	}
 
-	err := wait.PollUntilContextTimeout(context.Background(), 20*time.Second, dnsResolutionTimeout, false, func(context context.Context) (bool, error) {
-		_, err := net.LookupHost(hostname)
-		if err != nil {
-			e2e.Logf("[%v] Failed to resolve HTTP route's hostname %q: %v, retrying...", time.Now(), hostname, err)
-			return false, nil
-		}
-		return true, nil
-	})
-	o.Expect(err).NotTo(o.HaveOccurred(), "Timed out waiting for HTTP route's hostname %q to be resolved: %v", hostname, err)
+	// Create the http client to check the response status code.
+	client := makeHTTPClient(false, 10*time.Second, lbAddress)
 
 	// Wait for http route to respond, and when it does, check for the status code.
 	err = wait.PollUntilContextTimeout(context.Background(), 5*time.Second, 5*time.Minute, false, func(context context.Context) (bool, error) {

test/extended/router/grpc-interop.go

@@ -53,8 +53,8 @@ var _ = g.Describe("[sig-network-edge][Conformance][Area:Networking][Feature:Rou
 				g.Skip("Skip on platforms where the default router is not exposed by a load balancer service.")
 			}
 
-			defaultDomain, err := getDefaultIngressClusterDomainName(oc, time.Minute)
-			o.Expect(err).NotTo(o.HaveOccurred(), "failed to find default domain name")
+			baseDomain, err := getClusterBaseDomainName(oc, time.Minute)
+			o.Expect(err).NotTo(o.HaveOccurred(), "failed to find base domain name")
 
 			g.By("Locating the canary image reference")
 			image, err := getCanaryImage(oc)
@@ -196,7 +196,7 @@ var _ = g.Describe("[sig-network-edge][Conformance][Area:Networking][Feature:Rou
 			pemCrt2, err := certgen.MarshalCertToPEMString(tlsCrt2Data)
 			o.Expect(err).NotTo(o.HaveOccurred())
 
-			shardFQDN := oc.Namespace() + "." + defaultDomain
+			shardFQDN := oc.Namespace() + "." + baseDomain
 
 			g.By("Creating routes to test for gRPC interoperability")
 			routeType := oc.Namespace()
@@ -330,6 +330,15 @@ var _ = g.Describe("[sig-network-edge][Conformance][Area:Networking][Feature:Rou
 			o.Expect(shardService).NotTo(o.BeNil())
 			o.Expect(shardService.Status.LoadBalancer.Ingress).To(o.Not(o.BeEmpty()))
 
+			isDNSManaged, err := isDNSManaged(oc, time.Minute)
+			if err != nil {
+				e2e.Failf("Failed to get default ingresscontroller DNSManaged status: %v", err)
+			}
+			lbAddress := ""
+			if !isDNSManaged {
+				lbAddress = getLoadBalancerAddress(oc, "router-"+oc.Namespace())
+			}
+
 			testCases := []string{
 				"cancel_after_begin",
 				"cancel_after_first_response",
@@ -352,15 +361,15 @@ var _ = g.Describe("[sig-network-edge][Conformance][Area:Networking][Feature:Rou
 				routev1.TLSTerminationReencrypt,
 				routev1.TLSTerminationPassthrough,
 			} {
-				err := grpcExecTestCases(oc, routeType, 5*time.Minute, testCases...)
+				err := grpcExecTestCases(oc, routeType, 5*time.Minute, lbAddress, testCases...)
 				o.Expect(err).NotTo(o.HaveOccurred())
 			}
 		})
 	})
 })
 
 // grpcExecTestCases run gRPC interop test cases.
-func grpcExecTestCases(oc *exutil.CLI, routeType routev1.TLSTerminationType, timeout time.Duration, testCases ...string) error {
+func grpcExecTestCases(oc *exutil.CLI, routeType routev1.TLSTerminationType, timeout time.Duration, lbAddress string, testCases ...string) error {
 	host, err := getHostnameForRoute(oc, fmt.Sprintf("grpc-interop-%s", routeType))
 	if err != nil {
 		return err
@@ -371,6 +380,7 @@ func grpcExecTestCases(oc *exutil.CLI, routeType routev1.TLSTerminationType, tim
 		Port:     443,
 		UseTLS:   true,
 		Insecure: true,
+		Target:   lbAddress,
 	}
 
 	if routeType == "h2c" {

test/extended/router/grpc-interop/clientconn.go

@@ -1,9 +1,11 @@
 package grpc_interop
 
 import (
+	"context"
 	"crypto/tls"
 	"crypto/x509"
 	"errors"
+	"fmt"
 	"net"
 	"strconv"
 
@@ -17,6 +19,8 @@ type DialParams struct {
 	Host     string
 	Port     int
 	Insecure bool
+	// Target is the actual IP you want to dial instead of resolving hostname
+	Target string
 }
 
 func Dial(cfg DialParams) (*grpc.ClientConn, error) {
@@ -44,5 +48,18 @@ func Dial(cfg DialParams) (*grpc.ClientConn, error) {
 		opts = append(opts, grpc.WithInsecure())
 	}
 
+	if cfg.Target != "" {
+		dialer := func(ctx context.Context, addr string) (net.Conn, error) {
+			_, port, err := net.SplitHostPort(addr)
+			if err != nil {
+				return nil, fmt.Errorf("failed to split host:port from %q: %w", addr, err)
+			}
+			// Connect to targetIP:port regardless of hostname
+			dest := net.JoinHostPort(cfg.Target, port)
+			return (&net.Dialer{}).DialContext(ctx, "tcp", dest)
+		}
+		opts = append(opts, grpc.WithContextDialer(dialer))
+	}
+
 	return grpc.Dial(net.JoinHostPort(cfg.Host, strconv.Itoa(cfg.Port)), append(opts, grpc.WithBlock())...)
 }

test/extended/router/h2spec.go

@@ -80,9 +80,9 @@ var _ = g.Describe("[sig-network-edge][Conformance][Area:Networking][Feature:Rou
 				}
 			}
 
-			g.By("Getting the default domain")
-			defaultDomain, err := getDefaultIngressClusterDomainName(oc, time.Minute)
-			o.Expect(err).NotTo(o.HaveOccurred(), "failed to find default domain name")
+			g.By("Getting the base domain")
+			baseDomain, err := getClusterBaseDomainName(oc, time.Minute)
+			o.Expect(err).NotTo(o.HaveOccurred(), "failed to find base domain name")
 
 			g.By("Locating the router image reference")
 			routerImage, err := exutil.FindRouterImage(oc)
@@ -409,7 +409,7 @@ BFNBRELPe53ZdLKWpf2Sr96vRPRNw
 			e2e.ExpectNoError(e2epod.WaitForPodNameRunningInNamespace(context.TODO(), oc.KubeClient(), "h2spec-haproxy", oc.KubeFramework().Namespace.Name))
 			e2e.ExpectNoError(e2epod.WaitForPodNameRunningInNamespace(context.TODO(), oc.KubeClient(), "h2spec", oc.KubeFramework().Namespace.Name))
 
-			shardFQDN := oc.Namespace() + "." + defaultDomain
+			shardFQDN := oc.Namespace() + "." + baseDomain
 
 			// The new router shard is using a namespace
 			// selector so label this test namespace to