Add list all-tests command for aggregated test listing

pkg/cmd/openshift-tests/list/all_tests.go

@@ -0,0 +1,157 @@
+package list
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"sort"
+
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+	"gopkg.in/yaml.v3"
+	"k8s.io/cli-runtime/pkg/genericclioptions"
+	"k8s.io/kubectl/pkg/util/templates"
+
+	"github.com/openshift/origin/pkg/test/extensions"
+	"github.com/openshift/origin/pkg/testsuites"
+)
+
+// resolveSuiteQualifiers finds a suite by name, first checking origin's internal
+// suites, then checking suites advertised by the already-extracted extension binaries.
+func resolveSuiteQualifiers(ctx context.Context, suiteName string, binaries extensions.TestBinaries) ([]string, error) {
+	for _, s := range testsuites.InternalTestSuites() {
+		if s.Name == suiteName {
+			return s.Qualifiers, nil
+		}
+	}
+
+	extensionInfos, err := binaries.Info(ctx, 4)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get extension info: %w", err)
+	}
+	for _, e := range extensionInfos {
+		for _, s := range e.Suites {
+			if s.Name == suiteName {
+				return s.Qualifiers, nil
+			}
+		}
+	}
+
+	return nil, fmt.Errorf("suite %q not found", suiteName)
+}
+
+func NewListAllTestsCommand(streams genericclioptions.IOStreams) *cobra.Command {
+	var suiteName string
+
+	cmd := &cobra.Command{
+		Use:   "all-tests",
+		Short: "List tests from all extension binaries",
+		Long: templates.LongDesc(`
+		List all tests discovered from all extension binaries in the release payload.
+
+		Unlike 'list tests', which lists tests from a single extension component,
+		this command aggregates tests from all extension binaries — the same set
+		of tests that 'run' operates on.
+
+		Use --suite to filter tests by a suite's qualifiers. This works with both
+		origin-defined suites (like openshift/network/third-party) and suites
+		advertised by extension binaries.
+
+		This command does not require cluster access.
+		`),
+		SilenceUsage:  true,
+		SilenceErrors: true,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			ctx := context.Background()
+
+			const defaultBinaryParallelism = 10
+
+			extractionContext, extractionContextCancel := context.WithTimeout(ctx, 30*60*1e9)
+			defer extractionContextCancel()
+			cleanUpFn, allBinaries, _, err := extensions.ExtractAllTestBinaries(extractionContext, defaultBinaryParallelism, extensions.WithPayloadOnly())
+			if err != nil {
+				return fmt.Errorf("failed to extract test binaries: %w", err)
+			}
+			defer cleanUpFn()
+
+			infoContext, infoContextCancel := context.WithTimeout(ctx, 30*60*1e9)
+			defer infoContextCancel()
+			logrus.Infof("Fetching info from %d extension binaries", len(allBinaries))
+			if _, err := allBinaries.Info(infoContext, defaultBinaryParallelism); err != nil {
+				logrus.Warnf("Some extension binaries failed info fetch (they may require cluster access): %v", err)
+			}
+
+			// Filter to binaries that successfully returned info, since ListTests
+			// requires info to be populated. Binaries that failed info (e.g. due to
+			// missing cluster access) are skipped.
+			var availableBinaries extensions.TestBinaries
+			for _, b := range allBinaries {
+				if b.HasInfo() {
+					availableBinaries = append(availableBinaries, b)
+				}
+			}
+			logrus.Infof("%d of %d binaries available for listing", len(availableBinaries), len(allBinaries))
+
+			listContext, listContextCancel := context.WithTimeout(ctx, 10*60*1e9)
+			defer listContextCancel()
+
+			specs, err := availableBinaries.ListTests(listContext, defaultBinaryParallelism, nil)
+			if err != nil {
+				return fmt.Errorf("failed to list tests: %w", err)
+			}
+
+			logrus.Infof("Discovered %d total tests", len(specs))
+
+			if suiteName != "" {
+				qualifiers, err := resolveSuiteQualifiers(ctx, suiteName, availableBinaries)
+				if err != nil {
+					return err
+				}
+
+				specs, err = extensions.FilterWrappedSpecs(specs, qualifiers)
+				if err != nil {
+					return fmt.Errorf("failed to filter tests by suite qualifiers: %w", err)
+				}
+
+				logrus.Infof("Suite %q selected %d tests", suiteName, len(specs))
+			}
+
+			outputFormat, err := cmd.Flags().GetString("output")
+			if err != nil {
+				return errors.Wrapf(err, "error accessing flag output for command %s", cmd.Name())
+			}
+
+			sort.Slice(specs, func(i, j int) bool {
+				return specs[i].Name < specs[j].Name
+			})
+
+			switch outputFormat {
+			case "":
+				for _, spec := range specs {
+					fmt.Fprintln(streams.Out, spec.Name)
+				}
+			case "json":
+				data, err := json.MarshalIndent(specs, "", "  ")
+				if err != nil {
+					return fmt.Errorf("failed to marshal tests to JSON: %w", err)
+				}
+				fmt.Fprintln(streams.Out, string(data))
+			case "yaml":
+				data, err := yaml.Marshal(specs)
+				if err != nil {
+					return fmt.Errorf("failed to marshal tests to YAML: %w", err)
+				}
+				fmt.Fprintln(streams.Out, string(data))
+			default:
+				return fmt.Errorf("invalid output format: %s", outputFormat)
+			}
+
+			return nil
+		},
+	}
+
+	cmd.Flags().StringVar(&suiteName, "suite", "", "Filter tests by the qualifiers of the specified suite")
+	cmd.Flags().StringP("output", "o", "", "Output format; available options are 'yaml' and 'json'")
+	return cmd
+}

pkg/cmd/openshift-tests/list/root.go

@@ -22,6 +22,7 @@ func NewListCommand(streams genericclioptions.IOStreams, extensionRegistry *exte
 	oteListCmd.AddCommand(
 		NewListSuitesCommand(streams),
 		NewListExtensionsCommand(streams),
+		NewListAllTestsCommand(streams),
 	)
 
 	return oteListCmd

pkg/test/extensions/binary.go

@@ -171,6 +171,16 @@ type TestBinary struct {
 	info *Extension
 }
 
+// Name returns the base name of the binary.
+func (b *TestBinary) Name() string {
+	return filepath.Base(b.binaryPath)
+}
+
+// HasInfo returns true if Info() has been successfully called on this binary.
+func (b *TestBinary) HasInfo() bool {
+	return b.info != nil
+}
+
 // UnpermittedExtension describes a discovered non-payload extension that is not permitted by any TestExtensionAdmission.
 // Used to generate a synthetic skip test.
 type UnpermittedExtension struct {
@@ -233,10 +243,6 @@ var extensionBinaries = []TestBinary{
 		imageTag:   "cluster-cloud-controller-manager-operator",
 		binaryPath: "/usr/bin/cloud-controller-manager-aws-tests-ext.gz",
 	},
-	{
-		imageTag:   "cluster-cloud-controller-manager-operator",
-		binaryPath: "/usr/bin/cloud-controller-manager-operator-tests-ext.gz",
-	},
 	{
 		imageTag:   "cluster-config-operator",
 		binaryPath: "/usr/bin/cluster-config-operator-tests-ext.gz",
@@ -614,10 +620,31 @@ func (b *TestBinary) ListImages(ctx context.Context) (ImageSet, error) {
 	return result, nil
 }
 
+// ExtractionOption configures the behavior of ExtractAllTestBinaries.
+type ExtractionOption func(*extractionOptions)
+
+type extractionOptions struct {
+	payloadOnly bool
+}
+
+// WithPayloadOnly skips non-payload extension discovery, which requires cluster
+// access. When set, only payload test binaries are extracted. Registry auth must
+// be provided via REGISTRY_AUTH_FILE or CI cluster profile.
+func WithPayloadOnly() ExtractionOption {
+	return func(o *extractionOptions) {
+		o.payloadOnly = true
+	}
+}
+
 // ExtractAllTestBinaries determines the optimal release payload to use, and extracts all the external
 // test binaries from it (payload + permitted non-payload), and returns cleanup, binaries, and any
 // unpermitted non-payload extensions for synthetic skip tests.
-func ExtractAllTestBinaries(ctx context.Context, parallelism int) (func(), TestBinaries, []UnpermittedExtension, error) {
+func ExtractAllTestBinaries(ctx context.Context, parallelism int, opts ...ExtractionOption) (func(), TestBinaries, []UnpermittedExtension, error) {
+	var options extractionOptions
+	for _, opt := range opts {
+		opt(&options)
+	}
+
 	if len(os.Getenv("OPENSHIFT_SKIP_EXTERNAL_TESTS")) > 0 {
 		logrus.Warning("Using built-in tests only due to OPENSHIFT_SKIP_EXTERNAL_TESTS being set")
 		var internalBinaries []*TestBinary
@@ -649,8 +676,14 @@ func ExtractAllTestBinaries(ctx context.Context, parallelism int) (func(), TestB
 
 	defer os.RemoveAll(tmpDir)
 
-	oc := exutil.NewCLIWithoutNamespace("default")
-	registryAuthFilePath, err := DetermineRegistryAuthFilePath(tmpDir, oc)
+	var registryAuthFilePath string
+	var oc *exutil.CLI
+	if options.payloadOnly {
+		registryAuthFilePath, err = DetermineRegistryAuthFilePathWithoutCluster(tmpDir)
+	} else {
+		oc = exutil.NewCLIWithoutNamespace("default")
+		registryAuthFilePath, err = DetermineRegistryAuthFilePath(tmpDir, oc)
+	}
 	if err != nil {
 		return nil, nil, nil, fmt.Errorf("failed to determine registry auth file path: %w", err)
 	}
@@ -660,14 +693,18 @@ func ExtractAllTestBinaries(ctx context.Context, parallelism int) (func(), TestB
 		return nil, nil, nil, errors.WithMessage(err, "could not create external binary provider")
 	}
 
-	permitPatterns, err := DiscoverNonPayloadBinaryAdmission(ctx, oc.AdminConfig())
-	if err != nil {
-		logrus.Warnf("Skipping non-payload extension discovery (admission check failed): %v", err)
-		permitPatterns = nil
-	}
-	permittedNonPayload, unpermittedNonPayload, err := discoverNonPayloadExtensions(ctx, oc, permitPatterns)
-	if err != nil {
-		logrus.Warnf("Non-payload extension discovery failed: %v", err)
+	var permittedNonPayload []nonPayloadSource
+	var unpermittedNonPayload []UnpermittedExtension
+	if !options.payloadOnly {
+		permitPatterns, err := DiscoverNonPayloadBinaryAdmission(ctx, oc.AdminConfig())
+		if err != nil {
+			logrus.Warnf("Skipping non-payload extension discovery (admission check failed): %v", err)
+			permitPatterns = nil
+		}
+		permittedNonPayload, unpermittedNonPayload, err = discoverNonPayloadExtensions(ctx, oc, permitPatterns)
+		if err != nil {
+			logrus.Warnf("Non-payload extension discovery failed: %v", err)
+		}
 	}
 
 	var (

pkg/test/extensions/util.go

@@ -289,6 +289,28 @@ func ExtractReleaseImageStream(extractPath, releaseImage string,
 	return is, releaseImage, nil
 }
 
+// DetermineRegistryAuthFilePathWithoutCluster resolves registry auth without
+// cluster access. It checks REGISTRY_AUTH_FILE and the CI cluster profile path,
+// falling back to unauthenticated access if neither is available.
+func DetermineRegistryAuthFilePathWithoutCluster(tmpDir string) (string, error) {
+	registryAuthFilePath := os.Getenv("REGISTRY_AUTH_FILE")
+	if len(registryAuthFilePath) != 0 {
+		logrus.Infof("Using REGISTRY_AUTH_FILE environment variable: %v", registryAuthFilePath)
+		return registryAuthFilePath, nil
+	}
+
+	ciProfilePullSecretPath := "/run/secrets/ci.openshift.io/cluster-profile/pull-secret"
+	if _, err := os.Stat(ciProfilePullSecretPath); err == nil {
+		logrus.Infof("Detected %v; using cluster profile for image access", ciProfilePullSecretPath)
+		return ciProfilePullSecretPath, nil
+	} else if !os.IsNotExist(err) {
+		return "", fmt.Errorf("failed to check for CI pull secret at %s: %w", ciProfilePullSecretPath, err)
+	}
+
+	logrus.Warningf("No REGISTRY_AUTH_FILE or cluster profile pull-secret found; falling back to default container credentials")
+	return "", nil
+}
+
 func DetermineRegistryAuthFilePath(tmpDir string, oc *util.CLI) (string, error) {
 	// To extract binaries bearing external tests, we must inspect the release
 	// payload under tests as well as extract content from component images

test/extended/extension/payload_compliance.go

@@ -0,0 +1,53 @@
+package extension
+
+import (
+	"context"
+	"time"
+
+	g "github.com/onsi/ginkgo/v2"
+	o "github.com/onsi/gomega"
+
+	"github.com/openshift/origin/pkg/test/extensions"
+	"k8s.io/apimachinery/pkg/util/sets"
+)
+
+// knownInfoFailures lists extension binaries that are known to fail the
+// "info" command without cluster access. Each entry should have a tracking
+// issue for fixing the upstream binary. Remove entries as fixes land.
+var knownInfoFailures = sets.New[string](
+	"ovn-kubernetes-tests-ext",               // https://github.com/openshift/ovn-kubernetes/pull/3170
+	"cloud-controller-manager-aws-tests-ext", // https://github.com/openshift/cluster-cloud-controller-manager-operator/pull/458
+)
+
+var _ = g.Describe("[sig-ci] [OTE] Payload extension binaries [Suite:openshift/conformance/parallel]", func() {
+	defer g.GinkgoRecover()
+
+	g.It("should all respond to the info command", func(ctx context.Context) {
+		extractCtx, extractCancel := context.WithTimeout(ctx, 30*time.Minute)
+		defer extractCancel()
+
+		cleanUpFn, allBinaries, _, err := extensions.ExtractAllTestBinaries(extractCtx, 10, extensions.WithPayloadOnly())
+		o.Expect(err).NotTo(o.HaveOccurred(), "failed to extract test binaries from payload")
+		defer cleanUpFn()
+
+		var failures []string
+		for _, binary := range allBinaries {
+			binName := binary.Name()
+			infoCtx, infoCancel := context.WithTimeout(ctx, 10*time.Minute)
+			_, err := binary.Info(infoCtx)
+			infoCancel()
+
+			if err != nil {
+				if knownInfoFailures.Has(binName) {
+					g.GinkgoLogr.Info("Skipping known info failure", "binary", binName, "error", err)
+					continue
+				}
+				failures = append(failures, binName+": "+err.Error())
+			} else if knownInfoFailures.Has(binName) {
+				failures = append(failures, binName+": listed in knownInfoFailures but info succeeded — remove from exemption list")
+			}
+		}
+
+		o.Expect(failures).To(o.BeEmpty(), "extension binaries failed the OTE info contract")
+	})
+})

test/extended/router/config_manager.go

@@ -65,9 +65,9 @@ var _ = g.Describe("[sig-network][Feature:Router][apigroup:route.openshift.io]",
 	// in order to allow binding 80/443 without being root. Without the privilege,
 	// the capability does not take effect and haproxy fails to start.
 	oc = exutil.NewCLIWithPodSecurityLevel("router-config-manager", api.LevelPrivileged)
-	kubeClient = oc.AdminKubeClient()
 
 	g.BeforeEach(func() {
+		kubeClient = oc.AdminKubeClient()
 		ns = oc.Namespace()
 
 		routerImage, err := exutil.FindRouterImage(oc)

test/extended/router/config_manager_ingress.go

@@ -52,10 +52,10 @@ var _ = g.Describe("[sig-network-edge][Feature:Router][apigroup:route.openshift.
 
 	ctx := context.Background()
 	oc := exutil.NewCLIWithPodSecurityLevel("router-dcm-ingress", api.LevelPrivileged).AsAdmin()
-	kubeClient := oc.AdminKubeClient()
 
 	// variables updated on every new test
 	var (
+		kubeClient       kubernetes.Interface
 		execPod          execPodRef
 		controller       types.NamespacedName
 		routeSelectorSet labels.Set
@@ -69,6 +69,7 @@ var _ = g.Describe("[sig-network-edge][Feature:Router][apigroup:route.openshift.
 	})
 
 	g.BeforeEach(func() {
+		kubeClient = oc.AdminKubeClient()
 		// ingress controller need to be created in operator's namespace, ...
 		nsOperator := "openshift-ingress-operator"
 		controllerName := names.SimpleNameGenerator.GenerateName("e2e-dcm-")