diff --git a/Makefile b/Makefile index b37fe7789..e0c80eb06 100644 --- a/Makefile +++ b/Makefile @@ -57,11 +57,3 @@ test-e2e: hack/e2e.sh .PHONY: test-e2e - -operator-e2e-test: GO_TEST_PACKAGES :=./test/e2e/... -operator-e2e-test: GO_TEST_FLAGS += -v -operator-e2e-test: GO_TEST_FLAGS += -count=1 -operator-e2e-test: GO_TEST_FLAGS += -timeout 3h -operator-e2e-test: GO_TEST_FLAGS += -p 1 -operator-e2e-test: test-unit -.PHONY: operator-e2e-test diff --git a/test/e2e/operator_removal_test.go b/test/e2e/operator_removal_test.go deleted file mode 100644 index b9ff5fa8a..000000000 --- a/test/e2e/operator_removal_test.go +++ /dev/null @@ -1,241 +0,0 @@ -package e2e - -import ( - "context" - "testing" - "time" - - configv1 "github.com/openshift/api/config/v1" - operatorapi "github.com/openshift/api/operator/v1" - configclient "github.com/openshift/client-go/config/clientset/versioned" - operatorclient "github.com/openshift/client-go/operator/clientset/versioned" - clusteroperatorhelpers "github.com/openshift/library-go/pkg/config/clusteroperator/v1helpers" - "github.com/openshift/library-go/pkg/operator/v1helpers" - "github.com/openshift/vmware-vsphere-csi-driver-operator/test/library" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" -) - -const ( - clientName = "vsphere-operator-e2e" - storageCOName = "storage" - csiDriverName = "csi.vsphere.vmware.com" - disabledCondition = "VMwareVSphereControllerDisabled" - - csiDriverNameSpace = "openshift-cluster-csi-drivers" - csiControllerDeploymentName = "vmware-vsphere-csi-driver-controller" - csiNodeDaemonSetName = "vmware-vsphere-csi-driver-node" -) - -type disableConditionStatus int - -const ( - operatorDisabled = iota - operatorEnabled - clusterCSIDriverNotFound -) - -var ( - WaitPollInterval = time.Second - WaitPollTimeout = 10 * time.Minute -) - -// TestStorageRemoval tests the removal of the storage operator from the cluster -// -// by marking the ClusterCSIDriver as removed and waiting for the storage resources to be removed. -// The test does not verify removal of each storage resource but only deployment and daemonset. -// It also verifies if storage can be restored by marking managmentState as Managed -func TestStorageRemoval(t *testing.T) { - kubeConfig, err := library.NewClientConfigForTest(t) - if err != nil { - t.Fatalf("Failed to get kubeconfig: %v", err) - } - - kubeClient := kubernetes.NewForConfigOrDie(rest.AddUserAgent(kubeConfig, clientName)) - ocpOperatorClient := operatorclient.NewForConfigOrDie(rest.AddUserAgent(kubeConfig, clientName)) - - configClient := configclient.NewForConfigOrDie(rest.AddUserAgent(kubeConfig, clientName)) - t.Logf("Waiting for storage to be available") - waitForStorageAvailable(t, configClient, ocpOperatorClient, kubeClient) - t.Logf("Marking ClusterCSIDriver as removed") - err = makeClusterCSIDriverRemoved(context.TODO(), ocpOperatorClient) - - if err != nil { - restoreStorage(t, ocpOperatorClient, configClient, kubeClient) - t.Fatalf("Failed to update ClusterCSIDriver: %v", err) - } - t.Logf("removed storage operator from cluster") - err = waitForStorageResourceRemoval(t, ocpOperatorClient, kubeClient) - if err != nil { - restoreStorage(t, ocpOperatorClient, configClient, kubeClient) - t.Fatalf("Failed to wait for storage resource removal: %v", err) - } - time.Sleep(10 * time.Second) - restoreStorage(t, ocpOperatorClient, configClient, kubeClient) -} - -func restoreStorage(t *testing.T, client *operatorclient.Clientset, configClient *configclient.Clientset, kubeClient *kubernetes.Clientset) { - t.Log("Restoring storage operator to cluster") - err := makeClusterCSIDriverManaged(context.TODO(), client) - if err != nil { - t.Fatalf("Failed to update ClusterCSIDriver: %v", err) - } - waitForStorageAvailable(t, configClient, client, kubeClient) -} - -func waitForStorageResourceRemoval(t *testing.T, client *operatorclient.Clientset, kubeClient *kubernetes.Clientset) error { - // make sure storage to be removed - t.Logf("Waiting for storage resource to be removed") - return wait.PollUntilContextTimeout(context.TODO(), WaitPollInterval, WaitPollTimeout, false, func(pollContext context.Context) (bool, error) { - disabledConditionStatusVar, err := checkDisabledCondition(t, pollContext, client) - if err != nil { - return false, err - } - done := disabledConditionStatusVar == operatorDisabled - if done { - deploymentCreated, err := checkForDeploymentCreation(t, pollContext, kubeClient) - if err != nil { - return false, err - } - done = !deploymentCreated - } - - if done { - daemonsetCreated, err := checkForDaemonset(t, pollContext, kubeClient) - if err != nil { - return false, err - } - done = !daemonsetCreated - } - return done, nil - }) -} - -func waitForStorageAvailable(t *testing.T, client *configclient.Clientset, operatorClient *operatorclient.Clientset, kubeClient *kubernetes.Clientset) { - err := wait.PollUntilContextTimeout(context.TODO(), WaitPollInterval, WaitPollTimeout, false, func(pollContext context.Context) (bool, error) { - clusterOperator, err := client.ConfigV1().ClusterOperators().Get(pollContext, storageCOName, metav1.GetOptions{}) - if apierrors.IsNotFound(err) { - t.Log("ClusterOperator/storage does not yet exist.") - return false, nil - } - if err != nil { - t.Log("Unable to retrieve ClusterOperator/storage:", err) - return false, err - } - conditions := clusterOperator.Status.Conditions - available := clusteroperatorhelpers.IsStatusConditionPresentAndEqual(conditions, configv1.OperatorAvailable, configv1.ConditionTrue) - notProgressing := clusteroperatorhelpers.IsStatusConditionPresentAndEqual(conditions, configv1.OperatorProgressing, configv1.ConditionFalse) - notDegraded := clusteroperatorhelpers.IsStatusConditionPresentAndEqual(conditions, configv1.OperatorDegraded, configv1.ConditionFalse) - done := available && notProgressing && notDegraded - - if done { - disableConditionStatusVar, err := checkDisabledCondition(t, pollContext, operatorClient) - if err != nil { - return false, err - } - done = disableConditionStatusVar == operatorEnabled - } - - if done { - deploymentCreated, err := checkForDeploymentCreation(t, pollContext, kubeClient) - if err != nil { - return false, err - } - done = deploymentCreated - } - - if done { - daemonsetCreated, err := checkForDaemonset(t, pollContext, kubeClient) - if err != nil { - return false, err - } - done = daemonsetCreated - } - - t.Logf("ClusterOperator/storage: Available: %v Progressing: %v Degraded: %v\n", available, !notProgressing, !notDegraded) - return done, nil - }) - if err != nil { - t.Fatal(err) - } -} - -func checkDisabledCondition(t *testing.T, ctx context.Context, operatorClient *operatorclient.Clientset) (disableConditionStatus, error) { - clusterCSIDriver, err := operatorClient.OperatorV1().ClusterCSIDrivers().Get(ctx, csiDriverName, metav1.GetOptions{}) - if apierrors.IsNotFound(err) { - t.Log("cluster-csi-driver does not yet exist.") - return clusterCSIDriverNotFound, nil - } - if err != nil { - t.Log("Unable to retrieve cluster-csi-driver", err) - return clusterCSIDriverNotFound, err - } - disabledCondition := v1helpers.FindOperatorCondition(clusterCSIDriver.Status.Conditions, disabledCondition) - if disabledCondition != nil && disabledCondition.Status == operatorapi.ConditionTrue { - return operatorDisabled, nil - } - return operatorEnabled, nil -} - -func makeClusterCSIDriverRemoved(ctx context.Context, client *operatorclient.Clientset) error { - return wait.PollUntilContextTimeout(ctx, WaitPollInterval, WaitPollTimeout, false, func(pollContext context.Context) (bool, error) { - clusterCSIDriver, err := client.OperatorV1().ClusterCSIDrivers().Get(pollContext, csiDriverName, metav1.GetOptions{}) - if err != nil { - return false, err - } - - if clusterCSIDriver.Spec.ManagementState == operatorapi.Removed { - return true, nil - } - - clusterCSIDriver.Spec.ManagementState = operatorapi.Removed - client.OperatorV1().ClusterCSIDrivers().Update(pollContext, clusterCSIDriver, metav1.UpdateOptions{}) - return false, nil - }) -} - -func makeClusterCSIDriverManaged(ctx context.Context, client *operatorclient.Clientset) error { - return wait.PollUntilContextTimeout(ctx, WaitPollInterval, WaitPollTimeout, false, func(pollContext context.Context) (bool, error) { - clusterCSIDriver, err := client.OperatorV1().ClusterCSIDrivers().Get(pollContext, csiDriverName, metav1.GetOptions{}) - if err != nil { - return false, err - } - - if clusterCSIDriver.Spec.ManagementState == operatorapi.Managed { - return true, nil - } - - clusterCSIDriver.Spec.ManagementState = operatorapi.Managed - client.OperatorV1().ClusterCSIDrivers().Update(pollContext, clusterCSIDriver, metav1.UpdateOptions{}) - return false, nil - }) -} - -func checkForDeploymentCreation(t *testing.T, ctx context.Context, kubeClient *kubernetes.Clientset) (bool, error) { - deployment, err := kubeClient.AppsV1().Deployments(csiDriverNameSpace).Get(ctx, csiControllerDeploymentName, metav1.GetOptions{}) - if apierrors.IsNotFound(err) { - t.Log("Deployment does not yet exist.") - return false, nil - } - if err != nil { - t.Logf("Unable to retrieve deployment: %v", err) - return false, err - } - return deployment != nil, nil -} - -func checkForDaemonset(t *testing.T, ctx context.Context, kubeClient *kubernetes.Clientset) (bool, error) { - daemonset, err := kubeClient.AppsV1().DaemonSets(csiDriverNameSpace).Get(ctx, csiNodeDaemonSetName, metav1.GetOptions{}) - if apierrors.IsNotFound(err) { - t.Log("Daemonset does not yet exist.") - return false, nil - } - if err != nil { - t.Logf("Unable to retrieve daemonset: %v", err) - return false, err - } - return daemonset != nil, nil -} diff --git a/test/library/client.go b/test/library/client.go deleted file mode 100644 index ef9df36f9..000000000 --- a/test/library/client.go +++ /dev/null @@ -1,20 +0,0 @@ -package library - -import ( - "testing" - - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" - "k8s.io/client-go/tools/clientcmd/api" -) - -// NewClientConfigForTest returns a config configured to connect to the api server -func NewClientConfigForTest(t *testing.T) (*rest.Config, error) { - loader := clientcmd.NewDefaultClientConfigLoadingRules() - clientConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loader, &clientcmd.ConfigOverrides{ClusterInfo: api.Cluster{InsecureSkipTLSVerify: true}}) - config, err := clientConfig.ClientConfig() - if err == nil { - t.Logf("Found configuration for host %v", config.Host) - } - return config, err -}