fix: detangle code and unmount old vols on start (#43)

* chore: update node-driver-registrar to v2.13.0

* fix: remove post start hook

* fix: add function that unmounts rclone mounts

* chore: cleanup code

* chore: helm updates

Author: olevski

cmd/csi-rclone-plugin/main.go

@@ -4,10 +4,12 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"time"
 
-	"github.com/SwissDataScienceCenter/csi-rclone/pkg/kube"
 	"github.com/SwissDataScienceCenter/csi-rclone/pkg/rclone"
 	"github.com/spf13/cobra"
+	"k8s.io/klog"
+	mountUtils "k8s.io/mount-utils"
 )
 
 var (
@@ -21,54 +23,107 @@ func init() {
 
 func main() {
 
-	cmd := &cobra.Command{
+	root := &cobra.Command{
 		Use:   "rclone",
 		Short: "CSI based rclone driver",
 	}
 
 	runCmd := &cobra.Command{
 		Use:   "run",
 		Short: "Start the CSI driver.",
+	}
+	root.AddCommand(runCmd)
+
+	runNode := &cobra.Command{
+		Use:   "node",
+		Short: "Start the CSI driver node service - expected to run in a daemonset on every node.",
 		Run: func(cmd *cobra.Command, args []string) {
-			handle()
+			handleNode()
 		},
 	}
-	cmd.AddCommand(runCmd)
-
-	runCmd.PersistentFlags().StringVar(&nodeID, "nodeid", "", "node id")
-	runCmd.MarkPersistentFlagRequired("nodeid")
-
-	runCmd.PersistentFlags().StringVar(&endpoint, "endpoint", "", "CSI endpoint")
-	runCmd.MarkPersistentFlagRequired("endpoint")
+	runNode.PersistentFlags().StringVar(&nodeID, "nodeid", "", "node id")
+	runNode.MarkPersistentFlagRequired("nodeid")
+	runNode.PersistentFlags().StringVar(&endpoint, "endpoint", "", "CSI endpoint")
+	runNode.MarkPersistentFlagRequired("endpoint")
+	runCmd.AddCommand(runNode)
+	runController := &cobra.Command{
+		Use:   "controller",
+		Short: "Start the CSI driver controller.",
+		Run: func(cmd *cobra.Command, args []string) {
+			handleController()
+		},
+	}
+	runController.PersistentFlags().StringVar(&nodeID, "nodeid", "", "node id")
+	runController.MarkPersistentFlagRequired("nodeid")
+	runController.PersistentFlags().StringVar(&endpoint, "endpoint", "", "CSI endpoint")
+	runController.MarkPersistentFlagRequired("endpoint")
+	runCmd.AddCommand(runController)
 
 	versionCmd := &cobra.Command{
 		Use:   "version",
 		Short: "Prints information about this version of csi rclone plugin",
 		Run: func(cmd *cobra.Command, args []string) {
-			fmt.Printf(`csi-rclone plugin
-Version:    %s
-`, rclone.DriverVersion)
+			fmt.Printf("csi-rclone plugin Version: %s", rclone.DriverVersion)
 		},
 	}
-	cmd.AddCommand(versionCmd)
+	root.AddCommand(versionCmd)
 
-	cmd.ParseFlags(os.Args[1:])
-	if err := cmd.Execute(); err != nil {
+	root.ParseFlags(os.Args[1:])
+	if err := root.Execute(); err != nil {
 		fmt.Fprintf(os.Stderr, "%s", err.Error())
 		os.Exit(1)
 	}
 
 	os.Exit(0)
 }
 
-func handle() {
-	kubeClient, err := kube.GetK8sClient()
+func handleNode() {
+	err := unmountOldVols()
+	if err != nil {
+		klog.Warningf("There was an error when trying to unmount old volumes: %v", err)
+	}
+	d := rclone.NewDriver(nodeID, endpoint)
+	ns, err := rclone.NewNodeServer(d.CSIDriver)
 	if err != nil {
 		panic(err)
 	}
-	d := rclone.NewDriver(nodeID, endpoint, kubeClient)
+	d.WithNodeServer(ns)
 	err = d.Run()
 	if err != nil {
 		panic(err)
 	}
 }
+
+func handleController() {
+	d := rclone.NewDriver(nodeID, endpoint)
+	cs := rclone.NewControllerServer(d.CSIDriver)
+	d.WithControllerServer(cs)
+	err := d.Run()
+	if err != nil {
+		panic(err)
+	}
+}
+
+// unmountOldVols is used to unmount volumes after a restart on a node
+func unmountOldVols() error {
+	const mountType = "fuse.rclone"
+	const unmountTimeout = time.Second * 5
+	klog.Info("Checking for existing mounts")
+	mounter := mountUtils.Mounter{}
+	mounts, err := mounter.List()
+	if err != nil {
+		return err
+	}
+	for _, mount := range mounts {
+		if mount.Type != mountType {
+			continue
+		}
+		err := mounter.UnmountWithForce(mount.Path, unmountTimeout)
+		if err != nil {
+			klog.Warningf("Failed to unmount %s because of %v.", mount.Path, err)
+			continue
+		}
+		klog.Infof("Sucessfully unmounted %s", mount.Path)
+	}
+	return nil
+}

deploy/csi-rclone/templates/csi-controller-rclone.yaml

@@ -59,6 +59,7 @@ spec:
       - name: rclone
         args:
         - run
+        - controller
         - --nodeid=$(NODE_ID)
         - --endpoint=$(CSI_ENDPOINT)
         env:
@@ -93,7 +94,7 @@ spec:
           name: socket-dir
       - name: liveness-probe
         imagePullPolicy: Always
-        image: registry.k8s.io/sig-storage/livenessprobe:v2.11.0
+        image: registry.k8s.io/sig-storage/livenessprobe:v2.15.0
         args:
         - --csi-address=/csi/csi.sock
         volumeMounts:

deploy/csi-rclone/templates/csi-nodeplugin-rclone.yaml

@@ -60,6 +60,7 @@ spec:
       - name: rclone
         args:
         - run
+        - node
         - --nodeid=$(NODE_ID)
         - --endpoint=$(CSI_ENDPOINT)
         env:
@@ -77,14 +78,6 @@ spec:
           value: {{ .Values.logLevel | default "NOTICE" | quote }}
         image: {{ .Values.csiNodepluginRclone.rclone.image.repository }}:{{ .Values.csiNodepluginRclone.rclone.image.tag | default .Chart.AppVersion }}
         imagePullPolicy: {{ .Values.csiNodepluginRclone.rclone.imagePullPolicy }}
-        # TODO: check if necessary
-        lifecycle:
-          postStart:
-            exec:
-              command:
-              - /bin/sh
-              - -c
-              - mount -t fuse.rclone | while read -r mount; do umount $(echo $mount | awk {print $3}) ; done
         resources:
           {{- toYaml .Values.csiControllerRclone.rclone.resources | nindent 12 }}
         securityContext: {{- toYaml .Values.csiNodepluginRclone.rclone.containerSecurityContext

deploy/csi-rclone/values.yaml

@@ -31,7 +31,7 @@ csiNodepluginRclone:
   nodeDriverRegistrar:
     image:
       repository: registry.k8s.io/sig-storage/csi-node-driver-registrar
-      tag: v2.12.0
+      tag: v2.13.0
     imagePullPolicy: IfNotPresent
   rclone:
     containerSecurityContext:

devenv/nix/goModule.nix

@@ -5,7 +5,7 @@ let
     pname = "csi-rclone-pvc-1";
     version = "0.2.0";
     src = ../../.;
-    vendorHash = "sha256-q1tfnO5B6U9c+Ve+kpOfnWGvbdShgkPXvR7axsA7O5Y=";
+    vendorHash = "sha256-14l9ybbaHRs+rlcSABBwpUXe9TNlP1HoCNZCvdYiOmk=";
     # CGO = 0;
     # preBuild = ''
     #   whoami

go.mod

@@ -1,48 +1,52 @@
 module github.com/SwissDataScienceCenter/csi-rclone
 
-go 1.18
+go 1.23.0
+
+toolchain go1.23.4
 
 require (
 	github.com/container-storage-interface/spec v1.9.0
+	github.com/fernet/fernet-go v0.0.0-20240119011108-303da6aec611
+	github.com/google/uuid v1.4.0
 	github.com/kubernetes-csi/csi-test/v5 v5.2.0
 	github.com/kubernetes-csi/drivers v1.0.2
+	github.com/onsi/ginkgo/v2 v2.13.1
+	github.com/onsi/gomega v1.30.0
 	github.com/spf13/cobra v1.1.1
 	golang.org/x/net v0.17.0
 	google.golang.org/grpc v1.58.1
+	gopkg.in/ini.v1 v1.67.0
 	k8s.io/api v0.20.4
 	k8s.io/apimachinery v0.20.4
 	k8s.io/client-go v0.20.4
 	k8s.io/klog v1.0.0
-	k8s.io/kubernetes v1.20.4
-	k8s.io/utils v0.0.0-20231127182322-b307cd553661
+	k8s.io/mount-utils v0.32.1
+	k8s.io/utils v0.0.0-20241210054802-24370beab758
 )
 
 require (
-	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/fernet/fernet-go v0.0.0-20240119011108-303da6aec611 // indirect
-	github.com/go-logr/logr v1.3.0 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
 	github.com/gogo/protobuf v1.3.1 // indirect
 	github.com/golang/glog v1.1.0 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/gofuzz v1.1.0 // indirect
 	github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 // indirect
-	github.com/google/uuid v1.4.0 // indirect
 	github.com/googleapis/gnostic v0.4.1 // indirect
 	github.com/imdario/mergo v0.3.7 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.10 // indirect
 	github.com/kubernetes-csi/csi-lib-utils v0.3.1 // indirect
+	github.com/moby/sys/mountinfo v0.7.2 // indirect
+	github.com/moby/sys/userns v0.1.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.1 // indirect
-	github.com/onsi/ginkgo/v2 v2.13.1 // indirect
-	github.com/onsi/gomega v1.30.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/stretchr/testify v1.8.4 // indirect
 	golang.org/x/crypto v0.14.0 // indirect
 	golang.org/x/oauth2 v0.10.0 // indirect
-	golang.org/x/sys v0.14.0 // indirect
+	golang.org/x/sys v0.26.0 // indirect
 	golang.org/x/term v0.13.0 // indirect
 	golang.org/x/text v0.13.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
@@ -51,42 +55,9 @@ require (
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5 // indirect
 	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
-	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/klog/v2 v2.110.1 // indirect
+	k8s.io/klog/v2 v2.130.1 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.0.2 // indirect
 	sigs.k8s.io/yaml v1.2.0 // indirect
 )
-
-replace (
-	k8s.io/api => k8s.io/api v0.20.4
-	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.20.4
-	k8s.io/apimachinery => k8s.io/apimachinery v0.20.4
-	k8s.io/apiserver => k8s.io/apiserver v0.20.4
-	k8s.io/cli-runtime => k8s.io/cli-runtime v0.20.4
-	k8s.io/client-go => k8s.io/client-go v0.20.4
-	k8s.io/cloud-provider => k8s.io/cloud-provider v0.20.4
-	k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.20.4
-	k8s.io/code-generator => k8s.io/code-generator v0.20.4
-	k8s.io/component-base => k8s.io/component-base v0.20.4
-	k8s.io/component-helpers => k8s.io/component-helpers v0.20.4
-	k8s.io/controller-manager => k8s.io/controller-manager v0.20.4
-	k8s.io/cri-api => k8s.io/cri-api v0.20.4
-	k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.20.4
-	k8s.io/dynamic-resource-allocation => k8s.io/dynamic-resource-allocation v0.20.4
-	k8s.io/endpointslice => k8s.io/endpointslice v0.20.4
-	k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.20.4
-	k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.20.4
-	k8s.io/kube-proxy => k8s.io/kube-proxy v0.20.4
-	k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.20.4
-	k8s.io/kubectl => k8s.io/kubectl v0.20.4
-	k8s.io/kubelet => k8s.io/kubelet v0.20.4
-	k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.20.4
-	k8s.io/metrics => k8s.io/metrics v0.20.4
-	k8s.io/mount-utils => k8s.io/mount-utils v0.20.4
-	k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.20.4
-	k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.20.4
-	k8s.io/sample-cli-plugin => k8s.io/sample-cli-plugin v0.20.4
-	k8s.io/sample-controller => k8s.io/sample-controller v0.20.4
-)