Skip to content

Commit e306f9d

Browse files
author
jiuyu
committed
Feat: support Runtime Fuse UpdateStrategy - OnIdle
Signed-off-by: 玖宇 <guotongyu.gty@alibaba-inc.com>
1 parent 441d216 commit e306f9d

17 files changed

Lines changed: 336 additions & 4 deletions

File tree

api/v1alpha1/constant.go

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,3 +53,16 @@ const (
5353
// OnRuntimeDeletedCleanPolicy cleans fuse pod only when the cache runtime is deleted
5454
OnRuntimeDeletedCleanPolicy FuseCleanPolicy = "OnRuntimeDeleted"
5555
)
56+
57+
type FuseUpdateStrategy string
58+
59+
const (
60+
// NoneFuseUpdateStrategy is the default clean policy. It will be transformed to OnDeleteFuseUpdateStrategy automatically.
61+
NoneFuseUpdateStrategy FuseUpdateStrategy = ""
62+
63+
// OnDeleteFuseUpdateStrategy cleans fuse pod once th fuse pod on some node is deleted
64+
OnDeleteFuseUpdateStrategy FuseUpdateStrategy = "OnDelete"
65+
66+
// OnIdleFuseUpdateStrategy cleans fuse pod once th fuse pod on some node is in idle
67+
OnIdleFuseUpdateStrategy FuseUpdateStrategy = "OnIdle"
68+
)

api/v1alpha1/juicefsruntime_types.go

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,14 @@ type JuiceFSFuseSpec struct {
164164
// +optional
165165
CleanPolicy FuseCleanPolicy `json:"cleanPolicy,omitempty"`
166166

167+
// UpdateStrategy decides when to update Fuse pods.
168+
// Currently Fluid supports two UpdateStrategy: OnDelete and OnIdle
169+
// OnDelete update fuse pod by native daemonset once the fuse pod on some node is deleted
170+
// OnIdle update fuse pod once the fuse pod on some node is in idle
171+
// Defaults to OnDelete
172+
// +optional
173+
UpdateStrategy FuseUpdateStrategy `json:"updateStrategy,omitempty"`
174+
167175
// PodMetadata defines labels and annotations that will be propagated to JuiceFs's pods.
168176
// +optional
169177
PodMetadata PodMetadata `json:"podMetadata,omitempty"`

charts/fluid/fluid/crds/data.fluid.io_juicefsruntimes.yaml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -328,6 +328,14 @@ spec:
328328
More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
329329
type: object
330330
type: object
331+
updateStrategy:
332+
description: |-
333+
UpdateStrategy decides when to update Fuse pods.
334+
Currently Fluid supports two UpdateStrategy: OnDelete and OnIdle
335+
OnDelete update fuse pod by native daemonset once the fuse pod on some node is deleted
336+
OnIdle update fuse pod once the fuse pod on some node is in idle
337+
Defaults to OnDelete
338+
type: string
331339
volumeMounts:
332340
description: VolumeMounts specifies the volumes listed in ".spec.volumes"
333341
to mount into runtime component's filesystem.

charts/juicefs/templates/fuse/daemonset.yaml

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -143,13 +143,20 @@ spec:
143143
privileged: true
144144
{{- end }}
145145
lifecycle:
146+
{{- if .Values.fuse.postStartCmd }}
147+
postStart:
148+
exec:
149+
command: ["sh", "-c", "{{ .Values.fuse.postStartCmd }}"]
150+
{{- end }}
146151
preStop:
147152
exec:
148153
command: ["sh", "-c", "umount {{ .Values.fuse.mountPath }}"]
149154
volumeMounts:
150155
- name: juicefs-fuse-mount
151156
mountPath: {{ .Values.fuse.hostMountPath }}
152157
mountPropagation: Bidirectional
158+
- name: juicefs-fuse-meta
159+
mountPath: {{ .Values.fuse.hostMetaPath }}
153160
- mountPath: /root/script
154161
name: script
155162
{{- if .Values.fuse.volumeMounts }}
@@ -161,6 +168,10 @@ spec:
161168
hostPath:
162169
path: {{ .Values.fuse.hostMountPath }}
163170
type: DirectoryOrCreate
171+
- name: juicefs-fuse-meta
172+
hostPath:
173+
path: {{ .Values.fuse.hostMetaPath }}
174+
type: DirectoryOrCreate
164175
- name: script
165176
configMap:
166177
name: {{ template "juicefs.fullname" . }}-fuse-script

charts/juicefs/values.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,7 @@ fuse:
9595
mountPath: /mnt/jfs
9696
cacheDir: ""
9797
hostMountPath: /mnt/jfs
98+
hostMetaPath: /mnt/.meta
9899
command: "/usr/local/bin/juicefs mount /mnt/jfs"
99100
statCmd: "stat -c %i /mnt/jfs"
100101
updateStrategy:

config/crd/bases/data.fluid.io_juicefsruntimes.yaml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -328,6 +328,14 @@ spec:
328328
More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
329329
type: object
330330
type: object
331+
updateStrategy:
332+
description: |-
333+
UpdateStrategy decides when to update Fuse pods.
334+
Currently Fluid supports two UpdateStrategy: OnDelete and OnIdle
335+
OnDelete update fuse pod by native daemonset once the fuse pod on some node is deleted
336+
OnIdle update fuse pod once the fuse pod on some node is in idle
337+
Defaults to OnDelete
338+
type: string
331339
volumeMounts:
332340
description: VolumeMounts specifies the volumes listed in ".spec.volumes"
333341
to mount into runtime component's filesystem.

pkg/common/env_names.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@ const (
2525

2626
EnvRuntimeInfoCacheSize = "RUNTIMEINFO_CACHE_SIZE"
2727

28+
EnvRuntimeFuseImageVersion = "RUNTIMEINFO_FUSE_IMAGE_VERSION"
29+
2830
EnvEnableRuntimeInfoCache = "ENABLE_RUNTIMEINFO_CACHE"
2931

3032
EnvRuntimeInfoCacheTTL = "RUNTIMEINFO_CACHE_TTL"

pkg/common/label.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,9 @@ const (
8383
)
8484

8585
const (
86+
// i.e. fuse.runtime.fluid.io/image-version
87+
AnnotationRuntimeFuseImageVersion = "fuse.runtime." + LabelAnnotationPrefix + "image-version"
88+
8689
// i.e. controller.runtime.fluid.io/replicas
8790
RuntimeControllerReplicas = "controller.runtime." + LabelAnnotationPrefix + "replicas"
8891

pkg/csi/plugins/nodeserver.go

Lines changed: 89 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -291,10 +291,10 @@ func (ns *nodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstag
291291
}
292292
defer ns.locks.Release(volumeId)
293293

294-
// 1. get runtime namespace and name
294+
// 1. get runtime namespace and name from pvc
295295
// A nil volumeContext is passed because unlike csi.NodeStageVolumeRequest, csi.NodeUnstageVolumeRequest has
296296
// no volume context attribute.
297-
namespace, name, err := ns.getRuntimeNamespacedName(nil, volumeId)
297+
pvc, err := volume.GetPVCByVolumeId(ns.apiReader, volumeId)
298298
if err != nil {
299299
if utils.IgnoreNotFound(err) == nil {
300300
// For cases like the related persistent volume has been deleted, ignore it and return success
@@ -304,6 +304,13 @@ func (ns *nodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstag
304304
glog.Errorf("NodeUnstageVolume: can't get runtime namespace and name given (volumeContext: nil, volumeId: %s): %v", volumeId, err)
305305
return nil, errors.Wrapf(err, "NodeUnstageVolume: can't get namespace and name by volume id %s", volumeId)
306306
}
307+
namespace, name := pvc.Namespace, pvc.Name
308+
309+
// get latestFuseImageVersion from pvc annotations
310+
var latestFuseImageVersion string
311+
if pvc.Annotations != nil {
312+
latestFuseImageVersion = pvc.Annotations[common.AnnotationRuntimeFuseImageVersion]
313+
}
307314

308315
// 2. Check fuse clean policy. If clean policy is set to OnRuntimeDeleted, there is no
309316
// need to clean fuse eagerly.
@@ -318,6 +325,23 @@ func (ns *nodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstag
318325
}
319326

320327
var shouldCleanFuse bool
328+
defer func() {
329+
//if shouldCleanFuse == true, fuse pod will be deleted and recreate by NodeStage
330+
if !shouldCleanFuse {
331+
updateStrategy := runtimeInfo.GetFuseUpdateStrategy()
332+
glog.Infof("NodeUnstage GetFuseUpdateStrategy %v", updateStrategy)
333+
switch updateStrategy {
334+
case v1alpha1.OnIdleFuseUpdateStrategy:
335+
if checkIfFuseNeedUpdate(namespace, name, volumeId, latestFuseImageVersion, runtimeInfo) {
336+
if err := ns.recreateFusePodByNodeLabelModify(namespace, name, runtimeInfo); err != nil {
337+
glog.Warningf("NodeUnstageVolume: recreate fuse pod for %v/%v failed, err: %v", namespace, name, err)
338+
}
339+
cleanMetadataFuseImageVersionAfterFuseClean(namespace, name, runtimeInfo.GetRuntimeType())
340+
}
341+
}
342+
}
343+
}()
344+
321345
cleanPolicy := runtimeInfo.GetFuseCleanPolicy()
322346
glog.Infof("NodeUnstageVolume: Using %s clean policy for runtime %s in namespace %s", cleanPolicy, runtimeInfo.GetName(), runtimeInfo.GetNamespace())
323347
switch cleanPolicy {
@@ -696,3 +720,66 @@ func checkMountPathExists(ctx context.Context, mountPath string) error {
696720
}
697721
return nil
698722
}
723+
724+
func checkIfFuseNeedUpdate(namespace, name, volumeId, latestFuseImageVersion string, runtimeInfo base.RuntimeInfoInterface) (needUpdate bool) {
725+
if len(latestFuseImageVersion) == 0 {
726+
return
727+
}
728+
729+
inUse, err := checkMountInUse(volumeId)
730+
if err != nil {
731+
glog.Warningf("NodeUnstage checkMountInUse failed %v, skip to update fuse pod", err)
732+
return
733+
}
734+
glog.Infof("NodeUnstage checkMountInUse %v", inUse)
735+
if inUse {
736+
return
737+
}
738+
739+
currentImageVersion, found, err := utils.GetFuseImageVersionFromMetadata(namespace, name, runtimeInfo.GetRuntimeType())
740+
glog.Infof("NodeUnstage GetFuseImageVersionFromMetadata %v, %v, %v", currentImageVersion, found, err)
741+
if err != nil {
742+
glog.Warningf("NodeUnstage GetFuseImageVersionFromMetadata failed %v, skip to update fuse pod", err)
743+
return
744+
}
745+
if !found {
746+
return
747+
}
748+
749+
glog.Infof("NodeUnstage checkIfFuseNeedUpdate currentImageVersion: %v, latestFuseImageVersion: %v", currentImageVersion, latestFuseImageVersion)
750+
if currentImageVersion != latestFuseImageVersion {
751+
needUpdate = true
752+
return
753+
}
754+
return
755+
}
756+
757+
func (ns *nodeServer) recreateFusePodByNodeLabelModify(namespace, name string, runtimeInfo base.RuntimeInfoInterface) error {
758+
node, err := ns.getNode()
759+
if err != nil {
760+
glog.Errorf("NodeUnstageVolume: can't get node %s: %v", ns.nodeId, err)
761+
return errors.Wrapf(err, "NodeUnstageVolume: can't get node %s", ns.nodeId)
762+
}
763+
764+
fuseLabelKey := utils.GetFuseLabelName(namespace, name, runtimeInfo.GetOwnerDatasetUID())
765+
var labelsToModify common.LabelsToModify
766+
labelsToModify.Delete(fuseLabelKey)
767+
err = ns.patchNodeWithLabel(node, labelsToModify)
768+
if err != nil {
769+
glog.Errorf("NodeUnstageVolume: error when patching labels on node %s: %v", ns.nodeId, err)
770+
return errors.Wrapf(err, "NodeUnstageVolume: error when patching labels on node %s", ns.nodeId)
771+
}
772+
773+
return nil
774+
}
775+
776+
func cleanMetadataFuseImageVersionAfterFuseClean(namespace, name, runtimeType string) {
777+
filePath := utils.GetMetadataFuseImageVersion(namespace, name, runtimeType)
778+
err := os.Remove(filePath)
779+
if err != nil {
780+
glog.Errorf("Error deleting file %s: %v", filePath, err)
781+
return
782+
}
783+
784+
glog.Infof("File %s deleted successfully", filePath)
785+
}

pkg/ddc/base/runtime.go

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,16 +77,24 @@ type RuntimeInfoInterface interface {
7777

7878
SetFuseNodeSelector(nodeSelector map[string]string)
7979

80+
SetFuseName(fuseName string)
81+
8082
SetupFuseCleanPolicy(policy datav1alpha1.FuseCleanPolicy)
8183

84+
SetupFuseUpdateStrategy(strategy datav1alpha1.FuseUpdateStrategy)
85+
8286
SetupWithDataset(dataset *datav1alpha1.Dataset)
8387

8488
SetOwnerDatasetUID(alias types.UID)
8589

8690
GetFuseNodeSelector() (nodeSelector map[string]string)
8791

92+
GetFuseName() string
93+
8894
GetFuseCleanPolicy() datav1alpha1.FuseCleanPolicy
8995

96+
GetFuseUpdateStrategy() datav1alpha1.FuseUpdateStrategy
97+
9098
SetDeprecatedNodeLabel(deprecated bool)
9199

92100
IsDeprecatedNodeLabel() bool
@@ -142,11 +150,16 @@ type RuntimeInfo struct {
142150
}
143151

144152
type Fuse struct {
153+
Name string
154+
145155
NodeSelector map[string]string
146156

147157
// CleanPolicy decides when to clean fuse pods.
148158
CleanPolicy datav1alpha1.FuseCleanPolicy
149159

160+
// UpdateStrategy decides when to update fuse pods.
161+
UpdateStrategy datav1alpha1.FuseUpdateStrategy
162+
150163
// Metrics
151164
MetricsScrapeTarget mountModeSelector
152165
}
@@ -314,6 +327,16 @@ func (info *RuntimeInfo) SetFuseNodeSelector(nodeSelector map[string]string) {
314327
info.fuse.NodeSelector = nodeSelector
315328
}
316329

330+
// GetFuseNodeSelector gets the fuse deploy mode
331+
func (info *RuntimeInfo) GetFuseName() string {
332+
return info.fuse.Name
333+
}
334+
335+
// SetFuseNodeSelector setups the fuse deploy mode
336+
func (info *RuntimeInfo) SetFuseName(fuseName string) {
337+
info.fuse.Name = fuseName
338+
}
339+
317340
// GetFuseNodeSelector gets the fuse deploy mode
318341
func (info *RuntimeInfo) GetFuseNodeSelector() (nodeSelector map[string]string) {
319342
nodeSelector = info.fuse.NodeSelector
@@ -329,10 +352,23 @@ func (info *RuntimeInfo) SetupFuseCleanPolicy(policy datav1alpha1.FuseCleanPolic
329352
info.fuse.CleanPolicy = policy
330353
}
331354

355+
func (info *RuntimeInfo) SetupFuseUpdateStrategy(strategy datav1alpha1.FuseUpdateStrategy) {
356+
if strategy == datav1alpha1.NoneFuseUpdateStrategy {
357+
// Default to set the fuse clean policy to OnRuntimeDeleted
358+
info.fuse.UpdateStrategy = datav1alpha1.OnDeleteFuseUpdateStrategy
359+
return
360+
}
361+
info.fuse.UpdateStrategy = strategy
362+
}
363+
332364
func (info *RuntimeInfo) GetFuseCleanPolicy() datav1alpha1.FuseCleanPolicy {
333365
return info.fuse.CleanPolicy
334366
}
335367

368+
func (info *RuntimeInfo) GetFuseUpdateStrategy() datav1alpha1.FuseUpdateStrategy {
369+
return info.fuse.UpdateStrategy
370+
}
371+
336372
// SetDeprecatedNodeLabel set the DeprecatedNodeLabel
337373
func (info *RuntimeInfo) SetDeprecatedNodeLabel(deprecated bool) {
338374
info.deprecatedNodeLabel = deprecated
@@ -492,6 +528,7 @@ func GetRuntimeInfo(client client.Client, name, namespace string) (runtimeInfo R
492528
}
493529
runtimeInfo.SetFuseNodeSelector(juicefsRuntime.Spec.Fuse.NodeSelector)
494530
runtimeInfo.SetupFuseCleanPolicy(juicefsRuntime.Spec.Fuse.CleanPolicy)
531+
runtimeInfo.SetupFuseUpdateStrategy(juicefsRuntime.Spec.Fuse.UpdateStrategy)
495532
case common.ThinRuntime:
496533
thinRuntime, err := utils.GetThinRuntime(client, name, namespace)
497534
if err != nil {

0 commit comments

Comments
 (0)