juicefs: support extra options in dataload

charts/fluid-dataloader/juicefs/templates/configmap.yaml

@@ -38,7 +38,7 @@ data:
           local pod="${podNames[i]}"
 
           echo -e "juicefs warmup on $pod $targetPath starts"
-          /usr/local/bin/kubectl -n $ns exec -it $pod -- timeout $TIMEOUT /usr/local/bin/juicefs warmup $targetPath
+          /usr/local/bin/kubectl -n $ns exec -it $pod -- timeout $TIMEOUT /usr/local/bin/juicefs warmup $targetPath $OPTION
           echo -e "juicefs warmup on $pod $targetPath ends"
         done
         fi
@@ -47,7 +47,7 @@ data:
         then
           echo -e "juicefs warmup $targetPath starts"
           local pod="${podNames[0]}"
-          /usr/local/bin/kubectl -n $ns exec -it $pod -- timeout $TIMEOUT /usr/bin/juicefs warmup $targetPath
+          /usr/local/bin/kubectl -n $ns exec -it $pod -- timeout $TIMEOUT /usr/bin/juicefs warmup $targetPath $OPTION
           echo -e "juicefs warmup $targetPath ends"
         fi
     }

charts/fluid-dataloader/juicefs/templates/cronjob.yaml

@@ -111,6 +111,10 @@ spec:
                 - name: TIMEOUT
                   value: {{ $val | quote }}
                 {{- end }}
+                {{- if eq $key "option" }}
+                - name: OPTION
+                  value: {{ $val | quote }}
+                {{- end }}
                 {{- if eq $key "edition" }}
                 - name: EDITION
                   value: {{ $val | quote }}

charts/fluid-dataloader/juicefs/templates/job.yaml

@@ -124,6 +124,10 @@ spec:
             - name: TIMEOUT
               value: {{ $val | quote }}
             {{- end }}
+            {{- if eq $key "option" }}
+            - name: OPTION
+              value: {{ $val | quote }}
+            {{- end }}
             {{- if eq $key "edition" }}
             - name: EDITION
               value: {{ $val | quote }}

docs/zh/samples/juicefs/juicefs_data_accelerate.md

@@ -21,9 +21,12 @@ spec:
     namespace: default
   target:
     - path: /dir1
+  options:
+    threads: "50"
 ```
 
 其中：
 - `spec.dataset.name`：指定要加速数据访问的 Dataset 名称；
 - `spec.dataset.namespce`：指定要加速数据访问的 Dataset 的 namespace；
 - `spec.target.path`：指定要加速数据访问的数据路径，可以是目录或文件；target 是列表，可以填多个 path。
+- `spec.options`：指定缓存加速的参数，可用参数可参考 [JuiceFS 的缓存加速文档](https://juicefs.com/docs/zh/community/command_reference#warmup)，注意 key 和 value 均为字符串类型。

pkg/ddc/alluxio/delete_volume_test.go

@@ -38,17 +38,19 @@ type TestCase struct {
 }
 
 // newTestAlluxioEngine creates an instance of AlluxioEngine for testing purposes.
-// 
+//
 // Parameters:
-//   client     - A Kubernetes client used for interactions with the API server.
-//   name       - The name for the AlluxioEngine (used for associated PV/PVC resources in tests).
-//   namespace  - The namespace where the resources reside.
-//   withRunTime - A flag indicating whether the engine should be initialized with runtime information.
-//                 When true, an AlluxioRuntime instance and corresponding runtimeInfo are created;
-//                 when false, both runtime and runtimeInfo are set to nil to simulate a scenario without runtime support.
-// 
+//
+//	client     - A Kubernetes client used for interactions with the API server.
+//	name       - The name for the AlluxioEngine (used for associated PV/PVC resources in tests).
+//	namespace  - The namespace where the resources reside.
+//	withRunTime - A flag indicating whether the engine should be initialized with runtime information.
+//	              When true, an AlluxioRuntime instance and corresponding runtimeInfo are created;
+//	              when false, both runtime and runtimeInfo are set to nil to simulate a scenario without runtime support.
+//
 // Returns:
-//   A pointer to an AlluxioEngine instance configured according to the provided parameters.
+//
+//	A pointer to an AlluxioEngine instance configured according to the provided parameters.
 func newTestAlluxioEngine(client client.Client, name string, namespace string, withRunTime bool) *AlluxioEngine {
 	runTime := &datav1alpha1.AlluxioRuntime{}
 	runTimeInfo, _ := base.BuildRuntimeInfo(name, namespace, "alluxio")

pkg/ddc/alluxio/load_data_test.go

@@ -115,11 +115,11 @@ func TestGenerateDataLoadValueFile(t *testing.T) {
 //
 // Parameters:
 //   - t: A testing.T object used for managing test state and reporting test failures. It provides methods like Errorf and Fail
-//        to indicate test failures and log additional information.
+//     to indicate test failures and log additional information.
 //
 // Returns:
 //   - None (This is a test function, so it does not return any value. Its purpose is to validate the behavior of the code
-//           under test and report any issues via the testing.T object.)
+//     under test and report any issues via the testing.T object.)
 func Test_genDataLoadValue(t *testing.T) {
 	testCases := map[string]struct {
 		image         string

pkg/ddc/alluxio/transform_ufs_test.go

@@ -88,8 +88,8 @@ func TestTransformDatasetToVolume(t *testing.T) {
 	}
 }
 
-// TestTransformDatasetToPVC tests the transformation of Dataset specifications into Persistent Volume Claim (PVC) 
-// configurations in Alluxio volumes. It validates the correct conversion of Dataset mount configurations to 
+// TestTransformDatasetToPVC tests the transformation of Dataset specifications into Persistent Volume Claim (PVC)
+// configurations in Alluxio volumes. It validates the correct conversion of Dataset mount configurations to
 // corresponding UFSVolume structures with proper path mappings.
 //
 // Parameters:

pkg/ddc/alluxio/transform_volumes_test.go

@@ -248,9 +248,9 @@ func TestTransformWorkerVolumes(t *testing.T) {
 	}
 }
 
-// TestTransformFuseVolumes tests the behavior of the transformFuseVolumes method.  
-// It defines multiple test cases to verify whether the Volumes and VolumeMounts  
-// in an AlluxioRuntime object are correctly transformed into the Alluxio structure,  
+// TestTransformFuseVolumes tests the behavior of the transformFuseVolumes method.
+// It defines multiple test cases to verify whether the Volumes and VolumeMounts
+// in an AlluxioRuntime object are correctly transformed into the Alluxio structure,
 // and checks the error handling logic.
 func TestTransformFuseVolumes(t *testing.T) {
 	type testCase struct {

pkg/ddc/alluxio/ufs_test.go

@@ -239,20 +239,19 @@ func TestShouldCheckUFS(t *testing.T) {
 	}
 }
 
-
 // TestPrepareUFS tests the PrepareUFS method of AlluxioEngine.
 // This method prepares the underlying file system (UFS) by checking
 // the Alluxio master state, mounting UFS, and performing necessary
 // metadata synchronization.
 //
 // Test logic:
-// 1. Create multiple test cases to simulate different states of
-//    AlluxioRuntime, Dataset, and StatefulSet.
-// 2. Initialize AlluxioEngine and its dependencies using a fake client.
-// 3. Use Monkey Patching to mock the behavior of AlluxioFileUtils methods.
-// 4. Call e.PrepareUFS() and verify whether the UFS mounting process
-//    executes correctly.
-// 5. Assert that the returned errors match the expected outcomes.
+//  1. Create multiple test cases to simulate different states of
+//     AlluxioRuntime, Dataset, and StatefulSet.
+//  2. Initialize AlluxioEngine and its dependencies using a fake client.
+//  3. Use Monkey Patching to mock the behavior of AlluxioFileUtils methods.
+//  4. Call e.PrepareUFS() and verify whether the UFS mounting process
+//     executes correctly.
+//  5. Assert that the returned errors match the expected outcomes.
 //
 // Parameters:
 // - t *testing.T: The testing context provided by the Go testing framework.

pkg/ddc/alluxio/utils_test.go

@@ -184,6 +184,7 @@ func TestMountRootWithoutEnvSet(t *testing.T) {
 // This function checks whether a specified port is in the list of used ports.
 // Test cases include:
 //   - Checking if a port is in the list of used ports.
+//
 // Each test case calls the isPortInUsed function and verifies if the returned value matches the expected result.
 // If the returned value does not match the expected result, the test fails and outputs an error message.
 func Test_isPortInUsed(t *testing.T) {
@@ -368,29 +369,29 @@ func TestGetDataSetFileNum(t *testing.T) {
 	}
 }
 
-// TestGetRuntime tests the AlluxioEngine.getRuntime() method to verify it correctly retrieves 
+// TestGetRuntime tests the AlluxioEngine.getRuntime() method to verify it correctly retrieves
 // the AlluxioRuntime custom resource from the Kubernetes cluster.
-// 
+//
 // Test Structure:
 // - Defines a test table with multiple test cases (though currently only one example exists).
 // - Each test case specifies:
-//   * Input fields: Simulated AlluxioEngine instance configuration.
-//   * Expected output: The AlluxioRuntime object that should be returned.
-//   * Error expectation: Whether an error is expected during retrieval.
-// 
+//   - Input fields: Simulated AlluxioEngine instance configuration.
+//   - Expected output: The AlluxioRuntime object that should be returned.
+//   - Error expectation: Whether an error is expected during retrieval.
+//
 // Key Testing Components:
-// - Uses Kubernetes client-go testing utilities (fake client, scheme registration) to mock 
-//   API server interactions, avoiding real cluster dependencies.
-// - Validates both successful retrieval and error conditions.
-// - Checks deep equality between retrieved and expected objects to ensure metadata accuracy.
+//   - Uses Kubernetes client-go testing utilities (fake client, scheme registration) to mock
+//     API server interactions, avoiding real cluster dependencies.
+//   - Validates both successful retrieval and error conditions.
+//   - Checks deep equality between retrieved and expected objects to ensure metadata accuracy.
 //
 // Test Workflow for Each Case:
 // 1. Register required Kubernetes resource types (AlluxioRuntime, core v1) into the scheme.
 // 2. Initialize a fake client preloaded with the expected AlluxioRuntime object.
 // 3. Instantiate the AlluxioEngine with test-specific configurations and the fake client.
 // 4. Execute getRuntime() and validate:
-//    - Error behavior matches expectations
-//    - Retrieved object matches the expected object structure exactly
+//   - Error behavior matches expectations
+//   - Retrieved object matches the expected object structure exactly
 //
 // Edge Cases Covered (via additional test cases in TODO):
 // - Non-existent runtime
@@ -815,12 +816,12 @@ func TestGetFuseDaemonsetName(t *testing.T) {
 // namespace, and engine name parameters to validate the resulting filesystem path.
 //
 // Parameters:
-//  - t : *testing.T
-//    Testing framework handle for managing test state and reporting failures
+//   - t : *testing.T
+//     Testing framework handle for managing test state and reporting failures
 //
 // Returns:
-//  - None
-//    Failures are reported through t.Errorf
+//   - None
+//     Failures are reported through t.Errorf
 func TestGetMountPoint(t *testing.T) {
 	type fields struct {
 		name      string

pkg/ddc/jindo/load_data_test.go

@@ -554,7 +554,7 @@ func Test_genDataLoadValue(t *testing.T) {
 	}
 }
 
-// TestGenerateDataLoadValueFileWithRuntimeHDD tests the generation of data load value files 
+// TestGenerateDataLoadValueFileWithRuntimeHDD tests the generation of data load value files
 // when using a JindoRuntime with an HDD-based tiered storage configuration.
 //
 // This function ensures that the dataset and JindoRuntime configurations are correctly initialized

pkg/ddc/juicefs/data_load.go

@@ -179,12 +179,6 @@ func (j *JuiceFSEngine) genDataLoadValue(image string, cacheinfo map[string]stri
 	dataloadInfo.TargetPaths = targetPaths
 
 	options := map[string]string{}
-	// resolve spec options
-	if dataload.Spec.Options != nil {
-		for key, value := range dataload.Spec.Options {
-			options[key] = value
-		}
-	}
 
 	for key, value := range cacheinfo {
 		options[key] = value
@@ -201,9 +195,21 @@ func (j *JuiceFSEngine) genDataLoadValue(image string, cacheinfo map[string]stri
 	}
 	options["edition"] = cacheinfo[Edition]
 	options["runtimeName"] = j.name
-	if _, ok := options["timeout"]; !ok {
-		options["timeout"] = DefaultDataLoadTimeout
+	timeout := dataload.Spec.Options["timeout"]
+	delete(dataload.Spec.Options, "timeout")
+	if timeout == "" {
+		timeout = DefaultDataLoadTimeout
+	}
+	warmupOtions := []string{}
+	for k, v := range dataload.Spec.Options {
+		if v != "" {
+			warmupOtions = append(warmupOtions, fmt.Sprintf("--%s=%s", k, v))
+		} else {
+			warmupOtions = append(warmupOtions, fmt.Sprintf("--%s", k))
+		}
 	}
+	options["option"] = strings.Join(warmupOtions, " ")
+	options["timeout"] = timeout
 
 	dataloadInfo.Options = options
 

pkg/ddc/juicefs/data_load_test.go

@@ -17,6 +17,7 @@ limitations under the License.
 package juicefs
 
 import (
+	"encoding/json"
 	"os"
 	"path/filepath"
 	"reflect"
@@ -25,14 +26,15 @@ import (
 
 	"github.com/fluid-cloudnative/fluid/pkg/common"
 
-	datav1alpha1 "github.com/fluid-cloudnative/fluid/api/v1alpha1"
-	cdataload "github.com/fluid-cloudnative/fluid/pkg/dataload"
-	cruntime "github.com/fluid-cloudnative/fluid/pkg/runtime"
-	"github.com/fluid-cloudnative/fluid/pkg/utils/fake"
 	appsv1 "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+
+	datav1alpha1 "github.com/fluid-cloudnative/fluid/api/v1alpha1"
+	cdataload "github.com/fluid-cloudnative/fluid/pkg/dataload"
+	cruntime "github.com/fluid-cloudnative/fluid/pkg/runtime"
+	"github.com/fluid-cloudnative/fluid/pkg/utils/fake"
 )
 
 var valuesConfigMapData = `
@@ -561,7 +563,7 @@ func TestJuiceFSEngine_genDataLoadValue(t *testing.T) {
 					ImagePullSecrets: []v1.LocalObjectReference{},
 					Options: map[string]string{
 						// dataload spec options
-						"dl-opts-k-1": "dl-opts-v-1",
+						"option": "--dl-opts-k-1=dl-opts-v-1",
 						// cache info
 						Edition:         CommunityEdition,
 						"cache-info-k1": "cache-info-v1",
@@ -728,7 +730,7 @@ func TestJuiceFSEngine_genDataLoadValue(t *testing.T) {
 					},
 					Options: map[string]string{
 						// dataload spec options
-						"dl-opts-k-1": "dl-opts-v-1",
+						"option": "--dl-opts-k-1=dl-opts-v-1",
 						// cache info
 						Edition:         CommunityEdition,
 						"cache-info-k1": "cache-info-v1",
@@ -829,7 +831,7 @@ func TestJuiceFSEngine_genDataLoadValue(t *testing.T) {
 					},
 					Options: map[string]string{
 						// dataload spec options
-						"dl-opts-k-1": "dl-opts-v-1",
+						"option": "--dl-opts-k-1=dl-opts-v-1",
 						// cache info
 						Edition:         CommunityEdition,
 						"cache-info-k1": "cache-info-v1",
@@ -938,7 +940,7 @@ func TestJuiceFSEngine_genDataLoadValue(t *testing.T) {
 					},
 					Options: map[string]string{
 						// dataload spec options
-						"dl-opts-k-1": "dl-opts-v-1",
+						"option": "--dl-opts-k-1=dl-opts-v-1",
 						// cache info
 						Edition:         CommunityEdition,
 						"cache-info-k1": "cache-info-v1",
@@ -961,7 +963,9 @@ func TestJuiceFSEngine_genDataLoadValue(t *testing.T) {
 		}
 		got, _ := engine.genDataLoadValue(item.image, item.cacheInfo, item.pods, item.targetDataset, item.dataload)
 		if !reflect.DeepEqual(got, item.want) {
-			t.Errorf("case %s, got %v,want:%v", k, got, item.want)
+			gotJson, _ := json.Marshal(got)
+			wantJson, _ := json.Marshal(item.want)
+			t.Errorf("case %s, got %v\nwant:%v", k, string(gotJson), string(wantJson))
 		}
 	}
 }

pkg/ddc/juicefs/data_migrate_test.go

@@ -317,26 +317,26 @@ func TestJuiceFSEngine_genDataUrl_PVC(t *testing.T) {
 	}
 }
 
-//TestJuiceFSEngine_genDataUrl Test JuiceFSEngine's genDataUrl function
-//This function is used to generate URLs for data migration and supports the following two data sources:
+// TestJuiceFSEngine_genDataUrl Test JuiceFSEngine's genDataUrl function
+// This function is used to generate URLs for data migration and supports the following two data sources:
 // 1. ExternalStorage: Generate URLs with authentication information via URIs and encryption options.
 // 2. DataSet: Generate a URL in JuiceFS format from the dataset information and mount points.
 //
-//Parameter:
+// Parameter:
 // - t *testing. T: The context object of the test framework to report test results and errors.
 //
-//Test Cases:
-//- test-external: Tests externally stored URL generation to verify that the encryption option is correctly replaced.
-//- test-external-subpath: Tests the URL generation of the external storage subpath.
-//- test-external-subpath-file: tests the URL generation of the external storage subpath file.
-//- test-dataset: Test the URL generation of the dataset store to verify that the subpath is handled correctly.
-//- test-dataset-no-path: The URL generated by the test dataset store (no subpath).
-//- test-dataset-subpath-file: The URL of the test dataset storage subpath file is generated.
-//- test-dataset-subpath-file2: The URL of the test dataset storage subpath file (with mount point) is generated.
-//- test-dataset-subpath-file3: The URL of the test dataset storage subpath file (with mount points, no subpaths).
+// Test Cases:
+// - test-external: Tests externally stored URL generation to verify that the encryption option is correctly replaced.
+// - test-external-subpath: Tests the URL generation of the external storage subpath.
+// - test-external-subpath-file: tests the URL generation of the external storage subpath file.
+// - test-dataset: Test the URL generation of the dataset store to verify that the subpath is handled correctly.
+// - test-dataset-no-path: The URL generated by the test dataset store (no subpath).
+// - test-dataset-subpath-file: The URL of the test dataset storage subpath file is generated.
+// - test-dataset-subpath-file2: The URL of the test dataset storage subpath file (with mount point) is generated.
+// - test-dataset-subpath-file3: The URL of the test dataset storage subpath file (with mount points, no subpaths).
 //
-//Return value:
-//- No return value, test failure is reported via t.Errorf.
+// Return value:
+// - No return value, test failure is reported via t.Errorf.
 func TestJuiceFSEngine_genDataUrl(t *testing.T) {
 	juicefsSecret := &corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{

pkg/ddc/juicefs/datasetinfo_parser.go

@@ -63,6 +63,7 @@ func parseCacheInfoFromConfigMap(configMap *v1.ConfigMap) (cacheinfo map[string]
 //   - client: A Kubernetes client used to interact with the cluster.
 //   - name: The base name of the target ConfigMap.
 //   - namespace: The namespace where the ConfigMap is located.
+//
 // Returns:
 //   - A map containing file system information parsed from the ConfigMap.
 //   - An error if the ConfigMap retrieval or parsing fails.