Skip to content

Commit 7fea764

Browse files
CAICAIIsCAICAIIs
committed
fix: harden jindo multi-oss acceptance coverage
Signed-off-by: CAICAIIs <3360776475@qq.com>
1 parent 00e3a67 commit 7fea764

6 files changed

Lines changed: 160 additions & 46 deletions

File tree

‎pkg/ddc/jindocache/transform.go‎

Lines changed: 21 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -584,30 +584,29 @@ func (e *JindoCacheEngine) transformMaster(runtime *datav1alpha1.JindoRuntime, m
584584
e.Log.Error(err, "invalid encryptOption secret reference", "key", key, "mountPoint", mount.MountPoint)
585585
return err
586586
}
587-
if mountType == "oss" && ossBucketName != "" {
588-
if secretMountSupport {
589-
secretURI := buildBucketSecretURI(ossBucketName)
590-
if value.BucketSecretPaths == nil {
591-
value.BucketSecretPaths = map[string]string{}
592-
}
593-
value.BucketSecretPaths[ossBucketName] = secretURI
594-
595-
itemPath := ""
596-
switch key {
597-
case "fs.oss.accessKeyId":
598-
itemPath = ossBucketName + "/AccessKeyId"
599-
case "fs.oss.accessKeySecret":
600-
itemPath = ossBucketName + "/AccessKeySecret"
601-
}
602-
if itemPath != "" {
603-
value.SecretProjections, err = appendSecretProjection(value.SecretProjections, secretKeyRef.Name, secretKeyRef.Key, itemPath)
604-
if err != nil {
605-
return err
587+
if mountType == "oss" && ossBucketName != "" {
588+
if secretMountSupport {
589+
itemPath := ""
590+
switch key {
591+
case "fs.oss.accessKeyId":
592+
itemPath = ossBucketName + "/AccessKeyId"
593+
case "fs.oss.accessKeySecret":
594+
itemPath = ossBucketName + "/AccessKeySecret"
606595
}
596+
if itemPath != "" {
597+
secretURI := buildBucketSecretURI(ossBucketName)
598+
if value.BucketSecretPaths == nil {
599+
value.BucketSecretPaths = map[string]string{}
600+
}
601+
value.BucketSecretPaths[ossBucketName] = secretURI
602+
value.SecretProjections, err = appendSecretProjection(value.SecretProjections, secretKeyRef.Name, secretKeyRef.Key, itemPath)
603+
if err != nil {
604+
return err
605+
}
606+
e.Log.Info("Configure OSS bucket credential projection", "bucket", ossBucketName, "secretName", secretKeyRef.Name, "key", key)
607+
}
608+
continue
607609
}
608-
e.Log.Info("Configure OSS bucket credential projection", "bucket", ossBucketName, "secretName", secretKeyRef.Name, "key", key)
609-
continue
610-
}
611610

612611
secret, err := kubeclient.GetSecret(e.Client, secretKeyRef.Name, e.namespace)
613612
if err != nil {

‎pkg/ddc/jindocache/transform_test.go‎

Lines changed: 49 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1899,7 +1899,7 @@ func TestJindoCacheEngine_transformMasterRejectsConflictingSameBucketSecretProje
18991899
}
19001900
}
19011901

1902-
func TestJindoCacheEngine_transformMasterPreservesInlineOSSCredentials(t *testing.T) {
1902+
func TestJindoCacheEngine_transformMasterSupportsInlineOSSCredentialsCompatibility(t *testing.T) {
19031903
s := runtime.NewScheme()
19041904
s.AddKnownTypes(datav1alpha1.GroupVersion, &datav1alpha1.JindoRuntime{}, &datav1alpha1.Dataset{})
19051905
_ = corev1.AddToScheme(s)
@@ -1949,6 +1949,54 @@ func TestJindoCacheEngine_transformMasterPreservesInlineOSSCredentials(t *testin
19491949
}
19501950
}
19511951

1952+
func TestJindoCacheEngine_transformMasterIgnoresNonOSSCredentialEncryptOptionForBucketSecretProjection(t *testing.T) {
1953+
s := runtime.NewScheme()
1954+
s.AddKnownTypes(datav1alpha1.GroupVersion, &datav1alpha1.JindoRuntime{}, &datav1alpha1.Dataset{})
1955+
_ = corev1.AddToScheme(s)
1956+
1957+
engine := JindoCacheEngine{
1958+
name: "test",
1959+
namespace: "fluid",
1960+
Client: fake.NewFakeClientWithScheme(s),
1961+
Log: fake.NullLogger(),
1962+
runtime: &datav1alpha1.JindoRuntime{
1963+
Spec: datav1alpha1.JindoRuntimeSpec{
1964+
Fuse: datav1alpha1.JindoFuseSpec{},
1965+
},
1966+
},
1967+
}
1968+
1969+
dataset := &datav1alpha1.Dataset{
1970+
Spec: datav1alpha1.DatasetSpec{
1971+
Mounts: []datav1alpha1.Mount{{
1972+
MountPoint: "oss://bucket-a/data",
1973+
Name: "mount-a",
1974+
Options: map[string]string{
1975+
"fs.oss.endpoint": "oss-cn-shanghai.aliyuncs.com",
1976+
},
1977+
EncryptOptions: []datav1alpha1.EncryptOption{{
1978+
Name: "fs.oss.sessionToken",
1979+
ValueFrom: datav1alpha1.EncryptOptionSource{
1980+
SecretKeyRef: datav1alpha1.SecretKeySelector{Name: "secret-a", Key: "token"},
1981+
},
1982+
}},
1983+
}},
1984+
},
1985+
}
1986+
1987+
value := &Jindo{}
1988+
if err := engine.transformMaster(engine.runtime, "/test", value, dataset, true); err != nil {
1989+
t.Fatalf("transformMaster() error = %v", err)
1990+
}
1991+
1992+
if len(value.SecretProjections) != 0 {
1993+
t.Fatalf("expected no secret projections for non-AK/SK encryptOptions, got %d", len(value.SecretProjections))
1994+
}
1995+
if len(value.BucketSecretPaths) != 0 {
1996+
t.Fatalf("expected no bucket secret paths for non-AK/SK encryptOptions, got %#v", value.BucketSecretPaths)
1997+
}
1998+
}
1999+
19522000
func TestJindoCacheEngine_transformMasterReturnsErrorWhenReferencedSecretMissing(t *testing.T) {
19532001
s := runtime.NewScheme()
19542002
s.AddKnownTypes(datav1alpha1.GroupVersion, &datav1alpha1.JindoRuntime{}, &datav1alpha1.Dataset{})

‎pkg/ddc/jindofsx/transform.go‎

Lines changed: 21 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -491,30 +491,29 @@ func (e *JindoFSxEngine) transformMaster(runtime *datav1alpha1.JindoRuntime, met
491491
e.Log.Error(err, "invalid encryptOption secret reference", "key", key, "mountPoint", mount.MountPoint)
492492
return err
493493
}
494-
if mountType == "oss" && ossBucketName != "" {
495-
if secretMountSupport {
496-
secretURI := buildBucketSecretURI(ossBucketName)
497-
if value.BucketSecretPaths == nil {
498-
value.BucketSecretPaths = map[string]string{}
499-
}
500-
value.BucketSecretPaths[ossBucketName] = secretURI
501-
502-
itemPath := ""
503-
switch key {
504-
case "fs.oss.accessKeyId":
505-
itemPath = ossBucketName + "/AccessKeyId"
506-
case "fs.oss.accessKeySecret":
507-
itemPath = ossBucketName + "/AccessKeySecret"
508-
}
509-
if itemPath != "" {
510-
value.SecretProjections, err = appendSecretProjection(value.SecretProjections, secretKeyRef.Name, secretKeyRef.Key, itemPath)
511-
if err != nil {
512-
return err
494+
if mountType == "oss" && ossBucketName != "" {
495+
if secretMountSupport {
496+
itemPath := ""
497+
switch key {
498+
case "fs.oss.accessKeyId":
499+
itemPath = ossBucketName + "/AccessKeyId"
500+
case "fs.oss.accessKeySecret":
501+
itemPath = ossBucketName + "/AccessKeySecret"
502+
}
503+
if itemPath != "" {
504+
secretURI := buildBucketSecretURI(ossBucketName)
505+
if value.BucketSecretPaths == nil {
506+
value.BucketSecretPaths = map[string]string{}
507+
}
508+
value.BucketSecretPaths[ossBucketName] = secretURI
509+
value.SecretProjections, err = appendSecretProjection(value.SecretProjections, secretKeyRef.Name, secretKeyRef.Key, itemPath)
510+
if err != nil {
511+
return err
512+
}
513+
e.Log.Info("Configure OSS bucket credential projection", "bucket", ossBucketName, "secretName", secretKeyRef.Name, "key", key)
513514
}
515+
continue
514516
}
515-
e.Log.Info("Configure OSS bucket credential projection", "bucket", ossBucketName, "secretName", secretKeyRef.Name, "key", key)
516-
continue
517-
}
518517

519518
secret, err := kubeclient.GetSecret(e.Client, secretKeyRef.Name, e.namespace)
520519
if err != nil {

‎pkg/ddc/jindofsx/transform_master_test.go‎

Lines changed: 49 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -387,7 +387,7 @@ func TestJindoFSxEngine_transformMasterRejectsConflictingSameBucketSecretProject
387387
}
388388
}
389389

390-
func TestJindoFSxEngine_transformMasterPreservesInlineOSSCredentials(t *testing.T) {
390+
func TestJindoFSxEngine_transformMasterSupportsInlineOSSCredentialsCompatibility(t *testing.T) {
391391
s := runtime.NewScheme()
392392
s.AddKnownTypes(datav1alpha1.GroupVersion, &datav1alpha1.JindoRuntime{}, &datav1alpha1.Dataset{})
393393
_ = corev1.AddToScheme(s)
@@ -437,6 +437,54 @@ func TestJindoFSxEngine_transformMasterPreservesInlineOSSCredentials(t *testing.
437437
}
438438
}
439439

440+
func TestJindoFSxEngine_transformMasterIgnoresNonOSSCredentialEncryptOptionForBucketSecretProjection(t *testing.T) {
441+
s := runtime.NewScheme()
442+
s.AddKnownTypes(datav1alpha1.GroupVersion, &datav1alpha1.JindoRuntime{}, &datav1alpha1.Dataset{})
443+
_ = corev1.AddToScheme(s)
444+
445+
engine := JindoFSxEngine{
446+
name: "test",
447+
namespace: "fluid",
448+
Client: fake.NewFakeClientWithScheme(s),
449+
Log: fake.NullLogger(),
450+
runtime: &datav1alpha1.JindoRuntime{
451+
Spec: datav1alpha1.JindoRuntimeSpec{
452+
Fuse: datav1alpha1.JindoFuseSpec{},
453+
},
454+
},
455+
}
456+
457+
dataset := &datav1alpha1.Dataset{
458+
Spec: datav1alpha1.DatasetSpec{
459+
Mounts: []datav1alpha1.Mount{{
460+
MountPoint: "oss://bucket-a/data",
461+
Name: "mount-a",
462+
Options: map[string]string{
463+
"fs.oss.endpoint": "oss-cn-shanghai.aliyuncs.com",
464+
},
465+
EncryptOptions: []datav1alpha1.EncryptOption{{
466+
Name: "fs.oss.sessionToken",
467+
ValueFrom: datav1alpha1.EncryptOptionSource{
468+
SecretKeyRef: datav1alpha1.SecretKeySelector{Name: "secret-a", Key: "token"},
469+
},
470+
}},
471+
}},
472+
},
473+
}
474+
475+
value := &Jindo{}
476+
if err := engine.transformMaster(engine.runtime, "/test", value, dataset, true); err != nil {
477+
t.Fatalf("transformMaster() error = %v", err)
478+
}
479+
480+
if len(value.SecretProjections) != 0 {
481+
t.Fatalf("expected no secret projections for non-AK/SK encryptOptions, got %d", len(value.SecretProjections))
482+
}
483+
if len(value.BucketSecretPaths) != 0 {
484+
t.Fatalf("expected no bucket secret paths for non-AK/SK encryptOptions, got %#v", value.BucketSecretPaths)
485+
}
486+
}
487+
440488
func TestJindoFSxEngine_transformMasterUsesReferencedSecretKeysForNonOSSMounts(t *testing.T) {
441489
s := runtime.NewScheme()
442490
s.AddKnownTypes(datav1alpha1.GroupVersion, &datav1alpha1.JindoRuntime{}, &datav1alpha1.Dataset{})

‎test/gha-e2e/jindo/multi-oss-job.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,13 @@ spec:
1212
image: registry-cn-hongkong.ack.aliyuncs.com/acs/smartdata:6.9.1-202509151826
1313
imagePullPolicy: IfNotPresent
1414
resources:
15+
requests:
16+
cpu: "100m"
17+
memory: "128Mi"
18+
ephemeral-storage: "1Gi"
1519
limits:
20+
cpu: "500m"
21+
memory: "256Mi"
1622
ephemeral-storage: "5Gi"
1723
command: ["/bin/sh"]
1824
args:

‎test/gha-e2e/jindo/oss-emulator.yaml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,13 @@ spec:
3939
- name: emulator
4040
image: fluidcloudnative/oss-emulator:e2e
4141
imagePullPolicy: IfNotPresent
42+
resources:
43+
requests:
44+
cpu: "50m"
45+
memory: "64Mi"
46+
limits:
47+
cpu: "250m"
48+
memory: "128Mi"
4249
env:
4350
- name: BUCKET_NAME
4451
value: bucket-a
@@ -91,6 +98,13 @@ spec:
9198
- name: emulator
9299
image: fluidcloudnative/oss-emulator:e2e
93100
imagePullPolicy: IfNotPresent
101+
resources:
102+
requests:
103+
cpu: "50m"
104+
memory: "64Mi"
105+
limits:
106+
cpu: "250m"
107+
memory: "128Mi"
94108
env:
95109
- name: BUCKET_NAME
96110
value: bucket-b

0 commit comments

Comments
 (0)