Skip to content

Commit ff4bf82

Browse files
sudo-aminesudo-amine
authored
sync: support configurable deletion propagation (#180)
* sync: support configurable deletion propagation Use an annotation on kcp source objects to select the propagation policy for deleting service-cluster copies. Keep background propagation as the default and resolve kcp-origin related resources from their own annotations. Signed-off-by: Amine HADRI <amine.hadri.mba@gmail.com> * sync: rename shadowed copy variable in related-copies test golangci-lint's predeclared linter flags local variables named copy since it shadows the builtin. Rename to copyObj to fix the lint job. Signed-off-by: Amine HADRI <amine.hadri.mba@gmail.com> --------- Signed-off-by: Amine HADRI <amine.hadri.mba@gmail.com>
1 parent 45eaad7 commit ff4bf82

11 files changed

Lines changed: 653 additions & 14 deletions

File tree

‎docs/content/publish-resources/technical-details.md‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,19 @@ is the only real evidence in the kcp side that the Sync Agent is even doing thin
5858
(source) object is deleted, the corresponding local object is deleted as well. Once the local object
5959
is gone, the finalizer is removed from the source object.
6060

61+
By default, the local object is deleted with background propagation. A different policy can be
62+
selected by annotating the kcp object before deleting it:
63+
64+
```bash
65+
kubectl annotate <resource> <name> \
66+
syncagent.kcp.io/deletion-propagation-policy=foreground
67+
```
68+
69+
Supported values are `background`, `foreground`, and `orphan`. A missing or invalid value defaults
70+
to `background`. The annotation controls deletion in the service cluster independently of the
71+
policy used to delete the kcp object. For kcp-origin related resources, annotate each related object
72+
with the policy to use for its local copy.
73+
6174
### Phase 3: Ensure Object Existence
6275

6376
We have a source object and now need to create the destination. This chart shows what's happening.

‎internal/sync/deletion_policy.go‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
/*
2+
Copyright 2026 The KCP Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package sync
18+
19+
import (
20+
syncagentv1alpha1 "github.com/kcp-dev/api-syncagent/sdk/apis/syncagent/v1alpha1"
21+
22+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
23+
)
24+
25+
const (
26+
deletionPropagationPolicyAnnotation = "syncagent.kcp.io/deletion-propagation-policy"
27+
deletionPropagationBackground = "background"
28+
deletionPropagationForeground = "foreground"
29+
deletionPropagationOrphan = "orphan"
30+
)
31+
32+
// deletionPropagationPolicy resolves the policy specified for the
33+
// service-cluster copy. Missing or unsupported values mean background deletion.
34+
func deletionPropagationPolicy(obj metav1.Object) metav1.DeletionPropagation {
35+
switch obj.GetAnnotations()[deletionPropagationPolicyAnnotation] {
36+
case deletionPropagationForeground:
37+
return metav1.DeletePropagationForeground
38+
case deletionPropagationOrphan:
39+
return metav1.DeletePropagationOrphan
40+
default:
41+
return metav1.DeletePropagationBackground
42+
}
43+
}
44+
45+
func relatedDeletionPropagationPolicy(
46+
origin syncagentv1alpha1.RelatedResourceOrigin,
47+
obj metav1.Object,
48+
) metav1.DeletionPropagation {
49+
if origin != syncagentv1alpha1.RelatedResourceOriginKcp {
50+
return metav1.DeletePropagationBackground
51+
}
52+
53+
return deletionPropagationPolicy(obj)
54+
}
55+
56+
func normalizeDeletionPropagationPolicy(policy metav1.DeletionPropagation) metav1.DeletionPropagation {
57+
switch policy {
58+
case metav1.DeletePropagationForeground, metav1.DeletePropagationOrphan:
59+
return policy
60+
default:
61+
return metav1.DeletePropagationBackground
62+
}
63+
}

‎internal/sync/deletion_policy_test.go‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
/*
2+
Copyright 2026 The KCP Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package sync
18+
19+
import (
20+
"testing"
21+
22+
syncagentv1alpha1 "github.com/kcp-dev/api-syncagent/sdk/apis/syncagent/v1alpha1"
23+
24+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
25+
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
26+
)
27+
28+
func TestDeletionPropagationPolicy(t *testing.T) {
29+
testcases := []struct {
30+
name string
31+
policy string
32+
expected metav1.DeletionPropagation
33+
}{
34+
{
35+
name: "foreground annotation",
36+
policy: deletionPropagationForeground,
37+
expected: metav1.DeletePropagationForeground,
38+
},
39+
{
40+
name: "orphan annotation",
41+
policy: deletionPropagationOrphan,
42+
expected: metav1.DeletePropagationOrphan,
43+
},
44+
{
45+
name: "background annotation",
46+
policy: deletionPropagationBackground,
47+
expected: metav1.DeletePropagationBackground,
48+
},
49+
{
50+
name: "missing annotation defaults to background",
51+
expected: metav1.DeletePropagationBackground,
52+
},
53+
{
54+
name: "unknown annotation defaults to background",
55+
policy: "unknown",
56+
expected: metav1.DeletePropagationBackground,
57+
},
58+
}
59+
60+
for _, testcase := range testcases {
61+
t.Run(testcase.name, func(t *testing.T) {
62+
obj := &unstructured.Unstructured{}
63+
if testcase.policy != "" {
64+
obj.SetAnnotations(map[string]string{deletionPropagationPolicyAnnotation: testcase.policy})
65+
}
66+
67+
got := deletionPropagationPolicy(obj)
68+
if got != testcase.expected {
69+
t.Fatalf("expected %q, got %q", testcase.expected, got)
70+
}
71+
})
72+
}
73+
}
74+
75+
func TestRelatedDeletionPropagationPolicy(t *testing.T) {
76+
obj := &unstructured.Unstructured{}
77+
obj.SetAnnotations(map[string]string{
78+
deletionPropagationPolicyAnnotation: deletionPropagationForeground,
79+
})
80+
81+
if got := relatedDeletionPropagationPolicy(
82+
syncagentv1alpha1.RelatedResourceOriginKcp,
83+
obj,
84+
); got != metav1.DeletePropagationForeground {
85+
t.Fatalf("expected kcp-origin related object to use foreground, got %q", got)
86+
}
87+
88+
if got := relatedDeletionPropagationPolicy(
89+
syncagentv1alpha1.RelatedResourceOriginService,
90+
obj,
91+
); got != metav1.DeletePropagationBackground {
92+
t.Fatalf("expected service-origin related object to use background, got %q", got)
93+
}
94+
}

‎internal/sync/metadata.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,7 @@ func filterUnsyncableLabels(original labels.Set) labels.Set {
112112
var unsyncableAnnotations = sets.New(
113113
"kcp.io/cluster",
114114
"kubectl.kubernetes.io/last-applied-configuration",
115+
deletionPropagationPolicyAnnotation,
115116
remoteObjectNamespaceAnnotation,
116117
remoteObjectNameAnnotation,
117118
remoteObjectWorkspacePathAnnotation,

‎internal/sync/object_syncer.go‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ import (
3131

3232
corev1 "k8s.io/api/core/v1"
3333
apierrors "k8s.io/apimachinery/pkg/api/errors"
34+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
3435
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
3536
"k8s.io/apimachinery/pkg/runtime"
3637
"k8s.io/apimachinery/pkg/types"
@@ -77,6 +78,8 @@ type objectSyncer struct {
7778
// being deleted; used to clean up related resources when the primary object
7879
// is being deleted.
7980
forceDelete bool
81+
// deletionPropagationPolicy is used when deleting the destination object.
82+
deletionPropagationPolicy metav1.DeletionPropagation
8083
// useServerSideApply switches the syncer from client-side merge patches
8184
// (backed by a last-known-state secret) to Kubernetes Server-Side Apply
8285
// using a stable field manager. SSA preserves fields owned by other
@@ -623,8 +626,9 @@ func (s *objectSyncer) handleDeletion(ctx context.Context, log *zap.SugaredLogge
623626
if dest.object != nil {
624627
if dest.object.GetDeletionTimestamp() == nil {
625628
log.Debugw("Deleting destination object…", "dest-object", newObjectKey(dest.object, dest.clusterName, logicalcluster.None))
626-
s.recordEvent(ctx, source, dest, corev1.EventTypeNormal, "ObjectCleanup", "Object deletion has been started and will progress in the background.")
627-
if err := dest.client.Delete(ctx, dest.object); err != nil {
629+
s.recordEvent(ctx, source, dest, corev1.EventTypeNormal, "ObjectCleanup", "Object deletion has been started.")
630+
if err := dest.client.Delete(ctx, dest.object,
631+
ctrlruntimeclient.PropagationPolicy(normalizeDeletionPropagationPolicy(s.deletionPropagationPolicy))); err != nil {
628632
return false, fmt.Errorf("failed to delete destination object: %w", err)
629633
}
630634
}

‎internal/sync/object_syncer_test.go‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,11 @@ import (
2424

2525
"github.com/kcp-dev/logicalcluster/v3"
2626

27+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2728
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
2829
"k8s.io/client-go/tools/record"
2930
ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client"
31+
"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
3032
)
3133

3234
// fakeStateStore is a minimal ObjectStateStore for unit tests.
@@ -56,6 +58,81 @@ func makeUnstructuredWithStatus(name, namespace string, status map[string]interf
5658
return obj
5759
}
5860

61+
func TestHandleDeletionUsesPropagationPolicy(t *testing.T) {
62+
testcases := []struct {
63+
name string
64+
policy metav1.DeletionPropagation
65+
expected metav1.DeletionPropagation
66+
}{
67+
{
68+
name: "foreground",
69+
policy: metav1.DeletePropagationForeground,
70+
expected: metav1.DeletePropagationForeground,
71+
},
72+
{
73+
name: "orphan",
74+
policy: metav1.DeletePropagationOrphan,
75+
expected: metav1.DeletePropagationOrphan,
76+
},
77+
{
78+
name: "background",
79+
policy: metav1.DeletePropagationBackground,
80+
expected: metav1.DeletePropagationBackground,
81+
},
82+
{
83+
name: "background by default",
84+
expected: metav1.DeletePropagationBackground,
85+
},
86+
}
87+
88+
for _, testcase := range testcases {
89+
t.Run(testcase.name, func(t *testing.T) {
90+
destination := makeUnstructuredWithStatus("destination", "default", nil)
91+
92+
var got *metav1.DeletionPropagation
93+
destinationClient := newFakeClientBuilder().
94+
WithObjects(destination).
95+
WithInterceptorFuncs(interceptor.Funcs{
96+
Delete: func(ctx context.Context, client ctrlruntimeclient.WithWatch, obj ctrlruntimeclient.Object, options ...ctrlruntimeclient.DeleteOption) error {
97+
deleteOptions := &ctrlruntimeclient.DeleteOptions{}
98+
for _, option := range options {
99+
option.ApplyToDelete(deleteOptions)
100+
}
101+
got = deleteOptions.PropagationPolicy
102+
return client.Delete(ctx, obj, options...)
103+
},
104+
}).
105+
Build()
106+
107+
source := makeUnstructuredWithStatus("source", "default", nil)
108+
source.SetFinalizers([]string{deletionFinalizer})
109+
now := metav1.Now()
110+
source.SetDeletionTimestamp(&now)
111+
112+
syncer := objectSyncer{
113+
blockSourceDeletion: true,
114+
deletionPropagationPolicy: testcase.policy,
115+
eventObjSide: syncSideSource,
116+
}
117+
ctx := WithEventRecorder(t.Context(), record.NewFakeRecorder(1))
118+
119+
requeue, err := syncer.handleDeletion(ctx, zap.NewNop().Sugar(), syncSide{object: source}, syncSide{
120+
client: destinationClient,
121+
object: destination,
122+
})
123+
if err != nil {
124+
t.Fatalf("handleDeletion returned an error: %v", err)
125+
}
126+
if !requeue {
127+
t.Fatal("handleDeletion did not request a requeue")
128+
}
129+
if got == nil || *got != testcase.expected {
130+
t.Fatalf("expected %q propagation policy, got %v", testcase.expected, got)
131+
}
132+
})
133+
}
134+
}
135+
59136
func TestSyncObjectStatusForward(t *testing.T) {
60137
log := zap.NewNop().Sugar()
61138

‎internal/sync/syncer.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -209,6 +209,7 @@ func (s *ResourceSyncer) Process(ctx context.Context, remoteObj *unstructured.Un
209209
// object state; this allows the code to create meaningful patches and not overwrite
210210
// fields that were defaulted by the kube-apiserver or a mutating webhook
211211
stateStore := s.newObjectStateStore(sourceSide, destSide)
212+
deletionPolicy := deletionPropagationPolicy(remoteObj)
212213

213214
syncer := objectSyncer{
214215
// The primary object should be labelled with the agent name.
@@ -223,6 +224,8 @@ func (s *ResourceSyncer) Process(ctx context.Context, remoteObj *unstructured.Un
223224
// perform cleanup on the service cluster side when the source object
224225
// in kcp is deleted
225226
blockSourceDeletion: true,
227+
// Apply the service-cluster deletion policy specified on the kcp source.
228+
deletionPropagationPolicy: deletionPolicy,
226229
// use the configured mutations from the PublishedResource
227230
mutator: s.primaryMutator,
228231
// make sure the syncer can remember the current state of any object

‎internal/sync/syncer_related.go‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,7 @@ func (s *ResourceSyncer) processRelatedResource(ctx context.Context, log *zap.Su
118118
// remember which destination copies we (re)synced this pass, so a MatchOrigin prune can delete
119119
// the copies that no longer have a matching origin object.
120120
synced := sets.New[string]()
121+
deletionPolicies := map[string]metav1.DeletionPropagation{}
121122

122123
// We "forward" the deletion to the related objects only if the primary is already in deletion
123124
// and the related object either originated from the user (so on the service cluster we just
@@ -149,6 +150,9 @@ func (s *ResourceSyncer) processRelatedResource(ctx context.Context, log *zap.Su
149150
object: destObject,
150151
}
151152

153+
deletionPolicy := relatedDeletionPropagationPolicy(relRes.Origin, resolved.original)
154+
deletionPolicies[relatedCopyKey(resolved.destination.Namespace, resolved.destination.Name)] = deletionPolicy
155+
152156
// When status sync is enabled, include "status" in subresources so it is stripped from
153157
// the spec patch (avoiding a no-op write on resources that have a status subresource).
154158
// The status is then separately written via the status subresource endpoint by syncStatusForward.
@@ -192,6 +196,8 @@ func (s *ResourceSyncer) processRelatedResource(ctx context.Context, log *zap.Su
192196
eventObjSide: eventObjSide,
193197
// force deletion of related resources when the primary object is being deleted
194198
forceDelete: forceDelete,
199+
// Kcp-origin objects can request how their service-cluster copies are deleted.
200+
deletionPropagationPolicy: deletionPolicy,
195201
// propagate the SSA mode chosen for the primary syncer to keep
196202
// behavior consistent across the whole resource graph
197203
useServerSideApply: s.useServerSideApply,
@@ -237,7 +243,7 @@ func (s *ResourceSyncer) processRelatedResource(ctx context.Context, log *zap.Su
237243
// had already disappeared mid-life (which the loop can no longer resolve).
238244
selector := relatedCopySelector(primary, remote.clusterName, s.pubRes.Name, relRes.Identifier, s.agentName)
239245

240-
pruneRequeue, err := s.pruneRelatedCopies(ctx, log, dest, primary, projectedGVK, selector, nil, true)
246+
pruneRequeue, err := s.pruneRelatedCopies(ctx, log, dest, primary, projectedGVK, selector, nil, deletionPolicies, true)
241247
if err != nil {
242248
return false, fmt.Errorf("failed to tear down related copies: %w", err)
243249
}
@@ -283,7 +289,7 @@ func (s *ResourceSyncer) processRelatedResource(ctx context.Context, log *zap.Su
283289
}
284290
}
285291

286-
pruneRequeue, err := s.pruneRelatedCopies(ctx, log, dest, primary, projectedGVK, selector, synced, false)
292+
pruneRequeue, err := s.pruneRelatedCopies(ctx, log, dest, primary, projectedGVK, selector, synced, deletionPolicies, false)
287293
if err != nil {
288294
return false, fmt.Errorf("failed to prune related copies: %w", err)
289295
}
@@ -387,7 +393,7 @@ func (s *ResourceSyncer) rememberRelatedObjects(ctx context.Context, log *zap.Su
387393
// (mid-life prune). It only ever operates on the destination client, so origin objects are never
388394
// touched, and it only ever sees objects that carry our provenance labels, so hand-created objects
389395
// are never in scope.
390-
func (s *ResourceSyncer) pruneRelatedCopies(ctx context.Context, log *zap.SugaredLogger, dest syncSide, primary *unstructured.Unstructured, projectedGVK schema.GroupVersionKind, selector labels.Selector, keep sets.Set[string], deleteAll bool) (requeue bool, err error) {
396+
func (s *ResourceSyncer) pruneRelatedCopies(ctx context.Context, log *zap.SugaredLogger, dest syncSide, primary *unstructured.Unstructured, projectedGVK schema.GroupVersionKind, selector labels.Selector, keep sets.Set[string], deletionPolicies map[string]metav1.DeletionPropagation, deleteAll bool) (requeue bool, err error) {
391397
list := &unstructured.UnstructuredList{}
392398
list.SetAPIVersion(projectedGVK.GroupVersion().String())
393399
list.SetKind(projectedGVK.Kind + "List")
@@ -421,7 +427,9 @@ func (s *ResourceSyncer) pruneRelatedCopies(ctx context.Context, log *zap.Sugare
421427
}
422428

423429
log.Debugw("Pruning related object copy…", "namespace", item.GetNamespace(), "name", item.GetName())
424-
if err := dest.client.Delete(ctx, item); err != nil {
430+
policy := deletionPolicies[relatedCopyKey(item.GetNamespace(), item.GetName())]
431+
if err := dest.client.Delete(ctx, item,
432+
ctrlruntimeclient.PropagationPolicy(normalizeDeletionPropagationPolicy(policy))); err != nil {
425433
if apierrors.IsNotFound(err) {
426434
continue
427435
}

0 commit comments

Comments
 (0)