Skip to content

feat(perf): reduce event amplification via predicate and status upda… #521

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
julius-malcovsky wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

feat(perf): reduce event amplification via predicate and status upda… #521

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions api/internal/controller/api_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,10 +48,10 @@ func (r *ApiReconciler) SetupWithManager(mgr ctrl.Manager) error {
r.Controller = cc.NewController(&api.ApiHandler{}, r.Client, r.Recorder)

return ctrl.NewControllerManagedBy(mgr).
For(&apiapi.Api{}).
For(&apiapi.Api{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})).
Watches(&apiapi.Api{},
handler.EnqueueRequestsFromMapFunc(r.MapApiToApi),
builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}),
builder.WithPredicates(predicate.GenerationChangedPredicate{}),
).
WithOptions(controller.Options{
MaxConcurrentReconciles: cconfig.MaxConcurrentReconciles,
Expand Down
8 changes: 5 additions & 3 deletions api/internal/controller/apiexposure_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,14 +58,14 @@ func (r *ApiExposureReconciler) SetupWithManager(mgr ctrl.Manager) error {
r.Controller = cc.NewController(&apiexposure.ApiExposureHandler{}, r.Client, r.Recorder)

return ctrl.NewControllerManagedBy(mgr).
For(&apiv1.ApiExposure{}).
For(&apiv1.ApiExposure{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})).
Watches(&apiv1.Api{},
handler.EnqueueRequestsFromMapFunc(r.MapApiToApiExposure),
builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}),
builder.WithPredicates(predicate.GenerationChangedPredicate{}),
).
Watches(&apiv1.ApiExposure{},
handler.EnqueueRequestsFromMapFunc(r.MapApiExposureToApiExposure),
builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}),
builder.WithPredicates(predicate.GenerationChangedPredicate{}),
).
// Watch ApiSubscription with ResourceVersionChangedPredicate (not GenerationChangedPredicate)
// because we need to react to approval status changes, which update Status (not Spec).
Expand All @@ -78,6 +78,8 @@ func (r *ApiExposureReconciler) SetupWithManager(mgr ctrl.Manager) error {
handler.EnqueueRequestsFromMapFunc(r.MapRouteToApiExposure),
builder.WithPredicates(predicate.GenerationChangedPredicate{}),
).
// Watch Zone with ResourceVersionChangedPredicate because zone readiness is conveyed
// via status conditions (not spec), and exposures must react when a zone becomes ready.
Watches(&adminv1.Zone{},
handler.EnqueueRequestsFromMapFunc(r.MapZoneToApiExposure),
builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}),
Expand Down
7 changes: 5 additions & 2 deletions api/internal/controller/apisubscription_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,14 +65,17 @@ func (r *ApiSubscriptionReconciler) SetupWithManager(mgr ctrl.Manager) error {
r.Controller = cc.NewController(&apisubscription.ApiSubscriptionHandler{}, r.Client, r.Recorder)

return ctrl.NewControllerManagedBy(mgr).
For(&apiapi.ApiSubscription{}, builder.WithPredicates(predicate.ResourceVersionChangedPredicate{})).
For(&apiapi.ApiSubscription{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})).
// ApprovalRequest and Approval are watched without a predicate so that status
// changes (approval decisions) re-enqueue the parent ApiSubscription.
// GenerationChangedPredicate would miss these since decisions update Status, not Spec.
Owns(&approvalapi.ApprovalRequest{}).
Owns(&approvalapi.Approval{}).
Owns(&gatewayapi.ConsumeRoute{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})).
Owns(&apiapi.RemoteApiSubscription{}).
Watches(&apiapi.Api{},
handler.EnqueueRequestsFromMapFunc(r.MapApiToApiSubscription),
builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}),
builder.WithPredicates(predicate.GenerationChangedPredicate{}),
).
Watches(&apiapi.ApiExposure{},
handler.EnqueueRequestsFromMapFunc(r.MapApiExposureToApiSubscription),
Expand Down
20 changes: 17 additions & 3 deletions common/pkg/controller/controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ package controller

import (
"context"
"reflect"

apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
Expand Down Expand Up @@ -66,7 +67,10 @@ func (c *ControllerImpl[T]) Reconcile(ctx context.Context, req reconcile.Request
if changed, setupErr := FirstSetup(ctx, c.Client, object); setupErr != nil {
return HandleError(ctx, setupErr, object, c.Recorder), nil
} else if changed {
return reconcile.Result{}, nil
// Requeue explicitly so the full reconcile runs after the finalizer write,
// regardless of whether the watch event fires (finalizer updates do not bump
// Generation, so GenerationChangedPredicate would otherwise filter it out).
return reconcile.Result{Requeue: true}, nil
}

logger.V(1).Info("Fetched object")
Expand Down Expand Up @@ -118,9 +122,19 @@ func (c *ControllerImpl[T]) Reconcile(ctx context.Context, req reconcile.Request
c.Event(ctx, object, "Warning", "UnknownReady", "Resource has an unknown ready status")
}

// Snapshot conditions before stamping so we can detect real changes.
// StampObservedGeneration mutates ObservedGeneration on each condition in-place,
// so any generation advance will show up as a diff.
condsBefore := append([]metav1.Condition(nil), object.GetConditions()...)
StampObservedGeneration(object)
if err = c.Client.Status().Update(ctx, object); err != nil {
return HandleError(ctx, err, object, c.Recorder), nil

// Skip the status write when nothing actually changed. This prevents a
// ResourceVersion bump that would otherwise re-enqueue this controller
// (via For()) and cross-trigger downstream watchers on every periodic reconcile.
if !reflect.DeepEqual(condsBefore, object.GetConditions()) {
if err = c.Client.Status().Update(ctx, object); err != nil {
return HandleError(ctx, err, object, c.Recorder), nil
}
}

requeueAfter := config.RequeueWithJitter()
Expand Down
2 changes: 1 addition & 1 deletion common/pkg/controller/controller_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@ var _ = Describe("Controller", func() {

res, err := controller.Reconcile(ctx, req, &test.TestResource{})
Expect(err).ToNot(HaveOccurred())
Expect(res).To(Equal(reconcile.Result{}))
Expect(res).To(Equal(reconcile.Result{Requeue: true}))

Expect(k8sClient.Get(ctx, req.NamespacedName, obj)).To(Succeed())
Expect(controllerutil.ContainsFinalizer(obj, config.FinalizerName)).To(BeTrue())
Expand Down
4 changes: 3 additions & 1 deletion gateway/internal/controller/consumer_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,10 @@ import (
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/tools/record"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/builder"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller"
"sigs.k8s.io/controller-runtime/pkg/predicate"

gatewayv1 "github.com/telekom/controlplane/gateway/api/v1"
consumer_handler "github.com/telekom/controlplane/gateway/internal/handler/consumer"
Expand Down Expand Up @@ -43,7 +45,7 @@ func (r *ConsumerReconciler) SetupWithManager(mgr ctrl.Manager) error {
r.Controller = cc.NewController(&consumer_handler.ConsumerHandler{}, r.Client, r.Recorder)

return ctrl.NewControllerManagedBy(mgr).
For(&gatewayv1.Consumer{}).
For(&gatewayv1.Consumer{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})).
WithOptions(controller.Options{
MaxConcurrentReconciles: cconfig.MaxConcurrentReconciles,
RateLimiter: cc.NewRateLimiter(),
Expand Down
2 changes: 1 addition & 1 deletion gateway/internal/controller/consumeroute_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ func (r *ConsumeRouteReconciler) SetupWithManager(mgr ctrl.Manager) error {
r.Controller = cc.NewController(&consumeroute_handler.ConsumeRouteHandler{}, r.Client, r.Recorder)

return ctrl.NewControllerManagedBy(mgr).
For(&gatewayv1.ConsumeRoute{}).
For(&gatewayv1.ConsumeRoute{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})).
WithOptions(controller.Options{
MaxConcurrentReconciles: cconfig.MaxConcurrentReconciles,
RateLimiter: cc.NewRateLimiter(),
Expand Down
4 changes: 3 additions & 1 deletion gateway/internal/controller/gateway_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,10 @@ import (
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/tools/record"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/builder"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller"
"sigs.k8s.io/controller-runtime/pkg/predicate"

v1 "github.com/telekom/controlplane/gateway/api/v1"
handler "github.com/telekom/controlplane/gateway/internal/handler/gateway"
Expand Down Expand Up @@ -43,7 +45,7 @@ func (r *GatewayReconciler) SetupWithManager(mgr ctrl.Manager) error {
r.Controller = cc.NewController(&handler.GatewayHandler{}, r.Client, r.Recorder)

return ctrl.NewControllerManagedBy(mgr).
For(&v1.Gateway{}).
For(&v1.Gateway{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})).
WithOptions(controller.Options{
MaxConcurrentReconciles: cconfig.MaxConcurrentReconciles,
RateLimiter: cc.NewRateLimiter(),
Expand Down
2 changes: 1 addition & 1 deletion gateway/internal/controller/route_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ func (r *RouteReconciler) SetupWithManager(mgr ctrl.Manager) error {
r.Controller = cc.NewController(&routehandler.RouteHandler{}, r.Client, r.Recorder)

return ctrl.NewControllerManagedBy(mgr).
For(&gatewayv1.Route{}).
For(&gatewayv1.Route{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})).
Watches(&gatewayv1.ConsumeRoute{},
handler.EnqueueRequestsFromMapFunc(r.mapConsumeRouteToRoute),
builder.WithPredicates(predicate.GenerationChangedPredicate{})).
Expand Down
Loading