From 4523de24540f4b9956b383914ef7d2af55dc67e7 Mon Sep 17 00:00:00 2001 From: ravilr Date: Tue, 13 Feb 2024 14:07:54 -0800 Subject: [PATCH 1/2] Add jitter to poll interval Signed-off-by: ravilr --- cmd/provider/main.go | 10 +++++++++- internal/controller/kubernetes.go | 6 ++++-- internal/controller/object/object.go | 4 +++- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/cmd/provider/main.go b/cmd/provider/main.go index cb764550..8e6bf113 100644 --- a/cmd/provider/main.go +++ b/cmd/provider/main.go @@ -72,6 +72,14 @@ func main() { ctrl.SetLogger(zl) } + // configure the jitter to be the 10% of the poll interval + pollJitter := time.Duration(float64(*pollInterval) * 0.1) + log.Debug("Starting", + "sync-interval", syncInterval.String(), + "poll-interval", pollInterval.String(), + "poll-jitter", pollJitter.String(), + "max-reconcile-rate", *maxReconcileRate) + cfg, err := ctrl.GetConfig() kingpin.FatalIfError(err, "Cannot get API server rest config") @@ -130,7 +138,7 @@ func main() { // notice and remove when we drop support for v1alpha1. kingpin.FatalIfError(ctrl.NewWebhookManagedBy(mgr).For(&v1alpha1.Object{}).Complete(), "Cannot create Object webhook") - kingpin.FatalIfError(object.Setup(mgr, o, *sanitizeSecrets), "Cannot setup controller") + kingpin.FatalIfError(object.Setup(mgr, o, *sanitizeSecrets, pollJitter), "Cannot setup controller") kingpin.FatalIfError(mgr.Start(ctrl.SetupSignalHandler()), "Cannot start controller manager") } diff --git a/internal/controller/kubernetes.go b/internal/controller/kubernetes.go index 7b957c61..5fb3bf49 100644 --- a/internal/controller/kubernetes.go +++ b/internal/controller/kubernetes.go @@ -17,6 +17,8 @@ limitations under the License. package controller import ( + "time" + ctrl "sigs.k8s.io/controller-runtime" "github.com/crossplane/crossplane-runtime/pkg/controller" @@ -27,11 +29,11 @@ import ( // Setup creates all Template controllers with the supplied logger and adds them to // the supplied manager. -func Setup(mgr ctrl.Manager, o controller.Options, sanitizeSecrets bool) error { +func Setup(mgr ctrl.Manager, o controller.Options, sanitizeSecrets bool, pollJitter time.Duration) error { if err := config.Setup(mgr, o); err != nil { return err } - if err := object.Setup(mgr, o, sanitizeSecrets); err != nil { + if err := object.Setup(mgr, o, sanitizeSecrets, pollJitter); err != nil { return err } return nil diff --git a/internal/controller/object/object.go b/internal/controller/object/object.go index dc054ebc..a160872c 100644 --- a/internal/controller/object/object.go +++ b/internal/controller/object/object.go @@ -21,6 +21,7 @@ import ( "encoding/base64" "fmt" "strings" + "time" "github.com/pkg/errors" v1 "k8s.io/api/core/v1" @@ -91,7 +92,7 @@ const ( ) // Setup adds a controller that reconciles Object managed resources. -func Setup(mgr ctrl.Manager, o controller.Options, sanitizeSecrets bool) error { +func Setup(mgr ctrl.Manager, o controller.Options, sanitizeSecrets bool, pollJitter time.Duration) error { name := managed.ControllerName(v1alpha2.ObjectGroupKind) cps := []managed.ConnectionPublisher{managed.NewAPISecretPublisher(mgr.GetClient(), mgr.GetScheme())} @@ -112,6 +113,7 @@ func Setup(mgr ctrl.Manager, o controller.Options, sanitizeSecrets bool) error { }), managed.WithFinalizer(&objFinalizer{client: mgr.GetClient()}), managed.WithPollInterval(o.PollInterval), + managed.WithPollJitterHook(pollJitter), managed.WithLogger(o.Logger.WithValues("controller", name)), managed.WithRecorder(event.NewAPIRecorder(mgr.GetEventRecorderFor(name))), managed.WithConnectionPublishers(cps...), From 554eca3dc4f55a9fcfb627163007a921b9f7199a Mon Sep 17 00:00:00 2001 From: ravilr Date: Thu, 15 Feb 2024 19:16:09 -0800 Subject: [PATCH 2/2] make poll jitter percentage configurable through flag option Signed-off-by: ravilr --- cmd/provider/main.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cmd/provider/main.go b/cmd/provider/main.go index 8e6bf113..18019c12 100644 --- a/cmd/provider/main.go +++ b/cmd/provider/main.go @@ -54,6 +54,7 @@ func main() { debug = app.Flag("debug", "Run with debug logging.").Short('d').Bool() syncInterval = app.Flag("sync", "Controller manager sync period such as 300ms, 1.5h, or 2h45m").Short('s').Default("1h").Duration() pollInterval = app.Flag("poll", "Poll interval controls how often an individual resource should be checked for drift.").Default("1m").Duration() + pollJitterPercentage = app.Flag("poll-jitter-percentage", "Percentage of jitter to apply to poll interval. It cannot be negative, and must be less than 100.").Default("10").Uint() leaderElection = app.Flag("leader-election", "Use leader election for the controller manager.").Short('l').Default("false").Envar("LEADER_ELECTION").Bool() maxReconcileRate = app.Flag("max-reconcile-rate", "The number of concurrent reconciliations that may be running at one time.").Default("10").Int() enableManagementPolicies = app.Flag("enable-management-policies", "Enable support for Management Policies.").Default("true").Envar("ENABLE_MANAGEMENT_POLICIES").Bool() @@ -72,8 +73,10 @@ func main() { ctrl.SetLogger(zl) } - // configure the jitter to be the 10% of the poll interval - pollJitter := time.Duration(float64(*pollInterval) * 0.1) + if *pollJitterPercentage >= 100 { + kingpin.Fatalf("invalid --poll-jitter-percentage %v must be less than 100", *pollJitterPercentage) + } + pollJitter := time.Duration(float64(*pollInterval) * (float64(*pollJitterPercentage) / 100.0)) log.Debug("Starting", "sync-interval", syncInterval.String(), "poll-interval", pollInterval.String(),