From fa3c8d179d1de442343d8a76e23c66591abe395b Mon Sep 17 00:00:00 2001 From: Abdul Haseeb Date: Tue, 25 May 2021 09:39:55 +0500 Subject: [PATCH 1/4] Add operator health-check functionality --- .../slack-operator/templates/deployment.yaml | 16 +++++++++++++ charts/slack-operator/values.yaml | 3 +-- config/default/manager_auth_proxy_patch.yaml | 1 + config/manager/manager.yaml | 23 +++++++++++++------ main.go | 15 +++++++----- 5 files changed, 43 insertions(+), 15 deletions(-) diff --git a/charts/slack-operator/templates/deployment.yaml b/charts/slack-operator/templates/deployment.yaml index a9b4689..c72ac69 100644 --- a/charts/slack-operator/templates/deployment.yaml +++ b/charts/slack-operator/templates/deployment.yaml @@ -40,6 +40,7 @@ spec: - containerPort: 8443 name: https - args: + - --health-probe-bind-address=:8081 - --metrics-bind-address=127.0.0.1:8080 - --leader-elect command: @@ -53,7 +54,22 @@ spec: value: "{{ default true .Values.webhook.enabled }}" image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} + securityContext: + allowPrivilegeEscalation: false name: manager + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + name: manager + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 ports: - containerPort: 9443 name: webhook-server diff --git a/charts/slack-operator/values.yaml b/charts/slack-operator/values.yaml index 1d81698..f7d72a0 100644 --- a/charts/slack-operator/values.yaml +++ b/charts/slack-operator/values.yaml @@ -48,8 +48,7 @@ resources: podAnnotations: {} podSecurityContext: - {} - # fsGroup: 2000 + runAsNonRoot: true securityContext: {} diff --git a/config/default/manager_auth_proxy_patch.yaml b/config/default/manager_auth_proxy_patch.yaml index eb334fe..c050f35 100644 --- a/config/default/manager_auth_proxy_patch.yaml +++ b/config/default/manager_auth_proxy_patch.yaml @@ -22,5 +22,6 @@ spec: name: https - name: manager args: + - "--health-probe-bind-address=:8081" - "--metrics-bind-address=127.0.0.1:8080" - "--leader-elect" diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index ba1cd31..79adfe7 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -22,6 +22,8 @@ spec: labels: control-plane: controller-manager spec: + securityContext: + runAsNonRoot: true containers: - command: - /manager @@ -29,6 +31,20 @@ spec: - --leader-elect image: controller:latest name: manager + securityContext: + allowPrivilegeEscalation: false + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 resources: limits: cpu: 100m @@ -36,12 +52,5 @@ spec: requests: cpu: 100m memory: 20Mi - env: - - name: WATCH_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: CONFIG_SECRET_NAME - value: slack-secret serviceAccountName: controller-manager terminationGracePeriodSeconds: 10 diff --git a/main.go b/main.go index 9e28c9e..abdf085 100644 --- a/main.go +++ b/main.go @@ -52,8 +52,10 @@ func init() { func main() { var metricsAddr string var enableLeaderElection bool + var probeAddr string flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") + flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") @@ -72,12 +74,13 @@ func main() { } options := ctrl.Options{ - Scheme: scheme, - MetricsBindAddress: metricsAddr, - Port: 9443, - LeaderElection: enableLeaderElection, - LeaderElectionID: "957ea167.stakater.com", - Namespace: watchNamespace, // namespaced-scope when the value is not an empty string + Scheme: scheme, + MetricsBindAddress: metricsAddr, + Port: 9443, + HealthProbeBindAddress: probeAddr, + LeaderElection: enableLeaderElection, + LeaderElectionID: "957ea167.stakater.com", + Namespace: watchNamespace, // namespaced-scope when the value is not an empty string } // Add support for MultiNamespace set in WATCH_NAMESPACE (e.g ns1,ns2) From e0cd1a1d0c17f6bc5ea5430fd5a653a5a8f514a0 Mon Sep 17 00:00:00 2001 From: Abdul Haseeb Date: Tue, 25 May 2021 09:40:40 +0500 Subject: [PATCH 2/4] Bump operator-sdk to 1.7.2 --- .github/workflows/push.yml | 2 +- README.md | 2 +- .../bases/slack-operator.clusterserviceversion.yaml | 2 +- config/scorecard/patches/basic.config.yaml | 2 +- config/scorecard/patches/olm.config.yaml | 10 +++++----- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 2ab9a6a..ddf191a 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -8,7 +8,7 @@ on: env: DOCKER_FILE_PATH: Dockerfile GOLANG_VERSION: 1.16 - OPERATOR_SDK_VERSION: "1.7.1" + OPERATOR_SDK_VERSION: "1.7.2" KUSTOMIZE_VERSION: "3.8.7" KUBERNETES_VERSION: "1.20.2" KIND_VERSION: "0.10.0" diff --git a/README.md b/README.md index 56d29dc..04ea3cc 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ $ oc apply -f bundle/manifests ## Local Development -- [Operator-sdk v1.7.1](https://github.com/operator-framework/operator-sdk/releases/tag/v1.7.1) is required for local development. +- [Operator-sdk v1.7.2](https://github.com/operator-framework/operator-sdk/releases/tag/v1.7.2) is required for local development. 1. Create `slack-secret` secret 2. Run `make run ENABLE_WEBHOOKS=false WATCH_NAMESPACE=default OPERATOR_NAMESPACE=default` where `WATCH_NAMESPACE` denotes the namespaces that the operator is supposed to watch and `OPERATOR_NAMESPACE` is the namespace in which it's supposed to be deployed. diff --git a/config/manifests/bases/slack-operator.clusterserviceversion.yaml b/config/manifests/bases/slack-operator.clusterserviceversion.yaml index 58e5531..dcb59b8 100644 --- a/config/manifests/bases/slack-operator.clusterserviceversion.yaml +++ b/config/manifests/bases/slack-operator.clusterserviceversion.yaml @@ -4,7 +4,7 @@ metadata: annotations: alm-examples: '[]' capabilities: Basic Install - operators.operatorframework.io/builder: operator-sdk-v1.7.1 + operators.operatorframework.io/builder: operator-sdk-v1.7.2 operators.operatorframework.io/project_layout: go.kubebuilder.io/v3 name: slack-operator.v0.0.0 namespace: placeholder diff --git a/config/scorecard/patches/basic.config.yaml b/config/scorecard/patches/basic.config.yaml index 1c0f973..799dfa2 100644 --- a/config/scorecard/patches/basic.config.yaml +++ b/config/scorecard/patches/basic.config.yaml @@ -4,7 +4,7 @@ entrypoint: - scorecard-test - basic-check-spec - image: quay.io/operator-framework/scorecard-test:1.7.1 + image: quay.io/operator-framework/scorecard-test:1.7.2 labels: suite: basic test: basic-check-spec-test diff --git a/config/scorecard/patches/olm.config.yaml b/config/scorecard/patches/olm.config.yaml index a037adf..9ed3301 100644 --- a/config/scorecard/patches/olm.config.yaml +++ b/config/scorecard/patches/olm.config.yaml @@ -4,7 +4,7 @@ entrypoint: - scorecard-test - olm-bundle-validation - image: quay.io/operator-framework/scorecard-test:1.7.1 + image: quay.io/operator-framework/scorecard-test:1.7.2 labels: suite: olm test: olm-bundle-validation-test @@ -14,7 +14,7 @@ entrypoint: - scorecard-test - olm-crds-have-validation - image: quay.io/operator-framework/scorecard-test:1.7.1 + image: quay.io/operator-framework/scorecard-test:1.7.2 labels: suite: olm test: olm-crds-have-validation-test @@ -24,7 +24,7 @@ entrypoint: - scorecard-test - olm-crds-have-resources - image: quay.io/operator-framework/scorecard-test:1.7.1 + image: quay.io/operator-framework/scorecard-test:1.7.2 labels: suite: olm test: olm-crds-have-resources-test @@ -34,7 +34,7 @@ entrypoint: - scorecard-test - olm-spec-descriptors - image: quay.io/operator-framework/scorecard-test:1.7.1 + image: quay.io/operator-framework/scorecard-test:1.7.2 labels: suite: olm test: olm-spec-descriptors-test @@ -44,7 +44,7 @@ entrypoint: - scorecard-test - olm-status-descriptors - image: quay.io/operator-framework/scorecard-test:1.7.1 + image: quay.io/operator-framework/scorecard-test:1.7.2 labels: suite: olm test: olm-status-descriptors-test From 291a36ba833c38afeb65d5a144b9f1de70ecf3ef Mon Sep 17 00:00:00 2001 From: Abdul Haseeb Date: Tue, 25 May 2021 10:02:11 +0500 Subject: [PATCH 3/4] Update main.go with health check --- go.sum | 8 ++++++++ main.go | 16 +++++++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/go.sum b/go.sum index 98336c3..987665d 100644 --- a/go.sum +++ b/go.sum @@ -35,14 +35,21 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.11.1 h1:eVvIXUKiTgv++6YnWb42DUA1YL7qDugnKP0HljexdnQ= github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw= github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= +github.com/Azure/go-autorest/autorest/adal v0.9.5 h1:Y3bBUV4rTuxenJJs41HU3qmqsb+auo+a3Lz+PlJPpL0= github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= +github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/logger v0.2.0 h1:e4RVHVZKC5p6UANLJHkM4OfR1UKZPj8Wt8Pcx+3oqrE= github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -130,6 +137,7 @@ github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLi github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= diff --git a/main.go b/main.go index abdf085..cf3c911 100644 --- a/main.go +++ b/main.go @@ -25,9 +25,12 @@ import ( "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" - _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" + // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) + // to ensure that exec-entrypoint and run can make use of them. + _ "k8s.io/client-go/plugin/pkg/client/auth" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" slackv1alpha1 "github.com/stakater/slack-operator/api/v1alpha1" @@ -116,6 +119,17 @@ func main() { os.Exit(1) } } + + // Add health endpoints + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + setupLog.Error(err, "unable to set up health check") + os.Exit(1) + } + if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { + setupLog.Error(err, "unable to set up ready check") + os.Exit(1) + } + // +kubebuilder:scaffold:builder setupLog.Info("starting manager") From 914b15ef175f365d96facd1a9e9cee97feb4b4df Mon Sep 17 00:00:00 2001 From: Abdul Haseeb Date: Tue, 25 May 2021 11:58:57 +0500 Subject: [PATCH 4/4] Handle feedback --- config/manager/manager.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 79adfe7..7c718bb 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -52,5 +52,12 @@ spec: requests: cpu: 100m memory: 20Mi + env: + - name: WATCH_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: CONFIG_SECRET_NAME + value: slack-secret serviceAccountName: controller-manager terminationGracePeriodSeconds: 10