Skip to content

Commit

Permalink
Merge pull request #246 from turkenh/ssa-second-attempt
Browse files Browse the repository at this point in the history
Add Alpha Support for Server Side Apply (2nd attempt)
  • Loading branch information
turkenh authored Sep 17, 2024
2 parents 193a4c6 + 88580c6 commit c5b4b82
Show file tree
Hide file tree
Showing 48 changed files with 117,484 additions and 222 deletions.
5 changes: 4 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ NPROCS ?= 1
GO_TEST_PARALLEL := $(shell echo $$(( $(NPROCS) / 2 )))

GO_STATIC_PACKAGES = $(GO_PROJECT)/cmd/provider
GO_SUBDIRS += cmd internal apis
GO_SUBDIRS += cmd internal apis pkg
GO111MODULE = on
GOLANGCILINT_VERSION = 1.55.2
-include build/makelib/golang.mk
Expand Down Expand Up @@ -89,6 +89,9 @@ CROSSPLANE_NAMESPACE = crossplane-system
-include build/makelib/local.xpkg.mk
-include build/makelib/controlplane.mk

# TODO(turkenh): Add "examples/object/object-ssa-owner.yaml" to the list to test the SSA functionality as part of the e2e tests.
# The test is disabled for now because uptest clears the package cache when the provider restarted with the SSA flag.
# Enable after https://github.com/crossplane/uptest/issues/17 is fixed.
UPTEST_EXAMPLE_LIST ?= "examples/object/object.yaml,examples/object/object-watching.yaml"
uptest: $(UPTEST) $(KUBECTL) $(KUTTL)
@$(INFO) running automated tests
Expand Down
6 changes: 6 additions & 0 deletions cmd/provider/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ func main() {

enableManagementPolicies = app.Flag("enable-management-policies", "Enable support for Management Policies.").Default("true").Envar("ENABLE_MANAGEMENT_POLICIES").Bool()
enableWatches = app.Flag("enable-watches", "Enable support for watching resources.").Default("false").Envar("ENABLE_WATCHES").Bool()
enableServerSideApply = app.Flag("enable-server-side-apply", "Enable server side apply to sync object manifests to k8s API.").Default("false").Envar("ENABLE_SERVER_SIDE_APPLY").Bool()
)
kingpin.MustParse(app.Parse(os.Args[1:]))

Expand Down Expand Up @@ -160,6 +161,11 @@ func main() {
log.Info("Alpha feature enabled", "flag", features.EnableAlphaWatches)
}

if *enableServerSideApply {
o.Features.Enable(features.EnableAlphaServerSideApply)
log.Info("Alpha feature enabled", "flag", features.EnableAlphaServerSideApply)
}

// NOTE(lsviben): We are registering the conversion webhook with v1alpha1
// Object. As far as I can see and based on some tests, it doesn't matter
// which version we use here. Leaving it as v1alpha1 as it will be easy to
Expand Down
24 changes: 24 additions & 0 deletions examples/object/object-ssa-labeler.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
# Note: This example is for the alpha feature of server side apply.
# It requires the provider to be started with the --enable-server-side-apply flag.
apiVersion: kubernetes.crossplane.io/v1alpha2
kind: Object
metadata:
name: sample-service-labeler
spec:
# Note: This resource will only patch/update the manifest below.
# It will not delete or create the resource.
# As a limitation, it will not clean up the changes it made during its deletion.
# This requires the Server Side Apply feature to be enabled in the provider
# with the --enable-server-side-apply flag.
managementPolicies: ["Observe", "Update"]
forProvider:
manifest:
apiVersion: v1
kind: Service
metadata:
name: sample-service
namespace: default
labels:
another-key: another-value
providerConfigRef:
name: kubernetes-provider
29 changes: 29 additions & 0 deletions examples/object/object-ssa-owner.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
# Note: This example is for the alpha feature of server side apply.
# It requires the provider to be started with the --enable-server-side-apply flag.
apiVersion: kubernetes.crossplane.io/v1alpha2
kind: Object
metadata:
name: sample-service-owner
annotations:
uptest.upbound.io/pre-assert-hook: testhooks/enable-ssa.sh
uptest.upbound.io/post-assert-hook: testhooks/validate-ssa.sh
uptest.upbound.io/timeout: "60"
spec:
forProvider:
manifest:
apiVersion: v1
kind: Service
metadata:
name: sample-service
namespace: default
labels:
some-key: some-value
spec:
selector:
app.kubernetes.io/name: MyApp
ports:
- protocol: TCP
port: 80
targetPort: 9376
providerConfigRef:
name: kubernetes-provider
3 changes: 2 additions & 1 deletion examples/object/object-watching.yaml
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
---
# Note: This example is for the alpha feature of watching resources.
# It requires the provider to be started with the --enable-watches flag.
apiVersion: kubernetes.crossplane.io/v1alpha2
kind: Object
metadata:
Expand Down
5 changes: 5 additions & 0 deletions examples/object/testhooks/enable-ssa.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
#!/usr/bin/env bash
set -aeuo pipefail

echo "Enabling ssa feature for the provider"
${KUBECTL} patch deploymentruntimeconfig runtimeconfig-provider-kubernetes --type='json' -p='[{"op":"replace","path":"/spec/deploymentTemplate/spec/template/spec/containers/0/args", "value":["--debug", "--enable-server-side-apply"]}]'
25 changes: 25 additions & 0 deletions examples/object/testhooks/validate-ssa.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
#!/usr/bin/env bash
set -aeuo pipefail

# This script is used to validate the ssa feature, triggered by the
# uptest framework via `uptest.upbound.io/post-assert-hook`: https://github.com/crossplane/uptest/tree/e64457e2cce153ada54da686c8bf96143f3f6329?tab=readme-ov-file#hooks

LABELER_OBJECT="examples/object/object-ssa-labeler.yaml"
${KUBECTL} apply -f ${LABELER_OBJECT}
${KUBECTL} wait -f ${LABELER_OBJECT} --for condition=ready --timeout=1m

if ! ${KUBECTL} get service sample-service -o jsonpath='{.metadata.annotations}' | grep -v "last-applied-configuration"; then # This annotation should not be present when SSA is enabled
echo "SSA validation failed! Annotation 'last-applied-configuration' should not exist when SSA is enabled!"
#exit 1
fi
if ! (${KUBECTL} get service sample-service -o jsonpath='{.metadata.labels.some-key}' | grep -q "some-value" && ${KUBECTL} get service sample-service -o jsonpath='{.metadata.labels.another-key}' | grep -q "another-value"); then
echo "SSA validation failed! Labels 'some-key' and 'another-key' from both Objects should exist with values 'some-value' and 'another-value' respectively!"
#exit 1
fi
echo "Successfully validated the SSA feature!"

${KUBECTL} delete -f ${LABELER_OBJECT}

echo "Disabling SSA feature for the provider"
${KUBECTL} patch deploymentruntimeconfig runtimeconfig-provider-kubernetes --type='json' -p='[{"op":"replace","path":"/spec/deploymentTemplate/spec/template/spec/containers/0/args", "value":["--debug"]}]'

2 changes: 1 addition & 1 deletion examples/object/testhooks/validate-watching.sh
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ fi
echo "Enabling watch feature for the provider"
${KUBECTL} patch deploymentruntimeconfig runtimeconfig-provider-kubernetes --type='json' -p='[{"op":"replace","path":"/spec/deploymentTemplate/spec/template/spec/containers/0/args", "value":["--debug", "--enable-watches"]}]'

sleep 3
sleep 30

echo "Patching referenced secret"
${KUBECTL} patch secret bar --type='merge' -p='{"stringData":{"key":"new-value"}}'
Expand Down
7 changes: 4 additions & 3 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ toolchain go1.22.3
require (
github.com/Azure/kubelogin v0.1.1
github.com/alecthomas/kingpin/v2 v2.4.0
github.com/crossplane/crossplane-runtime v1.17.0-rc.0.0.20240509182037-b31be7747c60
github.com/crossplane/crossplane-runtime v1.17.0-rc.0.0.20240513123822-e50f51abfed2
github.com/crossplane/crossplane-tools v0.0.0-20240522174801-1ad3d4c87f21
github.com/google/cel-go v0.17.7
github.com/google/go-cmp v0.6.0
Expand All @@ -17,12 +17,15 @@ require (
github.com/upbound/up-sdk-go v0.3.1-0.20240517133145-e5da98257888
go.uber.org/zap v1.26.0
golang.org/x/oauth2 v0.20.0
golang.org/x/sync v0.6.0
k8s.io/api v0.29.3
k8s.io/apimachinery v0.29.3
k8s.io/client-go v0.29.3
k8s.io/kube-openapi v0.0.0-20240209001042-7a0d5b415232
k8s.io/utils v0.0.0-20240102154912-e7106e64919e
sigs.k8s.io/controller-runtime v0.17.1
sigs.k8s.io/controller-tools v0.14.0
sigs.k8s.io/structured-merge-diff/v4 v4.4.1
)

require (
Expand Down Expand Up @@ -104,8 +107,6 @@ require (
k8s.io/apiextensions-apiserver v0.29.1 // indirect
k8s.io/component-base v0.29.3 // indirect
k8s.io/klog/v2 v2.120.1 // indirect
k8s.io/kube-openapi v0.0.0-20240209001042-7a0d5b415232 // indirect
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
sigs.k8s.io/yaml v1.4.0 // indirect
)
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,8 @@ github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2y
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/crossplane/crossplane-runtime v1.17.0-rc.0.0.20240509182037-b31be7747c60 h1:subdCU8vHUDkaQAYBKKddCT2HpEK4A9fFEJ3nhGoTBc=
github.com/crossplane/crossplane-runtime v1.17.0-rc.0.0.20240509182037-b31be7747c60/go.mod h1:Pz2tdGVMF6KDGzHZOkvKro0nKc8EzK0sb/nSA7pH4Dc=
github.com/crossplane/crossplane-runtime v1.17.0-rc.0.0.20240513123822-e50f51abfed2 h1:/BYZb3VUwP05YEf6H9WMWe0yxMMrblhLMHO2gTc6tLY=
github.com/crossplane/crossplane-runtime v1.17.0-rc.0.0.20240513123822-e50f51abfed2/go.mod h1:Pz2tdGVMF6KDGzHZOkvKro0nKc8EzK0sb/nSA7pH4Dc=
github.com/crossplane/crossplane-tools v0.0.0-20240522174801-1ad3d4c87f21 h1:8wb7/zCbVPkeX68WbVESWJmSWQE5SZKzz0g9X4FlXRw=
github.com/crossplane/crossplane-tools v0.0.0-20240522174801-1ad3d4c87f21/go.mod h1:cN0Y7PFGQMM8mcagXVCbeQoKtipmFWQTPZYyziCPBUI=
github.com/dave/jennifer v1.7.0 h1:uRbSBH9UTS64yXbh4FrMHfgfY762RD+C7bUPKODpSJE=
Expand Down
31 changes: 31 additions & 0 deletions internal/controller/object/fake/mocks.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
package fake

import (
"context"

"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"

"github.com/crossplane-contrib/provider-kubernetes/apis/object/v1alpha2"
)

// A ResourceSyncer is a fake ResourceSyncer.
type ResourceSyncer struct {
GetObservedStateFn func(ctx context.Context, obj *v1alpha2.Object, current *unstructured.Unstructured) (*unstructured.Unstructured, error)
GetDesiredStateFn func(ctx context.Context, obj *v1alpha2.Object, manifest *unstructured.Unstructured) (*unstructured.Unstructured, error)
SyncResourceFn func(ctx context.Context, obj *v1alpha2.Object, desired *unstructured.Unstructured) (*unstructured.Unstructured, error)
}

// GetObservedState calls the GetObservedStateFn.
func (r *ResourceSyncer) GetObservedState(ctx context.Context, obj *v1alpha2.Object, current *unstructured.Unstructured) (*unstructured.Unstructured, error) {
return r.GetObservedStateFn(ctx, obj, current)
}

// GetDesiredState calls the GetDesiredStateFn.
func (r *ResourceSyncer) GetDesiredState(ctx context.Context, obj *v1alpha2.Object, manifest *unstructured.Unstructured) (*unstructured.Unstructured, error) {
return r.GetDesiredStateFn(ctx, obj, manifest)
}

// SyncResource calls the SyncResourceFn.
func (r *ResourceSyncer) SyncResource(ctx context.Context, obj *v1alpha2.Object, desired *unstructured.Unstructured) (*unstructured.Unstructured, error) {
return r.SyncResourceFn(ctx, obj, desired)
}
16 changes: 10 additions & 6 deletions internal/controller/object/indexes.go
Original file line number Diff line number Diff line change
Expand Up @@ -66,9 +66,9 @@ func IndexByProviderGVK(o client.Object) []string {
}

// Index the desired object.
// We don't expect errors here, as the getDesired function is already called
// We don't expect errors here, as the parseManifest function is already called
// in the reconciler and the desired object already validated.
d, _ := getDesired(obj)
d, _ := parseManifest(obj)
keys = append(keys, refKeyProviderGVK(obj.Spec.ProviderConfigReference.Name, d.GetKind(), d.GroupVersionKind().Group, d.GroupVersionKind().Version)) // unification is done by the informer.

// unification is done by the informer.
Expand Down Expand Up @@ -98,9 +98,9 @@ func IndexByProviderNamespacedNameGVK(o client.Object) []string {
}

// Index the desired object.
// We don't expect errors here, as the getDesired function is already called
// We don't expect errors here, as the parseManifest function is already called
// in the reconciler and the desired object already validated.
d, _ := getDesired(obj)
d, _ := parseManifest(obj)
keys = append(keys, refKeyProviderNamespacedNameGVK(obj.Spec.ProviderConfigReference.Name, d.GetNamespace(), d.GetName(), d.GetKind(), d.GetAPIVersion())) // unification is done by the informer.

return keys
Expand All @@ -123,8 +123,12 @@ func enqueueObjectsForReferences(ca cache.Cache, log logging.Logger) func(ctx co
}
// queue those Objects for reconciliation
for _, o := range objects.Items {
log.Info("Enqueueing Object because referenced resource changed", "name", o.GetName(), "referencedGVK", rGVK.String(), "referencedName", ev.Object.GetName(), "providerConfig", pc)
q.Add(reconcile.Request{NamespacedName: types.NamespacedName{Name: o.GetName()}})
// We only enqueue the Object if it has the Watch flag set to true.
// Not every referencing Object watches the referenced resource.
if o.Spec.Watch {
log.Info("Enqueueing Object because referenced resource changed", "name", o.GetName(), "referencedGVK", rGVK.String(), "referencedName", ev.Object.GetName(), "providerConfig", pc)
q.Add(reconcile.Request{NamespacedName: types.NamespacedName{Name: o.GetName()}})
}
}
}
}
Loading

0 comments on commit c5b4b82

Please sign in to comment.