Skip to content

Commit

Permalink
Cache hub template queries
Browse files Browse the repository at this point in the history 
This leverages the new caching functionality in go-template-utils so
that API queries are not duplicated.

As a side-effect, this should cause less reconciles because when
starting a watch due to a hub template, it does not cause an initial
reconcile merely from the watch being created.

Relates:
https://issues.redhat.com/browse/ACM-7402
https://issues.redhat.com/browse/ACM-7398

Signed-off-by: mprahl <[email protected]>
(cherry picked from commit df22cc2)
  • Loading branch information
@mprahl @openshift-ci
mprahl authored and openshift-ci[bot] committed Oct 17, 2023
1 parent d180162 commit 3eb01c9
Show file tree
Hide file tree
Showing 8 changed files with 160 additions and 175 deletions.
2 changes: 1 addition & 1 deletion controllers/propagator/encryption.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import (
"fmt"
"time"

"github.com/stolostron/go-template-utils/v3/pkg/templates"
"github.com/stolostron/go-template-utils/v4/pkg/templates"
corev1 "k8s.io/api/core/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
Expand Down
2 changes: 1 addition & 1 deletion controllers/propagator/encryption_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import (

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/stolostron/go-template-utils/v3/pkg/templates"
"github.com/stolostron/go-template-utils/v4/pkg/templates"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
Expand Down
179 changes: 83 additions & 96 deletions controllers/propagator/propagation.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,18 +13,16 @@ import (
"sync"
"time"

templates "github.com/stolostron/go-template-utils/v3/pkg/templates"
templates "github.com/stolostron/go-template-utils/v4/pkg/templates"
k8sdepwatches "github.com/stolostron/kubernetes-dependency-watches/client"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/json"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/record"
clusterv1 "open-cluster-management.io/api/cluster/v1"
clusterv1beta1 "open-cluster-management.io/api/cluster/v1beta1"
appsv1 "open-cluster-management.io/multicloud-operators-subscription/pkg/apis/apps/placementrule/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
Expand All @@ -36,16 +34,11 @@ import (
)

const (
startDelim = "{{hub"
stopDelim = "hub}}"
TemplateStartDelim = "{{hub"
TemplateStopDelim = "hub}}"
TriggerUpdateAnnotation = "policy.open-cluster-management.io/trigger-update"
)

var (
kubeConfig *rest.Config
kubeClient *kubernetes.Interface
)

type Propagator struct {
client.Client
Scheme *runtime.Scheme
Expand All @@ -54,24 +47,6 @@ type Propagator struct {
ReplicatedPolicyUpdates chan event.GenericEvent
}

func Initialize(kubeconfig *rest.Config, kubeclient *kubernetes.Interface) {
kubeConfig = kubeconfig
kubeClient = kubeclient
}

// getTemplateCfg returns the default policy template configuration.
func getTemplateCfg() templates.Config {
// (Encryption settings are set during the processTemplates method)
// Adding eight spaces to the indentation makes the usage of `indent N` be from the logical
// starting point of the resource object wrapped in the ConfigurationPolicy.
return templates.Config{
AdditionalIndentation: 8,
DisabledFunctions: []string{},
StartDelim: startDelim,
StopDelim: stopDelim,
}
}

// clusterDecision contains a single decision where the replicated policy
// should be processed and any overrides to the root policy
type clusterDecision struct {
Expand Down Expand Up @@ -432,23 +407,50 @@ func (r *RootPolicyReconciler) handleRootPolicy(instance *policiesv1.Policy) err
// a helper to quickly check if there are any templates in any of the policy templates
func policyHasTemplates(instance *policiesv1.Policy) bool {
for _, policyT := range instance.Spec.PolicyTemplates {
if templates.HasTemplate(policyT.ObjectDefinition.Raw, startDelim, false) {
if templates.HasTemplate(policyT.ObjectDefinition.Raw, TemplateStartDelim, false) {
return true
}
}

return false
}

type templateCtx struct {
ManagedClusterName string
ManagedClusterLabels map[string]string
}

func addManagedClusterLabels(clusterName string) func(templates.CachingQueryAPI, interface{}) (interface{}, error) {
return func(api templates.CachingQueryAPI, ctx interface{}) (interface{}, error) {
typedCtx, ok := ctx.(templateCtx)
if !ok {
return ctx, nil
}

managedClusterGVK := schema.GroupVersionKind{
Group: "cluster.open-cluster-management.io",
Version: "v1",
Kind: "ManagedCluster",
}

managedCluster, err := api.Get(managedClusterGVK, "", clusterName)
if err != nil {
return ctx, err
}

typedCtx.ManagedClusterLabels = managedCluster.GetLabels()

return typedCtx, nil
}
}

// Iterates through policy definitions and processes hub templates. A special annotation
// policy.open-cluster-management.io/trigger-update is used to trigger reprocessing of the templates
// and ensure that replicated-policies in the cluster are updated only if there is a change. This
// annotation is deleted from the replicated policies and not propagated to the cluster namespaces.
func (r *ReplicatedPolicyReconciler) processTemplates(
replicatedPlc *policiesv1.Policy, decision appsv1.PlacementDecision, rootPlc *policiesv1.Policy,
) (
map[k8sdepwatches.ObjectIdentifier]bool, error,
) {
) error {
log := log.WithValues(
"policyName", rootPlc.GetName(),
"policyNamespace", rootPlc.GetNamespace(),
Expand All @@ -457,7 +459,6 @@ func (r *ReplicatedPolicyReconciler) processTemplates(
log.V(1).Info("Processing templates")

annotations := replicatedPlc.GetAnnotations()
templateRefObjs := map[k8sdepwatches.ObjectIdentifier]bool{}

// handle possible nil map
if len(annotations) == 0 {
Expand All @@ -469,7 +470,7 @@ func (r *ReplicatedPolicyReconciler) processTemplates(
if boolDisable, err := strconv.ParseBool(disable); err == nil && boolDisable {
log.Info("Detected the disable-templates annotation. Will not process templates.")

return templateRefObjs, nil
return nil
}
}

Expand All @@ -480,29 +481,38 @@ func (r *ReplicatedPolicyReconciler) processTemplates(
replicatedPlc.SetAnnotations(annotations)
}

templateCfg := getTemplateCfg()
templateCfg.LookupNamespace = rootPlc.GetNamespace()
templateCfg.ClusterScopedAllowList = []templates.ClusterScopedObjectIdentifier{{
Group: "cluster.open-cluster-management.io",
Kind: "ManagedCluster",
Name: decision.ClusterName,
}}
plcGVK := replicatedPlc.GroupVersionKind()

tmplResolver, err := templates.NewResolver(kubeClient, kubeConfig, templateCfg)
if err != nil {
log.Error(err, "Error instantiating template resolver")
panic(err)
templateResolverOptions := templates.ResolveOptions{
ClusterScopedAllowList: []templates.ClusterScopedObjectIdentifier{
{
Group: "cluster.open-cluster-management.io",
Kind: "ManagedCluster",
Name: decision.ClusterName,
},
},
DisableAutoCacheCleanUp: true,
LookupNamespace: rootPlc.GetNamespace(),
Watcher: &k8sdepwatches.ObjectIdentifier{
Group: plcGVK.Group,
Version: plcGVK.Version,
Kind: plcGVK.Kind,
Namespace: replicatedPlc.GetNamespace(),
Name: replicatedPlc.GetName(),
},
}

var templateResult templates.TemplateResult

// A policy can have multiple policy templates within it, iterate and process each
for _, policyT := range replicatedPlc.Spec.PolicyTemplates {
if !templates.HasTemplate(policyT.ObjectDefinition.Raw, templateCfg.StartDelim, false) {
if !templates.HasTemplate(policyT.ObjectDefinition.Raw, TemplateStartDelim, false) {
continue
}

if !isConfigurationPolicy(policyT) {
// has Templates but not a configuration policy
err = k8serrors.NewBadRequest("Templates are restricted to only Configuration Policies")
err := k8serrors.NewBadRequest("Templates are restricted to only Configuration Policies")
log.Error(err, "Not a Configuration Policy")

r.Recorder.Event(rootPlc, "Warning", "PolicyPropagation",
Expand All @@ -513,51 +523,30 @@ func (r *ReplicatedPolicyReconciler) processTemplates(
),
)

return templateRefObjs, err
return err
}

log.V(1).Info("Found an object definition with templates")

templateContext := struct {
ManagedClusterName string
ManagedClusterLabels map[string]string
}{
ManagedClusterName: decision.ClusterName,
}
templateContext := templateCtx{ManagedClusterName: decision.ClusterName}

if strings.Contains(string(policyT.ObjectDefinition.Raw), "ManagedClusterLabels") {
templateRefObjs[k8sdepwatches.ObjectIdentifier{
Group: "cluster.open-cluster-management.io",
Version: "v1",
Kind: "ManagedCluster",
Namespace: "",
Name: decision.ClusterName,
}] = true

managedCluster := &clusterv1.ManagedCluster{}

err := r.Get(context.TODO(), types.NamespacedName{Name: decision.ClusterName}, managedCluster)
if err != nil {
log.Error(err, "Failed to get the ManagedCluster in order to use its labels in a hub template")
}

// if an error occurred, the ManagedClusterLabels will just be left empty
templateContext.ManagedClusterLabels = managedCluster.Labels
templateResolverOptions.ContextTransformers = append(
templateResolverOptions.ContextTransformers, addManagedClusterLabels(decision.ClusterName),
)
}

// Handle value encryption initialization
usesEncryption := templates.UsesEncryption(
policyT.ObjectDefinition.Raw, templateCfg.StartDelim, templateCfg.StopDelim,
)
usesEncryption := templates.UsesEncryption(policyT.ObjectDefinition.Raw, TemplateStartDelim, TemplateStopDelim)
// Initialize AES Key and initialization vector
if usesEncryption && !templateCfg.EncryptionEnabled {
if usesEncryption && !templateResolverOptions.EncryptionEnabled {
log.V(1).Info("Found an object definition requiring encryption. Handling encryption keys.")
// Get/generate the encryption key
encryptionKey, err := r.getEncryptionKey(decision.ClusterName)
if err != nil {
log.Error(err, "Failed to get/generate the policy encryption key")

return templateRefObjs, err
return err
}

// Get/generate the initialization vector
Expand All @@ -567,34 +556,25 @@ func (r *ReplicatedPolicyReconciler) processTemplates(
if err != nil {
log.Error(err, "Failed to get initialization vector")

return templateRefObjs, err
return err
}

// Set the initialization vector in the annotations
replicatedPlc.SetAnnotations(annotations)

// Set the EncryptionConfig with the retrieved key
templateCfg.EncryptionConfig = templates.EncryptionConfig{
templateResolverOptions.EncryptionConfig = templates.EncryptionConfig{
EncryptionEnabled: true,
AESKey: encryptionKey,
InitializationVector: initializationVector,
}

err = tmplResolver.SetEncryptionConfig(templateCfg.EncryptionConfig)
if err != nil {
log.Error(err, "Error setting encryption configuration")

return templateRefObjs, err
}
}

templateResult, tplErr := tmplResolver.ResolveTemplate(policyT.ObjectDefinition.Raw, templateContext)
var tplErr error

// Record the referenced objects in the template even if there is an error. This is because a change in the
// object could fix the error.
for _, refObj := range templateResult.ReferencedObjects {
templateRefObjs[refObj] = true
}
templateResult, tplErr = r.TemplateResolver.ResolveTemplate(
policyT.ObjectDefinition.Raw, templateContext, &templateResolverOptions,
)

if tplErr != nil {
log.Error(tplErr, "Failed to resolve templates")
Expand Down Expand Up @@ -635,7 +615,7 @@ func (r *ReplicatedPolicyReconciler) processTemplates(
}
}

return templateRefObjs, tplErr
return tplErr
}

policyT.ObjectDefinition.Raw = templateResult.ResolvedJSON
Expand All @@ -646,7 +626,7 @@ func (r *ReplicatedPolicyReconciler) processTemplates(

jsonErr := json.Unmarshal(templateResult.ResolvedJSON, policyTObjectUnstructured)
if jsonErr != nil {
return templateRefObjs, fmt.Errorf("failed to unmarshal the object definition to JSON: %w", jsonErr)
return fmt.Errorf("failed to unmarshal the object definition to JSON: %w", jsonErr)
}

policyTAnnotations := policyTObjectUnstructured.GetAnnotations()
Expand All @@ -663,17 +643,24 @@ func (r *ReplicatedPolicyReconciler) processTemplates(

updatedPolicyT, jsonErr := json.Marshal(policyTObjectUnstructured)
if jsonErr != nil {
return templateRefObjs, fmt.Errorf("failed to marshal the policy template to JSON: %w", jsonErr)
return fmt.Errorf("failed to marshal the policy template to JSON: %w", jsonErr)
}

policyT.ObjectDefinition.Raw = updatedPolicyT
}
}
}

if templateResult.CacheCleanUp != nil {
err := templateResult.CacheCleanUp()
if err != nil {
return err
}
}

log.V(1).Info("Successfully processed templates")

return templateRefObjs, nil
return nil
}

func isConfigurationPolicy(policyT *policiesv1.PolicyTemplate) bool {
Expand Down
Loading

0 comments on commit 3eb01c9

Please sign in to comment.