Cherrypick to support v1.26+

cmd/craned/app/manager.go

@@ -52,6 +52,7 @@ import (
 	"github.com/gocrane/crane/pkg/recommendation"
 	"github.com/gocrane/crane/pkg/server"
 	serverconfig "github.com/gocrane/crane/pkg/server/config"
+	"github.com/gocrane/crane/pkg/utils"
 	"github.com/gocrane/crane/pkg/utils/target"
 	"github.com/gocrane/crane/pkg/webhooks"
 )
@@ -143,7 +144,7 @@ func Run(ctx context.Context, opts *options.Options) error {
 		}
 	}()
 
-	initControllers(podOOMRecorder, mgr, opts, predictorMgr, historyDataSources[providers.PrometheusDataSource])
+	initControllers(ctx, podOOMRecorder, mgr, opts, predictorMgr, historyDataSources[providers.PrometheusDataSource])
 	// initialize custom collector metrics
 	initMetricCollector(mgr)
 	runAll(ctx, mgr, predictorMgr, dataSourceProviders[providers.PrometheusDataSource], opts)
@@ -256,6 +257,8 @@ func initDataSources(mgr ctrl.Manager, opts *options.Options) (map[providers.Dat
 			hybridDataSources[providers.PrometheusDataSource] = provider
 			realtimeDataSources[providers.PrometheusDataSource] = provider
 			historyDataSources[providers.PrometheusDataSource] = provider
+
+			utils.SetExtensionLabels(opts.DataSourcePromConfig.ExtensionLabels)
 		}
 	}
 	return realtimeDataSources, historyDataSources, hybridDataSources
@@ -266,7 +269,7 @@ func initPredictorManager(opts *options.Options, realtimeDataSources map[provide
 }
 
 // initControllers setup controllers with manager
-func initControllers(oomRecorder oom.Recorder, mgr ctrl.Manager, opts *options.Options, predictorMgr predictor.Manager, historyDataSource providers.History) {
+func initControllers(ctx context.Context, oomRecorder oom.Recorder, mgr ctrl.Manager, opts *options.Options, predictorMgr predictor.Manager, historyDataSource providers.History) {
 	discoveryClientSet, err := discovery.NewDiscoveryClientForConfig(mgr.GetConfig())
 	if err != nil {
 		klog.Exit(err, "Unable to create discover client")
@@ -417,6 +420,13 @@ func initControllers(oomRecorder oom.Recorder, mgr ctrl.Manager, opts *options.O
 		}).SetupWithManager(mgr); err != nil {
 			klog.Exit(err, "unable to create controller", "controller", "RecommendationTriggerController")
 		}
+
+		checker := recommendationctrl.Checker{
+			Client:          mgr.GetClient(),
+			MonitorInterval: opts.MonitorInterval,
+			OutDateInterval: opts.OutDateInterval,
+		}
+		checker.Run(ctx.Done())
 	}
 
 	// CnpController

cmd/craned/app/options/options.go

@@ -67,6 +67,12 @@ type Options struct {
 
 	// CacheUnstructured indicates whether to cache Unstructured objects. When enabled, it will speed up reading Unstructured objects, but will increase memory usage.
 	CacheUnstructured bool
+
+	// MonitorInterval is the interval for recommendation checker
+	MonitorInterval time.Duration
+
+	// OutDateInterval is the checking interval for identify a recommendation is outdated
+	OutDateInterval time.Duration
 }
 
 // NewOptions builds an empty options.
@@ -115,6 +121,7 @@ func (o *Options) AddFlags(flags *pflag.FlagSet) {
 	flags.StringVar(&o.DataSourcePromConfig.AdapterConfigMapKey, "prometheus-adapter-configmap-key", "", "prometheus adapter-configmap key")
 	flags.StringVar(&o.DataSourcePromConfig.AdapterConfig, "prometheus-adapter-config", "", "prometheus adapter-config path")
 	flags.StringVar(&o.DataSourcePromConfig.AdapterExtensionLabels, "prometheus-adapter-extension-labels", "", "prometheus adapter extension-labels for expressionQuery")
+	flags.StringVar(&o.DataSourcePromConfig.ExtensionLabels, "extension-labels", "", "extension-labels for every prometheus query")
 	flags.StringVar(&o.DataSourcePromConfig.Auth.Username, "prometheus-auth-username", "", "prometheus auth username")
 	flags.StringVar(&o.DataSourcePromConfig.Auth.Password, "prometheus-auth-password", "", "prometheus auth password")
 	flags.StringVar(&o.DataSourcePromConfig.Auth.BearerToken, "prometheus-auth-bearertoken", "", "prometheus auth bearertoken")
@@ -139,4 +146,6 @@ func (o *Options) AddFlags(flags *pflag.FlagSet) {
 	flags.IntVar(&o.OOMRecordMaxNumber, "oom-record-max-number", 10000, "Max number for oom records to store in configmap")
 	flags.IntVar(&o.TimeSeriesPredictionMaxConcurrentReconciles, "time-series-prediction-max-concurrent-reconciles", 10, "Max concurrent reconciles for TimeSeriesPrediction controller")
 	flags.BoolVar(&o.CacheUnstructured, "cache-unstructured", true, "whether to cache Unstructured objects. When enabled, it will speed up reading Unstructured objects but will increase memory usage")
+	flags.DurationVar(&o.MonitorInterval, "recommendation-monitor-interval", time.Hour, "interval for recommendation checker")
+	flags.DurationVar(&o.OutDateInterval, "recommendation-outdate-interval", 24*time.Hour, "interval for identify a recommendation is outdated")
 }

deploy/craned/rbac.yaml

@@ -1,13 +1,128 @@
 apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: craned
+  namespace: crane-system
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  - secrets
+  verbs:
+  - create
+- apiGroups:
+  - ""
+  resourceNames:
+  - craned
+  resources:
+  - configmaps
+  verbs:
+  - get
+  - patch
+  - update
+- apiGroups:
+  - ""
+  resourceNames:
+  - clusters-secret-store
+  resources:
+  - secrets
+  verbs:
+  - get
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - get
+  - patch
+  - update
+  - create
+---
+apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
   name: craned
 rules:
-  - apiGroups: [ '*' ]
-    resources: [ '*' ]
-    verbs: [ "*" ]
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  - pods
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - analysis.crane.io
+  resources:
+  - "*"
+  verbs:
+  - "*"
+- apiGroups:
+  - apps
+  resources:
+  - daemonsets
+  - deployments
+  - deployments/scale
+  - statefulsets
+  - statefulsets/scale
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+    - apps
+  resources:
+    - daemonsets/status
+    - deployments/status
+    - deployments/scale
+    - statefulsets/status
+    - statefulsets/scale
+  verbs:
+    - update
+- apiGroups:
+  - autoscaling
+  resources:
+  - horizontalpodautoscalers
+  verbs:
+  - '*'
+- apiGroups:
+  - autoscaling.crane.io
+  resources:
+  - '*'
+  verbs:
+  - '*'
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+  - update
+- apiGroups:
+  - prediction.crane.io
+  resources:
+  - '*'
+  verbs:
+  - '*'
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: craned
+  namespace: crane-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: craned
+subjects:
+- kind: ServiceAccount
+  name: craned
+  namespace: crane-system
 ---
-
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
@@ -17,6 +132,6 @@ roleRef:
   kind: ClusterRole
   name: craned
 subjects:
-  - kind: ServiceAccount
-    name: craned
-    namespace: crane-system
+- kind: ServiceAccount
+  name: craned
+  namespace: crane-system

pkg/controller/ehpa/hpa.go

@@ -208,10 +208,30 @@ func (c *EffectiveHPAController) GetHPAMetrics(ctx context.Context, ehpa *autosc
 			var metricIdentifier string
 			var averageValue *resource.Quantity
 			switch metric.Type {
-			case autoscalingv2.ResourceMetricSourceType:
-				metricIdentifier = utils.GetMetricIdentifier(metric, metric.Resource.Name.String())
+			case autoscalingv2.ResourceMetricSourceType, autoscalingv2.ContainerResourceMetricSourceType:
+				var averageUtilization *int32
+				var containerName string
+				if metric.Resource != nil {
+					if metric.Resource.Target.AverageUtilization != nil {
+						averageUtilization = metric.Resource.Target.AverageUtilization
+					}
+					if metric.Resource.Target.AverageValue != nil {
+						averageValue = metric.Resource.Target.AverageValue
+					}
+				}
+				if metric.ContainerResource != nil {
+					containerName = metric.ContainerResource.Container
+					if metric.ContainerResource.Target.AverageUtilization != nil {
+						averageUtilization = metric.ContainerResource.Target.AverageUtilization
+					}
+					if metric.ContainerResource.Target.AverageValue != nil {
+						averageValue = metric.ContainerResource.Target.AverageValue
+					}
+				}
+
 				// When use AverageUtilization in EffectiveHorizontalPodAutoscaler's metricSpec, convert to AverageValue
-				if metric.Resource.Target.AverageUtilization != nil {
+				if averageUtilization != nil {
+					metricName := utils.GetMetricName(metric)
 					scale, _, err := utils.GetScale(ctx, c.RestMapper, c.ScaleClient, ehpa.Namespace, ehpa.Spec.ScaleTargetRef)
 					if err != nil {
 						return nil, err
@@ -231,24 +251,21 @@ func (c *EffectiveHPAController) GetHPAMetrics(ctx context.Context, ehpa *autosc
 						return nil, fmt.Errorf("failed to get available pods. ")
 					}
 
-					requests, err := utils.CalculatePodRequests(availablePods, metric.Resource.Name)
+					requests, err := utils.CalculatePodRequests(availablePods, v1.ResourceName(metricName), containerName)
 					if err != nil {
 						return nil, err
 					}
 
-					value := int64((float64(requests) * float64(*metric.Resource.Target.AverageUtilization) / 100) / float64(len(availablePods)))
+					value := int64((float64(requests) * float64(*averageUtilization) / 100) / float64(len(availablePods)))
 					averageValue = resource.NewMilliQuantity(value, resource.DecimalSI)
-				} else {
-					averageValue = metric.Resource.Target.AverageValue
 				}
 			case autoscalingv2.ExternalMetricSourceType:
-				metricIdentifier = utils.GetMetricIdentifier(metric, metric.External.Metric.Name)
 				averageValue = metric.External.Target.AverageValue
 			case autoscalingv2.PodsMetricSourceType:
-				metricIdentifier = utils.GetMetricIdentifier(metric, metric.Pods.Metric.Name)
 				averageValue = metric.Pods.Target.AverageValue
 			}
 
+			metricIdentifier = utils.GetPredictionMetricIdentifier(metric)
 			if metricIdentifier == "" {
 				continue
 			}

pkg/controller/ehpa/predict.go

@@ -133,33 +133,20 @@ func (c *EffectiveHPAController) NewPredictionObject(ehpa *autoscalingapi.Effect
 
 	var predictionMetrics []predictionapi.PredictionMetric
 	for _, metric := range ehpa.Spec.Metrics {
-		var metricName string
-		//get metricIdentifier by metric.Type and metricName
-		var metricIdentifier string
-		switch metric.Type {
-		case autoscalingv2.ResourceMetricSourceType:
-			metricName = metric.Resource.Name.String()
-			metricIdentifier = utils.GetMetricIdentifier(metric, metric.Resource.Name.String())
-		case autoscalingv2.ExternalMetricSourceType:
-			metricName = metric.External.Metric.Name
-			metricIdentifier = utils.GetMetricIdentifier(metric, metric.External.Metric.Name)
-		case autoscalingv2.PodsMetricSourceType:
-			metricName = metric.Pods.Metric.Name
-			metricIdentifier = utils.GetMetricIdentifier(metric, metric.Pods.Metric.Name)
-		}
-
+		metricIdentifier := utils.GetPredictionMetricIdentifier(metric)
 		if metricIdentifier == "" {
 			continue
 		}
 
+		metricName := utils.GetMetricName(metric)
 		//get matchLabels
 		var matchLabels []string
 		var metricRule *prometheus_adapter.MetricRule
 
 		// Supreme priority: annotation
 		expressionQuery := utils.GetExpressionQueryAnnotation(metricIdentifier, ehpa.Annotations)
 		if expressionQuery == "" {
-			var nameReg string
+			var podNameReg string
 			// get metricRule from prometheus-adapter
 			switch metric.Type {
 			case autoscalingv2.ResourceMetricSourceType:
@@ -169,7 +156,23 @@ func (c *EffectiveHPAController) NewPredictionObject(ehpa *autoscalingapi.Effect
 						klog.Errorf("Got MetricRulesResource prometheus-adapter-resource Failed MetricName[%s]", metricName)
 					} else {
 						klog.V(4).Infof("Got MetricRulesResource prometheus-adapter-resource MetricMatches[%s] SeriesName[%s]", metricRule.MetricMatches, metricRule.SeriesName)
-						nameReg = utils.GetPodNameReg(ehpa.Spec.ScaleTargetRef.Name, ehpa.Spec.ScaleTargetRef.Kind)
+						podNameReg = utils.GetPodNameReg(ehpa.Spec.ScaleTargetRef.Name, ehpa.Spec.ScaleTargetRef.Kind)
+					}
+				}
+			case autoscalingv2.ContainerResourceMetricSourceType:
+				if len(mrs.MetricRulesResource) > 0 {
+					metricRule = prometheus_adapter.MatchMetricRule(mrs.MetricRulesResource, metricName)
+					if metricRule == nil {
+						klog.Errorf("Got MetricRulesResource prometheus-adapter-resource Failed MetricName[%s]", metricName)
+					} else {
+						klog.V(4).Infof("Got MetricRulesResource prometheus-adapter-resource MetricMatches[%s] SeriesName[%s]", metricRule.MetricMatches, metricRule.SeriesName)
+						podNameReg = utils.GetPodNameReg(ehpa.Spec.ScaleTargetRef.Name, ehpa.Spec.ScaleTargetRef.Kind)
+						// Compared to ResourceMetricSourceType, there is an additional container-name field
+						containerLabel := "container"
+						if metricRule.ContainerLabel != "" {
+							containerLabel = metricRule.ContainerLabel
+						}
+						matchLabels = append(matchLabels, utils.MapSortToArray(map[string]string{containerLabel: metric.ContainerResource.Container})...)
 					}
 				}
 			case autoscalingv2.PodsMetricSourceType:
@@ -179,12 +182,9 @@ func (c *EffectiveHPAController) NewPredictionObject(ehpa *autoscalingapi.Effect
 						klog.Errorf("Got MetricRulesCustomer prometheus-adapter-customer Failed MetricName[%s]", metricName)
 					} else {
 						klog.V(4).Infof("Got MetricRulesCustomer prometheus-adapter-customer MetricMatches[%s] SeriesName[%s]", metricRule.MetricMatches, metricRule.SeriesName)
-						nameReg = utils.GetPodNameReg(ehpa.Spec.ScaleTargetRef.Name, ehpa.Spec.ScaleTargetRef.Kind)
-
+						podNameReg = utils.GetPodNameReg(ehpa.Spec.ScaleTargetRef.Name, ehpa.Spec.ScaleTargetRef.Kind)
 						if metric.Pods.Metric.Selector != nil {
-							for _, i := range utils.MapSortToArray(metric.Pods.Metric.Selector.MatchLabels) {
-								matchLabels = append(matchLabels, i)
-							}
+							matchLabels = append(matchLabels, utils.MapSortToArray(metric.Pods.Metric.Selector.MatchLabels)...)
 						}
 					}
 				}
@@ -196,18 +196,16 @@ func (c *EffectiveHPAController) NewPredictionObject(ehpa *autoscalingapi.Effect
 					} else {
 						klog.V(4).Infof("Got MetricRulesExternal prometheus-adapter-external MetricMatches[%s] SeriesName[%s]", metricRule.MetricMatches, metricRule.SeriesName)
 						if metric.External.Metric.Selector != nil {
-							for _, i := range utils.MapSortToArray(metric.External.Metric.Selector.MatchLabels) {
-								matchLabels = append(matchLabels, i)
-							}
+							matchLabels = append(matchLabels, utils.MapSortToArray(metric.External.Metric.Selector.MatchLabels)...)
 						}
 					}
 				}
 			}
 
 			if metricRule != nil {
-				// Second priority: get default expressionQuery
+				// Second priority: get prometheus-adapter expressionQuery
 				var err error
-				expressionQuery, err = metricRule.QueryForSeries(ehpa.Namespace, nameReg, append(mrs.ExtensionLabels, matchLabels...))
+				expressionQuery, err = metricRule.QueryForSeries(ehpa.Namespace, podNameReg, append(mrs.ExtensionLabels, matchLabels...))
 				if err != nil {
 					klog.Errorf("Got promSelector prometheus-adapter %v %v", metricRule, err)
 				} else {