Adding Operational Metrics for Cache RCAs (#481)

* Adding Operational Metrics for Cache
opendistro-for-elasticsearch · Oct 19, 2020 · 378e0cb · 378e0cb
1 parent 4c78b48
commit 378e0cb
Show file tree

Hide file tree

Showing 5 changed files with 19 additions and 3 deletions.
diff --git a/...endistro/elasticsearch/performanceanalyzer/decisionmaker/deciders/CacheHealthDecider.java b/...endistro/elasticsearch/performanceanalyzer/decisionmaker/deciders/CacheHealthDecider.java
@@ -121,8 +121,6 @@ private <R extends BaseClusterRca> List<Action> getActionsFromRca(
   }
 
   private void configureModifyCacheActionPriority() {
-    // Assigning shard request cache higher priority over field data cache
-    // TODO: Modify as per the performance test results
     this.modifyCacheActionPriorityList.add(ResourceEnum.SHARD_REQUEST_CACHE);
     this.modifyCacheActionPriorityList.add(ResourceEnum.FIELD_DATA_CACHE);
   }

diff --git a/...pendistro/elasticsearch/performanceanalyzer/rca/framework/metrics/RcaVerticesMetrics.java b/...pendistro/elasticsearch/performanceanalyzer/rca/framework/metrics/RcaVerticesMetrics.java
@@ -32,7 +32,13 @@ public enum RcaVerticesMetrics implements MeasurementSet {
   NUM_HIGH_HEAP_CLUSTER_RCA_TRIGGERED(
       "HighHeapClusterRcaCount", "count", Collections.singletonList(Statistics.COUNT)),
   YOUNG_GEN_RCA_NAMED_COUNT(
-      "YoungGenRcaNamedCount", "namedCount", Collections.singletonList(Statistics.NAMED_COUNTERS));
+      "YoungGenRcaNamedCount", "namedCount", Collections.singletonList(Statistics.NAMED_COUNTERS)),
+  NUM_FIELD_DATA_CACHE_RCA_TRIGGERED(
+      "FieldDataCacheRcaCount", "count", Collections.singletonList(Statistics.COUNT)),
+  NUM_SHARD_REQUEST_CACHE_RCA_TRIGGERED(
+          "ShardRequestCacheCount", "count", Collections.singletonList(Statistics.COUNT)),
+  CLUSTER_RCA_NAMED_COUNT(
+          "ClusterRcaNamedCount", "namedCount", Collections.singletonList(Statistics.NAMED_COUNTERS));
 
   /** What we want to appear as the metric name. */
   private String name;

diff --git a/...n/opendistro/elasticsearch/performanceanalyzer/rca/store/rca/cache/FieldDataCacheRca.java b/...n/opendistro/elasticsearch/performanceanalyzer/rca/store/rca/cache/FieldDataCacheRca.java
@@ -19,6 +19,7 @@
 import static com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.store.rca.cache.CacheUtil.getCacheMaxSize;
 import static com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.store.rca.cache.CacheUtil.isSizeThresholdExceeded;
 
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.PerformanceAnalyzerApp;
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.grpc.FlowUnitMessage;
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.grpc.Resource;
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.metricsdb.MetricsDB;
@@ -33,6 +34,7 @@
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.summaries.HotResourceSummary;
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.summaries.ResourceUtil;
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.core.RcaConf;
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.metrics.RcaVerticesMetrics;
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.util.InstanceDetails;
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.scheduler.FlowUnitOperationArgWrapper;
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.store.rca.cluster.NodeKey;
@@ -121,6 +123,8 @@ public ResourceFlowUnit<HotNodeSummary> operate() {
             if (cacheEvictionCollector.isUnhealthy(currTimestamp) && exceedsSizeThreshold) {
                 context = new ResourceContext(Resources.State.UNHEALTHY);
                 nodeSummary.appendNestedSummary(cacheEvictionCollector.generateSummary(currTimestamp));
+                PerformanceAnalyzerApp.RCA_VERTICES_METRICS_AGGREGATOR.updateStat(
+                        RcaVerticesMetrics.NUM_FIELD_DATA_CACHE_RCA_TRIGGERED, instanceDetails.getInstanceId().toString(), 1);
             } else {
                 context = new ResourceContext(Resources.State.HEALTHY);
             }

diff --git a/...pendistro/elasticsearch/performanceanalyzer/rca/store/rca/cache/ShardRequestCacheRca.java b/...pendistro/elasticsearch/performanceanalyzer/rca/store/rca/cache/ShardRequestCacheRca.java
@@ -20,6 +20,7 @@
 import static com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.store.rca.cache.CacheUtil.getCacheMaxSize;
 import static com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.store.rca.cache.CacheUtil.isSizeThresholdExceeded;
 
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.PerformanceAnalyzerApp;
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.grpc.FlowUnitMessage;
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.grpc.Resource;
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.metricsdb.MetricsDB;
@@ -35,6 +36,7 @@
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.summaries.HotResourceSummary;
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.summaries.ResourceUtil;
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.core.RcaConf;
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.metrics.RcaVerticesMetrics;
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.util.InstanceDetails;
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.scheduler.FlowUnitOperationArgWrapper;
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.store.rca.cluster.NodeKey;
@@ -145,6 +147,8 @@ public ResourceFlowUnit operate() {
                     && exceedsSizeThreshold) {
                 context = new ResourceContext(Resources.State.UNHEALTHY);
                 nodeSummary.appendNestedSummary(cacheEvictionCollector.generateSummary(currTimestamp));
+                PerformanceAnalyzerApp.RCA_VERTICES_METRICS_AGGREGATOR.updateStat(
+                        RcaVerticesMetrics.NUM_SHARD_REQUEST_CACHE_RCA_TRIGGERED, instanceDetails.getInstanceId().toString(), 1);
             } else {
                 context = new ResourceContext(Resources.State.HEALTHY);
             }

diff --git a/...on/opendistro/elasticsearch/performanceanalyzer/rca/store/rca/cluster/BaseClusterRca.java b/...on/opendistro/elasticsearch/performanceanalyzer/rca/store/rca/cluster/BaseClusterRca.java
@@ -15,13 +15,15 @@
 
 package com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.store.rca.cluster;
 
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.PerformanceAnalyzerApp;
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.Rca;
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.Resources;
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.Resources.State;
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.contexts.ResourceContext;
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.flow_units.ResourceFlowUnit;
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.summaries.HotClusterSummary;
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.summaries.HotNodeSummary;
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.metrics.RcaVerticesMetrics;
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.util.InstanceDetails;
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.scheduler.FlowUnitOperationArgWrapper;
 import com.google.common.annotations.VisibleForTesting;
@@ -168,6 +170,8 @@ private ResourceFlowUnit<HotClusterSummary> generateFlowUnit() {
       for (HotNodeSummary nodeSummary : unhealthyNodeSummaries) {
         clusterSummary.appendNestedSummary(nodeSummary);
       }
+      PerformanceAnalyzerApp.RCA_VERTICES_METRICS_AGGREGATOR.updateStat(
+              RcaVerticesMetrics.CLUSTER_RCA_NAMED_COUNT, this.getClass().getName(), 1);
       return new ResourceFlowUnit<>(timestamp, new ResourceContext(Resources.State.UNHEALTHY), clusterSummary, true);
     }
     else {