diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/ExternalMonitoringConfiguration.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/ExternalMonitoringConfiguration.java index 44d9816ded..e3a828a463 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/ExternalMonitoringConfiguration.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/ExternalMonitoringConfiguration.java @@ -55,7 +55,8 @@ public GraphiteClient graphiteClient(@Qualifier("monitoringRestTemplate") RestTe public VictoriaMetricsMetricsProvider prometheusMetricsProvider(PrometheusClient prometheusClient, PrometheusMonitoringClientProperties properties) { return new VictoriaMetricsMetricsProvider(prometheusClient, - properties.getConsumersMetricsPrefix(), properties.getFrontendMetricsPrefix()); + properties.getConsumersMetricsPrefix(), properties.getFrontendMetricsPrefix(), + properties.getAdditionalFilters()); } @Bean diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/PrometheusMonitoringClientProperties.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/PrometheusMonitoringClientProperties.java index 4c0977f192..cdf4ce2cdb 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/PrometheusMonitoringClientProperties.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/PrometheusMonitoringClientProperties.java @@ -3,6 +3,7 @@ public class PrometheusMonitoringClientProperties extends ExternalMonitoringClientProperties { private String consumersMetricsPrefix = "hermes_consumers"; private String frontendMetricsPrefix = "hermes_frontend"; + private String additionalFilters = ""; public String getConsumersMetricsPrefix() { return consumersMetricsPrefix; @@ -19,4 +20,12 @@ public String getFrontendMetricsPrefix() { public void setFrontendMetricsPrefix(String frontendMetricsPrefix) { this.frontendMetricsPrefix = frontendMetricsPrefix; } + + public String getAdditionalFilters() { + return additionalFilters; + } + + public void setAdditionalFilters(String additionalFilters) { + this.additionalFilters = additionalFilters; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/prometheus/VictoriaMetricsMetricsProvider.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/prometheus/VictoriaMetricsMetricsProvider.java index 29209b9dde..0b44d97cd4 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/prometheus/VictoriaMetricsMetricsProvider.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/prometheus/VictoriaMetricsMetricsProvider.java @@ -28,15 +28,17 @@ public class VictoriaMetricsMetricsProvider implements MonitoringSubscriptionMet private final String consumersMetricsPrefix; private final String frontendMetricsPrefix; + private final String additionalFilters; private final String subscriptionMetricsToQuery; private final String topicMetricsToQuery; private final PrometheusClient prometheusClient; public VictoriaMetricsMetricsProvider(PrometheusClient prometheusClient, String consumersMetricsPrefix, - String frontendMetricsPrefix) { + String frontendMetricsPrefix, String additionalFilters) { this.prometheusClient = prometheusClient; this.consumersMetricsPrefix = consumersMetricsPrefix.isEmpty() ? "" : consumersMetricsPrefix + "_"; this.frontendMetricsPrefix = frontendMetricsPrefix.isEmpty() ? "" : frontendMetricsPrefix + "_"; + this.additionalFilters = additionalFilters; this.subscriptionMetricsToQuery = Stream.of(SUBSCRIPTION_DELIVERED, SUBSCRIPTION_TIMEOUTS, SUBSCRIPTION_THROUGHPUT, SUBSCRIPTION_OTHER_ERRORS, SUBSCRIPTION_BATCHES, SUBSCRIPTION_STATUS_CODES) @@ -56,9 +58,9 @@ public MonitoringSubscriptionMetrics subscriptionMetrics(SubscriptionName subscr https://docs.victoriametrics.com/MetricsQL.html. Basic PromQL does not support `keep_metric_names` param. */ String queryFormat = "sum by (__name__,group,topic,subscription,status_code)" - + "(irate({__name__=~'%s',group='%s',topic='%s',subscription='%s'}[1m]) keep_metric_names)"; + + "(irate({__name__=~'%s',group='%s',topic='%s',subscription='%s', %s}[1m]) keep_metric_names)"; String query = String.format(queryFormat, subscriptionMetricsToQuery, subscriptionName.getTopicName().getGroupName(), - subscriptionName.getTopicName().getName(), subscriptionName.getName()); + subscriptionName.getTopicName().getName(), subscriptionName.getName(), additionalFilters); MonitoringMetricsContainer prometheusMetricsContainer = prometheusClient.readMetrics(query); return MonitoringSubscriptionMetricsProvider .metricsBuilder() @@ -80,8 +82,9 @@ public MonitoringTopicMetrics topicMetrics(TopicName topicName) { https://docs.victoriametrics.com/MetricsQL.html. Basic PromQL does not support `keep_metric_names` param. */ String queryFormat = "sum by (__name__, group, topic) (irate({__name__=~'%s', group='%s', " - + "topic='%s'}[1m]) keep_metric_names)"; - String query = String.format(queryFormat, topicMetricsToQuery, topicName.getGroupName(), topicName.getName()); + + "topic='%s', %s}[1m]) keep_metric_names)"; + String query = String.format(queryFormat, topicMetricsToQuery, topicName.getGroupName(), topicName.getName(), + additionalFilters); MonitoringMetricsContainer prometheusMetricsContainer = prometheusClient.readMetrics(query); return MonitoringTopicMetricsProvider .metricsBuilder() diff --git a/hermes-management/src/test/groovy/pl/allegro/tech/hermes/management/infrastructure/metrics/HybridPrometheusBasedSubscriptionMetricsRepositoryTest.groovy b/hermes-management/src/test/groovy/pl/allegro/tech/hermes/management/infrastructure/metrics/HybridPrometheusBasedSubscriptionMetricsRepositoryTest.groovy index e5581587f0..7c7ef12876 100644 --- a/hermes-management/src/test/groovy/pl/allegro/tech/hermes/management/infrastructure/metrics/HybridPrometheusBasedSubscriptionMetricsRepositoryTest.groovy +++ b/hermes-management/src/test/groovy/pl/allegro/tech/hermes/management/infrastructure/metrics/HybridPrometheusBasedSubscriptionMetricsRepositoryTest.groovy @@ -22,7 +22,8 @@ class HybridPrometheusBasedSubscriptionMetricsRepositoryTest extends Specificati private SubscriptionLagSource lagSource = new NoOpSubscriptionLagSource() - private VictoriaMetricsMetricsProvider prometheusMetricsProvider = new VictoriaMetricsMetricsProvider(client, "hermes_consumers", "hermes_frontend"); + private VictoriaMetricsMetricsProvider prometheusMetricsProvider = new VictoriaMetricsMetricsProvider( + client, "hermes_consumers", "hermes_frontend", "service=~'hermes'"); private HybridSubscriptionMetricsRepository repository = new HybridSubscriptionMetricsRepository(prometheusMetricsProvider, summedSharedCounter, zookeeperPaths, lagSource) @@ -34,7 +35,7 @@ class HybridPrometheusBasedSubscriptionMetricsRepositoryTest extends Specificati "|hermes_consumers_subscription_other_errors_total" + "|hermes_consumers_subscription_batches_total" + "|hermes_consumers_subscription_http_status_codes_total'," + - "group='group',topic='topic',subscription='subscription'}[1m]) keep_metric_names)" + "group='group',topic='topic',subscription='subscription', service=~'hermes'}[1m]) keep_metric_names)" def "should read subscription metrics from multiple places"() { given: diff --git a/hermes-management/src/test/groovy/pl/allegro/tech/hermes/management/infrastructure/metrics/HybridPrometheusBasedTopicMetricsRepositoryTest.groovy b/hermes-management/src/test/groovy/pl/allegro/tech/hermes/management/infrastructure/metrics/HybridPrometheusBasedTopicMetricsRepositoryTest.groovy index 0833b55a15..c3dcad72ad 100644 --- a/hermes-management/src/test/groovy/pl/allegro/tech/hermes/management/infrastructure/metrics/HybridPrometheusBasedTopicMetricsRepositoryTest.groovy +++ b/hermes-management/src/test/groovy/pl/allegro/tech/hermes/management/infrastructure/metrics/HybridPrometheusBasedTopicMetricsRepositoryTest.groovy @@ -21,7 +21,7 @@ class HybridPrometheusBasedTopicMetricsRepositoryTest extends Specification { private SubscriptionRepository subscriptionRepository = Mock(SubscriptionRepository) private VictoriaMetricsMetricsProvider prometheusMetricsProvider = new VictoriaMetricsMetricsProvider(client, - "hermes_consumers", "hermes_frontend") + "hermes_consumers", "hermes_frontend", "service='hermes'") private HybridTopicMetricsRepository repository = new HybridTopicMetricsRepository(prometheusMetricsProvider, summedSharedCounter, zookeeperPaths, subscriptionRepository) @@ -31,7 +31,7 @@ class HybridPrometheusBasedTopicMetricsRepositoryTest extends Specification { String query = "sum by (__name__, group, topic) (irate({__name__=~'hermes_frontend_topic_requests_total" + "|hermes_consumers_subscription_delivered_total" + "|hermes_frontend_topic_throughput_bytes_total', group='group', " + - "topic='topic'}[1m]) keep_metric_names)" + "topic='topic', service='hermes'}[1m]) keep_metric_names)" TopicName topic = new TopicName('group', 'topic') client.readMetrics(query) >> MonitoringMetricsContainer.createEmpty()