Use official instrumentation:opentelemetry-runtime-metrics rather tha…

…n custom code to collect JVM metrics + refactor metering instrument reconfiguration
jenkinsci · Mar 27, 2023 · ff0304a · ff0304a
1 parent 9e446a6
commit ff0304a
Show file tree

Hide file tree

Showing 17 changed files with 562 additions and 494 deletions.
diff --git a/pom.xml b/pom.xml
@@ -128,6 +128,11 @@
             <artifactId>opentelemetry-resources</artifactId>
             <version>${opentelemetry-instrumentation-alpha.version}</version>
         </dependency>
+        <dependency>
+            <groupId>io.opentelemetry.instrumentation</groupId>
+            <artifactId>opentelemetry-runtime-metrics</artifactId>
+            <version>${opentelemetry-instrumentation-alpha.version}</version>
+        </dependency>
         <dependency>
             <groupId>io.opentelemetry.contrib</groupId>
             <artifactId>opentelemetry-aws-resources</artifactId>

diff --git a/src/main/java/io/jenkins/plugins/opentelemetry/AbstractOtelComponent.java b/src/main/java/io/jenkins/plugins/opentelemetry/AbstractOtelComponent.java
diff --git a/src/main/java/io/jenkins/plugins/opentelemetry/OpenTelemetrySdkProvider.java b/src/main/java/io/jenkins/plugins/opentelemetry/OpenTelemetrySdkProvider.java
@@ -9,6 +9,7 @@
 import com.google.common.base.Preconditions;
 import hudson.Extension;
 import hudson.ExtensionList;
+import io.jenkins.plugins.opentelemetry.opentelemetry.ClosingOpenTelemetry;
 import io.jenkins.plugins.opentelemetry.opentelemetry.GlobalOpenTelemetrySdk;
 import io.jenkins.plugins.opentelemetry.opentelemetry.autoconfigure.ConfigPropertiesUtils;
 import io.opentelemetry.api.GlobalOpenTelemetry;
@@ -46,7 +47,7 @@ public class OpenTelemetrySdkProvider {
      */
     public static final String DEFAULT_OTEL_JAVA_DISABLED_RESOURCE_PROVIDERS = ProcessResourceProvider.class.getName();
 
-    protected transient OpenTelemetry openTelemetry;
+    protected transient ClosingOpenTelemetry openTelemetry;
     protected transient OpenTelemetrySdk openTelemetrySdk;
     @Nonnull
     protected final transient TracerDelegate tracer = new TracerDelegate();
@@ -113,6 +114,7 @@ public void shutdown() {
             LOGGER.log(Level.FINE, () -> "Shutdown Otel SDK on components: " + ExtensionList.lookup(OtelComponent.class).stream().sorted().map(e -> e.getClass().getName()).collect(Collectors.joining(", ")));
 
             ExtensionList.lookup(OtelComponent.class).stream().sorted().forEachOrdered(OtelComponent::beforeSdkShutdown);
+            this.openTelemetry.close();
             this.openTelemetrySdk.getSdkTracerProvider().shutdown();
             this.openTelemetrySdk.getSdkMeterProvider().shutdown();
             this.openTelemetrySdk.getSdkLoggerProvider().shutdown();
@@ -130,14 +132,17 @@ public void initialize(@Nonnull OpenTelemetryConfiguration configuration) {
             initializeNoOp();
         }
         LOGGER.log(Level.FINE, () -> "Initialize Otel SDK on components: " + ExtensionList.lookup(OtelComponent.class).stream().sorted().map(e -> e.getClass().getName()).collect(Collectors.joining(", ")));
-        ExtensionList.lookup(OtelComponent.class).stream().sorted().forEachOrdered(otelComponent -> otelComponent.afterSdkInitialized(meter, otelLogger, eventEmitter, tracer, config));
+        ExtensionList.lookup(OtelComponent.class).stream().sorted().forEachOrdered(otelComponent -> {
+            otelComponent.afterSdkInitialized(meter, otelLogger, eventEmitter, tracer, config);
+            otelComponent.afterSdkInitialized(openTelemetry, config);
+        });
     }
 
     public void initializeOtlp(@Nonnull OpenTelemetryConfiguration configuration) {
 
         AutoConfiguredOpenTelemetrySdkBuilder sdkBuilder = AutoConfiguredOpenTelemetrySdk.builder();
         // PROPERTIES
-        sdkBuilder.addPropertiesSupplier(() -> configuration.toOpenTelemetryProperties());
+        sdkBuilder.addPropertiesSupplier(configuration::toOpenTelemetryProperties);
 
         // RESOURCE
         sdkBuilder.addResourceCustomizer((resource, configProperties) -> {
@@ -155,7 +160,7 @@ public void initializeOtlp(@Nonnull OpenTelemetryConfiguration configuration) {
         this.openTelemetrySdk = autoConfiguredOpenTelemetrySdk.getOpenTelemetrySdk();
         this.resource = autoConfiguredOpenTelemetrySdk.getResource();
         this.config = autoConfiguredOpenTelemetrySdk.getConfig();
-        this.openTelemetry = this.openTelemetrySdk;
+        this.openTelemetry = new ClosingOpenTelemetry(this.openTelemetrySdk);
         String opentelemetryPluginVersion = OtelUtils.getOpentelemetryPluginVersion();
         this.tracer.setDelegate(openTelemetry.getTracerProvider()
             .tracerBuilder(GlobalOpenTelemetrySdk.INSTRUMENTATION_NAME)
@@ -185,7 +190,7 @@ public void initializeNoOp() {
         this.openTelemetrySdk = null;
         this.resource = Resource.getDefault();
         this.config = ConfigPropertiesUtils.emptyConfig();
-        this.openTelemetry = OpenTelemetry.noop();
+        this.openTelemetry = ClosingOpenTelemetry.noop();
         GlobalOpenTelemetry.resetForTest(); // hack for testing in Intellij cause by DiskUsageMonitoringInitializer
         GlobalLoggerProvider.resetForTest();
         GlobalEventEmitterProvider.resetForTest();

diff --git a/src/main/java/io/jenkins/plugins/opentelemetry/OtelComponent.java b/src/main/java/io/jenkins/plugins/opentelemetry/OtelComponent.java
@@ -5,84 +5,55 @@
 
 package io.jenkins.plugins.opentelemetry;
 
+import io.opentelemetry.api.OpenTelemetry;
 import io.opentelemetry.api.events.EventEmitter;
 import io.opentelemetry.api.metrics.Meter;
-import io.opentelemetry.api.metrics.ObservableDoubleGauge;
 import io.opentelemetry.api.metrics.ObservableLongCounter;
-import io.opentelemetry.api.metrics.ObservableLongGauge;
 import io.opentelemetry.api.metrics.ObservableLongUpDownCounter;
 import io.opentelemetry.api.trace.Tracer;
 import io.opentelemetry.sdk.autoconfigure.spi.ConfigProperties;
 
-import java.util.ArrayList;
-import java.util.List;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
 /**
  * Interface for components that want to be notified when the Otel SDK has been initialized or will be shutdown.
- *
+ * <p>
  * The life cycle of consumers of the OpenTelemetry SDK (consumers of {@link io.opentelemetry.api.trace.TracerProvider},
  * {@link io.opentelemetry.api.metrics.MeterProvider}, and {@link io.opentelemetry.sdk.logs.SdkLoggerProvider}) can NOT
  * use the Jenkins life cycle because those consumers of the Otel SDK need to perform initialization tasks after the
  * Otel SDK has been initialized and have to shut down things before the Otel SDK is shutdown due to a reconfiguration.
- *
+ * <p>
  * Used by components that create counters...
  */
 public interface OtelComponent extends Comparable<OtelComponent>{
 
     /**
      * Invoked soon after the Otel SDK has been initialized.
+     * Created {@link AutoCloseable} metering instruments don't have to be closed by Otel components, the OpenTelemetry
+     * plugin takes care of this  (eg {@link ObservableLongUpDownCounter}, {@link ObservableLongCounter}...)
      *
      * @param meter            {@link Meter} of the newly initialized Otel SDK
      * @param otelLogger       {@link io.opentelemetry.api.logs.Logger} of the newly initialized Otel SDK
      * @param eventEmitter
      * @param tracer           {@link Tracer} of the newly initialized Otel SDK
      * @param configProperties {@link ConfigProperties} of the newly initialized Otel SDK
      */
-    void afterSdkInitialized(Meter meter, io.opentelemetry.api.logs.Logger otelLogger, EventEmitter eventEmitter, Tracer tracer, ConfigProperties configProperties);
+    default void afterSdkInitialized(Meter meter, io.opentelemetry.api.logs.Logger otelLogger, EventEmitter eventEmitter, Tracer tracer, ConfigProperties configProperties) {}
 
     /**
-     * Invoked just before the Otel SDK is shutdown
+     * Invoked soon after the Otel SDK has been initialized.
+     * Created {@link AutoCloseable} metering instruments don't have to be closed by Otel components, the OpenTelemetry
+     * plugin takes care of this  (eg {@link ObservableLongUpDownCounter}, {@link ObservableLongCounter}...)
+     *
+     * @param openTelemetry
+     * @param configProperties {@link ConfigProperties} of the newly initialized Otel SDK
      */
-    void beforeSdkShutdown();
+    default void afterSdkInitialized(OpenTelemetry openTelemetry, ConfigProperties configProperties) {}
 
     /**
-     * Helper for {@link OtelComponent} implementations to manage the created metric instruments
+     * Invoked just before the Otel SDK is shutdown.
+     * Created {@link AutoCloseable} metering instruments don't have to be closed by Otel components, the OpenTelemetry
+     * plugin takes care of this  (eg {@link ObservableLongUpDownCounter}, {@link ObservableLongCounter}...)
      */
-    class State {
-        private final static Logger logger = Logger.getLogger(State.class.getName());
-        private final List<AutoCloseable> instruments = new ArrayList<>();
-
-        public void registerInstrument(ObservableLongCounter instrument) {
-            instruments.add(instrument);
-        }
-
-        public void registerInstrument(ObservableLongGauge instrument) {
-            instruments.add(instrument);
-        }
-
-        public void registerInstrument(ObservableLongUpDownCounter instrument) {
-            instruments.add(instrument);
-        }
-
-        public void registerInstrument(ObservableDoubleGauge instrument) {
-            instruments.add(instrument);
-        }
-
-        public void closeInstruments() {
-            List<AutoCloseable> instruments = this.instruments;
-            this.instruments.clear(); // reset the list of instruments for reuse
-            for (AutoCloseable instrument : instruments) {
-                try {
-                    instrument.close();
-                } catch (Exception e) {
-                    // should never happen, Otel instruments override the #close method to indicate they don't throw exceptions
-                    logger.log(Level.INFO, "Exception closing instrument " + instrument, e);
-                }
-            }
-        }
-    }
+    default void beforeSdkShutdown() {}
 
     /**
      * @return the ordinal of this otel component to execute step handlers in predictable order. The smallest ordinal is handled first.

diff --git a/src/main/java/io/jenkins/plugins/opentelemetry/computer/MonitoringComputerListener.java b/src/main/java/io/jenkins/plugins/opentelemetry/computer/MonitoringComputerListener.java
@@ -40,8 +40,6 @@ public class MonitoringComputerListener extends ComputerListener implements Otel
 
     private LongCounter failureAgentCounter;
 
-    private OtelComponent.State state = new State();
-
     @Override
     public void afterSdkInitialized(Meter meter, io.opentelemetry.api.logs.Logger otelLogger, EventEmitter eventEmitter, Tracer tracer, ConfigProperties configProperties) {
         final Jenkins jenkins = Jenkins.get();
@@ -66,24 +64,21 @@ public void afterSdkInitialized(Meter meter, io.opentelemetry.api.logs.Logger ot
                 LOGGER.log(Level.WARNING, "Failure getting attributes for Jenkins Controller computer " + controllerComputer, e);
             }
         }
-        state.registerInstrument(
         meter.gaugeBuilder(JenkinsSemanticMetrics.JENKINS_AGENTS_OFFLINE)
             .ofLongs()
             .setDescription("Number of offline agents")
             .setUnit("1")
-            .buildWithCallback(valueObserver -> valueObserver.record(this.getOfflineAgentsCount())));
-        state.registerInstrument(
+            .buildWithCallback(valueObserver -> valueObserver.record(this.getOfflineAgentsCount()));
             meter.gaugeBuilder(JenkinsSemanticMetrics.JENKINS_AGENTS_ONLINE)
             .ofLongs()
             .setDescription("Number of online agents")
             .setUnit("1")
-            .buildWithCallback(valueObserver -> valueObserver.record(this.getOnlineAgentsCount())));
-        state.registerInstrument(
+            .buildWithCallback(valueObserver -> valueObserver.record(this.getOnlineAgentsCount()));
             meter.gaugeBuilder(JenkinsSemanticMetrics.JENKINS_AGENTS_TOTAL)
             .ofLongs()
             .setDescription("Number of agents")
             .setUnit("1")
-            .buildWithCallback(valueObserver -> valueObserver.record(this.getAgentsCount())));
+            .buildWithCallback(valueObserver -> valueObserver.record(this.getAgentsCount()));
         failureAgentCounter = meter.counterBuilder(JenkinsSemanticMetrics.JENKINS_AGENTS_LAUNCH_FAILURE)
             .setDescription("Number of ComputerLauncher failures")
             .setUnit("1")
@@ -136,11 +131,6 @@ public void onLaunchFailure(Computer computer, TaskListener taskListener) throws
         LOGGER.log(Level.FINE, () -> "onLaunchFailure(" + computer + "): ");
     }
 
-    @Override
-    public void beforeSdkShutdown() {
-        state.closeInstruments();
-    }
-
     private static class GetComputerAttributes extends MasterToSlaveCallable<Map<String, String>, IOException> {
         @Override
         public Map<String, String> call() throws IOException {

diff --git a/src/main/java/io/jenkins/plugins/opentelemetry/init/DiskUsageMonitoringInitializer.java b/src/main/java/io/jenkins/plugins/opentelemetry/init/DiskUsageMonitoringInitializer.java
@@ -8,7 +8,7 @@
 import com.cloudbees.simplediskusage.DiskItem;
 import com.cloudbees.simplediskusage.QuickDiskUsagePlugin;
 import hudson.Extension;
-import io.jenkins.plugins.opentelemetry.AbstractOtelComponent;
+import io.jenkins.plugins.opentelemetry.OtelComponent;
 import io.jenkins.plugins.opentelemetry.semconv.JenkinsSemanticMetrics;
 import io.opentelemetry.api.events.EventEmitter;
 import io.opentelemetry.api.metrics.Meter;
@@ -27,7 +27,7 @@
  * Capture disk usage metrics relying on the {@link QuickDiskUsagePlugin}
  */
 @Extension(dynamicLoadable = YesNoMaybe.YES, optional = true)
-public class DiskUsageMonitoringInitializer extends AbstractOtelComponent {
+public class DiskUsageMonitoringInitializer implements OtelComponent {
 
     private final static Logger LOGGER = Logger.getLogger(DiskUsageMonitoringInitializer.class.getName());
 
@@ -39,12 +39,11 @@ public class DiskUsageMonitoringInitializer extends AbstractOtelComponent {
 
     @Override
     public void afterSdkInitialized(Meter meter, io.opentelemetry.api.logs.Logger otelLogger, EventEmitter eventEmitter, Tracer tracer, ConfigProperties configProperties) {
-        registerInstrument(
             meter.gaugeBuilder(JenkinsSemanticMetrics.JENKINS_DISK_USAGE_BYTES)
                 .ofLongs()
                 .setDescription("Disk usage of first level folder in JENKINS_HOME.")
                 .setUnit("byte")
-                .buildWithCallback(valueObserver -> valueObserver.record(calculateDiskUsageInBytes())));
+                .buildWithCallback(valueObserver -> valueObserver.record(calculateDiskUsageInBytes()));
 
         LOGGER.log(Level.FINE, () -> "Start monitoring Jenkins controller disk usage...");
     }

diff --git a/src/main/java/io/jenkins/plugins/opentelemetry/init/GitHubClientMonitoring.java b/src/main/java/io/jenkins/plugins/opentelemetry/init/GitHubClientMonitoring.java
@@ -7,7 +7,7 @@
 
 import com.google.common.base.Preconditions;
 import hudson.Extension;
-import io.jenkins.plugins.opentelemetry.AbstractOtelComponent;
+import io.jenkins.plugins.opentelemetry.OtelComponent;
 import io.opentelemetry.api.common.Attributes;
 import io.opentelemetry.api.common.AttributesBuilder;
 import io.opentelemetry.api.events.EventEmitter;
@@ -38,7 +38,7 @@
  * field of the {@link Connector} class because we have not found any public API to observe the state of this GitHub client.
  */
 @Extension(dynamicLoadable = YesNoMaybe.YES, optional = true)
-public class GitHubClientMonitoring extends AbstractOtelComponent {
+public class GitHubClientMonitoring implements OtelComponent {
     private final static Logger logger = Logger.getLogger(GitHubClientMonitoring.class.getName());
 
     private final Field gitHub_clientField;
@@ -87,7 +87,6 @@ public GitHubClientMonitoring() {
 
     @Override
     public void afterSdkInitialized(Meter meter, io.opentelemetry.api.logs.Logger otelLogger, EventEmitter eventEmitter, Tracer tracer, ConfigProperties configProperties) {
-        registerInstrument(
             meter.gaugeBuilder(GITHUB_API_RATE_LIMIT_REMAINING_REQUESTS)
                 .ofLongs()
                 .setDescription("GitHub Repository API rate limit remaining requests")
@@ -139,7 +138,7 @@ public void afterSdkInitialized(Meter meter, io.opentelemetry.api.logs.Logger ot
                         }
                     });
 
-                }));
+                });
         logger.log(Level.FINE, () -> "Start monitoring Jenkins GitHub client...");
     }
 }