#1850 disable internal metric service, add micrometer, add Prometheus…

…MetricsPlugin

engine/src/main/java/com/arcadedb/GlobalConfiguration.java

@@ -307,8 +307,15 @@ This setting is intended as a safety measure against excessive resource consumpt
   SERVER_MODE("arcadedb.server.mode", SCOPE.SERVER, "Server mode between 'development', 'test' and 'production'", String.class,
       "development", Set.of((Object[]) new String[] { "development", "test", "production" })),
 
+  // Metrics
   SERVER_METRICS("arcadedb.serverMetrics", SCOPE.SERVER, "True to enable metrics", Boolean.class, true),
 
+  SERVER_METRICS_LOGGING("arcadedb.serverMetrics.logging", SCOPE.SERVER, "True to enable metrics logging", Boolean.class, true),
+
+  SERVER_METRICS_BACKENDS("arcadedb.serverMetrics.backends", SCOPE.SERVER, "Comma separated list of metrics backends to enable",
+      String.class, "", Set.of("prometheus", "elasticsearch", "datadog", "jmx")),
+
+  //paths
   SERVER_ROOT_PATH("arcadedb.server.rootPath", SCOPE.SERVER,
       "Root path in the file system where the server is looking for files. By default is the current directory", String.class,
       null),
@@ -389,7 +396,7 @@ This setting is intended as a safety measure against excessive resource consumpt
       String.class, ""),
 
   HA_QUORUM("arcadedb.ha.quorum", SCOPE.SERVER,
-      "Default quorum between 'none', one, two, three, 'majority' and 'all' servers. Default is majority", String.class, "majority",
+      "Default quorum between 'none', one, two, three, 'majority' and 'all' servers. Default is majority",String.class  , "majority",
       Set.of(new String[] { "none", "one", "two", "three", "majority", "all" })),
 
   HA_QUORUM_TIMEOUT("arcadedb.ha.quorumTimeout", SCOPE.SERVER, "Timeout waiting for the quorum", Long.class, 10000),

engine/src/main/java/com/arcadedb/utility/CodeUtils.java

@@ -28,7 +28,9 @@
  */
 public class CodeUtils {
 
-  public static void executeIgnoringExceptions(final CallableNoReturn callback, final String errorMessage,
+  public static void executeIgnoringExceptions(
+      final CallableNoReturn callback,
+      final String errorMessage,
       final boolean logException) {
     try {
       callback.call();

metrics/pom.xml

@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+    Copyright © 2021-present Arcade Data Ltd ([email protected])
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+    SPDX-FileCopyrightText: 2021-present Arcade Data Ltd ([email protected])
+    SPDX-License-Identifier: Apache-2.0
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>com.arcadedb</groupId>
+        <artifactId>arcadedb-parent</artifactId>
+        <version>24.11.2-SNAPSHOT</version>
+        <relativePath>../pom.xml</relativePath>
+    </parent>
+
+    <artifactId>arcadedb-metrics</artifactId>
+    <packaging>jar</packaging>
+
+    <properties>
+        <micrometer.version>1.13.6</micrometer.version>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.arcadedb</groupId>
+            <artifactId>arcadedb-server</artifactId>
+            <version>${project.parent.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>io.micrometer</groupId>
+            <artifactId>micrometer-registry-prometheus</artifactId>
+            <version>${micrometer.version}</version>
+        </dependency>
+
+    </dependencies>
+</project>

metrics/src/main/java/com/arcadedb/metrics/prometheus/PrometheusMetricsPlugin.java

@@ -0,0 +1,54 @@
+package com.arcadedb.metrics.prometheus;
+
+import com.arcadedb.ContextConfiguration;
+import com.arcadedb.GlobalConfiguration;
+import com.arcadedb.log.LogManager;
+import com.arcadedb.server.ArcadeDBServer;
+import com.arcadedb.server.ServerPlugin;
+import com.arcadedb.server.http.HttpServer;
+import io.micrometer.core.instrument.Metrics;
+import io.micrometer.prometheusmetrics.PrometheusConfig;
+import io.micrometer.prometheusmetrics.PrometheusMeterRegistry;
+import io.undertow.io.Sender;
+import io.undertow.server.handlers.PathHandler;
+import io.undertow.util.Headers;
+
+import java.util.logging.Level;
+
+public class PrometheusMetricsPlugin implements ServerPlugin {
+
+  private PrometheusMeterRegistry registry;
+  private boolean                 enabled;
+
+  @Override
+  public void configure(ArcadeDBServer arcadeDBServer, ContextConfiguration configuration) {
+    enabled = configuration.getValueAsBoolean(GlobalConfiguration.SERVER_METRICS);
+    registry = new PrometheusMeterRegistry(PrometheusConfig.DEFAULT);
+    Metrics.addRegistry(registry);
+  }
+
+  @Override
+  public void startService() {
+    if (enabled) {
+      LogManager.instance().log(this, Level.INFO, "Prometheus backend metrics enabled");
+    }
+  }
+
+  @Override
+  public void registerAPI(final HttpServer httpServer, final PathHandler routes) {
+    if (!enabled)
+      return;
+
+    routes.addExactPath("/prometheus", exchange -> {
+      String response = registry.scrape();
+      exchange.setStatusCode(200);
+      exchange.getResponseHeaders().put(Headers.CONTENT_LENGTH, response.getBytes().length);
+      exchange.getResponseSender().send(response);
+
+    });
+
+    LogManager.instance().log(this, Level.INFO, "Prometheus backend metrics http handler configured");
+
+  }
+
+}

package/pom.xml

@@ -114,6 +114,11 @@
             <artifactId>arcadedb-server</artifactId>
             <version>${project.parent.version}</version>
         </dependency>
+        <dependency>
+            <groupId>com.arcadedb</groupId>
+            <artifactId>arcadedb-metrics</artifactId>
+            <version>${project.parent.version}</version>
+        </dependency>
         <dependency>
             <groupId>com.arcadedb</groupId>
             <artifactId>arcadedb-studio</artifactId>

pom.xml

@@ -115,6 +115,7 @@
         <module>engine</module>
         <module>network</module>
         <module>server</module>
+        <module>metrics</module>
         <module>integration</module>
         <module>console</module>
         <module>gremlin</module>

server/pom.xml

@@ -36,6 +36,7 @@
         <metrics.version>4.2.19</metrics.version>
         <slf4j.version>2.0.16</slf4j.version>
         <undertow-core.version>2.3.18.Final</undertow-core.version>
+        <micrometer.version>1.13.6</micrometer.version>
     </properties>
 
     <profiles>
@@ -96,5 +97,11 @@
             <artifactId>slf4j-jdk14</artifactId>
             <version>${slf4j.version}</version>
         </dependency>
+        <dependency>
+            <groupId>io.micrometer</groupId>
+            <artifactId>micrometer-core</artifactId>
+            <version>${micrometer.version}</version>
+        </dependency>
+
     </dependencies>
 </project>

server/src/main/java/com/arcadedb/server/ArcadeDBServer.java

@@ -40,20 +40,37 @@
 import com.arcadedb.server.ha.HAServer;
 import com.arcadedb.server.ha.ReplicatedDatabase;
 import com.arcadedb.server.http.HttpServer;
-import com.arcadedb.server.monitor.DefaultServerMetrics;
-import com.arcadedb.server.monitor.ServerMetrics;
 import com.arcadedb.server.monitor.ServerMonitor;
 import com.arcadedb.server.security.ServerSecurity;
 import com.arcadedb.server.security.ServerSecurityException;
 import com.arcadedb.server.security.ServerSecurityUser;
 import com.arcadedb.utility.CodeUtils;
 import com.arcadedb.utility.FileUtils;
-
-import java.io.*;
-import java.lang.reflect.*;
-import java.util.*;
-import java.util.concurrent.*;
-import java.util.logging.*;
+import io.micrometer.core.instrument.Metrics;
+import io.micrometer.core.instrument.binder.jvm.ClassLoaderMetrics;
+import io.micrometer.core.instrument.binder.jvm.JvmGcMetrics;
+import io.micrometer.core.instrument.binder.jvm.JvmMemoryMetrics;
+import io.micrometer.core.instrument.binder.jvm.JvmThreadMetrics;
+import io.micrometer.core.instrument.binder.system.ProcessorMetrics;
+import io.micrometer.core.instrument.logging.LoggingMeterRegistry;
+import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.lang.reflect.InvocationTargetException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.logging.Level;
 
 import static com.arcadedb.engine.ComponentFile.MODE.READ_ONLY;
 import static com.arcadedb.engine.ComponentFile.MODE.READ_WRITE;
@@ -75,7 +92,7 @@ public enum STATUS {OFFLINE, STARTING, ONLINE, SHUTTING_DOWN}
   private final       ConcurrentMap<String, ServerDatabase> databases                            = new ConcurrentHashMap<>();
   private final       List<ReplicationCallback>             testEventListeners                   = new ArrayList<>();
   private volatile    STATUS                                status                               = STATUS.OFFLINE;
-  private             ServerMetrics                         serverMetrics                        = new DefaultServerMetrics();
+  //  private             ServerMetrics                         serverMetrics                        = new DefaultServerMetrics();
   private             ServerMonitor                         serverMonitor;
 
   static {
@@ -133,12 +150,21 @@ public synchronized void start() {
 
     // START METRICS & CONNECTED JMX REPORTER
     if (configuration.getValueAsBoolean(GlobalConfiguration.SERVER_METRICS)) {
-      if (serverMetrics != null)
-        serverMetrics.stop();
-      serverMetrics = new DefaultServerMetrics();
+      Metrics.addRegistry(new SimpleMeterRegistry());
+
+      new ClassLoaderMetrics().bindTo(Metrics.globalRegistry);
+      new JvmMemoryMetrics().bindTo(Metrics.globalRegistry);
+      new JvmGcMetrics().bindTo(Metrics.globalRegistry);
+      new ProcessorMetrics().bindTo(Metrics.globalRegistry);
+      new JvmThreadMetrics().bindTo(Metrics.globalRegistry);
+//      new JvmThreadDeadlockMetrics().bindTo(registry);
+
+      if (configuration.getValueAsBoolean(GlobalConfiguration.SERVER_METRICS_LOGGING)) {
+        LogManager.instance().log(this, Level.INFO, "- Logging metrics enabled...");
+        Metrics.addRegistry(new LoggingMeterRegistry());
+      }
       LogManager.instance().log(this, Level.INFO, "- Metrics Collection Started...");
     }
-
     security = new ServerSecurity(this, configuration, serverRootPath + "/config");
     security.startService();
 
@@ -295,8 +321,8 @@ public synchronized void stop() {
 
     CodeUtils.executeIgnoringExceptions(() -> {
       LogManager.instance().log(this, Level.INFO, "- Stop JMX Metrics");
-      serverMetrics.stop();
-      serverMetrics = new DefaultServerMetrics();
+//      serverMetrics.stop();
+//      serverMetrics = new DefaultServerMetrics();
     }, "Error on stopping JMX Metrics", false);
 
     LogManager.instance().log(this, Level.INFO, "ArcadeDB Server is down");
@@ -319,9 +345,9 @@ public Collection<ServerPlugin> getPlugins() {
     return Collections.unmodifiableCollection(plugins.values());
   }
 
-  public ServerMetrics getServerMetrics() {
-    return serverMetrics;
-  }
+//  public ServerMetrics getServerMetrics() {
+//    return serverMetrics;
+//  }
 
   public ServerDatabase getDatabase(final String databaseName) {
     return getDatabase(databaseName, false, true);

server/src/main/java/com/arcadedb/server/event/FileServerEventLog.java

@@ -28,10 +28,16 @@
 import com.arcadedb.server.security.ServerSecurityException;
 import com.arcadedb.utility.FileUtils;
 
-import java.io.*;
-import java.text.*;
-import java.util.*;
-import java.util.logging.*;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.Date;
+import java.util.List;
+import java.util.logging.Level;
 
 public class FileServerEventLog implements ServerEventLog {
   private final static String           FILE_PREFIX = "server-event-log-";
@@ -108,15 +114,15 @@ public void reportEvent(final EVENT_TYPE eventType, final String component, fina
     if (newFileName == null)
       return;
 
-    final JSONObject json = new JSONObject();
     try {
+      final JSONObject json = new JSONObject();
       json.put("time", dateFormat.format(new Date()));
       json.put("type", eventType);
       json.put("component", component);
       json.put("db", databaseName);
       json.put("message", message);
 
-      FileUtils.appendContentToFile(newFileName, json.toString() + "\n");
+      FileUtils.appendContentToFile(newFileName, json + "\n");
     } catch (IOException e) {
       LogManager.instance().log(this, Level.SEVERE, "Error on writing into server event log file %s", e, newFileName);
     }

server/src/main/java/com/arcadedb/server/ha/HAServer.java

@@ -144,18 +144,6 @@ public ForwardedMessage() {
     }
   }
 
-//  private static class RemovedServerInfo {
-//    String serverName;
-//    long   joinedOn;
-//    long   leftOn;
-//
-//    public RemovedServerInfo(final String remoteServerName, final long joinedOn) {
-//      this.serverName = remoteServerName;
-//      this.joinedOn = joinedOn;
-//      this.leftOn = System.currentTimeMillis();
-//    }
-//  }
-
   public HAServer(final ArcadeDBServer server, final ContextConfiguration configuration) {
     if (!configuration.getValueAsBoolean(GlobalConfiguration.TX_WAL))
       throw new ConfigurationException("Cannot start HA service without using WAL. Please enable the TX_WAL setting.");

server/src/main/java/com/arcadedb/server/http/handler/DeleteDropUserHandler.java

@@ -20,6 +20,7 @@
 
 import com.arcadedb.server.http.HttpServer;
 import com.arcadedb.server.security.ServerSecurityUser;
+import io.micrometer.core.instrument.Metrics;
 import io.undertow.server.HttpServerExchange;
 
 import java.util.*;
@@ -48,7 +49,8 @@ public ExecutionResponse execute(final HttpServerExchange exchange, final Server
     if (userName == null)
       return new ExecutionResponse(400, "{ \"error\" : \"User name parameter is null\"}");
 
-    httpServer.getServer().getServerMetrics().meter("http.drop-user").hit();
+//    httpServer.getServer().getServerMetrics().meter("http.drop-user").hit();
+    Metrics.counter("http.drop-user").increment(); ;
 
     final boolean result = httpServer.getServer().getSecurity().dropUser(userName);
     if (!result)

server/src/main/java/com/arcadedb/server/http/handler/GetDatabasesHandler.java

@@ -22,6 +22,7 @@
 import com.arcadedb.serializer.json.JSONObject;
 import com.arcadedb.server.http.HttpServer;
 import com.arcadedb.server.security.ServerSecurityUser;
+import io.micrometer.core.instrument.Metrics;
 import io.undertow.server.HttpServerExchange;
 
 import java.util.*;
@@ -42,7 +43,8 @@ protected ExecutionResponse execute(final HttpServerExchange exchange, final Ser
 
     final JSONObject result = createResult(user, null).put("result", new JSONArray(installedDatabases));
 
-    httpServer.getServer().getServerMetrics().meter("http.list-databases").hit();
+//    httpServer.getServer().getServerMetrics().meter("http.list-databases").hit();
+    Metrics.counter("http.list-databases").increment(); ;
 
     return new ExecutionResponse(200, result.toString());
   }