feat(server): unified metrics endpoint

src/shadowbox/infrastructure/prometheus_scraper.ts

@@ -37,7 +37,11 @@ interface QueryResult {
   error: string;
 }
 
-export class PrometheusClient {
+export interface PrometheusClient {
+  query(query: string): Promise<QueryResultData>;
+}
+
+export class ApiPrometheusClient implements PrometheusClient {
   constructor(private address: string) {}
 
   query(query: string): Promise<QueryResultData> {
@@ -101,7 +105,7 @@ async function spawnPrometheusSubprocess(
   prometheusEndpoint: string
 ): Promise<child_process.ChildProcess> {
   logging.info('======== Starting Prometheus ========');
-  logging.info(`${binaryFilename} ${processArgs.map(a => `"${a}"`).join(' ')}`);
+  logging.info(`${binaryFilename} ${processArgs.map((a) => `"${a}"`).join(' ')}`);
   const runProcess = child_process.spawn(binaryFilename, processArgs);
   runProcess.on('error', (error) => {
     logging.error(`Error spawning Prometheus: ${error}`);

src/shadowbox/server/api.yml

@@ -127,7 +127,60 @@ paths:
       responses:
         '204':
           description: Access key limit deleted successfully.
-
+  /server/metrics:
+    get:
+      tags: Server
+      parameters:
+        - in: query
+          name: since
+          description: the range of time to return data for
+          schema:
+            type: string
+      responses:
+        '200':
+          description: Display server metric information
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  server:
+                    type: array
+                    items:
+                      type: object
+                      properties:
+                        location:
+                          type: string
+                        asn:
+                          type: number
+                        asOrg:
+                          type: string
+                        tunnelTime:
+                          type: object
+                          properties:
+                            seconds: number
+                        dataTransferred:
+                          type: object
+                          properties:
+                            bytes: number
+                  accessKeys:
+                    type: array
+                    items:
+                      type: object
+                      properties:
+                        accessKeyId:
+                          type: string
+                        tunnelTime:
+                          type: object
+                          properties:
+                            seconds: number
+                        dataTransferred:
+                          type: object
+                          properties:
+                            bytes: number
+              examples:
+                '0':
+                  value: '{"server":[{"location":"US","asn":null,"asOrg":null,"tunnelTime":{"seconds":100},"dataTransferred":{"bytes":100}}],"accessKeys":[{"accessKeyId":"0","tunnelTime":{"seconds":100},"dataTransferred":{"bytes":100}}]}'
   /name:
     put:
       description: Renames the server

src/shadowbox/server/main.ts

@@ -24,7 +24,7 @@ import {RealClock} from '../infrastructure/clock';
 import {PortProvider} from '../infrastructure/get_port';
 import * as json_config from '../infrastructure/json_config';
 import * as logging from '../infrastructure/logging';
-import {PrometheusClient, startPrometheus} from '../infrastructure/prometheus_scraper';
+import {ApiPrometheusClient, startPrometheus} from '../infrastructure/prometheus_scraper';
 import {RolloutTracker} from '../infrastructure/rollout';
 import * as version from './version';
 
@@ -197,7 +197,7 @@ async function main() {
     prometheusEndpoint
   );
 
-  const prometheusClient = new PrometheusClient(prometheusEndpoint);
+  const prometheusClient = new ApiPrometheusClient(prometheusEndpoint);
   if (!serverConfig.data().portForNewAccessKeys) {
     serverConfig.data().portForNewAccessKeys = await portProvider.reserveNewPort();
     serverConfig.write();

src/shadowbox/server/manager_metrics.spec.ts

@@ -13,9 +13,105 @@
 // limitations under the License.
 
 import {PrometheusManagerMetrics} from './manager_metrics';
+import {PrometheusClient, QueryResultData} from '../infrastructure/prometheus_scraper';
 import {FakePrometheusClient} from './mocks/mocks';
 
+export class QueryMapPrometheusClient implements PrometheusClient {
+  constructor(private queryMap: {[query: string]: QueryResultData}) {}
+
+  async query(_query: string): Promise<QueryResultData> {
+    return this.queryMap[_query];
+  }
+}
+
 describe('PrometheusManagerMetrics', () => {
+  it('getServerMetrics', async (done) => {
+    const managerMetrics = new PrometheusManagerMetrics(
+      new QueryMapPrometheusClient({
+        'sum(increase(shadowsocks_data_bytes_per_location{dir=~"c<p|p>t"}[0h])) by (location, asn, asorg)':
+          {
+            resultType: 'vector',
+            result: [
+              {
+                metric: {
+                  location: 'US',
+                  asn: '49490',
+                  asorg: 'null',
+                },
+                value: [null, '1000'],
+              },
+            ],
+          },
+        'sum(increase(shadowsocks_tunnel_time_seconds_per_location[0h])) by (location, asn, asorg)':
+          {
+            resultType: 'vector',
+            result: [
+              {
+                metric: {
+                  location: 'US',
+                  asn: '49490',
+                  asorg: 'null',
+                },
+                value: [null, '1000'],
+              },
+            ],
+          },
+        'sum(increase(shadowsocks_data_bytes{dir=~"c<p|p>t"}[0h])) by (access_key)': {
+          resultType: 'vector',
+          result: [
+            {
+              metric: {
+                access_key: '0',
+              },
+              value: [null, '1000'],
+            },
+          ],
+        },
+        'sum(increase(shadowsocks_tunnel_time_seconds[0h])) by (access_key)': {
+          resultType: 'vector',
+          result: [
+            {
+              metric: {
+                access_key: '0',
+              },
+              value: [null, '1000'],
+            },
+          ],
+        },
+      })
+    );
+
+    const serverMetrics = await managerMetrics.getServerMetrics({hours: 0});
+
+    expect(JSON.stringify(serverMetrics, null, 2)).toEqual(`{
+  "server": [
+    {
+      "location": "US",
+      "asn": 49490,
+      "asOrg": "null",
+      "dataTransferred": {
+        "bytes": 1000
+      },
+      "tunnelTime": {
+        "seconds": 1000
+      }
+    }
+  ],
+  "accessKeys": [
+    {
+      "accessKeyId": 0,
+      "dataTransferred": {
+        "bytes": 1000
+      },
+      "tunnelTime": {
+        "seconds": 1000
+      }
+    }
+  ]
+}`);
+    done();
+  });
+
   it('getOutboundByteTransfer', async (done) => {
     const managerMetrics = new PrometheusManagerMetrics(
       new FakePrometheusClient({'access-key-1': 1000, 'access-key-2': 10000})

src/shadowbox/server/manager_metrics.ts

@@ -15,8 +15,40 @@
 import {PrometheusClient} from '../infrastructure/prometheus_scraper';
 import {DataUsageByUser, DataUsageTimeframe} from '../model/metrics';
 
+interface ServerMetricsTimeframe {
+  hours: number;
+}
+
+interface Duration {
+  seconds: number;
+}
+
+interface Data {
+  bytes: number;
+}
+
+interface ServerMetricsServerEntry {
+  location: string;
+  asn: number;
+  asOrg: string;
+  tunnelTime: Duration;
+  dataTransferred: Data;
+}
+
+interface ServerMetricsAccessKeyEntry {
+  accessKeyId: number;
+  tunnelTime: Duration;
+  dataTransferred: Data;
+}
+
+interface ServerMetrics {
+  server: ServerMetricsServerEntry[];
+  accessKeys: ServerMetricsAccessKeyEntry[];
+}
+
 export interface ManagerMetrics {
   getOutboundByteTransfer(timeframe: DataUsageTimeframe): Promise<DataUsageByUser>;
+  getServerMetrics(timeframe: ServerMetricsTimeframe): Promise<ServerMetrics>;
 }
 
 // Reads manager metrics from a Prometheus instance.
@@ -40,4 +72,71 @@ export class PrometheusManagerMetrics implements ManagerMetrics {
     }
     return {bytesTransferredByUserId: usage};
   }
+
+  async getServerMetrics({hours}: ServerMetricsTimeframe): Promise<ServerMetrics> {
+    const dataTransferredByLocation = await this.prometheusClient.query(
+      `sum(increase(shadowsocks_data_bytes_per_location{dir=~"c<p|p>t"}[${hours}h])) by (location, asn, asorg)`
+    );
+    const tunnelTimeByLocation = await this.prometheusClient.query(
+      `sum(increase(shadowsocks_tunnel_time_seconds_per_location[${hours}h])) by (location, asn, asorg)`
+    );
+    const dataTransferredByAccessKey = await this.prometheusClient.query(
+      `sum(increase(shadowsocks_data_bytes{dir=~"c<p|p>t"}[${hours}h])) by (access_key)`
+    );
+    const tunnelTimeByAccessKey = await this.prometheusClient.query(
+      `sum(increase(shadowsocks_tunnel_time_seconds[${hours}h])) by (access_key)`
+    );
+
+    const server = [];
+    const tunnelTimeByLocationMap = tunnelTimeByLocation.result.reduce((map, entry) => {
+      map.set(
+        `${entry.metric['location']},${entry.metric['asn']},${entry.metric['asorg']}`,
+        parseFloat(entry.value[1])
+      );
+
+      return map;
+    }, new Map());
+    for (const entry of dataTransferredByLocation.result) {
+      const result: ServerMetricsServerEntry = {
+        location: entry.metric['location'],
+        asn: parseInt(entry.metric['asn']),
+        asOrg: entry.metric['asorg'],
+        dataTransferred: {
+          bytes: parseFloat(entry.value[1]),
+        },
+        tunnelTime: {
+          seconds: tunnelTimeByLocationMap.get(
+            `${entry.metric['location']},${entry.metric['asn']},${entry.metric['asorg']}`
+          ),
+        },
+      };
+
+      server.push(result);
+    }
+
+    const accessKeys = [];
+    const tunnelTimeByAccessKeyMap = tunnelTimeByAccessKey.result.reduce((map, entry) => {
+      map.set(entry.metric['access_key'], parseFloat(entry.value[1]));
+
+      return map;
+    }, new Map());
+    for (const entry of dataTransferredByAccessKey.result) {
+      const result: ServerMetricsAccessKeyEntry = {
+        accessKeyId: parseInt(entry.metric['access_key']),
+        dataTransferred: {
+          bytes: parseFloat(entry.value[1]),
+        },
+        tunnelTime: {
+          seconds: tunnelTimeByAccessKeyMap.get(entry.metric['access_key']),
+        },
+      };
+
+      accessKeys.push(result);
+    }
+
+    return {
+      server,
+      accessKeys,
+    };
+  }
 }