feat: load balance test location in WPT (DELO-4766)

app.js

@@ -32,7 +32,7 @@ wpt(app);
 
 // catch 404 and forward to error handler
 app.all('*', function (req, res) {
-  res.send('what???', 404);
+  res.status(404).send('what???');
 });
 
 app.use(function (req, res, next) {

config/default.js

@@ -35,7 +35,12 @@ const conf = {
       "lcp": process.env.LCP_PATH || "data.median.firstView.largestPaints",
       "lcpURL": process.env.LCP_URL_PATH || "data.median.firstView.LargestContentfulPaintImageURL"
     },
-    timeout: process.env.WTP_TIMEOUT || 30000
+    timeout: process.env.WTP_TIMEOUT || 30000,
+    "locationSelector": {
+      "cacheTtl": process.env.WTP_LS_CACHE_TTL || 10,
+      "updateTimeout": process.env.WTP_LS_UPDATE_TIMEOUT || 20,
+      "defaultLocation": process.env.WTP_LS_DEFAULT_LOCATION || "IAD_US_01"
+    }
   },
   "cloudinary": {
     "cloudName": process.env.CLOUDINARY_NAME,

package.json

@@ -9,15 +9,16 @@
   },
   "dependencies": {
     "async": "^3.2.6",
+    "async-mutex": "^0.5.0",
     "body-parser": "~1.20.3",
     "bytes": "^3.1.2",
     "cloudinary": "1.41.3",
     "config": "^3.3.12",
     "cookie-parser": "~1.4.7",
-    "debug": "~4.3.7",
-    "dotenv": "^16.4.5",
-    "express": "~4.21.1",
-    "got": "^14.4.3",
+    "debug": "~4.4.0",
+    "dotenv": "^16.4.7",
+    "express": "~4.21.2",
+    "got": "^14.4.5",
     "lodash": "^4.17.21",
     "rollbar": "^2.26.4",
     "valid-url": "^1.0.9"
@@ -26,8 +27,8 @@
     "chai": "^4.5.0",
     "chai-http": "^4.4.0",
     "husky": "^8.0.3",
-    "mocha": "^10.7.3",
-    "nock": "^13.5.5",
+    "mocha": "^10.8.2",
+    "nock": "^13.5.6",
     "patch-package": "^8.0.0",
     "sinon": "^16.1.3",
     "sinon-chai": "^3.7.0"

routes/wpt.js

@@ -2,6 +2,7 @@
 const express = require('express');
 const validUrl = require('valid-url');
 const apiCaller = require('../wtp/apiCaller');
+const locationSelector = require("../wtp/locationSelector");
 const logger = require('../logger').logger;
 const {LOG_LEVEL_INFO, LOG_LEVEL_WARNING, LOG_LEVEL_ERROR, LOG_LEVEL_CRITICAL, LOG_LEVEL_DEBUG} = require('../logger');
 const path = require('path');
@@ -81,7 +82,19 @@ const wtp = (app) => {
   app.get('/version', (req, res) => {
     const packageJson = require('../package.json');
     res.json({version: packageJson.version});
+  });
+
+  app.get('/locations', async (req, res) => {
+    let locations = locationSelector.cachedAllLocations;
+    res.json({locations});
   })
+
+  app.get('/locations/current', async (req, res) => {
+    let location = locationSelector.location;
+    let lastUpdated = new Date(locationSelector.lastUpdated || 0).toISOString();
+    res.json({location, lastUpdated});
+  })
+
 };
 
 module.exports = wtp;

wtp/apiCaller.js

@@ -4,7 +4,6 @@
 
 "use strict";
 
-
 const path = require('path');
 const got = (...args) => import('got').then(({default: got}) => got(...args));
 const config = require('config');
@@ -16,7 +15,8 @@ const {truncateString} = require('../util/strings');
 const RESULTS_URL = 'https://www.webpagetest.org/jsonResult.php';
 const RUN_TEST_URL = 'http://www.webpagetest.org/runtest.php';
 const GET_TEST_STATUS = 'http://www.webpagetest.org/testStatus.php';
-
+const locationSelector = require('./locationSelector');
+const apiKeys = require('./apiKey');
 
 const getTestResults = async (testId, quality, cb) => {
   let options = {
@@ -58,11 +58,8 @@ const getTestResults = async (testId, quality, cb) => {
   }
 };
 
-
 const runWtpTest = async (url, mobile, cb) => {
   //logger.info('Running new test ' + url);
-  const apiKeys = config.get('wtp.apiKey').split(',');
-  const apiKey = apiKeys[Math.floor(Math.random() * apiKeys.length)];
   let options = {
     method: "POST",
     url: RUN_TEST_URL,
@@ -72,12 +69,12 @@ const runWtpTest = async (url, mobile, cb) => {
             width: config.get('wtp.viewportWidth'),
             height: config.get('wtp.viewportHeight'),
             custom: config.get('wtp.imageScript'),
-            location: 'Dulles:Chrome.Native', // Native means no speed shaping in browser, full speed ahead
-            mobile: (mobile) ? 1 : 0,
+            location: await locationSelector.getLocation() + ':Chrome.Native', // Native means no speed shaping in browser, full speed ahead
+      mobile: (mobile) ? 1 : 0,
             fvonly: 1, // first view only
             timeline: 1 // workaround for WPT sometimes hanging on getComputedStyle()
           },
-    headers: { 'User-Agent': 'WebSpeedTest', 'X-WPT-API-KEY': apiKey },
+    headers: { 'User-Agent': 'WebSpeedTest', 'X-WPT-API-KEY': apiKeys.getRandom() },
     throwHttpErrors: false
   };
   let response;

wtp/apiKey.js

@@ -0,0 +1,13 @@
+"use strict";
+
+const config = require('config');
+
+function getRandom() {
+  const apiKeys = config.get('wtp.apiKey').split(',');
+  const apiKey = apiKeys[Math.floor(Math.random() * apiKeys.length)];
+  return apiKey;
+}
+
+module.exports = {
+  getRandom: getRandom
+};

wtp/locationSelector.js

@@ -0,0 +1,136 @@
+const got = (...args) => import('got').then(({default: got}) => got(...args));
+const config = require('config');
+const {Mutex, withTimeout, E_TIMEOUT} = require('async-mutex');
+const apiKeys = require('./apiKey');
+const path = require("path");
+const logger = require('../logger').logger;
+
+const GET_LOCATIONS = 'http://www.webpagetest.org/getLocations.php?f=json';
+
+class LocationSelector {
+    constructor() {
+        if (!LocationSelector.instance) {
+            this.cachedAllLocations = [];
+            this.location = config.get('wtp.locationSelector.defaultLocation');
+            this.lastUpdated = null;
+            this.mutex = withTimeout(new Mutex(), config.get('wtp.locationSelector.updateTimeout') * 1000);
+            LocationSelector.instance = this;
+        }
+        return LocationSelector.instance;
+    }
+
+    isExpired() {
+        const now = Date.now();
+        return (!this.lastUpdated || (now - this.lastUpdated) > config.get('wtp.locationSelector.cacheTtl') * 1000);
+    }
+
+    async fetchLocations() {
+        let options = {
+            method: "GET",
+            url: GET_LOCATIONS,
+            headers: {'User-Agent': 'WebSpeedTest', 'X-WPT-API-KEY': apiKeys.getRandom()},
+        };
+        let response;
+        let rollBarMsg = {};
+        try {
+            response = await got(options);
+            const {statusCode, body} = response;
+            let bodyJson = JSON.parse(body);
+            rollBarMsg = {thirdPartyErrorCode: response.statusCode, file: path.basename((__filename))};
+            if (statusCode !== 200) {
+                rollBarMsg.thirdPartyErrorBody = bodyJson;
+                logger.error('WPT returned bad status', rollBarMsg);
+                return;
+            }
+            return bodyJson.data;
+        } catch (error) {
+            logger.critical('Error fetching WTP locations', JSON.stringify(error, Object.getOwnPropertyNames(error)));
+        }
+    };
+
+    getLocationScore(loc) {
+        // no instances running, hopefully they will be spin up for our request?
+        if (this.getLocationCapacity(loc) == 0) {
+            return 1;
+        }
+
+        let metrics = loc.PendingTests;
+        return (metrics.HighPriority + metrics.Testing) / (metrics.Idle + metrics.Testing)
+    }
+
+    getLocationCapacity(loc) {
+        return loc.PendingTests.Idle + loc.PendingTests.Testing;
+    }
+
+    getBestLocationId(locations) {
+        let selected = locations.reduce((acc, cur) => {
+            // if nothing to compare to, use current value
+            if (!acc) {
+                return cur;
+            }
+
+            // if acc less loaded
+            if (acc.score < cur.score) {
+                return acc;
+            }
+
+            // if cur less loaded
+            if (acc.score > cur.score) {
+                return cur;
+            }
+
+            // if same load on acc and cur
+            // then choose the one with bigger capacity (Idle + Testing)
+            return this.getLocationCapacity(acc) > this.getLocationCapacity(cur) ? acc : cur;
+        });
+
+        return selected.location;
+    }
+
+    async updateLocations() {
+        const newLocations = await this.fetchLocations();
+        if (!newLocations) {
+            return
+        }
+
+        const filtered = Object.keys(newLocations)
+            .filter(key => key.includes("_US_"))    // we only want US-based instances
+            .reduce((arr, key) => {
+                return [...arr, newLocations[key]];
+            }, []);
+
+        if (filtered.length === 0) {
+            return
+        }
+
+        // enrich locations with our internal score
+        filtered.forEach((loc) => {
+            loc.score = this.getLocationScore(loc);
+        });
+
+        this.location = this.getBestLocationId(filtered);
+        this.cachedAllLocations = filtered;
+        this.lastUpdated = Date.now();
+    };
+
+    async getLocation() {
+        if (this.isExpired()) {
+            try {
+                await this.mutex.runExclusive(async () => {
+                    if (this.isExpired()) {
+                        await this.updateLocations();
+                    }
+                });
+            } catch (e) {
+                if (e === E_TIMEOUT) {
+                    logger.error('Locations update is taking too long', e);
+                }
+            }
+        }
+
+        return this.location;
+    }
+}
+
+const instance = new LocationSelector();
+module.exports = instance;