-
Notifications
You must be signed in to change notification settings - Fork 4
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: load balance test location in WPT (DELO-4766) (#95)
This feature introduces load-balancing across multiple WPT locations. The goal is to always choose least loaded region. Please note that US-based regions are taken into consideration only. The following configuration options are introduced: * `WTP_LS_CACHE_TTL` (defaults: `10` sec) - how long to cache locations and selected location; `0` means we check best location on every test run * `WTP_LS_DEFAULT_LOCATION` (defaults: `IAD_US_01` - which location to use in case there is a problem with API and there is nothing selected yet * `WTP_LS_UPDATE_TIMEOUT` - (defaults: `20` sec) timeout for updating locations Also, for better insight there are two new endpoints (both responses served from local cache): * `/locations` - prints out all locations we consider along with their metrics * `/locations/current` - identifier of the location we currently use as the best one
- Loading branch information
1 parent
01447ed
commit 1b5be14
Showing
8 changed files
with
299 additions
and
105 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,13 @@ | ||
| "use strict"; | ||
|
|
||
| const config = require('config'); | ||
|
|
||
| function getRandom() { | ||
| const apiKeys = config.get('wtp.apiKey').split(','); | ||
| const apiKey = apiKeys[Math.floor(Math.random() * apiKeys.length)]; | ||
| return apiKey; | ||
| } | ||
|
|
||
| module.exports = { | ||
| getRandom: getRandom | ||
| }; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,136 @@ | ||
| const got = (...args) => import('got').then(({default: got}) => got(...args)); | ||
| const config = require('config'); | ||
| const {Mutex, withTimeout, E_TIMEOUT} = require('async-mutex'); | ||
| const apiKeys = require('./apiKey'); | ||
| const path = require("path"); | ||
| const logger = require('../logger').logger; | ||
|
|
||
| const GET_LOCATIONS = 'http://www.webpagetest.org/getLocations.php?f=json'; | ||
|
|
||
| class LocationSelector { | ||
| constructor() { | ||
| if (!LocationSelector.instance) { | ||
| this.cachedAllLocations = []; | ||
| this.location = config.get('wtp.locationSelector.defaultLocation'); | ||
| this.lastUpdated = null; | ||
| this.mutex = withTimeout(new Mutex(), config.get('wtp.locationSelector.updateTimeout') * 1000); | ||
| LocationSelector.instance = this; | ||
| } | ||
| return LocationSelector.instance; | ||
| } | ||
|
|
||
| isExpired() { | ||
| const now = Date.now(); | ||
| return (!this.lastUpdated || (now - this.lastUpdated) > config.get('wtp.locationSelector.cacheTtl') * 1000); | ||
| } | ||
|
|
||
| async fetchLocations() { | ||
| let options = { | ||
| method: "GET", | ||
| url: GET_LOCATIONS, | ||
| headers: {'User-Agent': 'WebSpeedTest', 'X-WPT-API-KEY': apiKeys.getRandom()}, | ||
| }; | ||
| let response; | ||
| let rollBarMsg = {}; | ||
| try { | ||
| response = await got(options); | ||
| const {statusCode, body} = response; | ||
| let bodyJson = JSON.parse(body); | ||
| rollBarMsg = {thirdPartyErrorCode: response.statusCode, file: path.basename((__filename))}; | ||
| if (statusCode !== 200) { | ||
| rollBarMsg.thirdPartyErrorBody = bodyJson; | ||
| logger.error('WPT returned bad status', rollBarMsg); | ||
| return; | ||
| } | ||
| return bodyJson.data; | ||
| } catch (error) { | ||
| logger.critical('Error fetching WTP locations', JSON.stringify(error, Object.getOwnPropertyNames(error))); | ||
| } | ||
| }; | ||
|
|
||
| getLocationScore(loc) { | ||
| // no instances running, hopefully they will be spin up for our request? | ||
| if (this.getLocationCapacity(loc) == 0) { | ||
| return 1; | ||
| } | ||
|
|
||
| let metrics = loc.PendingTests; | ||
| return (metrics.HighPriority + metrics.Testing) / (metrics.Idle + metrics.Testing) | ||
| } | ||
|
|
||
| getLocationCapacity(loc) { | ||
| return loc.PendingTests.Idle + loc.PendingTests.Testing; | ||
| } | ||
|
|
||
| getBestLocationId(locations) { | ||
| let selected = locations.reduce((acc, cur) => { | ||
| // if nothing to compare to, use current value | ||
| if (!acc) { | ||
| return cur; | ||
| } | ||
|
|
||
| // if acc less loaded | ||
| if (acc.score < cur.score) { | ||
| return acc; | ||
| } | ||
|
|
||
| // if cur less loaded | ||
| if (acc.score > cur.score) { | ||
| return cur; | ||
| } | ||
|
|
||
| // if same load on acc and cur | ||
| // then choose the one with bigger capacity (Idle + Testing) | ||
| return this.getLocationCapacity(acc) > this.getLocationCapacity(cur) ? acc : cur; | ||
| }); | ||
|
|
||
| return selected.location; | ||
| } | ||
|
|
||
| async updateLocations() { | ||
| const newLocations = await this.fetchLocations(); | ||
| if (!newLocations) { | ||
| return | ||
| } | ||
|
|
||
| const filtered = Object.keys(newLocations) | ||
| .filter(key => key.includes("_US_")) // we only want US-based instances | ||
| .reduce((arr, key) => { | ||
| return [...arr, newLocations[key]]; | ||
| }, []); | ||
|
|
||
| if (filtered.length === 0) { | ||
| return | ||
| } | ||
|
|
||
| // enrich locations with our internal score | ||
| filtered.forEach((loc) => { | ||
| loc.score = this.getLocationScore(loc); | ||
| }); | ||
|
|
||
| this.location = this.getBestLocationId(filtered); | ||
| this.cachedAllLocations = filtered; | ||
| this.lastUpdated = Date.now(); | ||
| }; | ||
|
|
||
| async getLocation() { | ||
| if (this.isExpired()) { | ||
| try { | ||
| await this.mutex.runExclusive(async () => { | ||
| if (this.isExpired()) { | ||
| await this.updateLocations(); | ||
| } | ||
| }); | ||
| } catch (e) { | ||
| if (e === E_TIMEOUT) { | ||
| logger.error('Locations update is taking too long', e); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| return this.location; | ||
| } | ||
| } | ||
|
|
||
| const instance = new LocationSelector(); | ||
| module.exports = instance; |
Oops, something went wrong.