feat: Add a rate limit feature

The fuzzy-tester library is designed to have the ability to run lots of
tests very quickly against a Pelias server. This is nice when the server
can handle it, but it often can't.

Some Pelias instances also have rate limits, and it is nice to respect
them.

This change adds a `-r` command line flag that takes an integer value
and uses it to set a per-second rate-limit on HTTP requests.

README.md

@@ -34,6 +34,7 @@ fuzzy-tester -t dev
 * `-o` Select an output mode. Valid values are `terminal` (default), `csv`, `json`, and `autocomplete` (see below)
 * `-q` Enable quiet mode. Only test failures (not successes) are printed
 * `-t` Select a test 'type' to filter on. This is a free-text value that can be added to each test, to allow running a subset of tests
+* `-r` Set a limit of the number of requests that can be sent per second when running tests. This is useful to avoid overloading a small Pelias server
 
 ## Test Case Files
 Test-cases are expected to live in `test_cases/`, and are split into test

bin/fuzzy-tester

@@ -27,7 +27,7 @@ var analyze_results = require( '../lib/analyze_results' );
   var urls = gather_test_urls(config, testSuites);
 
   // request all urls
-  request_urls(urls, function processResponses(responses) {
+  request_urls(config, urls, function processResponses(responses) {
     // responses is a simple object where the keys are urls, and the value is the
     // entire response from fetching that url
     // test cases can have many URLs (in autocomplete mode), and the same url

lib/processArguments.js

@@ -47,6 +47,11 @@ function setUpCommander() {
     endpointHelp,
     'prod'
   )
+  .option(
+    '-r, --rate <int>',
+    'The maxium number of requests per second (RPS) to allow',
+    100
+  )
   .option(
     '-o, --output <type>',
     outputGeneratorHelp,
@@ -113,6 +118,7 @@ function getConfig() {
       url: apiUrl,
       name: commander.endpoint
     },
+    rate: commander.rate,
     outputGenerator: outputGenerator,
     testType: commander.testType,
     testSuites: testSuites,

lib/request_urls.js

@@ -25,12 +25,13 @@ function shouldRetryRequest(res) {
   return false;
 }
 
-function request_urls(urls, callback) {
+function request_urls(config, urls, callback) {
   var total_length = urls.length;
   var responses = {};
   var agent = new http.Agent({keepAlive: true, maxSockets: 1});
   var intervalId;
-  var test_interval = new ExponentialBackoff(1, 5, 1, 20000);
+  const interval = 1000 / config.rate; // the number of miliseconds to delay between requests
+  var test_interval = new ExponentialBackoff(interval, 5, interval, 20000);
   var delay = test_interval.getBackoff();
 
   var getOneUrl = function (){