From 6d9849dc8e716ea213eeede36657cc86e79d3297 Mon Sep 17 00:00:00 2001
From: Yingge He <157551214+yinggeh@users.noreply.github.com>
Date: Fri, 17 May 2024 02:02:35 +0800
Subject: [PATCH] Multiple Model Configurations (#7185)

Add `--mode-config-name` option when starting Triton server. Allow users to create multiple configurations and select a custom configuration other than the default `model/config.pbtxt`.
---
 docs/user_guide/model_configuration.md |  80 +++++++++++++-
 docs/user_guide/model_repository.md    |  15 ++-
 qa/L0_custom_model_config/test.sh      | 145 +++++++++++++++++++++++++
 qa/L0_lifecycle/lifecycle_test.py      |  90 ++++++++++++++-
 qa/L0_lifecycle/test.sh                |  66 +++++++++++
 src/command_line_parser.cc             |  21 +++-
 src/command_line_parser.h              |   3 +
 7 files changed, 414 insertions(+), 6 deletions(-)
 create mode 100755 qa/L0_custom_model_config/test.sh

diff --git a/docs/user_guide/model_configuration.md b/docs/user_guide/model_configuration.md
index 023fcf259b..e8165081c1 100644
--- a/docs/user_guide/model_configuration.md
+++ b/docs/user_guide/model_configuration.md
@@ -1,5 +1,5 @@
 <!--
-# Copyright 2018-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright 2018-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -321,6 +321,84 @@ configuration](#minimal-model-configuration). You must still provide
 the optional portions of the model configuration by editing the
 config.pbtxt file.
 
+## Custom Model Configuration
+
+Sometimes when multiple devices running Triton instances that share one
+model repository, it is necessary to have models configured differently
+on each platform in order to achieve the best performance. Triton allows
+users to pick the custom model configuration name by setting `--model-config-name` option.
+
+For example, when running `./tritonserver --model-repository=</path/to/model/repository> --model-config-name=h100`,
+the server will search the custom configuration file `h100.pbtxt` under
+`/path/to/model/repository/<model-name>/configs` directory for each model
+that is loaded. If `h100.pbtxt` exists, it will be used as the configuration
+for this model. Otherwise, the default configuration `/path/to/model/repository/<model-name>/config.pbtxt`
+or [auto-generated model configuration](#auto-generated-model-configuration)
+will be selected based on the settings.
+
+Custom model configuration also works with `Explicit` and `Poll` model
+control modes. Users may delete or add new custom configurations and the
+server will pick the configuration file for each loaded model dynamically.
+
+Note: custom model configuration name should not contain any space character.
+
+Example 1: --model-config-name=h100
+```
+.
+└── model_repository/
+    ├── model_a/
+    │   ├── configs/
+    │   │   ├── v100.pbtxt
+    │   │   └── **h100.pbtxt**
+    │   └── config.pbtxt
+    ├── model_b/
+    │   ├── configs/
+    │   │   └── v100.pbtxt
+    │   └── **config.pbtxt**
+    └── model_c/
+        ├── configs/
+        │   └── config.pbtxt
+        └── **config.pbtxt**
+```
+
+Example 2: --model-config-name=config
+```
+.
+└── model_repository/
+    ├── model_a/
+    │   ├── configs/
+    │   │   ├── v100.pbtxt
+    │   │   └── h100.pbtxt
+    │   └── **config.pbtxt**
+    ├── model_b/
+    │   ├── configs/
+    │   │   └── v100.pbtxt
+    │   └── **config.pbtxt**
+    └── model_c/
+        ├── configs/
+        │   └── **config.pbtxt**
+        └── config.pbtxt
+```
+
+Example 3: --model-config-name not set
+```
+.
+└── model_repository/
+    ├── model_a/
+    │   ├── configs/
+    │   │   ├── v100.pbtxt
+    │   │   └── h100.pbtxt
+    │   └── **config.pbtxt**
+    ├── model_b/
+    │   ├── configs/
+    │   │   └── v100.pbtxt
+    │   └── **config.pbtxt**
+    └── model_c/
+        ├── configs/
+        │   └── config.pbtxt
+        └── **config.pbtxt**
+```
+
 ### Default Max Batch Size and Dynamic Batcher
 
 When a model is using the auto-complete feature, a default maximum
diff --git a/docs/user_guide/model_repository.md b/docs/user_guide/model_repository.md
index 494efba8e7..952334dadd 100644
--- a/docs/user_guide/model_repository.md
+++ b/docs/user_guide/model_repository.md
@@ -1,5 +1,5 @@
 <!--
-# Copyright 2018-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright 2018-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -57,6 +57,8 @@ The corresponding repository layout must be:
     <model-name>/
       [config.pbtxt]
       [<output-labels-file> ...]
+      [configs]/
+        [<custom-config-file> ...]
       <version>/
         <model-definition-file>
       <version>/
@@ -65,6 +67,8 @@ The corresponding repository layout must be:
     <model-name>/
       [config.pbtxt]
       [<output-labels-file> ...]
+      [configs]/
+        [<custom-config-file> ...]
       <version>/
         <model-definition-file>
       <version>/
@@ -83,10 +87,15 @@ config.pbtxt is required while for others it is optional. See
 Configuration](model_configuration.md#auto-generated-model-configuration)
 for more information.
 
+Each <model-name> directory may include an optional sub-directory configs.
+Within the configs directory there must be zero or more <custom-config-file>
+with .pbtxt file extension. For more information about how the custom model
+configuration is handled by Triton see [Custom Model Configuration](https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_configuration.md#custom-model-configuration).
+
 Each <model-name> directory must have at least one numeric
-sub-directory representing a version of the model.  For more
+sub-directory representing a version of the model. For more
 information about how the model versions are handled by Triton see
-[Model Versions](#model-versions).  Each model is executed by a
+[Model Versions](#model-versions). Each model is executed by a
 specific
 [backend](https://github.com/triton-inference-server/backend/blob/main/README.md).
 Within each version sub-directory there must be the files required by
diff --git a/qa/L0_custom_model_config/test.sh b/qa/L0_custom_model_config/test.sh
new file mode 100755
index 0000000000..d839cacbd5
--- /dev/null
+++ b/qa/L0_custom_model_config/test.sh
@@ -0,0 +1,145 @@
+#!/bin/bash
+# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+REPO_VERSION=${NVIDIA_TRITON_SERVER_VERSION}
+if [ "$#" -ge 1 ]; then
+    REPO_VERSION=$1
+fi
+if [ -z "$REPO_VERSION" ]; then
+    echo -e "Repository version must be specified"
+    echo -e "\n***\n*** Test Failed\n***"
+    exit 1
+fi
+if [ ! -z "$TEST_REPO_ARCH" ]; then
+    REPO_VERSION=${REPO_VERSION}_${TEST_REPO_ARCH}
+fi
+
+export CUDA_VISIBLE_DEVICES=0
+
+DATADIR="/data/inferenceserver/${REPO_VERSION}"
+CLIENT_LOG="./client.log"
+SERVER_LOG="./inference_server.log"
+
+SERVER=/opt/tritonserver/bin/tritonserver
+source ../common/util.sh
+
+RET=0
+rm -fr *.log
+
+rm -fr models && mkdir models
+cp -r $DATADIR/qa_model_repository/savedmodel_nobatch_float32_float32_float32 models/.
+mkdir models/savedmodel_nobatch_float32_float32_float32/configs
+
+test_custom_config()
+{
+    VERSION=$@
+
+    run_server
+    if [ "$SERVER_PID" == "0" ]; then
+        echo -e "\n***\n*** Failed to start $SERVER\n***"
+        cat $SERVER_LOG
+        exit 1
+    fi
+
+    set +e
+    code=`curl -s -w %{http_code} -o ./curl.out localhost:8000/v2/models/savedmodel_nobatch_float32_float32_float32/config`
+    set -e
+    if [ "$code" != "200" ]; then
+        cat $out.out
+        echo -e "\n***\n*** Test Failed to GET model configuration\n***"
+        RET=1
+    fi
+
+    matches=`grep -o "\"version_policy\":{\"specific\":{\"versions\":\[$VERSION\]}}" curl.out | wc -l`
+    if [ $matches -ne 1 ]; then
+        cat curl.out
+        echo -e "\n***\n*** Expected 1 version_policy:specific:versions, got $matches\n***"
+        RET=1
+    fi
+
+    kill $SERVER_PID
+    wait $SERVER_PID
+}
+
+# Prepare the file structure
+VERSION_DEFAULT="1,3"
+VERSION_H100="1"
+VERSION_V100="2"
+VERSION_CUSTOM="3"
+
+# Distinguish configs with different model versions
+(cd models/savedmodel_nobatch_float32_float32_float32 && \
+     sed -i "s/^version_policy:.*/version_policy: { specific: { versions: [$VERSION_DEFAULT] }}/" config.pbtxt)
+(cd models/savedmodel_nobatch_float32_float32_float32 && \
+     cp config.pbtxt configs/h100.pbtxt && \
+     sed -i "s/^version_policy:.*/version_policy: { specific: { versions: [$VERSION_H100] }}/" configs/h100.pbtxt)
+(cd models/savedmodel_nobatch_float32_float32_float32 && \
+     cp config.pbtxt configs/v100.pbtxt && \
+     sed -i "s/^version_policy:.*/version_policy: { specific: { versions: [$VERSION_V100] }}/" configs/v100.pbtxt)
+(cd models/savedmodel_nobatch_float32_float32_float32 && \
+     cp config.pbtxt configs/config.pbtxt && \
+     sed -i "s/^version_policy:.*/version_policy: { specific: { versions: [$VERSION_CUSTOM] }}/" configs/config.pbtxt)
+
+# Test default model config
+SERVER_ARGS="--model-repository=`pwd`/models"
+test_custom_config $VERSION_DEFAULT
+
+# Test model-config-name=h100
+SERVER_ARGS="--model-repository=`pwd`/models --model-config-name=h100"
+test_custom_config $VERSION_H100
+
+# Test model-config-name=v100
+SERVER_ARGS="--model-repository=`pwd`/models --model-config-name=v100"
+test_custom_config $VERSION_V100
+
+# Test model-config-name=config
+SERVER_ARGS="--model-repository=`pwd`/models --model-config-name=config"
+test_custom_config $VERSION_CUSTOM
+
+# Test model-config-name=h200. Expect fall back to default config since h200 config does not exist.
+SERVER_ARGS="--model-repository=`pwd`/models --model-config-name=h200"
+test_custom_config $VERSION_DEFAULT
+
+# Test model-config-name=
+SERVER_ARGS="--model-repository=`pwd`/models --model-config-name="
+run_server
+if [ "$SERVER_PID" != "0" ]; then
+    echo -e "\n***\n*** Failed: $SERVER started successfully when it was expected to fail\n***"
+    cat $SERVER_LOG
+    RET=1
+
+    kill $SERVER_PID
+    wait $SERVER_PID
+fi
+
+if [ $RET -eq 0 ]; then
+    echo -e "\n***\n*** Test Passed\n***"
+else
+    echo -e "\n***\n*** Test Failed\n***"
+fi
+
+exit $RET
diff --git a/qa/L0_lifecycle/lifecycle_test.py b/qa/L0_lifecycle/lifecycle_test.py
index 4797ab7465..a2bfc067bc 100755
--- a/qa/L0_lifecycle/lifecycle_test.py
+++ b/qa/L0_lifecycle/lifecycle_test.py
@@ -1259,7 +1259,7 @@ def test_dynamic_file_delete(self):
                     model_version=1,
                 )
                 self.assertTrue(
-                    False, "expected error for unavailable model " + graphdef_name
+                    False, "expected error for unavailable model " + model_name
                 )
             except Exception as ex:
                 self.assertIn("Request for unknown model", ex.message())
@@ -3405,6 +3405,94 @@ def test_shutdown_with_live_connection(self):
             "exit timeout countdown restart detected",
         )
 
+    def test_add_custom_config(self):
+        models_base = ("savedmodel",)
+        models = list()
+        for m in models_base:
+            models.append(tu.get_model_name(m, np.float32, np.float32, np.float32))
+
+        # Make sure savedmodel and plan are in the status
+        for model_name in models:
+            try:
+                for triton_client in (
+                    httpclient.InferenceServerClient("localhost:8000", verbose=True),
+                    grpcclient.InferenceServerClient("localhost:8001", verbose=True),
+                ):
+                    self.assertTrue(triton_client.is_server_live())
+                    self.assertTrue(triton_client.is_server_ready())
+                    self.assertTrue(triton_client.is_model_ready(model_name, "1"))
+                    self.assertFalse(triton_client.is_model_ready(model_name, "2"))
+                    self.assertTrue(triton_client.is_model_ready(model_name, "3"))
+            except Exception as ex:
+                self.assertTrue(False, "unexpected error {}".format(ex))
+
+        # Add custom model configuration, which cause model to be
+        # re-loaded and use custom config inside configs folder, which
+        # means that version policy will change and only version 2 will
+        # be available.
+        for base_name, model_name in zip(models_base, models):
+            shutil.copyfile(
+                "config.pbtxt.custom." + base_name,
+                "models/" + model_name + "/configs/custom.pbtxt",
+            )
+
+        time.sleep(5)  # wait for models to reload
+        for model_name in models:
+            try:
+                for triton_client in (
+                    httpclient.InferenceServerClient("localhost:8000", verbose=True),
+                    grpcclient.InferenceServerClient("localhost:8001", verbose=True),
+                ):
+                    self.assertTrue(triton_client.is_server_live())
+                    self.assertTrue(triton_client.is_server_ready())
+                    self.assertFalse(triton_client.is_model_ready(model_name, "1"))
+                    self.assertTrue(triton_client.is_model_ready(model_name, "2"))
+                    self.assertFalse(triton_client.is_model_ready(model_name, "3"))
+            except Exception as ex:
+                self.assertTrue(False, "unexpected error {}".format(ex))
+
+    def test_delete_custom_config(self):
+        models_base = ("savedmodel",)
+        models = list()
+        for m in models_base:
+            models.append(tu.get_model_name(m, np.float32, np.float32, np.float32))
+
+        # Make sure savedmodel and plan are in the status
+        for model_name in models:
+            try:
+                for triton_client in (
+                    httpclient.InferenceServerClient("localhost:8000", verbose=True),
+                    grpcclient.InferenceServerClient("localhost:8001", verbose=True),
+                ):
+                    self.assertTrue(triton_client.is_server_live())
+                    self.assertTrue(triton_client.is_server_ready())
+                    self.assertFalse(triton_client.is_model_ready(model_name, "1"))
+                    self.assertTrue(triton_client.is_model_ready(model_name, "2"))
+                    self.assertFalse(triton_client.is_model_ready(model_name, "3"))
+            except Exception as ex:
+                self.assertTrue(False, "unexpected error {}".format(ex))
+
+        # Delete custom model configuration, which cause model to be
+        # re-loaded and use default config, which means that version
+        # policy will be changed and so only version 1, 3 will be available
+        for model_name in models:
+            os.remove("models/" + model_name + "/configs/custom.pbtxt")
+
+        time.sleep(5)  # wait for models to reload
+        for model_name in models:
+            try:
+                for triton_client in (
+                    httpclient.InferenceServerClient("localhost:8000", verbose=True),
+                    grpcclient.InferenceServerClient("localhost:8001", verbose=True),
+                ):
+                    self.assertTrue(triton_client.is_server_live())
+                    self.assertTrue(triton_client.is_server_ready())
+                    self.assertTrue(triton_client.is_model_ready(model_name, "1"))
+                    self.assertFalse(triton_client.is_model_ready(model_name, "2"))
+                    self.assertTrue(triton_client.is_model_ready(model_name, "3"))
+            except Exception as ex:
+                self.assertTrue(False, "unexpected error {}".format(ex))
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/qa/L0_lifecycle/test.sh b/qa/L0_lifecycle/test.sh
index d40f953b7f..9236fdabfb 100755
--- a/qa/L0_lifecycle/test.sh
+++ b/qa/L0_lifecycle/test.sh
@@ -2131,6 +2131,72 @@ set -e
 kill $SERVER_PID
 wait $SERVER_PID
 
+LOG_IDX=$((LOG_IDX+1))
+
+# LifeCycleTest.test_add_custom_config
+rm -fr models config.pbtxt.*
+mkdir models
+for i in savedmodel; do
+    cp -r $DATADIR/qa_model_repository/${i}_float32_float32_float32 models/.
+    mkdir models/${i}_float32_float32_float32/configs
+    sed 's/^version_policy:.*/version_policy: { specific: { versions: [2] }}/' \
+        $DATADIR/qa_model_repository/${i}_float32_float32_float32/config.pbtxt > config.pbtxt.custom.${i}
+done
+
+SERVER_ARGS="--model-repository=`pwd`/models --repository-poll-secs=1 \
+             --model-control-mode=poll --exit-timeout-secs=5 \
+             --model-config-name=custom"
+SERVER_LOG="./inference_server_$LOG_IDX.log"
+run_server
+if [ "$SERVER_PID" == "0" ]; then
+    echo -e "\n***\n*** Failed to start $SERVER\n***"
+    cat $SERVER_LOG
+    exit 1
+fi
+
+rm -f $CLIENT_LOG
+set +e
+python $LC_TEST LifeCycleTest.test_add_custom_config >>$CLIENT_LOG 2>&1
+check_unit_test
+set -e
+
+kill $SERVER_PID
+wait $SERVER_PID
+
+LOG_IDX=$((LOG_IDX+1))
+
+# LifeCycleTest.test_delete_custom_config
+rm -fr models config.pbtxt.*
+mkdir models
+for i in savedmodel; do
+    cp -r $DATADIR/qa_model_repository/${i}_float32_float32_float32 models/.
+    mkdir models/${i}_float32_float32_float32/configs
+    sed 's/^version_policy:.*/version_policy: { specific: { versions: [2] }}/' \
+        $DATADIR/qa_model_repository/${i}_float32_float32_float32/config.pbtxt \
+        > models/${i}_float32_float32_float32/configs/custom.pbtxt
+done
+
+SERVER_ARGS="--model-repository=`pwd`/models --repository-poll-secs=1 \
+             --model-control-mode=poll --exit-timeout-secs=5 \
+             --model-config-name=custom"
+SERVER_LOG="./inference_server_$LOG_IDX.log"
+run_server
+if [ "$SERVER_PID" == "0" ]; then
+    echo -e "\n***\n*** Failed to start $SERVER\n***"
+    cat $SERVER_LOG
+    exit 1
+fi
+
+rm -f $CLIENT_LOG
+set +e
+python $LC_TEST LifeCycleTest.test_delete_custom_config >>$CLIENT_LOG 2>&1
+check_unit_test
+set -e
+
+kill $SERVER_PID
+wait $SERVER_PID
+
+
 if [ $RET -eq 0 ]; then
   echo -e "\n***\n*** Test Passed\n***"
 else
diff --git a/src/command_line_parser.cc b/src/command_line_parser.cc
index 57f90595a4..b7c367cedb 100644
--- a/src/command_line_parser.cc
+++ b/src/command_line_parser.cc
@@ -354,6 +354,7 @@ enum TritonOptionId {
   OPTION_MODEL_CONTROL_MODE,
   OPTION_POLL_REPO_SECS,
   OPTION_STARTUP_MODEL,
+  OPTION_CUSTOM_MODEL_CONFIG_NAME,
   OPTION_RATE_LIMIT,
   OPTION_RATE_LIMIT_RESOURCE,
   OPTION_PINNED_MEMORY_POOL_BYTE_SIZE,
@@ -440,6 +441,13 @@ TritonParser::SetupOptions()
        "Specifying --load-model=* in conjunction with another --load-model "
        "argument will result in error. Note that this option will only take "
        "effect if --model-control-mode=explicit is true."});
+  model_repo_options_.push_back(
+      {OPTION_CUSTOM_MODEL_CONFIG_NAME, "model-config-name", Option::ArgStr,
+       "The custom configuration name for models to load."
+       "The name should not contain any space character."
+       "For example: --model-config-name=h100. "
+       "If --model-config-name is not set, Triton will use the default "
+       "config.pbtxt."});
   model_repo_options_.push_back(
       {OPTION_MODEL_LOAD_THREAD_COUNT, "model-load-thread-count",
        Option::ArgInt,
@@ -1011,6 +1019,11 @@ TritonServerParameters::BuildTritonServerOptions()
         TRITONSERVER_ServerOptionsSetStartupModel(loptions, model.c_str()),
         "setting startup model");
   }
+  THROW_IF_ERR(
+      ParseException,
+      TRITONSERVER_ServerOptionsSetModelConfigName(
+          loptions, model_config_name_.c_str()),
+      "setting custom model configuration name for models");
   THROW_IF_ERR(
       ParseException,
       TRITONSERVER_ServerOptionsSetRateLimiterMode(loptions, rate_limit_mode_),
@@ -1353,7 +1366,6 @@ TritonParser::Parse(int argc, char** argv)
           lparams.http_forward_header_pattern_ =
               std::move(case_insensitive_prefix + optarg);
           break;
-          break;
         case OPTION_HTTP_THREAD_COUNT:
           lparams.http_thread_cnt_ = ParseOption<int>(optarg);
           break;
@@ -1591,6 +1603,13 @@ TritonParser::Parse(int argc, char** argv)
         case OPTION_STARTUP_MODEL:
           lparams.startup_models_.insert(optarg);
           break;
+        case OPTION_CUSTOM_MODEL_CONFIG_NAME:
+          if (std::strlen(optarg) == 0) {
+            throw ParseException(
+                "Error: empty argument for --model-config-name");
+          }
+          lparams.model_config_name_ = optarg;
+          break;
         case OPTION_MODEL_CONTROL_MODE: {
           std::string mode_str(optarg);
           std::transform(
diff --git a/src/command_line_parser.h b/src/command_line_parser.h
index 0131c99ff2..bf0cb72d3e 100644
--- a/src/command_line_parser.h
+++ b/src/command_line_parser.h
@@ -134,6 +134,9 @@ struct TritonServerParameters {
   uint32_t model_load_thread_count_{4};
   uint32_t model_load_retry_count_{0};
   std::map<int, double> load_gpu_limit_;
+  // Custom model configuration file. Fall back to default config.pbtxt if not
+  // set.
+  std::string model_config_name_;
 
   // Rate limiter configuration
   // FIXME: Once the rate limiter implementation is complete make