Add Python-based backends CI

.gitignore

@@ -5,3 +5,4 @@
 __pycache__
 tmp
 *.log
+test_results.txt

qa/L0_backend_python/python_based_backends/python_based_backends_test.py

@@ -0,0 +1,216 @@
+# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import json
+import queue
+import sys
+import unittest
+from functools import partial
+from random import randint
+
+import numpy as np
+import tritonclient.grpc as grpcclient
+from tritonclient.utils import *
+
+sys.path.append("../../common")
+from test_util import TestResultCollector
+
+
+class UserData:
+    def __init__(self):
+        self._completed_requests = queue.Queue()
+
+
+def callback(user_data, result, error):
+    if error:
+        user_data._completed_requests.put(error)
+    else:
+        user_data._completed_requests.put(result)
+
+
+def create_vllm_request(
+    prompt,
+    request_id,
+    stream,
+    sampling_parameters,
+    model_name,
+    send_parameters_as_tensor=True,
+):
+    inputs = []
+
+    inputs.append(grpcclient.InferInput("text_input", [1], "BYTES"))
+    inputs[-1].set_data_from_numpy(np.array([prompt.encode("utf-8")], dtype=np.object_))
+
+    inputs.append(grpcclient.InferInput("stream", [1], "BOOL"))
+    inputs[-1].set_data_from_numpy(np.array([stream], dtype=bool))
+
+    if send_parameters_as_tensor:
+        sampling_parameters_data = np.array(
+            [json.dumps(sampling_parameters).encode("utf-8")], dtype=np.object_
+        )
+        inputs.append(grpcclient.InferInput("sampling_parameters", [1], "BYTES"))
+        inputs[-1].set_data_from_numpy(sampling_parameters_data)
+
+    outputs = [grpcclient.InferRequestedOutput("text_output")]
+
+    return {
+        "model_name": model_name,
+        "inputs": inputs,
+        "outputs": outputs,
+        "request_id": str(request_id),
+        "parameters": sampling_parameters,
+    }
+
+
+class PythonBasedBackendsTest(TestResultCollector):
+    def setUp(self):
+        self.triton_client = grpcclient.InferenceServerClient(url="localhost:8001")
+        self.vllm_model_1 = "vllm_opt_1"
+        self.vllm_model_2 = "vllm_opt_2"
+        self.add_sub_model_1 = "add"
+        self.add_sub_model_2 = "sub"
+        self.python_model = "add_sub"
+        self.pytorch_model = "add_sub_pytorch"
+
+        self.triton_client.load_model(self.vllm_model_1)
+        self.triton_client.load_model(self.vllm_model_2)
+        self.triton_client.load_model(self.add_sub_model_1)
+        self.triton_client.load_model(self.add_sub_model_2)
+        self.triton_client.load_model(self.python_model)
+        self.triton_client.load_model(self.pytorch_model)
+
+    def test_vllm_models(self):
+        self.assertTrue(self.triton_client.is_model_ready(self.vllm_model_1))
+        self._test_vllm_model(self.vllm_model_1)
+
+        self.assertTrue(self.triton_client.is_model_ready(self.vllm_model_2))
+        self._test_vllm_model(self.vllm_model_2)
+
+    def test_add_sub_models(self):
+        self.assertTrue(self.triton_client.is_model_ready(self.add_sub_model_1))
+        self._test_add_sub_model(model_name=self.add_sub_model_1, single_output=True)
+
+        self.assertTrue(self.triton_client.is_model_ready(self.add_sub_model_2))
+        self._test_add_sub_model(model_name=self.add_sub_model_2, single_output=True)
+
+    def test_python_model(self):
+        self.assertTrue(self.triton_client.is_model_ready(self.python_model))
+        self._test_add_sub_model(model_name=self.python_model, shape=[16])
+
+    def test_pytorh_model(self):
+        self.assertTrue(self.triton_client.is_model_ready(self.pytorch_model))
+        self._test_add_sub_model(model_name=self.pytorch_model)
+
+    def _test_vllm_model(self, model_name, send_parameters_as_tensor=True):
+        user_data = UserData()
+        stream = False
+        prompts = [
+            "The most dangerous animal is",
+            "The capital of France is",
+            "The future of AI is",
+        ]
+        number_of_vllm_reqs = len(prompts)
+        sampling_parameters = {"temperature": "0.1", "top_p": "0.95"}
+
+        self.triton_client.start_stream(callback=partial(callback, user_data))
+        for i in range(number_of_vllm_reqs):
+            request_data = create_vllm_request(
+                prompts[i],
+                i,
+                stream,
+                sampling_parameters,
+                model_name,
+                send_parameters_as_tensor,
+            )
+            self.triton_client.async_stream_infer(
+                model_name=model_name,
+                request_id=request_data["request_id"],
+                inputs=request_data["inputs"],
+                outputs=request_data["outputs"],
+                parameters=sampling_parameters,
+            )
+
+        for i in range(number_of_vllm_reqs):
+            result = user_data._completed_requests.get()
+            self.assertIsNot(type(result), InferenceServerException)
+
+            output = result.as_numpy("text_output")
+            self.assertIsNotNone(output)
+
+        self.triton_client.stop_stream()
+
+    def _test_add_sub_model(self, model_name, shape=[4], single_output=False):
+        input0_data = np.random.rand(*shape).astype(np.float32)
+        input1_data = np.random.rand(*shape).astype(np.float32)
+
+        inputs = [
+            grpcclient.InferInput(
+                "INPUT0", input0_data.shape, np_to_triton_dtype(input0_data.dtype)
+            ),
+            grpcclient.InferInput(
+                "INPUT1", input1_data.shape, np_to_triton_dtype(input1_data.dtype)
+            ),
+        ]
+
+        inputs[0].set_data_from_numpy(input0_data)
+        inputs[1].set_data_from_numpy(input1_data)
+
+        if single_output:
+            outputs = [grpcclient.InferRequestedOutput("OUTPUT")]
+
+        else:
+            outputs = [
+                grpcclient.InferRequestedOutput("OUTPUT0"),
+                grpcclient.InferRequestedOutput("OUTPUT1"),
+            ]
+
+        response = self.triton_client.infer(
+            model_name, inputs, request_id=str(randint(10, 99)), outputs=outputs
+        )
+
+        if single_output:
+            if model_name == "add":
+                self.assertTrue(
+                    np.allclose(input0_data + input1_data, response.as_numpy("OUTPUT"))
+                )
+            else:
+                self.assertTrue(
+                    np.allclose(input0_data - input1_data, response.as_numpy("OUTPUT"))
+                )
+        else:
+            self.assertTrue(
+                np.allclose(input0_data + input1_data, response.as_numpy("OUTPUT0"))
+            )
+            self.assertTrue(
+                np.allclose(input0_data - input1_data, response.as_numpy("OUTPUT1"))
+            )
+
+    def tearDown(self):
+        self.triton_client.close()
+
+
+if __name__ == "__main__":
+    unittest.main()

qa/L0_backend_python/python_based_backends/test.sh

@@ -0,0 +1,125 @@
+#!/bin/bash
+# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+source ../../common/util.sh
+
+TRITON_DIR=${TRITON_DIR:="/opt/tritonserver"}
+SERVER=${TRITON_DIR}/bin/tritonserver
+BACKEND_DIR=${TRITON_DIR}/backends
+QA_MODELS_PATH="../../python_models"
+MODEL_REPOSITORY="`pwd`/models"
+SERVER_ARGS="--model-repository=${MODEL_REPOSITORY} --backend-directory=${BACKEND_DIR} --model-control-mode=explicit --log-verbose=1"
+SERVER_LOG="./python_based_backends_server.log"
+CLIENT_LOG="./python_based_backends_client.log"
+TEST_RESULT_FILE="./test_results.txt"
+CLIENT_PY="./python_based_backends_test.py"
+GEN_PYTORCH_MODEL_PY="../../common/gen_qa_pytorch_model.py"
+EXPECTED_NUM_TESTS=4
+RET=0
+
+# Setup vllm_backend
+pip3 install vllm
+mkdir -p ${BACKEND_DIR}/vllm
+wget -P ${BACKEND_DIR}/vllm https://raw.githubusercontent.com/triton-inference-server/vllm_backend/main/src/model.py
+
+rm -rf ${MODEL_REPOSITORY}
+
+# Setup vllm backend models
+mkdir -p ${MODEL_REPOSITORY}/vllm_opt_1/1/
+wget -P ${MODEL_REPOSITORY}/vllm_opt_1/1/ https://raw.githubusercontent.com/triton-inference-server/vllm_backend/main/samples/model_repository/vllm_model/1/model.json
+wget -P ${MODEL_REPOSITORY}/vllm_opt_1 https://raw.githubusercontent.com/triton-inference-server/vllm_backend/main/samples/model_repository/vllm_model/config.pbtxt
+sed -i 's/"gpu_memory_utilization": 0.5/"gpu_memory_utilization": 0.25/' ${MODEL_REPOSITORY}/vllm_opt_1/1/model.json
+
+cp -r ${MODEL_REPOSITORY}/vllm_opt_1/ ${MODEL_REPOSITORY}/vllm_opt_2/
+
+# Setup add_sub backend and models
+mkdir -p ${BACKEND_DIR}/add_sub
+cp ${QA_MODELS_PATH}/python_based_backends/add_sub_backend/model.py ${BACKEND_DIR}/add_sub/model.py
+
+mkdir -p ${MODEL_REPOSITORY}/add/1/
+cp ${QA_MODELS_PATH}/python_based_backends/add_sub_backend/add_model.json ${MODEL_REPOSITORY}/add/1/model.json
+cp ${QA_MODELS_PATH}/python_based_backends/add_sub_backend/config.pbtxt ${MODEL_REPOSITORY}/add/
+
+mkdir -p ${MODEL_REPOSITORY}/sub/1/
+cp ${QA_MODELS_PATH}/python_based_backends/add_sub_backend/sub_model.json ${MODEL_REPOSITORY}/sub/1/model.json
+cp ${QA_MODELS_PATH}/python_based_backends/add_sub_backend/config.pbtxt ${MODEL_REPOSITORY}/sub/
+
+# Setup python backend model
+mkdir -p ${MODEL_REPOSITORY}/add_sub/1
+cp ${QA_MODELS_PATH}/add_sub/model.py ${MODEL_REPOSITORY}/add_sub/1/
+cp ${QA_MODELS_PATH}/add_sub/config.pbtxt ${MODEL_REPOSITORY}/add_sub/
+
+# Setup pytorch backend model
+cp ${GEN_PYTORCH_MODEL_PY} ./gen_qa_pytorch_model.py
+GEN_PYTORCH_MODEL_PY=./gen_qa_pytorch_model.py
+
+set +e
+python3 ${GEN_PYTORCH_MODEL_PY} -m ${MODEL_REPOSITORY}
+
+if [ $? -ne 0 ]; then
+    echo -e "\n***\n*** Running ${GEN_PYTORCH_MODEL_PY} FAILED. \n***"
+    exit 1
+fi
+set -e
+
+run_server
+if [ "$SERVER_PID" == "0" ]; then
+    cat $SERVER_LOG
+    echo -e "\n***\n*** Failed to start $SERVER\n***"
+    exit 1
+fi
+
+set +e
+python3 $CLIENT_PY -v > $CLIENT_LOG 2>&1
+
+if [ $? -ne 0 ]; then
+    echo -e "\n***\n*** Running $CLIENT_PY FAILED. \n***"
+    RET=1
+else
+    check_test_results $TEST_RESULT_FILE $EXPECTED_NUM_TESTS
+    if [ $? -ne 0 ]; then
+        echo -e "\n***\n*** Test Result Verification FAILED.\n***"
+        RET=1
+    fi
+fi
+set -e
+
+kill $SERVER_PID
+wait $SERVER_PID
+rm -rf ${MODEL_REPOSITORY} ${GEN_PYTORCH_MODEL_PY}
+
+if [ $RET -eq 1 ]; then
+    cat $CLIENT_LOG
+    cat $SERVER_LOG
+    echo -e "\n***\n*** Python-based Backends test FAILED. \n***"
+else
+    echo -e "\n***\n*** Python-based Backends test PASSED. \n***"
+fi
+
+collect_artifacts_from_subdir
+
+exit $RET

qa/L0_backend_python/test.sh

@@ -395,7 +395,7 @@ fi
 # Disable variants test for Jetson since already built without GPU Tensor support
 # Disable decoupled test because it uses GPU tensors
 if [ "$TEST_JETSON" == "0" ]; then
-    SUBTESTS="ensemble io bls decoupled variants"
+    SUBTESTS="ensemble io bls decoupled variants python_based_backends"
     for TEST in ${SUBTESTS}; do
         # Run each subtest in a separate virtual environment to avoid conflicts
         # between dependencies.