Tests' re-org (#39)

Author: oandreeva-nv

ci/L0_backend_vllm/test.sh

@@ -26,7 +26,7 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 RET=0
-SUBTESTS="accuracy_test request_cancellation enabled_stream vllm_backend multi_lora"
+SUBTESTS="accuracy_test request_cancellation enabled_stream vllm_backend"
 
 python3 -m pip install --upgrade pip && pip3 install tritonclient[grpc]
 

ci/L0_backend_vllm/vllm_backend/test.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -30,7 +30,7 @@ source ../../common/util.sh
 TRITON_DIR=${TRITON_DIR:="/opt/tritonserver"}
 SERVER=${TRITON_DIR}/bin/tritonserver
 BACKEND_DIR=${TRITON_DIR}/backends
-SERVER_ARGS="--model-repository=$(pwd)/models --backend-directory=${BACKEND_DIR} --model-control-mode=explicit --log-verbose=1"
+SERVER_ARGS="--model-repository=$(pwd)/models --backend-directory=${BACKEND_DIR} --model-control-mode=explicit --load-model=vllm_opt --log-verbose=1"
 SERVER_LOG="./vllm_backend_server.log"
 CLIENT_LOG="./vllm_backend_client.log"
 TEST_RESULT_FILE='test_results.txt'
@@ -50,6 +50,13 @@ function assert_curl_success {
 
 rm -rf models && mkdir -p models
 cp -r ${SAMPLE_MODELS_REPO}/vllm_model models/vllm_opt
+# `vllm_opt`` model will be loaded on server start and stay loaded throughout
+# unittesting. To test vllm model load/unload we use a dedicated
+# `vllm_load_test`. To ensure that vllm's memory profiler will not error out
+# on `vllm_load_test` load, we reduce "gpu_memory_utilization" for `vllm_opt`,
+# so that at least 60% of GPU memory was available for other models.
+sed -i 's/"gpu_memory_utilization": 0.5/"gpu_memory_utilization": 0.4/' models/vllm_opt/1/model.json
+cp -r models/vllm_opt models/vllm_load_test
 
 mkdir -p models/add_sub/1/
 wget -P models/add_sub/1/ https://raw.githubusercontent.com/triton-inference-server/python_backend/main/examples/add_sub/model.py
@@ -96,7 +103,7 @@ wait $SERVER_PID
 SERVER_ARGS="--model-repository=$(pwd)/models --backend-directory=${BACKEND_DIR} --backend-config=python,default-max-batch-size=8"
 SERVER_LOG="./vllm_test_cmdline_server.log"
 
-rm -rf ./models/vllm_invalid_1 ./models/vllm_invalid_2
+rm -rf ./models/vllm_invalid_1 ./models/vllm_invalid_2 ./models/vllm_load_test
 
 run_server
 if [ "$SERVER_PID" == "0" ]; then

ci/L0_backend_vllm/vllm_backend/vllm_backend_test.py

@@ -1,4 +1,4 @@
-# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -48,21 +48,23 @@ def setUp(self):
         self.triton_client = grpcclient.InferenceServerClient(url="localhost:8001")
         self.vllm_model_name = "vllm_opt"
         self.python_model_name = "add_sub"
+        self.vllm_load_test = "vllm_load_test"
 
     def test_vllm_triton_backend(self):
         # Load both vllm and add_sub models
-        self.triton_client.load_model(self.vllm_model_name)
-        self.assertTrue(self.triton_client.is_model_ready(self.vllm_model_name))
+        self.triton_client.load_model(self.vllm_load_test)
+        self.assertTrue(self.triton_client.is_model_ready(self.vllm_load_test))
         self.triton_client.load_model(self.python_model_name)
         self.assertTrue(self.triton_client.is_model_ready(self.python_model_name))
 
         # Unload vllm model and test add_sub model
-        self.triton_client.unload_model(self.vllm_model_name)
-        self.assertFalse(self.triton_client.is_model_ready(self.vllm_model_name))
+        self.triton_client.unload_model(self.vllm_load_test)
+        self.assertFalse(self.triton_client.is_model_ready(self.vllm_load_test))
         self._test_python_model()
 
         # Load vllm model and unload add_sub model
-        self.triton_client.load_model(self.vllm_model_name)
+        self.triton_client.load_model(self.vllm_load_test)
+        self.assertTrue(self.triton_client.is_model_ready(self.vllm_load_test))
         self.triton_client.unload_model(self.python_model_name)
         self.assertFalse(self.triton_client.is_model_ready(self.python_model_name))
 
@@ -72,14 +74,17 @@ def test_vllm_triton_backend(self):
             sampling_parameters=SAMPLING_PARAMETERS,
             stream=False,
             send_parameters_as_tensor=True,
+            model_name=self.vllm_load_test,
         )
         self._test_vllm_model(
             prompts=PROMPTS,
             sampling_parameters=SAMPLING_PARAMETERS,
             stream=False,
             send_parameters_as_tensor=False,
+            model_name=self.vllm_load_test,
         )
-        self.triton_client.unload_model(self.vllm_model_name)
+        self.triton_client.unload_model(self.vllm_load_test)
+        self.assertFalse(self.triton_client.is_model_ready(self.vllm_load_test))
 
     def test_model_with_invalid_attributes(self):
         model_name = "vllm_invalid_1"
@@ -97,7 +102,6 @@ def test_exclude_input_in_output_default(self):
         in non-streaming mode.
         Expected result: prompt is returned with diffs.
         """
-        self.triton_client.load_model(self.vllm_model_name)
         prompts = [
             "The capital of France is",
         ]
@@ -112,15 +116,13 @@ def test_exclude_input_in_output_default(self):
             send_parameters_as_tensor=True,
             expected_output=expected_output,
         )
-        self.triton_client.unload_model(self.vllm_model_name)
 
     def test_exclude_input_in_output_false(self):
         """
         Verifying behavior for `exclude_input_in_output` = False
         in non-streaming mode.
         Expected result: prompt is returned with diffs.
         """
-        self.triton_client.load_model(self.vllm_model_name)
         # Test vllm model and unload vllm model
         prompts = [
             "The capital of France is",
@@ -137,15 +139,13 @@ def test_exclude_input_in_output_false(self):
             exclude_input_in_output=False,
             expected_output=expected_output,
         )
-        self.triton_client.unload_model(self.vllm_model_name)
 
     def test_exclude_input_in_output_true(self):
         """
         Verifying behavior for `exclude_input_in_output` = True
         in non-streaming mode.
         Expected result: only diffs are returned.
         """
-        self.triton_client.load_model(self.vllm_model_name)
         # Test vllm model and unload vllm model
         prompts = [
             "The capital of France is",
@@ -162,7 +162,6 @@ def test_exclude_input_in_output_true(self):
             exclude_input_in_output=True,
             expected_output=expected_output,
         )
-        self.triton_client.unload_model(self.vllm_model_name)
 
     def _test_vllm_model(
         self,
@@ -172,6 +171,7 @@ def _test_vllm_model(
         send_parameters_as_tensor,
         exclude_input_in_output=None,
         expected_output=None,
+        model_name="vllm_opt",
     ):
         user_data = UserData()
         number_of_vllm_reqs = len(prompts)
@@ -183,12 +183,12 @@ def _test_vllm_model(
                 i,
                 stream,
                 sampling_parameters,
-                self.vllm_model_name,
+                model_name,
                 send_parameters_as_tensor,
                 exclude_input_in_output=exclude_input_in_output,
             )
             self.triton_client.async_stream_infer(
-                model_name=self.vllm_model_name,
+                model_name=model_name,
                 request_id=request_data["request_id"],
                 inputs=request_data["inputs"],
                 outputs=request_data["outputs"],

ci/L0_backend_vllm/multi_lora/download.py renamed to ci/L0_multi_gpu/multi_lora/download.py

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -25,61 +25,24 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-source ../common/util.sh
-
-TRITON_DIR=${TRITON_DIR:="/opt/tritonserver"}
-SERVER=${TRITON_DIR}/bin/tritonserver
-BACKEND_DIR=${TRITON_DIR}/backends
-SERVER_ARGS="--model-repository=`pwd`/models --backend-directory=${BACKEND_DIR} --model-control-mode=explicit --log-verbose=1"
-SERVER_LOG="./vllm_multi_gpu_test_server.log"
-CLIENT_LOG="./vllm_multi_gpu_test_client.log"
-TEST_RESULT_FILE='test_results.txt'
-CLIENT_PY="./vllm_multi_gpu_test.py"
-SAMPLE_MODELS_REPO="../../samples/model_repository"
-EXPECTED_NUM_TESTS=1
-
-rm -rf models && mkdir -p models
-cp -r ${SAMPLE_MODELS_REPO}/vllm_model models/vllm_opt
-sed -i '3s/^/    "tensor_parallel_size": 2,\n/' models/vllm_opt/1/model.json
-
-python3 -m pip install --upgrade pip && pip3 install tritonclient[grpc] nvidia-ml-py3
-
 RET=0
+SUBTESTS="vllm_backend multi_lora"
 
-run_server
-if [ "$SERVER_PID" == "0" ]; then
-    cat $SERVER_LOG
-    echo -e "\n***\n*** Failed to start $SERVER\n***"
-    exit 1
-fi
+python3 -m pip install --upgrade pip && pip3 install tritonclient[grpc]
 
-set +e
-python3 $CLIENT_PY -v > $CLIENT_LOG 2>&1
+for TEST in ${SUBTESTS}; do
+    (cd ${TEST} && bash -ex test.sh && cd ..)
 
-if [ $? -ne 0 ]; then
-    cat $CLIENT_LOG
-    echo -e "\n***\n*** Running $CLIENT_PY FAILED. \n***"
-    RET=1
-else
-    check_test_results $TEST_RESULT_FILE $EXPECTED_NUM_TESTS
     if [ $? -ne 0 ]; then
-        cat $CLIENT_LOG
-        echo -e "\n***\n*** Test Result Verification FAILED.\n***"
+        echo "Subtest ${TEST} FAILED"
         RET=1
     fi
-fi
-set -e
-
-kill $SERVER_PID
-wait $SERVER_PID
-rm -rf models/
+done
 
-if [ $RET -eq 1 ]; then
-    cat $CLIENT_LOG
-    cat $SERVER_LOG
-    echo -e "\n***\n*** Multi GPU Utilization test FAILED. \n***"
+if [ $RET -eq 0 ]; then
+    echo -e "\n***\n*** vLLM Multi-GPU Tests Passed\n***"
 else
-    echo -e "\n***\n*** Multi GPU Utilization test PASSED. \n***"
+    echo -e "\n***\n*** vLLM Multi-GPU Tests FAILED\n***"
 fi
 
 exit $RET

ci/L0_backend_vllm/multi_lora/multi_lora_test.py renamed to ci/L0_multi_gpu/multi_lora/multi_lora_test.py

@@ -0,0 +1,85 @@
+#!/bin/bash
+# Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+source ../../common/util.sh
+
+TRITON_DIR=${TRITON_DIR:="/opt/tritonserver"}
+SERVER=${TRITON_DIR}/bin/tritonserver
+BACKEND_DIR=${TRITON_DIR}/backends
+SERVER_ARGS="--model-repository=`pwd`/models --backend-directory=${BACKEND_DIR} --model-control-mode=explicit --log-verbose=1"
+SERVER_LOG="./vllm_multi_gpu_test_server.log"
+CLIENT_LOG="./vllm_multi_gpu_test_client.log"
+TEST_RESULT_FILE='test_results.txt'
+CLIENT_PY="./vllm_multi_gpu_test.py"
+SAMPLE_MODELS_REPO="../../../samples/model_repository"
+EXPECTED_NUM_TESTS=1
+
+rm -rf models && mkdir -p models
+cp -r ${SAMPLE_MODELS_REPO}/vllm_model models/vllm_opt
+sed -i '3s/^/    "tensor_parallel_size": 2,\n/' models/vllm_opt/1/model.json
+
+python3 -m pip install --upgrade pip && pip3 install tritonclient[grpc] nvidia-ml-py3
+
+RET=0
+
+run_server
+if [ "$SERVER_PID" == "0" ]; then
+    cat $SERVER_LOG
+    echo -e "\n***\n*** Failed to start $SERVER\n***"
+    exit 1
+fi
+
+set +e
+python3 $CLIENT_PY -v > $CLIENT_LOG 2>&1
+
+if [ $? -ne 0 ]; then
+    cat $CLIENT_LOG
+    echo -e "\n***\n*** Running $CLIENT_PY FAILED. \n***"
+    RET=1
+else
+    check_test_results $TEST_RESULT_FILE $EXPECTED_NUM_TESTS
+    if [ $? -ne 0 ]; then
+        cat $CLIENT_LOG
+        echo -e "\n***\n*** Test Result Verification FAILED.\n***"
+        RET=1
+    fi
+fi
+set -e
+
+kill $SERVER_PID
+wait $SERVER_PID
+rm -rf models/
+
+if [ $RET -eq 1 ]; then
+    cat $CLIENT_LOG
+    cat $SERVER_LOG
+    echo -e "\n***\n*** Multi GPU Utilization test FAILED. \n***"
+else
+    echo -e "\n***\n*** Multi GPU Utilization test PASSED. \n***"
+fi
+
+exit $RET

ci/L0_backend_vllm/multi_lora/test.sh renamed to ci/L0_multi_gpu/multi_lora/test.sh

@@ -1,4 +1,4 @@
-# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -32,7 +32,7 @@
 import tritonclient.grpc as grpcclient
 from tritonclient.utils import *
 
-sys.path.append("../common")
+sys.path.append("../../common")
 from test_util import TestResultCollector, UserData, callback, create_vllm_request