Setup models

Signed-off-by: Rafal <[email protected]>

Author: Rafal

benchmarks/resnet50/model_repository/dali/config.pbtxt

@@ -0,0 +1,52 @@
+# The MIT License (MIT)
+#
+# Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of
+# this software and associated documentation files (the "Software"), to deal in
+# the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+# the Software, and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+name: "dali"
+backend: "dali"
+max_batch_size: 128
+input [
+  {
+    name: "DALI_INPUT_0"
+    data_type: TYPE_UINT8
+    dims: [ -1 ]
+    allow_ragged_batch: true
+  }
+]
+
+output [
+  {
+    name: "DALI_OUTPUT_0"
+    data_type: TYPE_FP32
+    dims: [ 224, 224, 3 ]
+  }
+]
+
+parameters: [
+  {
+    key: "num_threads"
+    value: { string_value: "4" }
+  }
+]
+
+dynamic_batching {
+  preferred_batch_size: [ 64, 128 ]
+  max_queue_delay_microseconds: 1
+}

benchmarks/resnet50/model_repository/dali/pipeline.py

@@ -0,0 +1,54 @@
+# Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import nvidia.dali as dali
+import nvidia.dali.types as types
+
+
+def parse_args():
+    import argparse
+    parser = argparse.ArgumentParser(description="Serialize the pipeline and save it to a file")
+    parser.add_argument('file_path', type=str, help='The path where to save the serialized pipeline')
+    return parser.parse_args()
+
+
+def preprocessing(images):
+  images = dali.fn.decoders.image(images, device="mixed", output_type=types.RGB)
+  images = dali.fn.resize(images, resize_x=224, resize_y=224)
+  return dali.fn.crop_mirror_normalize(images,
+                                       dtype=types.FLOAT,
+                                       output_layout="HWC",
+                                       crop=(224, 224),
+                                       mean=[0.485 * 255, 0.456 * 255, 0.406 * 255],
+                                       std=[0.229 * 255, 0.224 * 255, 0.225 * 255])
+
+@dali.pipeline_def(batch_size=1, num_threads=1, device_id=0)
+def pipe():
+    images = dali.fn.external_source(device="cpu", name="DALI_INPUT_0", no_copy=True)
+    return preprocessing(images)
+
+
+def main(filename):
+    pipe().serialize(filename=filename)
+
+
+if __name__ == '__main__':
+    args = parse_args()
+    main(args.file_path)

benchmarks/resnet50/model_repository/dali_trt_resnet50/config.pbtxt

@@ -0,0 +1,76 @@
+# The MIT License (MIT)
+#
+# Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+name: "dali_trt_resnet50"
+platform: "ensemble"
+max_batch_size: 128
+input [
+  {
+    name: "input"
+    data_type: TYPE_UINT8
+    dims: [ -1 ]
+  }
+]
+output [
+  {
+    name: "classes"
+    data_type: TYPE_INT32
+    dims: [ 1 ]
+  },
+  {
+    name: "probabilities"
+    data_type: TYPE_FP32
+    dims: [ 1001 ]
+  }
+]
+ensemble_scheduling {
+  step [
+    {
+      model_name: "dali"
+      model_version: -1
+      input_map {
+        key: "DALI_INPUT_0"
+        value: "input"
+      }
+      output_map {
+        key: "DALI_OUTPUT_0"
+        value: "preprocessed_image"
+      }
+    },
+    {
+      model_name: "resnet50_trt"
+      model_version: -1
+      input_map {
+        key: "input_tensor:0"
+        value: "preprocessed_image"
+      }
+      output_map {
+        key: "classes"
+        value: "classes"
+      }
+      output_map {
+        key: "probabilities"
+        value: "probabilities"
+      }
+    }
+  ]
+}

benchmarks/resnet50/model_repository/resnet50_onnx/config.pbtxt

@@ -0,0 +1,34 @@
+# The MIT License (MIT)
+#
+# Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of
+# this software and associated documentation files (the "Software"), to deal in
+# the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+# the Software, and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+platform: "onnxruntime_onnx"
+backend: 'onnxruntime'
+max_batch_size: 128
+
+instance_group {
+  count: 2
+  kind: KIND_GPU
+}
+dynamic_batching {
+  preferred_batch_size: 64
+  preferred_batch_size: 128
+  max_queue_delay_microseconds: 1
+}

benchmarks/resnet50/model_repository/resnet50_tf/config.pbtxt

@@ -0,0 +1,43 @@
+# The MIT License (MIT)
+#
+# Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of
+# this software and associated documentation files (the "Software"), to deal in
+# the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+# the Software, and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+name: "resnet50_tf"
+platform: "tensorflow_savedmodel"
+max_batch_size: 128
+
+# input [
+#   {
+#     name: "input_tensor:0"
+#     data_type: TYPE_FP32
+#     dims: [244, 244, 3]
+#   }
+# ]
+
+
+instance_group {
+  count: 2
+  kind: KIND_GPU
+}
+dynamic_batching {
+  preferred_batch_size: 64
+  preferred_batch_size: 128
+  max_queue_delay_microseconds: 1
+}

benchmarks/resnet50/model_repository/resnet50_trt/config.pbtxt

@@ -0,0 +1,33 @@
+# The MIT License (MIT)
+#
+# Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of
+# this software and associated documentation files (the "Software"), to deal in
+# the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+# the Software, and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+platform: "tensorrt_plan"
+max_batch_size: 128
+
+instance_group {
+  count: 2
+  kind: KIND_GPU
+}
+dynamic_batching {
+  preferred_batch_size: 64
+  preferred_batch_size: 128
+  max_queue_delay_microseconds: 1
+}

benchmarks/resnet50/scripts/download_checkpoint.sh

@@ -0,0 +1,31 @@
+#!/bin/bash -e
+
+# Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# Download checkpoint
+
+mkdir -p ${CHECKPOINT_DIR}
+if [ -f "${CHECKPOINT_DIR}/checkpoint" ]; then 
+  echo "Checkpoint already downloaded."
+else
+  echo "Downloading checkpoint ..."
+  wget --content-disposition https://api.ngc.nvidia.com/v2/models/nvidia/rn50_tf_amp_ckpt/versions/20.06.0/zip -O \
+    rn50_tf_amp_ckpt_20.06.0.zip || {
+    echo "ERROR: Failed to download checkpoint from NGC"
+    exit 1
+  }
+  unzip rn50_tf_amp_ckpt_20.06.0.zip -d ${CHECKPOINT_DIR}
+  rm rn50_tf_amp_ckpt_20.06.0.zip
+  echo "ok"
+fi

benchmarks/resnet50/scripts/model-loader.py

@@ -0,0 +1,55 @@
+# The MIT License (MIT)
+#
+# Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of
+# this software and associated documentation files (the "Software"), to deal in
+# the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+# the Software, and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+import tritonclient.grpc as t_client
+import argparse
+import sys
+
+def get_args():
+  parser = argparse.ArgumentParser(description='Load or unload a model in Triton server.')
+  parser.add_argument('action', action='store', choices=['load', 'unload', 'reload'])
+  parser.add_argument('-u', '--url', required=False, action='store', default='localhost:8001', help='Server url.')
+  parser.add_argument('-m', '--model', required=True, action='store', help='Model name.')
+  return parser.parse_args()
+
+
+def main(args):
+  client = t_client.InferenceServerClient(url=args.url)
+  if args.action in ['reload', 'unload']:
+    try:
+      client.unload_model(args.model)
+      print('Successfully unloaded model', args.model)
+    except:
+      print('Could not unload model', args.model)
+      sys.exit(1)
+
+  if args.action in ['reload', 'load']:
+    try:
+      client.load_model(args.model)
+      print('Successfully loaded model', args.model)
+    except:
+      print('Could not load model', args.model)
+      sys.exit(1)
+
+
+if __name__ == '__main__':
+  args = get_args()
+  main(args)