Add test for optional internal tensor within an ensemble (#6663) (#6676)

* Add test for optional internal tensor within an ensemble

* Fix up

qa/L0_optional_input/models/optional_connecting_tensor/config.pbtxt

@@ -0,0 +1,98 @@
+# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+platform: "ensemble"
+max_batch_size: 4
+input [
+  {
+    name: "INPUT0"
+    data_type: TYPE_FP32
+    dims: [ 1 ]
+    optional: true
+  },
+  {
+    name: "INPUT1"
+    data_type: TYPE_FP32
+    dims: [ 1 ]
+    optional: true
+  }
+]
+output [
+  {
+    name: "OUTPUT0"
+    data_type: TYPE_FP32
+    dims: [ 1 ]
+  },
+  {
+    name: "OUTPUT1"
+    data_type: TYPE_FP32
+    dims: [ 1 ]
+  }
+]
+ensemble_scheduling {
+  step [
+    {
+      model_name: "optional_identity"
+      model_version: -1
+      input_map {
+        key: "INPUT0"
+        value: "INPUT0"
+      }
+      input_map {
+        key: "INPUT1"
+        value: "INPUT1"
+      }
+      output_map {
+        key: "OUTPUT0"
+        value: "internal_output0"
+      }
+      output_map {
+        key: "OUTPUT1"
+        value: "internal_output1"
+      }
+    },
+    {
+      model_name: "optional_identity"
+      model_version: -1
+      input_map {
+        key: "INPUT0"
+        value: "internal_output0"
+      }
+      input_map {
+        key: "INPUT1"
+        value: "internal_output1"
+      }
+      output_map {
+        key: "OUTPUT0"
+        value: "OUTPUT0"
+      }
+      output_map {
+        key: "OUTPUT1"
+        value: "OUTPUT1"
+      }
+    }
+  ]
+}

qa/L0_optional_input/models/optional_identity/1/model.py

@@ -0,0 +1,46 @@
+# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import triton_python_backend_utils as pb_utils
+
+
+class TritonPythonModel:
+    def execute(self, requests):
+        """
+        Identity model in Python backend.
+        """
+        responses = []
+        for request in requests:
+            for tidx in ("0", "1"):
+                input_tensor = pb_utils.get_input_tensor_by_name(
+                    request, "INPUT" + tidx
+                )
+                if input_tensor is not None:
+                    out_tensor = pb_utils.Tensor(
+                        "OUTPUT" + tidx, input_tensor.as_numpy()
+                    )
+                    responses.append(pb_utils.InferenceResponse([out_tensor]))
+        return responses

qa/L0_optional_input/models/optional_identity/config.pbtxt

@@ -0,0 +1,53 @@
+# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+backend: "python"
+max_batch_size: 4
+input [
+  {
+    name: "INPUT0"
+    data_type: TYPE_FP32
+    dims: [ 1 ]
+    optional: true
+  },
+  {
+    name: "INPUT1"
+    data_type: TYPE_FP32
+    dims: [ 1 ]
+    optional: true
+  }
+]
+output [
+  {
+    name: "OUTPUT0"
+    data_type: TYPE_FP32
+    dims: [ 1 ]
+  },
+  {
+    name: "OUTPUT1"
+    data_type: TYPE_FP32
+    dims: [ 1 ]
+  }
+]

qa/L0_optional_input/optional_input_test.py

@@ -142,13 +142,8 @@ def check_status(self, model_name, batch_exec, request_cnt, infer_cnt):
             stats = self.triton_client_.get_inference_statistics(model_name, "1")
             self.assertEqual(len(stats.model_stats), 1, "expect 1 model stats")
             actual_exec_cnt = stats.model_stats[0].execution_count
-            if actual_exec_cnt == exec_cnt:
+            if stats.model_stats[0].execution_count > 0:
                 break
-            print(
-                "WARNING: expect {} executions, got {} (attempt {})".format(
-                    exec_cnt, actual_exec_cnt, i
-                )
-            )
             time.sleep(1)
 
         self.assertEqual(
@@ -411,6 +406,40 @@ def test_ensemble_optional_pipeline(self):
         except Exception as ex:
             self.assertTrue(False, "unexpected error {}".format(ex))
 
+    def test_ensemble_optional_connecting_tensor(self):
+        # The ensemble is a special case of pipelining models with optional
+        # inputs, where the request will only produce a subset of inputs
+        # for the second model while the ensemble graph connects all inputs of
+        # the second model (which is valid because the not-provided inputs
+        # are marked optional). See 'config.pbtxt' for detail.
+        self.model_name_ = "optional_connecting_tensor"
+
+        # Provide all inputs, send requests that don't form preferred batch
+        # so all requests should be returned after the queue delay
+        try:
+            provided_inputs = ("INPUT0",)
+            inputs = []
+            outputs = []
+            for provided_input in provided_inputs:
+                inputs.append(self.inputs_[provided_input])
+                outputs.append(self.outputs_[provided_input])
+
+            triton_client = grpcclient.InferenceServerClient("localhost:8001")
+            results = triton_client.infer(
+                model_name=self.model_name_, inputs=inputs, outputs=outputs
+            )
+
+            expected = self.input_data_["INPUT0"]
+            output_data = results.as_numpy("OUTPUT0")
+            self.assertTrue(
+                np.array_equal(output_data, expected),
+                "{}, {}, expected: {}, got {}".format(
+                    self.model_name_, "OUTPUT0", expected, output_data
+                ),
+            )
+        except Exception as ex:
+            self.assertTrue(False, "unexpected error {}".format(ex))
+
 
 if __name__ == "__main__":
     unittest.main()

qa/L0_optional_input/test.sh

@@ -41,6 +41,7 @@ rm -fr *.log
 mkdir -p ./models/identity_2_float32/1
 mkdir -p ./models/ensemble_identity_2_float32/1
 mkdir -p ./models/pipeline_identity_2_float32/1
+mkdir -p ./models/optional_connecting_tensor/1
 
 # Basic test cases
 TEST_CASES=${TEST_CASES:="test_all_inputs \
@@ -51,8 +52,9 @@ TEST_CASES=${TEST_CASES:="test_all_inputs \
                             test_ensemble_optional_same_input \
                             test_ensemble_optional_mix_inputs \
                             test_ensemble_optional_mix_inputs_2 \
-                            test_ensemble_optional_pipeline"}
-
+                            test_ensemble_optional_pipeline \
+                            test_ensemble_optional_connecting_tensor"}
+RET=0
 for i in $TEST_CASES ; do
     # Restart server for every test to clear model stats
     run_server
@@ -62,8 +64,6 @@ for i in $TEST_CASES ; do
         exit 1
     fi
 
-    RET=0
-
     echo "Test: $i" >>$TEST_LOG
 
     set +e