diff --git a/qa/L0_optional_input/models/optional_connecting_tensor/config.pbtxt b/qa/L0_optional_input/models/optional_connecting_tensor/config.pbtxt new file mode 100644 index 0000000000..afc4ebc00f --- /dev/null +++ b/qa/L0_optional_input/models/optional_connecting_tensor/config.pbtxt @@ -0,0 +1,98 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +platform: "ensemble" +max_batch_size: 4 +input [ + { + name: "INPUT0" + data_type: TYPE_FP32 + dims: [ 1 ] + optional: true + }, + { + name: "INPUT1" + data_type: TYPE_FP32 + dims: [ 1 ] + optional: true + } +] +output [ + { + name: "OUTPUT0" + data_type: TYPE_FP32 + dims: [ 1 ] + }, + { + name: "OUTPUT1" + data_type: TYPE_FP32 + dims: [ 1 ] + } +] +ensemble_scheduling { + step [ + { + model_name: "optional_identity" + model_version: -1 + input_map { + key: "INPUT0" + value: "INPUT0" + } + input_map { + key: "INPUT1" + value: "INPUT1" + } + output_map { + key: "OUTPUT0" + value: "internal_output0" + } + output_map { + key: "OUTPUT1" + value: "internal_output1" + } + }, + { + model_name: "optional_identity" + model_version: -1 + input_map { + key: "INPUT0" + value: "internal_output0" + } + input_map { + key: "INPUT1" + value: "internal_output1" + } + output_map { + key: "OUTPUT0" + value: "OUTPUT0" + } + output_map { + key: "OUTPUT1" + value: "OUTPUT1" + } + } + ] +} diff --git a/qa/L0_optional_input/models/optional_identity/1/model.py b/qa/L0_optional_input/models/optional_identity/1/model.py new file mode 100644 index 0000000000..c736ecc3bd --- /dev/null +++ b/qa/L0_optional_input/models/optional_identity/1/model.py @@ -0,0 +1,46 @@ +# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import triton_python_backend_utils as pb_utils + + +class TritonPythonModel: + def execute(self, requests): + """ + Identity model in Python backend. + """ + responses = [] + for request in requests: + for tidx in ("0", "1"): + input_tensor = pb_utils.get_input_tensor_by_name( + request, "INPUT" + tidx + ) + if input_tensor is not None: + out_tensor = pb_utils.Tensor( + "OUTPUT" + tidx, input_tensor.as_numpy() + ) + responses.append(pb_utils.InferenceResponse([out_tensor])) + return responses diff --git a/qa/L0_optional_input/models/optional_identity/config.pbtxt b/qa/L0_optional_input/models/optional_identity/config.pbtxt new file mode 100644 index 0000000000..0c73fd7ca5 --- /dev/null +++ b/qa/L0_optional_input/models/optional_identity/config.pbtxt @@ -0,0 +1,53 @@ +# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +backend: "python" +max_batch_size: 4 +input [ + { + name: "INPUT0" + data_type: TYPE_FP32 + dims: [ 1 ] + optional: true + }, + { + name: "INPUT1" + data_type: TYPE_FP32 + dims: [ 1 ] + optional: true + } +] +output [ + { + name: "OUTPUT0" + data_type: TYPE_FP32 + dims: [ 1 ] + }, + { + name: "OUTPUT1" + data_type: TYPE_FP32 + dims: [ 1 ] + } +] diff --git a/qa/L0_optional_input/optional_input_test.py b/qa/L0_optional_input/optional_input_test.py index 308efebf45..c1fd114d6b 100755 --- a/qa/L0_optional_input/optional_input_test.py +++ b/qa/L0_optional_input/optional_input_test.py @@ -142,13 +142,8 @@ def check_status(self, model_name, batch_exec, request_cnt, infer_cnt): stats = self.triton_client_.get_inference_statistics(model_name, "1") self.assertEqual(len(stats.model_stats), 1, "expect 1 model stats") actual_exec_cnt = stats.model_stats[0].execution_count - if actual_exec_cnt == exec_cnt: + if stats.model_stats[0].execution_count > 0: break - print( - "WARNING: expect {} executions, got {} (attempt {})".format( - exec_cnt, actual_exec_cnt, i - ) - ) time.sleep(1) self.assertEqual( @@ -411,6 +406,40 @@ def test_ensemble_optional_pipeline(self): except Exception as ex: self.assertTrue(False, "unexpected error {}".format(ex)) + def test_ensemble_optional_connecting_tensor(self): + # The ensemble is a special case of pipelining models with optional + # inputs, where the request will only produce a subset of inputs + # for the second model while the ensemble graph connects all inputs of + # the second model (which is valid because the not-provided inputs + # are marked optional). See 'config.pbtxt' for detail. + self.model_name_ = "optional_connecting_tensor" + + # Provide all inputs, send requests that don't form preferred batch + # so all requests should be returned after the queue delay + try: + provided_inputs = ("INPUT0",) + inputs = [] + outputs = [] + for provided_input in provided_inputs: + inputs.append(self.inputs_[provided_input]) + outputs.append(self.outputs_[provided_input]) + + triton_client = grpcclient.InferenceServerClient("localhost:8001") + results = triton_client.infer( + model_name=self.model_name_, inputs=inputs, outputs=outputs + ) + + expected = self.input_data_["INPUT0"] + output_data = results.as_numpy("OUTPUT0") + self.assertTrue( + np.array_equal(output_data, expected), + "{}, {}, expected: {}, got {}".format( + self.model_name_, "OUTPUT0", expected, output_data + ), + ) + except Exception as ex: + self.assertTrue(False, "unexpected error {}".format(ex)) + if __name__ == "__main__": unittest.main() diff --git a/qa/L0_optional_input/test.sh b/qa/L0_optional_input/test.sh index 351be38d4d..8bfd113d32 100755 --- a/qa/L0_optional_input/test.sh +++ b/qa/L0_optional_input/test.sh @@ -41,6 +41,7 @@ rm -fr *.log mkdir -p ./models/identity_2_float32/1 mkdir -p ./models/ensemble_identity_2_float32/1 mkdir -p ./models/pipeline_identity_2_float32/1 +mkdir -p ./models/optional_connecting_tensor/1 # Basic test cases TEST_CASES=${TEST_CASES:="test_all_inputs \ @@ -51,8 +52,9 @@ TEST_CASES=${TEST_CASES:="test_all_inputs \ test_ensemble_optional_same_input \ test_ensemble_optional_mix_inputs \ test_ensemble_optional_mix_inputs_2 \ - test_ensemble_optional_pipeline"} - + test_ensemble_optional_pipeline \ + test_ensemble_optional_connecting_tensor"} +RET=0 for i in $TEST_CASES ; do # Restart server for every test to clear model stats run_server @@ -62,8 +64,6 @@ for i in $TEST_CASES ; do exit 1 fi - RET=0 - echo "Test: $i" >>$TEST_LOG set +e