Merge pull request #10 from NVIDIA-ISAAC-ROS/release-dp3.1

Isaac ROS 0.31.0 (DP3.1)
NVIDIA-ISAAC-ROS · May 26, 2023 · f8384ac · f8384ac
2 parents 843ecc5 + 156c5c2
commit f8384ac
Show file tree

Hide file tree

Showing 13 changed files with 149 additions and 127 deletions.
diff --git a/README.md b/README.md
@@ -36,10 +36,11 @@ This package is powered by [NVIDIA Isaac Transport for ROS (NITROS)](https://dev
 
 The following table summarizes the per-platform performance statistics of sample graphs that use this package, with links included to the full benchmark output. These benchmark configurations are taken from the [Isaac ROS Benchmark](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark#list-of-isaac-ros-benchmarks) collection, based on the [`ros2_benchmark`](https://github.com/NVIDIA-ISAAC-ROS/ros2_benchmark) framework.
 
-| Sample Graph                                                                                                                            | Input Size | AGX Orin                                                                                                                                 | Orin NX                                                                                                                                 | x86_64 w/ RTX 3060 Ti                                                                                                                             |
-| --------------------------------------------------------------------------------------------------------------------------------------- | ---------- | ---------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------- |
-| [DNN Stereo Disparity Node](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/scripts//isaac_ros_ess_node.py)   | 1080p      | [63.6 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_ess_node-agx_orin.json)<br>2.6 ms | [24.5 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_ess_node-orin_nx.json)<br>3.0 ms | [131 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_ess_node-x86_64_rtx_3060Ti.json)<br>0.73 ms |
-| [DNN Stereo Disparity Graph](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/scripts//isaac_ros_ess_graph.py) | 1080p      | [52.7 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_ess_graph-agx_orin.json)<br>20 ms | [20.8 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_ess_graph-orin_nx.json)<br>50 ms | [116 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_ess_graph-x86_64_rtx_3060Ti.json)<br>8.7 ms |
+| Sample Graph                                                                                                                            | Input Size | AGX Orin                                                                                                                                 | Orin NX                                                                                                                                 | x86_64 w/ RTX 4060 Ti                                                                                                                     |
+| --------------------------------------------------------------------------------------------------------------------------------------- | ---------- | ---------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------- |
+| [DNN Stereo Disparity Node](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/scripts//isaac_ros_ess_node.py)   | 1080p      | [63.6 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_ess_node-agx_orin.json)<br>3.5 ms | [24.8 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_ess_node-orin_nx.json)<br>5.0 ms | [173 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_ess_node-nuc_4060ti.json)<br>3.4 ms |
+| [DNN Stereo Disparity Graph](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/scripts//isaac_ros_ess_graph.py) | 1080p      | [52.7 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_ess_graph-agx_orin.json)<br>21 ms | [20.8 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_ess_graph-orin_nx.json)<br>50 ms | [156 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_ess_graph-nuc_4060ti.json)<br>10 ms |
+
 
 ## Table of Contents
 
@@ -81,7 +82,7 @@ The following table summarizes the per-platform performance statistics of sample
 
 ## Latest Update
 
-Update 2023-04-05: Source available GXF extensions
+Update 2023-05-25: Upgraded model (1.1.0).
 
 ## Supported Platforms
 
@@ -143,7 +144,7 @@ To simplify development, we strongly recommend leveraging the Isaac ROS Dev Dock
 
    ```bash
    cd /workspaces/isaac_ros-dev/src/isaac_ros_dnn_stereo_disparity/resources && \
-   wget 'https://api.ngc.nvidia.com/v2/models/nvidia/isaac/dnn_stereo_disparity/versions/1.0.0/files/ess.etlt'
+   wget 'https://api.ngc.nvidia.com/v2/models/nvidia/isaac/dnn_stereo_disparity/versions/1.1.0/files/ess.etlt'
    ```
 
 6. Convert the encrypted model (`.etlt`) to a TensorRT engine plan:
@@ -302,6 +303,7 @@ Make sure that the ROS bag has a reasonable size and publish rate.
 
 | Date       | Changes                                    |
 | ---------- | ------------------------------------------ |
+| 2023-05-25 | Upgraded model (1.1.0)                     |
 | 2023-04-05 | Source available GXF extensions            |
 | 2022-10-19 | Updated OSS licensing                      |
 | 2022-08-31 | Update to be compatible with JetPack 5.0.2 |

diff --git a/isaac_ros_ess/config/ess_inference.yaml b/isaac_ros_ess/config/ess_inference.yaml
@@ -1,6 +1,6 @@
 %YAML 1.2
 # SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES
-# Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -173,7 +173,7 @@ components:
 - name: disparity_in
   type: nvidia::gxf::DoubleBufferReceiver
   parameters:
-    capacity: 1
+    capacity: 2
     policy: 0
 - type: nvidia::gxf::MessageAvailableSchedulingTerm
   parameters:
@@ -182,7 +182,7 @@ components:
 - name: left_cam_receiver
   type: nvidia::gxf::DoubleBufferReceiver
   parameters:
-    capacity: 1
+    capacity: 2
     policy: 0
 - type: nvidia::gxf::MessageAvailableSchedulingTerm
   parameters:
@@ -191,7 +191,7 @@ components:
 - name: right_cam_receiver
   type: nvidia::gxf::DoubleBufferReceiver
   parameters:
-    capacity: 1
+    capacity: 2
     policy: 0
 - type: nvidia::gxf::MessageAvailableSchedulingTerm
   parameters:
@@ -221,7 +221,7 @@ components:
     min_disparity: 0
     max_disparity: -1
 ---
-name: vault
+name: sink
 components:
 - name: signal
   type: nvidia::gxf::DoubleBufferReceiver
@@ -232,12 +232,10 @@ components:
   parameters:
     receiver: signal
     min_size: 1
-- name: vault
-  type: nvidia::gxf::Vault
+- name: sink
+  type: nvidia::isaac_ros::MessageRelay
   parameters:
     source: signal
-    max_waiting_count: 1
-    drop_waiting: false
 ---
 name: connections
 components:
@@ -264,12 +262,17 @@ components:
 - type: nvidia::gxf::Connection
   parameters:
     source: disparity_compositor/data_transmitter
-    target: vault/signal
+    target: sink/signal
 ---
 components:
 - name: clock
   type: nvidia::gxf::RealtimeClock
-- type: nvidia::gxf::GreedyScheduler
+- type: nvidia::gxf::MultiThreadScheduler
   parameters:
     clock: clock
     stop_on_deadlock: false
+    check_recession_period_ms: 1
+    worker_thread_number: 2
+- type: nvidia::gxf::JobStatistics
+  parameters:
+    clock: clock
diff --git a/isaac_ros_ess/gxf/ess/CMakeLists.txt b/isaac_ros_ess/gxf/ess/CMakeLists.txt
@@ -35,15 +35,36 @@ find_package(TENSORRT)
 
 # Create extension
 add_library(gxf_cvcore_ess SHARED
+  TensorOps.cpp
   ESS.cpp
   extensions/ess/ESS.hpp
-  ESSRegistry.cpp
 
+  extensions/tensor_ops/CameraModel.cpp
+  extensions/tensor_ops/CameraModel.hpp
+  extensions/tensor_ops/ConvertColorFormat.cpp
+  extensions/tensor_ops/ConvertColorFormat.hpp
+  extensions/tensor_ops/CropAndResize.cpp
+  extensions/tensor_ops/CropAndResize.hpp
+  extensions/tensor_ops/Frame3D.cpp
+  extensions/tensor_ops/Frame3D.hpp
   extensions/tensor_ops/ImageAdapter.cpp
   extensions/tensor_ops/ImageAdapter.hpp
   extensions/tensor_ops/ImageUtils.cpp
   extensions/tensor_ops/ImageUtils.hpp
-
+  extensions/tensor_ops/InterleavedToPlanar.cpp
+  extensions/tensor_ops/InterleavedToPlanar.hpp
+  extensions/tensor_ops/Normalize.cpp
+  extensions/tensor_ops/Normalize.hpp
+  extensions/tensor_ops/Reshape.cpp
+  extensions/tensor_ops/Reshape.hpp
+  extensions/tensor_ops/Resize.cpp
+  extensions/tensor_ops/Resize.hpp
+  extensions/tensor_ops/TensorOperator.cpp
+  extensions/tensor_ops/TensorOperator.hpp
+  extensions/tensor_ops/TensorStream.cpp
+  extensions/tensor_ops/TensorStream.hpp
+  extensions/tensor_ops/Undistort.cpp
+  extensions/tensor_ops/Undistort.hpp
   extensions/tensor_ops/detail/ImageAdapterTensorImpl.cpp
   extensions/tensor_ops/detail/ImageAdapterTensorImpl.hpp
   extensions/tensor_ops/detail/ImageAdapterVideoBufferImpl.cpp

diff --git a/isaac_ros_ess/gxf/ess/ESSRegistry.cpp b/isaac_ros_ess/gxf/ess/ESSRegistry.cpp
diff --git a/isaac_ros_ess/gxf/ess/TensorOps.cpp b/isaac_ros_ess/gxf/ess/TensorOps.cpp
@@ -0,0 +1,95 @@
+// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES
+// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// SPDX-License-Identifier: Apache-2.0
+#include "extensions/ess/ESS.hpp"
+#include "extensions/tensor_ops/CameraModel.hpp"
+#include "extensions/tensor_ops/ConvertColorFormat.hpp"
+#include "extensions/tensor_ops/CropAndResize.hpp"
+#include "extensions/tensor_ops/Frame3D.hpp"
+#include "extensions/tensor_ops/ImageAdapter.hpp"
+#include "extensions/tensor_ops/InterleavedToPlanar.hpp"
+#include "extensions/tensor_ops/Normalize.hpp"
+#include "extensions/tensor_ops/Reshape.hpp"
+#include "extensions/tensor_ops/Resize.hpp"
+#include "extensions/tensor_ops/TensorOperator.hpp"
+#include "extensions/tensor_ops/TensorStream.hpp"
+#include "extensions/tensor_ops/Undistort.hpp"
+
+#include "gxf/std/extension_factory_helper.hpp"
+
+GXF_EXT_FACTORY_BEGIN()
+GXF_EXT_FACTORY_SET_INFO(
+  0x6eae64ff97a94d9b, 0xb324f85e6a98a75a, "NvCvTensorOpsExtension",
+  "Generic CVCORE tensor_ops interfaces", "Nvidia_Gxf", "3.1.0", "LICENSE");
+
+GXF_EXT_FACTORY_ADD(
+  0xd073a92344ba4b82, 0xbd0f18f4996048e8, nvidia::cvcore::tensor_ops::CameraModel,
+  nvidia::gxf::Component,
+  "Construct Camera distortion model / Camera intrinsic compatible with CVCORE");
+
+GXF_EXT_FACTORY_ADD(
+  0x6c9419223e4b4c2c, 0x899a4d65279c6508, nvidia::cvcore::tensor_ops::Frame3D,
+  nvidia::gxf::Component,
+  "Construct Camera extrinsic compatible with CVCORE");
+
+GXF_EXT_FACTORY_ADD(
+  0xd94385e5b35b4635, 0x9adb0d214a3865f7, nvidia::cvcore::tensor_ops::TensorStream,
+  nvidia::gxf::Component, "Wrapper of CVCORE ITensorOperatorStream/ITensorOperatorContext");
+
+GXF_EXT_FACTORY_ADD(
+  0xd0c4ddad486a4a92, 0xb69c8a5304b205ea, nvidia::cvcore::tensor_ops::ImageAdapter,
+  nvidia::gxf::Component, "Utility component for conversion between message and cvcore image type");
+
+GXF_EXT_FACTORY_ADD(
+  0xadebc792bd0b4a57, 0x99c1405fd2ea0728, nvidia::cvcore::tensor_ops::StreamUndistort,
+  nvidia::gxf::Codelet, "Codelet for stream image undistortion in tensor_ops");
+
+GXF_EXT_FACTORY_ADD(
+  0xa58141ac7eca4ea6, 0x9b545446fe379a12, nvidia::cvcore::tensor_ops::Resize, nvidia::gxf::Codelet,
+  "Codelet for image resizing in tensor_ops");
+
+GXF_EXT_FACTORY_ADD(
+  0xeb8b5f5b36d44b49, 0x81f959fd28e6f678, nvidia::cvcore::tensor_ops::StreamResize,
+  nvidia::gxf::Codelet, "Codelet for stream image resizing in tensor_ops");
+
+GXF_EXT_FACTORY_ADD(
+  0x4a7ff422de3841bd, 0x9e743ac10d9294b7, nvidia::cvcore::tensor_ops::CropAndResize,
+  nvidia::gxf::Codelet, "Codelet for crop and resizing operation in tensor_ops");
+
+GXF_EXT_FACTORY_ADD(
+  0x7018f0b9034c462c, 0xa9fbaf7ee012974a, nvidia::cvcore::tensor_ops::Normalize,
+  nvidia::gxf::Codelet,
+  "Codelet for image normalization in tensor_ops");
+
+GXF_EXT_FACTORY_ADD(
+  0x269d4237f3c3479e, 0xbcca9ecc44c71a71, nvidia::cvcore::tensor_ops::InterleavedToPlanar,
+  nvidia::gxf::Codelet, "Codelet for convert interleaved image to planar image in tensor_ops");
+
+GXF_EXT_FACTORY_ADD(
+  0xfc4d7b4d8fcc4dab, 0xa286056e0fcafa79, nvidia::cvcore::tensor_ops::ConvertColorFormat,
+  nvidia::gxf::Codelet, "Codelet for image color conversion in tensor_ops");
+
+GXF_EXT_FACTORY_ADD(
+  0x5ab4a4d8f7a34553, 0xa90be52660b076fe, nvidia::cvcore::tensor_ops::StreamConvertColorFormat,
+  nvidia::gxf::Codelet, "Codelet for stream image color conversion in tensor_ops");
+
+GXF_EXT_FACTORY_ADD(
+  0x26789b7d5a8d4e85, 0x86b845ec5f4cd12b, nvidia::cvcore::tensor_ops::Reshape, nvidia::gxf::Codelet,
+  "Codelet for image reshape in tensor_ops");
+GXF_EXT_FACTORY_ADD(
+  0x1aa1eea914344aff, 0x97fddaaedb594121, nvidia::cvcore::ESS, nvidia::gxf::Codelet,
+  "ESS GXF Extension");
+GXF_EXT_FACTORY_END()
diff --git a/isaac_ros_ess/gxf/ess/cvcore/include/cv/tensor_ops/Errors.h b/isaac_ros_ess/gxf/ess/cvcore/include/cv/tensor_ops/Errors.h
@@ -22,6 +22,7 @@
 
 namespace cvcore { namespace tensor_ops {
 
+// TODO: Add doxygen style comments for each of the error codes
 enum class TensorOpsErrorCode : std::int32_t
 {
     SUCCESS = 0,

diff --git a/isaac_ros_ess/gxf/ess/cvcore/src/core/utility/ProfileUtils.cpp b/isaac_ros_ess/gxf/ess/cvcore/src/core/utility/ProfileUtils.cpp
@@ -66,6 +66,7 @@ std::string GetCPUName()
 #else
 std::string GetCPUName()
 {
+    // TODO: this will only work on linux platform
     std::ifstream cpuInfo("/proc/cpuinfo");
     if (!cpuInfo.good())
     {

diff --git a/isaac_ros_ess/gxf/ess/cvcore/src/tensor_ops/DBScan.cpp b/isaac_ros_ess/gxf/ess/cvcore/src/tensor_ops/DBScan.cpp
@@ -180,7 +180,7 @@ void DBScan::doClusterAndMerge(Array<BBox> &input, Array<BBox> &output, BBoxMerg
     doCluster(input, clusters);
     output.setSize(m_numClusters);
 
-    // merge bboxes based on different modes
+    // merge bboxes based on different modes (TODO: might add mininum/average in the future)
     if (type == MAXIMUM)
     {
         MergeMaximumBBoxes(input, clusters, output);

diff --git a/isaac_ros_ess/gxf/ess/cvcore/src/tensor_ops/vpi/VPITensorOperators.cpp b/isaac_ros_ess/gxf/ess/cvcore/src/tensor_ops/vpi/VPITensorOperators.cpp
@@ -101,7 +101,7 @@ std::error_code CreateVPIImageWrapperImpl(VPIImage &vpiImg, VPIImageData &imgdat
     return make_error_code(vpiStatus);
 }
 
-// helper function to wrap VPI image for planar image types
+// helper function to wrap VPI image for planar image types (TODO: not supported in vpi so far)
 template<ImageType T, typename std::enable_if<IsPlanarImage<T>::value>::type * = nullptr>
 std::error_code CreateVPIImageWrapperImpl(VPIImage &vpiImg, VPIImageData &imgdata, const Image<T> &cvcoreImage, VPIBackend backend)
 {
@@ -224,6 +224,7 @@ std::error_code DestroyVPIImageWrapper(VPIImage &image, VPIImageData &imageWrap)
 }
 std::error_code VPITensorStream::Status() noexcept
 {
+    // TODO Needs to be updated for supporting non blocking function calls.
     return ErrorCode::SUCCESS;
 }
 

diff --git a/isaac_ros_ess/gxf/ess/cvcore/src/trtbackend/TRTBackend.cpp b/isaac_ros_ess/gxf/ess/cvcore/src/trtbackend/TRTBackend.cpp
@@ -458,6 +458,7 @@ void TRTImpl::loadFromMemoryPointer(void *engine)
 
 void TRTImpl::setupIO(int batchSize)
 {
+    // @TODO: use getBindingDimensions to avoid re-setting the IO.
     m_bindingsCount = m_inferenceEngine->getNbBindings();
     for (int i = 0; i < m_bindingsCount; i++)
     {
@@ -542,6 +543,7 @@ void TRTBackend::infer(void **buffer)
 
 void TRTBackend::infer(void **buffer, int batchSize, cudaStream_t stream)
 {
+    //@TODO: fix kMin, kOpt, kMax batch size in SetupProfile() call and then add a check here.
     m_pImpl->setupIO(batchSize);
 
     bool success = true;