Audio PR - Augmentation support [ Downmix and ToDecibels ]

CHANGELOG.md

@@ -17,6 +17,11 @@
 * Pytorch iterator for Audio
 * Python audio unit test, and support to verify outputs
 * rocDecode for HW decode
+* Support for Audio augmentation - PreEmphasis filter
+* Support for reading from file lists in file reader
+* Support for Audio augmentation - Spectrogram
+* Support for Audio augmentation - ToDecibels
+* Support for downmixing audio channels during decoding
 
 ### Optimizations
 

rocAL/include/api/rocal_api_augmentation.h

@@ -1098,4 +1098,69 @@ extern "C" RocalTensor ROCAL_API_CALL rocalSSDRandomCrop(RocalContext context, R
                                                          RocalTensorLayout output_layout = ROCAL_NONE,
                                                          RocalTensorOutputType output_datatype = ROCAL_UINT8);
 
+/*! \brief Applies preemphasis filter to the input data.
+ * \ingroup group_rocal_augmentations
+ * \param [in] context Rocal context
+ * \param [in] input Input Rocal tensor
+ * \param [in] is_output Sets to True if the output tensor is part of the graph output
+ * \param [in] preemph_coeff Preemphasis coefficient
+ * \param [in] preemph_border_type Border value policy. Possible values are "zero", "clamp", "reflect".
+ * \param [in] output_datatype The data type of the output tensor
+ * \return RocalTensor
+ */
+extern "C" RocalTensor ROCAL_API_CALL rocalPreEmphasisFilter(RocalContext context,
+                                                             RocalTensor input,
+                                                             bool is_output,
+                                                             RocalFloatParam preemph_coeff = NULL,
+                                                             RocalAudioBorderType preemph_border_type = RocalAudioBorderType::ROCAL_CLAMP,
+                                                             RocalTensorOutputType output_datatype = ROCAL_FP32);
+
+/*! \brief Produces a spectrogram from a 1D audio signal.
+ * \ingroup group_rocal_augmentations
+ * \param [in] context Rocal context
+ * \param [in] input Input Rocal tensor
+ * \param [in] is_output is the output tensor part of the graph output
+ * \param [in] window_fn values of the window function
+ * \param [in] center_windows boolean value to specify whether extracted windows should be padded so that the window function is centered at multiples of window_step
+ * \param [in] reflect_padding Indicates the padding policy when sampling outside the bounds of the audio data
+ * \param [in] spectrogram_layout output spectrogram layout
+ * \param [in] power Exponent of the magnitude of the spectrum
+ * \param [in] nfft Size of the Fast Fourier transform (FFT)
+ * \param [in] window_length Window size in the number of samples
+ * \param [in] window_step Step between the Short-time Fourier transform (STFT) windows in number of samples
+ * \param [in] output_datatype the data type of the output tensor
+ * \return RocalTensor
+ */
+extern "C" RocalTensor ROCAL_API_CALL rocalSpectrogram(RocalContext context,
+                                                       RocalTensor input,
+                                                       bool is_output,
+                                                       std::vector<float> &window_fn,
+                                                       bool center_windows,
+                                                       bool reflect_padding,
+                                                       int power,
+                                                       int nfft,
+                                                       int window_length = 512,
+                                                       int window_step = 256,
+                                                       RocalTensorLayout output_layout = ROCAL_NFT,
+                                                       RocalTensorOutputType output_datatype = ROCAL_FP32);
+
+/*! \brief A
+ * \ingroup group_rocal_augmentations
+ * \param [in] p_context Rocal context
+ * \param [in] p_input Input Rocal tensor
+ * \param [in] is_output is the output tensor part of the graph output
+ * \param[in] cutoff_db minimum or cut-off ratio in dB
+ * \param[in] multiplier factor by which the logarithm is multiplied
+ * \param[in] reference_magnitude Reference magnitude which if not provided uses maximum value of input as reference
+ * \param [in] rocal_tensor_output_type the data type of the output tensor
+ * \return RocalTensor
+ */
+extern "C" RocalTensor ROCAL_API_CALL rocalToDecibels(RocalContext p_context,
+                                                      RocalTensor p_input,
+                                                      bool is_output,
+                                                      float cutoff_db,
+                                                      float multiplier,
+                                                      float reference_magnitude,
+                                                      RocalTensorOutputType rocal_tensor_output_type);
+
 #endif  // MIVISIONX_ROCAL_API_AUGMENTATION_H

rocAL/include/api/rocal_api_data_loaders.h

@@ -882,7 +882,8 @@ extern "C" RocalTensor ROCAL_API_CALL rocalJpegExternalFileSource(RocalContext p
 /*! Creates Audio file reader and decoder. It allocates the resources and objects required to read and decode audio files stored on the file systems. It has internal sharding capability to load/decode in parallel if user wants.
  * If the files are not in standard audio compression formats they will be ignored, Currently wav format is supported
  * \param [in] context Rocal context
- * \param [in] source_path A NULL terminated char string pointing to the location of files on the disk
+ * \param [in] source_path A NULL terminated char string pointing to the location on the disk
+ * \param [in] source_file_list_path A char string pointing to the file list location on the disk
  * \param [in] shard_count Defines the parallelism level by internally sharding the input dataset and load/decode using multiple decoder/loader instances. Using shard counts bigger than 1 improves the load/decode performance if compute resources (CPU cores) are available.
  * \param [in] is_output Boolean variable to enable the audio to be part of the output.
  * \param [in] shuffle Boolean variable to shuffle the dataset.
@@ -892,6 +893,7 @@ extern "C" RocalTensor ROCAL_API_CALL rocalJpegExternalFileSource(RocalContext p
  */
 extern "C" RocalTensor ROCAL_API_CALL rocalAudioFileSource(RocalContext context,
                                                            const char* source_path,
+                                                           const char* source_file_list_path,
                                                            unsigned shard_count,
                                                            bool is_output,
                                                            bool shuffle = false,
@@ -901,7 +903,8 @@ extern "C" RocalTensor ROCAL_API_CALL rocalAudioFileSource(RocalContext context,
 /*! Creates Audio file reader and decoder. It allocates the resources and objects required to read and decode audio files stored on the file systems. It has internal sharding capability to load/decode in parallel is user wants.
  * If the files are not in standard audio compression formats they will be ignored.
  * \param [in] context Rocal context
- * \param [in] source_path A NULL terminated char string pointing to the location of files on the disk
+ * \param [in] source_path A NULL terminated char string pointing to the location on the disk
+ * \param [in] source_file_list_path A char string pointing to the file list location on the disk
  * \param [in] shard_id Shard id for this loader
  * \param [in] shard_count Defines the parallelism level by internally sharding the input dataset and load/decode using multiple decoder/loader instances. Using shard counts bigger than 1 improves the load/decode performance if compute resources (CPU cores) are available.
  * \param [in] is_output Boolean variable to enable the audio to be part of the output.
@@ -912,6 +915,7 @@ extern "C" RocalTensor ROCAL_API_CALL rocalAudioFileSource(RocalContext context,
  */
 extern "C" RocalTensor ROCAL_API_CALL rocalAudioFileSourceSingleShard(RocalContext p_context,
                                                                       const char* source_path,
+                                                                      const char* source_file_list_path,
                                                                       unsigned shard_id,
                                                                       unsigned shard_count,
                                                                       bool is_output,

rocAL/include/api/rocal_api_meta_data.h

@@ -36,9 +36,10 @@ THE SOFTWARE.
  * \ingroup group_rocal_meta_data
  * \param [in] rocal_context rocal context
  * \param [in] source_path path to the folder that contains the dataset or metadata file
+ * \param file_list_path is the path to file list that contains the file names and its corresponding labels
  * \return RocalMetaData object, can be used to inquire about the rocal's output (processed) tensors
  */
-extern "C" RocalMetaData ROCAL_API_CALL rocalCreateLabelReader(RocalContext rocal_context, const char* source_path);
+extern "C" RocalMetaData ROCAL_API_CALL rocalCreateLabelReader(RocalContext rocal_context, const char* source_path, const char* file_list_path = "");
 
 /*! \brief creates video label reader
  * \ingroup group_rocal_meta_data

rocAL/include/api/rocal_api_types.h

@@ -218,9 +218,20 @@ enum RocalTensorLayout {
     /*! \brief AMD ROCAL_NFCHW
      */
     ROCAL_NFCHW = 3,
+    /*! \brief AMD ROCAL_NHW
+     */
+    ROCAL_NHW = 4,
+    /*! \brief AMD ROCAL_NFT
+     * Spectrogram Layout FT
+     */
+    ROCAL_NFT = 5,
+    /*! \brief AMD ROCAL_NTF
+     * Spectrogram Layout TF
+     */
+    ROCAL_NTF = 6,
     /*! \brief AMD ROCAL_NONE
      */
-    ROCAL_NONE = 4  // Layout for generic tensors (Non-Image or Non-Video)
+    ROCAL_NONE = 7  // Layout for generic tensors (Non-Image or Non-Video)
 };
 
 /*! \brief rocAL Tensor Output Type enum
@@ -238,7 +249,13 @@ enum RocalTensorOutputType {
     ROCAL_UINT8 = 2,
     /*! \brief AMD ROCAL_INT8
      */
-    ROCAL_INT8 = 3
+    ROCAL_INT8 = 3,
+    /*! \brief AMD ROCAL_UINT32
+     */
+    ROCAL_UINT32 = 4,
+    /*! \brief AMD ROCAL_INT32
+     */
+    ROCAL_INT32 = 5
 };
 
 /*! \brief rocAL Decoder Type enum
@@ -377,6 +394,21 @@ enum RocalExternalSourceMode {
     ROCAL_EXTSOURCE_RAW_UNCOMPRESSED = 2,
 };
 
+/*! \brief rocAL Audio Border Type enum
+ * \ingroup group_rocal_types
+ */
+enum RocalAudioBorderType {
+    /*! \brief AMD ROCAL_ZERO
+     */
+    ROCAL_ZERO = 0,
+    /*! \brief AMD ROCAL_CLAMP
+     */
+    ROCAL_CLAMP = 1,
+    /*! \brief AMD ROCAL_REFLECT
+     */
+    ROCAL_REFLECT = 2
+};
+
 /*! \brief Tensor Last Batch Policies
  *  \ingroup group_rocal_types
  */

rocAL/include/augmentations/audio_augmentations/node_downmix.h

@@ -0,0 +1,34 @@
+/*
+Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#pragma once
+#include "pipeline/graph.h"
+#include "pipeline/node.h"
+class DownmixNode : public Node {
+   public:
+    DownmixNode(const std::vector<Tensor *> &inputs, const std::vector<Tensor *> &outputs);
+    DownmixNode() = delete;
+
+   protected:
+    void create_node() override;
+    void update_node() override;
+};

rocAL/include/augmentations/audio_augmentations/node_preemphasis_filter.h

@@ -0,0 +1,44 @@
+/*
+Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#pragma once
+#include "pipeline/graph.h"
+#include "pipeline/node.h"
+#include "parameters/parameter_factory.h"
+#include "parameters/parameter_vx.h"
+#include "rocal_api_types.h"
+
+class PreemphasisFilterNode : public Node {
+    public:
+        PreemphasisFilterNode(const std::vector<Tensor *> &inputs, const std::vector<Tensor *> &outputs);
+        PreemphasisFilterNode() = delete;
+        void init(FloatParam *preemph_coeff, RocalAudioBorderType preemph_border);
+
+    protected:
+        void create_node() override;
+        void update_node() override;
+
+    private:
+        ParameterVX<float> _preemph_coeff;
+        constexpr static float PREEMPH_COEFF_RANGE[2] = {0.97, 0.97};
+        RocalAudioBorderType _preemph_border;
+};

rocAL/include/augmentations/audio_augmentations/node_spectrogram.h

@@ -0,0 +1,58 @@
+/*
+Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#pragma once
+#include "pipeline/graph.h"
+#include "pipeline/node.h"
+#include "rocal_api_types.h"
+
+/// @brief Generates hann window for spectrogram
+/// @param output 
+/// @param window_size 
+inline void hann_window(float *output, int window_size) {
+    double a = (2.0 * M_PI) / window_size;
+    for (int t = 0; t < window_size; t++) {
+        double phase = a * (t + 0.5);
+        output[t] = (0.5 * (1.0 - std::cos(phase)));
+    }
+}
+
+class SpectrogramNode : public Node {
+   public:
+    SpectrogramNode(const std::vector<Tensor *> &inputs, const std::vector<Tensor *> &outputs);
+    SpectrogramNode() = delete;
+    void init(bool is_center_windows, bool is_reflect_padding, int power, int nfft,
+              int window_length, int window_step, std::vector<float> &window_fn);
+
+   protected:
+    void create_node() override;
+    void update_node() override;
+
+   private:
+    std::vector<float> _window_fn;
+    int _power = 2;
+    int _nfft = 2048;
+    int _window_length = 512;
+    int _window_step = 256;
+    bool _is_center_windows = true;
+    bool _is_reflect_padding = true;
+};

rocAL/include/augmentations/audio_augmentations/node_to_decibels.h

@@ -0,0 +1,41 @@
+/*
+Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#pragma once
+#include "pipeline/graph.h"
+#include "pipeline/node.h"
+
+class ToDecibelsNode : public Node {
+   public:
+    ToDecibelsNode(const std::vector<Tensor *> &inputs, const std::vector<Tensor *> &outputs);
+    ToDecibelsNode() = delete;
+    void init(float cutoff_db, float multiplier, float reference_magnitude);
+
+   protected:
+    void create_node() override;
+    void update_node() override;
+
+   private:
+    float _cutoff_db = -200.0;
+    float _multiplier = 10.0;
+    float _reference_magnitude = 0.0;
+};

rocAL/include/augmentations/augmentations_nodes.h

@@ -55,3 +55,6 @@ THE SOFTWARE.
 #include "augmentations/node_copy.h"
 #include "augmentations/node_nop.h"
 #include "augmentations/node_sequence_rearrange.h"
+#include "augmentations/audio_augmentations/node_preemphasis_filter.h"
+#include "augmentations/audio_augmentations/node_spectrogram.h"
+#include "augmentations/audio_augmentations/node_to_decibels.h"

rocAL/include/pipeline/commons.h

@@ -46,6 +46,9 @@ enum class RocalTensorlayout {
     NCHW,
     NFHWC,
     NFCHW,
+    NHW,
+    NFT,
+    NTF,
     NONE
 };
 

rocAL/include/pipeline/master_graph.h

@@ -110,7 +110,7 @@ class MasterGraph {
     std::shared_ptr<T> meta_add_node(std::shared_ptr<M> node);
     Tensor *create_tensor(const TensorInfo &info, bool is_output);
     Tensor *create_loader_output_tensor(const TensorInfo &info);
-    std::vector<rocalTensorList *> create_label_reader(const char *source_path, MetaDataReaderType reader_type);
+    std::vector<rocalTensorList *> create_label_reader(const char *source_path, MetaDataReaderType reader_type, const char *file_list_path = "");
     std::vector<rocalTensorList *> create_video_label_reader(const char *source_path, MetaDataReaderType reader_type, unsigned sequence_length, unsigned frame_step, unsigned frame_stride, bool file_list_frame_num = true);
     std::vector<rocalTensorList *> create_coco_meta_data_reader(const char *source_path, bool is_output, MetaDataReaderType reader_type, MetaDataType label_type, bool ltrb_bbox = true, bool is_box_encoder = false,
                                                                 bool avoid_class_remapping = false, bool aspect_ratio_grouping = false, bool is_box_iou_matcher = false, float sigma = 0.0, unsigned pose_output_width = 0, unsigned pose_output_height = 0);