From 98994c6b81751c41ed64f7a32c2cd7b044abc24d Mon Sep 17 00:00:00 2001 From: xaviliz Date: Mon, 16 Jun 2025 10:19:03 +0200 Subject: [PATCH 01/60] Add initial script for building onnx-runtime. --- packaging/build_config.sh | 1 + packaging/debian_3rdparty/build_onnx.sh | 34 +++++++++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 packaging/debian_3rdparty/build_onnx.sh diff --git a/packaging/build_config.sh b/packaging/build_config.sh index 5ce5fffc3..b44590efb 100755 --- a/packaging/build_config.sh +++ b/packaging/build_config.sh @@ -28,6 +28,7 @@ CHROMAPRINT_VERSION=1.4.3 QT_SOURCE_URL=https://download.qt.io/archive/qt/4.8/4.8.4/qt-everywhere-opensource-src-4.8.4.tar.gz GAIA_VERSION=2.4.6-86-ged433ed TENSORFLOW_VERSION=2.5.0 +LIBONNXRUNTIME_VERSION=1.21.1 FFMPEG_AUDIO_FLAGS=" diff --git a/packaging/debian_3rdparty/build_onnx.sh b/packaging/debian_3rdparty/build_onnx.sh new file mode 100644 index 000000000..aa2c60d65 --- /dev/null +++ b/packaging/debian_3rdparty/build_onnx.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -e +. ../build_config.sh + +# rm -rf tmp +# mkdir tmp +cd tmp + +# Prerequisites: python>=3.10 +# cmake>=3.28 + +echo "Building onnxruntime $LIBONNXRUNTIME_VERSION" + +#curl -SLO "https://github.com/microsoft/onnxruntime/archive/refs/tags/v$LIBONNXRUNTIME_VERSION.tar.gz" +#! this file has an issue https://github.com/microsoft/onnxruntime/issues/24861 +#! it is fixed manually for testing https://github.com/microsoft/onnxruntime/commit/f57db79743c4d1a3553aa05cf95bcd10966030e6 +#! should be done in three-steps: first, downloading the package, editing the deps.txt file and running the script +#! in the main branch it is fixed, it should be replaced when a release is available. + +#tar -xf v$LIBONNXRUNTIME_VERSION.tar.gz +cd onnxruntime-$LIBONNXRUNTIME_VERSION + +python3 -m pip install cmake + +# compile library with cmake +./build.sh \ + --config RelWithDebInfo \ + --build_shared_lib \ + --parallel \ + --compile_no_warning_as_error \ + --skip_submodule_sync + +cd ../.. +# rm -fr tmp From 4e92dc06019b78d2be1a100bbfc5052da1b5ecb6 Mon Sep 17 00:00:00 2001 From: Xavier Lizarraga Date: Thu, 19 Jun 2025 15:56:14 +0200 Subject: [PATCH 02/60] Modify wscript for onnx flag. --- src/wscript | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/wscript b/src/wscript index ab86dc469..dcb8138c9 100644 --- a/src/wscript +++ b/src/wscript @@ -21,7 +21,9 @@ lib_map = { 'FFTW': 'fftw3f', 'LIBCHROMAPRINT': 'libchromaprint', 'GAIA2': 'gaia2', - 'TENSORFLOW': 'tensorflow'} + 'TENSORFLOW': 'tensorflow', + 'ONNXRUNTIME': 'onnxruntime', + } def options(ctx): @@ -200,6 +202,10 @@ def configure(ctx): if 'tensorflow' in ctx.env.CHECK_LIBS: ctx.check_cfg(package=lib_map['TENSORFLOW'], uselib_store='TENSORFLOW', args=check_cfg_args, mandatory=True) + + if 'onnxruntime' in ctx.env.CHECK_LIBS: + ctx.check_cfg(package=lib_map['ONNXRUNTIME'], uselib_store='ONNXRUNTIME', + args=check_cfg_args, mandatory=True) # needed by ffmpeg for the INT64_C macros ctx.env.DEFINES += ['__STDC_CONSTANT_MACROS'] From d673012508b9aae61f8e450799f8bed703862d88 Mon Sep 17 00:00:00 2001 From: xaviliz Date: Fri, 20 Jun 2025 12:09:27 +0200 Subject: [PATCH 03/60] Define algos for onnx-runtime and create --with-onnx flag. --- src/wscript | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/wscript b/src/wscript index dcb8138c9..05a146c26 100644 --- a/src/wscript +++ b/src/wscript @@ -57,6 +57,9 @@ def options(ctx): ctx.add_option('--with-tensorflow', action='store_true', dest='WITH_TENSORFLOW', default=False, help='build with Tensorflow support') + ctx.add_option('--with-onnx', action='store_true', + dest='WITH_ONNXRUNTIME', default=False, + help='build with Onnx-Runtime support') ctx.add_option('--lightweight', action='store', dest='LIGHTWEIGHT', default=False, help='build lightweight version with specified dependencies (comma separated: =' + ','.join(default_libs) + ')') @@ -112,6 +115,9 @@ def configure(ctx): if ctx.env.WITH_TENSORFLOW: ctx.env.CHECK_LIBS.append('tensorflow') + if ctx.env.WITH_ONNXRUNTIME: + ctx.env.CHECK_LIBS.append('onnxruntime') + if ctx.env.IGNORE_ALGOS: for a in ctx.env.IGNORE_ALGOS.split(","): a = a.strip() @@ -202,7 +208,7 @@ def configure(ctx): if 'tensorflow' in ctx.env.CHECK_LIBS: ctx.check_cfg(package=lib_map['TENSORFLOW'], uselib_store='TENSORFLOW', args=check_cfg_args, mandatory=True) - + if 'onnxruntime' in ctx.env.CHECK_LIBS: ctx.check_cfg(package=lib_map['ONNXRUNTIME'], uselib_store='ONNXRUNTIME', args=check_cfg_args, mandatory=True) @@ -347,6 +353,17 @@ def configure(ctx): print(' The following algorithms will be ignored: %s' % algos) ctx.env.ALGOIGNORE += algos + + algos = [ 'OnnxPredict' ] + if has('onnxruntime'): + print('- OnnxRuntime detected!') + print(' The following algorithms will be included: %s\n' % algos) + ctx.env.USE_LIBS += ' ONNXRUNTIME' + else: + print('- Essentia is configured without Onnx-Runtime.') + print(' The following algorithms will be ignored: %s' % algos) + ctx.env.ALGOIGNORE += algos + lel = len(ctx.env.EXAMPLE_LIST) if lel: print('- Compiling %s example%s' % (lel, "" if lel == 1 else "s")) From 4246f5bc676c03cc23b1f8e1dcc90c8985b1900d Mon Sep 17 00:00:00 2001 From: xaviliz Date: Fri, 20 Jun 2025 12:10:14 +0200 Subject: [PATCH 04/60] Support --with-onnx --- wscript | 1 + 1 file changed, 1 insertion(+) diff --git a/wscript b/wscript index a48d304c8..f9ed031bc 100644 --- a/wscript +++ b/wscript @@ -92,6 +92,7 @@ def configure(ctx): ctx.env.PKG_CONFIG_PATH = ctx.options.PKG_CONFIG_PATH ctx.env.WITH_GAIA = ctx.options.WITH_GAIA ctx.env.WITH_TENSORFLOW = ctx.options.WITH_TENSORFLOW + ctx.env.WITH_ONNXRUNTIME = ctx.options.WITH_ONNXRUNTIME ctx.env.LIGHTWEIGHT = ctx.options.LIGHTWEIGHT ctx.env.EXAMPLES = ctx.options.EXAMPLES ctx.env.EXAMPLE_LIST = [] From 9720e28bb0aa2fce13966d28795fcd454f5fc697 Mon Sep 17 00:00:00 2001 From: xaviliz Date: Mon, 23 Jun 2025 12:00:49 +0200 Subject: [PATCH 05/60] Rename library to be detected by pkg-config --- src/wscript | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/wscript b/src/wscript index 05a146c26..42b5127fb 100644 --- a/src/wscript +++ b/src/wscript @@ -22,7 +22,7 @@ lib_map = { 'LIBCHROMAPRINT': 'libchromaprint', 'GAIA2': 'gaia2', 'TENSORFLOW': 'tensorflow', - 'ONNXRUNTIME': 'onnxruntime', + 'ONNXRUNTIME': 'libonnxruntime', } @@ -211,7 +211,7 @@ def configure(ctx): if 'onnxruntime' in ctx.env.CHECK_LIBS: ctx.check_cfg(package=lib_map['ONNXRUNTIME'], uselib_store='ONNXRUNTIME', - args=check_cfg_args, mandatory=True) + args=['libonnxruntime >= 1.21.1'] + check_cfg_args, mandatory=True) # needed by ffmpeg for the INT64_C macros ctx.env.DEFINES += ['__STDC_CONSTANT_MACROS'] From d60385ebc0abb060024119ad63f51b2fe0ce5d70 Mon Sep 17 00:00:00 2001 From: xaviliz Date: Mon, 23 Jun 2025 14:07:49 +0200 Subject: [PATCH 06/60] Add initial onnxpredict.cpp files. --- .../machinelearning/onnxpredict.cpp | 419 ++++++++++++++++++ src/algorithms/machinelearning/onnxpredict.h | 150 +++++++ 2 files changed, 569 insertions(+) create mode 100644 src/algorithms/machinelearning/onnxpredict.cpp create mode 100644 src/algorithms/machinelearning/onnxpredict.h diff --git a/src/algorithms/machinelearning/onnxpredict.cpp b/src/algorithms/machinelearning/onnxpredict.cpp new file mode 100644 index 000000000..09ab5fe50 --- /dev/null +++ b/src/algorithms/machinelearning/onnxpredict.cpp @@ -0,0 +1,419 @@ +/* + * Copyright (C) 2006-2021 Music Technology Group - Universitat Pompeu Fabra + * + * This file is part of Essentia + * + * Essentia is free software: you can redistribute it and/or modify it under + * the terms of the GNU Affero General Public License as published by the Free + * Software Foundation (FSF), either version 3 of the License, or (at your + * option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the Affero GNU General Public License + * version 3 along with this program. If not, see http://www.gnu.org/licenses/ + */ + +#include "onnxpredict.h" + +using namespace std; +using namespace essentia; +using namespace standard; + +const char* OnnxPredict::name = "OnnxPredict"; +const char* OnnxPredict::category = "Machine Learning"; +const char* OnnxPredict::description = DOC("This algorithm runs a Tensorflow graph and stores the desired output tensors in a pool.\n" +"The Tensorflow graph should be stored in Protocol Buffer (.pb) binary format [1] or as a SavedModel [2], and should contain both the architecture and the weights of the model.\n" +"The parameter `inputs` should contain a list with the names of the input nodes that feed the model. The input Pool should contain the tensors corresponding to each input node stored using Essetia tensors. " +"The pool namespace for each input tensor has to match the input node's name.\n" +"In the same way, the `outputs` parameter should contain the names of the tensors to save. These tensors will be stored inside the output pool under a namespace that matches the tensor's name. " +"TensorFlow nodes return tensors identified as `nodeName:n`, where `n` goes from 0 to the number of outputs of the node - 1. " +"The first output tensor of a node can be referred implicitly (e.g., `nodeName`) or explicitly (e.g., `nodeName:0`), and the subsequent tensors require the specific index (e.g., nodeName:1, nodeName:2). " +"In TensorFlow 2 SavedModels all the outputs of the model are contained in the `PartitionedCall` node (e.g., `PartitionedCall:0`, `PartitionedCall:1`). " +"You can use TenorFlow's `saved_model_cli` to find the relation of outputs and `PartitionedCall` indices, for example:\n" +"\n" +">>> saved_model_cli show --dir SavedModel/ --all\n" +">>> ...\n" +">>> The given SavedModel SignatureDef contains the following output(s):\n" +">>> outputs['activations'] tensor_info:\n" +">>> dtype: DT_FLOAT\n" +">>> shape: (1, 400)\n" +">>> name: PartitionedCall:0\n" +">>> outputs['embeddings'] tensor_info:\n" +">>> dtype: DT_FLOAT\n" +">>> shape: (1, 1280)\n" +">>> name: PartitionedCall:1\n" +">>> ...\n" +"\n" +"To print a list with all the available nodes in the graph set the first element of `outputs` as an empty string (i.e., \"\")." +"\n" +"This algorithm is a wrapper for the Tensorflow C API [3]. The first time it is configured with a non-empty `graphFilename` it will try to load the contained graph and to attach a Tensorflow session to it. " +"The reset method deletes the current session (and the resources attached to it) and creates a new one relying on the available graph. " +"By reconfiguring the algorithm the graph is reloaded and the reset method is called.\n" +"\n" +"References:\n" +" [1] TensorFlow - An open source machine learning library for research and production.\n" +" https://www.tensorflow.org/extend/tool_developers/#protocol_buffers\n\n" +" [2] TensorFlow - An open source machine learning library for research and production.\n" +" https://www.tensorflow.org/guide/saved_model\n\n" +" [3] TensorFlow - An open source machine learning library for research and production.\n" +" https://www.tensorflow.org/api_docs/cc/"); + + +static void DeallocateBuffer(void* data, size_t) { + free(data); +} + + +void OnnxPredict::configure() { + _savedModel = parameter("savedModel").toString(); + _graphFilename = parameter("graphFilename").toString(); + + if ((_savedModel.empty()) and (_graphFilename.empty()) and (_isConfigured)) { + E_WARNING("OnnxPredict: You are trying to update a valid configuration with invalid parameters. " + "If you want to update the configuration specify a valid `graphFilename` or `savedModel` parameter."); + }; + + // Do not do anything if we did not get a non-empty model name. + if ((_savedModel.empty()) and (_graphFilename.empty())) return; + + _tags = parameter("tags").toVectorString(); + + _inputNames = parameter("inputs").toVectorString(); + _outputNames = parameter("outputs").toVectorString(); + _isTraining = parameter("isTraining").toBool(); + _isTrainingName = parameter("isTrainingName").toString(); + _squeeze = parameter("squeeze").toBool(); + + (_isTrainingName == "") ? _isTrainingSet = false : _isTrainingSet = true; + + _nInputs = _inputNames.size(); + _nOutputs = _outputNames.size(); + + // Allocate input and output tensors. + _inputTensors.resize(_nInputs + int(_isTrainingSet)); + _inputNodes.resize(_nInputs + int(_isTrainingSet)); + + _outputTensors.resize(_nOutputs); + _outputNodes.resize(_nOutputs); + + TF_DeleteGraph(_graph); + _graph = TF_NewGraph(); + + openGraph(); + + _isConfigured = true; + reset(); + + // If the first output name is empty just print out the list of nodes and return. + if (_outputNames[0] == "") { + E_INFO(availableNodesInfo()); + + return; + } + + // Initialize the list of input and output node names. + for (size_t i = 0; i < _nInputs; i++) { + _inputNodes[i] = graphOperationByName(_inputNames[i]); + } + + for (size_t i = 0; i < _nOutputs; i++) { + _outputNodes[i] = graphOperationByName(_outputNames[i]); + } + + // Add isTraining flag if needed. + if (_isTrainingSet) { + const int64_t dims[1] = {}; + TF_Tensor *isTraining = TF_AllocateTensor(TF_BOOL, dims, 0, 1); + void* isTrainingValue = TF_TensorData(isTraining); + + if (isTrainingValue == NULL) { + TF_DeleteTensor(isTraining); + throw EssentiaException("OnnxPredict: Error generating training phase flag"); + } + + memcpy(isTrainingValue, &_isTraining, sizeof(bool)); + + _inputTensors[_nInputs] = isTraining; + _inputNodes[_nInputs] = graphOperationByName(_isTrainingName); + } +} + + +void OnnxPredict::openGraph() { + // Prioritize savedModel when both are specified. + if (!_savedModel.empty()) { + std::vector tags_c; + tags_c.reserve(_tags.size()); + for (size_t i = 0; i < _tags.size(); i++) { + tags_c.push_back(const_cast(_tags[i].c_str())); + } + + TF_LoadSessionFromSavedModel(_sessionOptions, _runOptions, + _savedModel.c_str(), &tags_c[0], (int)tags_c.size(), + _graph, NULL, _status); + + if (TF_GetCode(_status) != TF_OK) { + throw EssentiaException("OnnxPredict: Error importing SavedModel specified in the `savedModel` parameter. ", TF_Message(_status)); + } + + E_INFO("OnnxPredict: Successfully loaded SavedModel: `" << _savedModel << "`"); + + return; + } + + if (!_graphFilename.empty()) { + // First we load and initialize the model. + const auto f = fopen(_graphFilename.c_str(), "rb"); + if (f == NULL) { + throw EssentiaException( + "OnnxPredict: could not open the Tensorflow graph file."); + } + + fseek(f, 0, SEEK_END); + const auto fsize = ftell(f); + fseek(f, 0, SEEK_SET); + + // Graph size sanity check. + if (fsize < 1) { + fclose(f); + throw(EssentiaException("OnnxPredict: Graph file is empty.")); + } + + // Reserve memory and read the graph. + const auto data = malloc(fsize); + fread(data, fsize, 1, f); + fclose(f); + + TF_Buffer* buffer = TF_NewBuffer(); + buffer->data = data; + buffer->length = fsize; + buffer->data_deallocator = DeallocateBuffer; + + TF_GraphImportGraphDef(_graph, buffer, _options, _status); + + TF_DeleteBuffer(buffer); + + if (TF_GetCode(_status) != TF_OK) { + throw EssentiaException("OnnxPredict: Error importing graph. ", TF_Message(_status)); + } + + E_INFO("OnnxPredict: Successfully loaded graph file: `" << _graphFilename << "`"); + } +} + + +void OnnxPredict::reset() { + if (!_isConfigured) return; + + TF_CloseSession(_session, _status); + if (TF_GetCode(_status) != TF_OK) { + throw EssentiaException("OnnxPredict: Error closing session. ", TF_Message(_status)); + } + + TF_DeleteSession(_session, _status); + if (TF_GetCode(_status) != TF_OK) { + throw EssentiaException("OnnxPredict: Error deleting session. ", TF_Message(_status)); + } + + _session = TF_NewSession(_graph, _sessionOptions, _status); + if (TF_GetCode(_status) != TF_OK) { + throw EssentiaException("OnnxPredict: Error creating new session after reset. ", TF_Message(_status)); + } +} + + +void OnnxPredict::compute() { + if (!_isConfigured) { + throw EssentiaException("OnnxPredict: This algorithm is not configured. To configure this algorithm you " + "should specify a valid `graphFilename` or `savedModel` as input parameter."); + } + + const Pool& poolIn = _poolIn.get(); + Pool& poolOut = _poolOut.get(); + + // Parse the input tensors from the pool into Tensorflow tensors. + for (size_t i = 0; i < _nInputs; i++) { + const Tensor& inputData = + poolIn.value >(_inputNames[i]); + _inputTensors[i] = TensorToTF(inputData); + } + + // Initialize output tensors. + for (size_t i = 0; i < _nOutputs; i++) { + _outputTensors[i] = NULL; + } + + // Run the Tensorflow session. + TF_SessionRun(_session, + NULL, // Run options. + &_inputNodes[0], // Input node names. + &_inputTensors[0], // input tensor values. + _nInputs + (int)_isTrainingSet, // Number of inputs. + &_outputNodes[0], // Output node names. + &_outputTensors[0], // Output tensor values. + _nOutputs, // Number of outputs. + NULL, 0, // Target operations, number of targets. + NULL, // Run metadata. + _status // Output status. + ); + + if (TF_GetCode(_status) != TF_OK) { + throw EssentiaException("OnnxPredict: Error running the Tensorflow session. ", TF_Message(_status)); + } + + // Copy the desired tensors into the output pool. + for (size_t i = 0; i < _nOutputs; i++) { + poolOut.set(_outputNames[i], TFToTensor(_outputTensors[i], _outputNodes[i])); + } + + // Deallocate tensors. + for (size_t i = 0; i < _nInputs; i++) { + TF_DeleteTensor(_inputTensors[i]); + } + + for (size_t i = 0; i < _nOutputs; i++) { + TF_DeleteTensor(_outputTensors[i]); + } +} + + +TF_Tensor* OnnxPredict::TensorToTF( + const Tensor& tensorIn) { + int dims = 1; + vector shape; + + // With squeeze, the Batch dimension is the only one allowed to be singleton + shape.push_back((int64_t)tensorIn.dimension(0)); + + if (_squeeze) { + for(int i = 1; i < tensorIn.rank(); i++) { + if (tensorIn.dimension(i) > 1) { + shape.push_back((int64_t)tensorIn.dimension(i)); + dims++; + } + } + + // There should be at least 2 dimensions (batch, data) + if (dims == 1) { + shape.push_back((int64_t) 1); + dims++; + } + + } else { + dims = tensorIn.rank(); + for(int i = 1; i < dims; i++) { + shape.push_back((int64_t)tensorIn.dimension(i)); + } + } + + TF_Tensor* tensorOut = TF_AllocateTensor( + TF_FLOAT, &shape[0], dims, + (size_t)tensorIn.size() * sizeof(Real)); + + if (tensorOut == NULL) { + throw EssentiaException("OnnxPredict: Error generating input tensor."); + } + + // Get a pointer to the data and fill the tensor. + void* tensorData = TF_TensorData(tensorOut); + + if (tensorData == NULL) { + TF_DeleteTensor(tensorOut); + throw EssentiaException("OnnxPredict: Error generating input tensors data."); + } + + memcpy(tensorData, tensorIn.data(), + std::min(tensorIn.size() * sizeof(Real), + TF_TensorByteSize(tensorOut))); + + return tensorOut; +} + + +const Tensor OnnxPredict::TFToTensor( + const TF_Tensor* tensor, TF_Output node) { + const Real* outputData = static_cast(TF_TensorData(tensor)); + + // Get the output tensor's shape. + size_t outNDims = TF_GraphGetTensorNumDims(_graph, node, _status); + + if (TF_GetCode(_status) != TF_OK) { + throw EssentiaException("OnnxPredict: Error getting the output tensor's shape. ", TF_Message(_status)); + } + + // Create and array to store the shape of the tensor. + array shape {1, 1, 1, 1}; + shape[0] = (int)TF_Dim(tensor, 0); + + // We are assuming one of the following cases: + // 1 - outNDims = 2 -> Batch + Feats + // 2 - outNDims = 3 -> Batch + Timestamps + Feats + // 3 - outNDims = 4 -> Batch + Channels + Timestamps + Feats + size_t idx = 1; + for (size_t i = shape.size() - outNDims + 1; i < shape.size(); i++, idx++) { + shape[i] = (int)TF_Dim(tensor, idx); + } + + // Return a const reference to the data in Eigen format. + return TensorMap(outputData, shape); +} + + +TF_Output OnnxPredict::graphOperationByName(const string nodeName) { + int index = 0; + const char* name = nodeName.c_str(); + string newNodeName; + + // TensorFlow operations (or nodes from the graph perspective) return tensors named , where n goes + // from 0 to the number of outputs. The first output tensor of a node can be extracted implicitly (nodeName) + // or explicitly (nodeName:0). To extract the subsequent tensors, the index has to be specified (nodeName:1, + // nodeName:2, ...). + string::size_type n = nodeName.find(':'); + if (n != string::npos) { + try { + newNodeName = nodeName.substr(0, n); + name = newNodeName.c_str(); + index = stoi(nodeName.substr(n + 1, nodeName.size())); + + } catch (const invalid_argument& ) { + throw EssentiaException("OnnxPredict: `" + nodeName + "` is not a valid node name. Make sure that all " + "your inputs and outputs follow the pattern `nodeName:n`, where `n` in an integer that " + "goes from 0 to the number of outputs of the node - 1."); + } + + } + + TF_Operation* oper = TF_GraphOperationByName(_graph, name); + + TF_Output output = {oper, index}; + + if (output.oper == NULL) { + throw EssentiaException("OnnxPredict: '" + string(nodeName) + + "' is not a valid node name of this graph.\n" + + availableNodesInfo()); + } + + int n_outputs = TF_OperationNumOutputs(oper); + if (index > n_outputs - 1) { + throw EssentiaException("OnnxPredict: Asked for the output with index `" + to_string(index) + + "`, but the node `" + name + "` only has `" + to_string(n_outputs) + "` outputs."); + } + + return output; +} + +vector OnnxPredict::nodeNames() { + size_t pos = 0; + TF_Operation *oper; + vector nodeNames; + + while ((oper = TF_GraphNextOperation(_graph, &pos)) != NULL) { + nodeNames.push_back(string(TF_OperationName(oper))); + } + + return nodeNames; +} diff --git a/src/algorithms/machinelearning/onnxpredict.h b/src/algorithms/machinelearning/onnxpredict.h new file mode 100644 index 000000000..39881ec2d --- /dev/null +++ b/src/algorithms/machinelearning/onnxpredict.h @@ -0,0 +1,150 @@ +/* + * Copyright (C) 2006-2021 Music Technology Group - Universitat Pompeu Fabra + * + * This file is part of Essentia + * + * Essentia is free software: you can redistribute it and/or modify it under + * the terms of the GNU Affero General Public License as published by the Free + * Software Foundation (FSF), either version 3 of the License, or (at your + * option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the Affero GNU General Public License + * version 3 along with this program. If not, see http://www.gnu.org/licenses/ + */ + +#ifndef ESSENTIA_ONNXPREDICT_H +#define ESSENTIA_ONNXPREDICT_H + +#include "algorithm.h" +#include "pool.h" +#include + + +namespace essentia { +namespace standard { + +class OnnxPredict : public Algorithm { + + protected: + Input _poolIn; + Output _poolOut; + + std::string _graphFilename; + std::vector _inputNames; + std::vector _outputNames; + + std::vector _inputTensors; + std::vector _outputTensors; + + std::vector _inputNodes; + std::vector _outputNodes; + + size_t _nInputs; + size_t _nOutputs; + + TF_Graph* _graph; + TF_Status* _status; + TF_ImportGraphDefOptions* _options; + TF_SessionOptions* _sessionOptions; + TF_Session* _session; + + std::string _savedModel; + std::vector _tags; + TF_Buffer* _runOptions; + + bool _isTraining; + bool _isTrainingSet; + std::string _isTrainingName; + + bool _squeeze; + + bool _isConfigured; + + void openGraph(); + TF_Tensor* TensorToTF(const Tensor& tensorIn); + const Tensor TFToTensor(const TF_Tensor* tensor, TF_Output node); + TF_Output graphOperationByName(const std::string nodeName); + std::vector nodeNames(); + + inline std::string availableNodesInfo() { + std::vector nodes = nodeNames(); + std::string info = "OnnxPredict: Available node names are:\n"; + for (std::vector::const_iterator i = nodes.begin(); i != nodes.end() - 1; ++i) info += *i + ", "; + return info + nodes.back() + ".\n\nReconfigure this algorithm with valid node names as inputs and outputs before starting the processing."; + } + + public: + OnnxPredict() : _graph(TF_NewGraph()), _status(TF_NewStatus()), + _options(TF_NewImportGraphDefOptions()), _sessionOptions(TF_NewSessionOptions()), + _session(TF_NewSession(_graph, _sessionOptions, _status)), _runOptions(NULL), + _isConfigured(false) { + declareInput(_poolIn, "poolIn", "the pool where to get the feature tensors"); + declareOutput(_poolOut, "poolOut", "the pool where to store the output tensors"); + } + + ~OnnxPredict(){ + TF_CloseSession(_session, _status); + TF_DeleteSessionOptions(_sessionOptions); + TF_DeleteSession(_session, _status); + TF_DeleteImportGraphDefOptions(_options); + TF_DeleteStatus(_status); + TF_DeleteGraph(_graph); + TF_DeleteBuffer(_runOptions); + } + + void declareParameters() { + const char* defaultTagsC[] = { "serve" }; + std::vector defaultTags = arrayToVector(defaultTagsC); + + declareParameter("graphFilename", "the name of the file from which to load the TensorFlow graph", "", ""); + declareParameter("savedModel", "the name of the TensorFlow SavedModel. Overrides parameter `graphFilename`", "", ""); + declareParameter("tags", "the tags of the savedModel", "", defaultTags); + declareParameter("inputs", "will look for these namespaces in poolIn. Should match the names of the input nodes in the Tensorflow graph", "", Parameter::VECTOR_STRING); + declareParameter("outputs", "will save the tensors on the graph nodes named after `outputs` to the same namespaces in the output pool. Set the first element of this list as an empty array to print all the available nodes in the graph", "", Parameter::VECTOR_STRING); + declareParameter("isTraining", "run the model in training mode (normalized with statistics of the current batch) instead of inference mode (normalized with moving statistics). This only applies to some models", "{true,false}", false); + declareParameter("isTrainingName", "the name of an additional input node indicating whether the model is to be run in a training mode (for models with a training mode, leave it empty otherwise)", "", ""); + declareParameter("squeeze", "remove singleton dimensions of the inputs tensors. Does not apply to the batch dimension", "{true,false}", true); + } + + void configure(); + void compute(); + void reset(); + + static const char* name; + static const char* category; + static const char* description; +}; + +} //namespace standard +} //namespace essentia + + +#include "streamingalgorithmwrapper.h" + +namespace essentia { +namespace streaming { + +class OnnxPredict : public StreamingAlgorithmWrapper { + + protected: + Sink _poolIn; + Source _poolOut; + + public: + OnnxPredict() { + declareAlgorithm("OnnxPredict"); + declareInput(_poolIn, TOKEN, "poolIn"); + declareOutput(_poolOut, TOKEN, "poolOut"); + _poolOut.setBufferType(BufferUsage::forSingleFrames); + } +}; + +} //namespace standard +} //namespace essentia + +#endif // ESSENTIA_ONNXPREDICT_H From 3bb07a92deb09c9aff987e1371566022b92b140c Mon Sep 17 00:00:00 2001 From: xaviliz Date: Tue, 8 Jul 2025 13:23:50 +0200 Subject: [PATCH 07/60] Provide a constructor and solve some errors. --- .../machinelearning/onnxpredict.cpp | 201 +++++++++++++----- src/algorithms/machinelearning/onnxpredict.h | 84 ++++++-- 2 files changed, 205 insertions(+), 80 deletions(-) diff --git a/src/algorithms/machinelearning/onnxpredict.cpp b/src/algorithms/machinelearning/onnxpredict.cpp index 09ab5fe50..f28d790be 100644 --- a/src/algorithms/machinelearning/onnxpredict.cpp +++ b/src/algorithms/machinelearning/onnxpredict.cpp @@ -25,7 +25,9 @@ using namespace standard; const char* OnnxPredict::name = "OnnxPredict"; const char* OnnxPredict::category = "Machine Learning"; -const char* OnnxPredict::description = DOC("This algorithm runs a Tensorflow graph and stores the desired output tensors in a pool.\n" + +// TODO: update description with ONNX +const char* OnnxPredict::description = DOC("This algorithm runs a Onnx graph and stores the desired output tensors in a pool.\n" "The Tensorflow graph should be stored in Protocol Buffer (.pb) binary format [1] or as a SavedModel [2], and should contain both the architecture and the weights of the model.\n" "The parameter `inputs` should contain a list with the names of the input nodes that feed the model. The input Pool should contain the tensors corresponding to each input node stored using Essetia tensors. " "The pool namespace for each input tensor has to match the input node's name.\n" @@ -69,29 +71,27 @@ static void DeallocateBuffer(void* data, size_t) { void OnnxPredict::configure() { - _savedModel = parameter("savedModel").toString(); _graphFilename = parameter("graphFilename").toString(); - if ((_savedModel.empty()) and (_graphFilename.empty()) and (_isConfigured)) { + if ((_graphFilename.empty()) and (_isConfigured)) { E_WARNING("OnnxPredict: You are trying to update a valid configuration with invalid parameters. " - "If you want to update the configuration specify a valid `graphFilename` or `savedModel` parameter."); + "If you want to update the configuration specify a valid `graphFilename` parameter."); }; // Do not do anything if we did not get a non-empty model name. - if ((_savedModel.empty()) and (_graphFilename.empty())) return; + if (_graphFilename.empty()) return; - _tags = parameter("tags").toVectorString(); - _inputNames = parameter("inputs").toVectorString(); - _outputNames = parameter("outputs").toVectorString(); + _inputs = parameter("inputs").toVectorString(); + _outputs = parameter("outputs").toVectorString(); _isTraining = parameter("isTraining").toBool(); _isTrainingName = parameter("isTrainingName").toString(); _squeeze = parameter("squeeze").toBool(); (_isTrainingName == "") ? _isTrainingSet = false : _isTrainingSet = true; - _nInputs = _inputNames.size(); - _nOutputs = _outputNames.size(); + _nInputs = _inputs.size(); + _nOutputs = _outputs.size(); // Allocate input and output tensors. _inputTensors.resize(_nInputs + int(_isTrainingSet)); @@ -100,31 +100,79 @@ void OnnxPredict::configure() { _outputTensors.resize(_nOutputs); _outputNodes.resize(_nOutputs); - TF_DeleteGraph(_graph); - _graph = TF_NewGraph(); + /* Set graph optimization level + // https://onnxruntime.ai/docs/performance/model-optimizations/graph-optimizations.html + session_options.SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_EXTENDED); + + // To enable model serialization after graph optimization set this + session_options.SetOptimizedModelFilePath("optimized_file_path");*/ + - openGraph(); + // DONE: reset graph and create a new empty one + /*Ort::ReleaseGraph(_graph); + _graph = Ort::Graph(); + + openGraph();*/ + + + _env = Ort::Env(ORT_LOGGING_LEVEL_WARNING, "default"); + //_session = Ort::Session{env, _graphFilename.c_str(), options_ort}; + + try{ + _session = Ort::Session(_env, _graphFilename.c_str(), _sessionOptions); + } + catch (Ort::Exception oe) { + std::cout << "ONNX exception caught: " << oe.what() << ". Code: " << oe.GetOrtErrorCode() << ".\n"; + //return -1; + } _isConfigured = true; reset(); // If the first output name is empty just print out the list of nodes and return. - if (_outputNames[0] == "") { + if (_outputs[0] == "") { E_INFO(availableNodesInfo()); return; } + + try { + memory_info = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault); + } + catch (Ort::Exception oe) { + std::cout << "ONNX exception caught: " << oe.what() << ". Code: " << oe.GetOrtErrorCode() << ".\n"; + //return -1; + } - // Initialize the list of input and output node names. + + /* Initialize the list of input and output node names. for (size_t i = 0; i < _nInputs; i++) { - _inputNodes[i] = graphOperationByName(_inputNames[i]); + _inputNodes[i] = graphOperationByName(_inputs[i]); } for (size_t i = 0; i < _nOutputs; i++) { - _outputNodes[i] = graphOperationByName(_outputNames[i]); + _outputNodes[i] = graphOperationByName(_outputs[i]); + }*/ + + + // print name/shape of inputs + auto input_names = _session.GetInputNames(); + auto input_shapes = _session.GetInputShapes(); + cout << "Input Node Name/Shape (" << input_names.size() << "):" << endl; + for (size_t i = 0; i < input_names.size(); i++) { + cout << "\t" << input_names[i] << " : " << print_shape(input_shapes[i]) << endl; + } + + // print name/shape of outputs + auto output_names = _session.GetOutputNames(); + auto output_shapes = _session.GetOutputShapes(); + cout << "Output Node Name/Shape (" << output_names.size() << "):" << endl; + for (size_t i = 0; i < output_names.size(); i++) { + cout << "\t" << output_names[i] << " : " << print_shape(output_shapes[i]) << endl; } + - // Add isTraining flag if needed. + /* Add isTraining flag if needed. if (_isTrainingSet) { const int64_t dims[1] = {}; TF_Tensor *isTraining = TF_AllocateTensor(TF_BOOL, dims, 0, 1); @@ -140,32 +188,37 @@ void OnnxPredict::configure() { _inputTensors[_nInputs] = isTraining; _inputNodes[_nInputs] = graphOperationByName(_isTrainingName); } + */ } - +/* void OnnxPredict::openGraph() { - // Prioritize savedModel when both are specified. - if (!_savedModel.empty()) { - std::vector tags_c; - tags_c.reserve(_tags.size()); - for (size_t i = 0; i < _tags.size(); i++) { - tags_c.push_back(const_cast(_tags[i].c_str())); - } - TF_LoadSessionFromSavedModel(_sessionOptions, _runOptions, - _savedModel.c_str(), &tags_c[0], (int)tags_c.size(), - _graph, NULL, _status); + if (!_graphFilename.empty()) { + + // Create a ModelProto instance + onnx::ModelProto model; + + // Open file input stream + std::ifstream input(model_path, std::ios::binary); + if (!input.is_open()) { + std::cerr << "Error opening model file: " << model_path << std::endl; + return 1; + } - if (TF_GetCode(_status) != TF_OK) { - throw EssentiaException("OnnxPredict: Error importing SavedModel specified in the `savedModel` parameter. ", TF_Message(_status)); + // Parse the binary ONNX model into the ModelProto + if (!model.ParseFromIstream(&input)) { + std::cerr << "Failed to parse ONNX model." << std::endl; + return 1; } - E_INFO("OnnxPredict: Successfully loaded SavedModel: `" << _savedModel << "`"); + // Access the graph + onnx::GraphProto* graph = model.mutable_graph(); - return; - } - - if (!_graphFilename.empty()) { + // Print some basic info + std::cout << "Model graph name: " << graph->name() << std::endl; + std::cout << "Number of nodes: " << graph->node_size() << std::endl; + // First we load and initialize the model. const auto f = fopen(_graphFilename.c_str(), "rb"); if (f == NULL) { @@ -187,6 +240,8 @@ void OnnxPredict::openGraph() { const auto data = malloc(fsize); fread(data, fsize, 1, f); fclose(f); + + // TODO: this should be updated to ONNX. Maybe it is not needed? TF_Buffer* buffer = TF_NewBuffer(); buffer->data = data; @@ -202,52 +257,80 @@ void OnnxPredict::openGraph() { } E_INFO("OnnxPredict: Successfully loaded graph file: `" << _graphFilename << "`"); + + } } +*/ void OnnxPredict::reset() { if (!_isConfigured) return; + + // TODO: update this part using ONNX functionalities - TF_CloseSession(_session, _status); - if (TF_GetCode(_status) != TF_OK) { - throw EssentiaException("OnnxPredict: Error closing session. ", TF_Message(_status)); - } + // this should close the session + // Ort::ReleaseSession(_session); // error: no type named 'ReleaseSession' in namespace 'Ort' - TF_DeleteSession(_session, _status); - if (TF_GetCode(_status) != TF_OK) { - throw EssentiaException("OnnxPredict: Error deleting session. ", TF_Message(_status)); + + if (Ort::Status::GetErrorCode(_status) != Ort::Status::IsOK()) { + throw EssentiaException("OnnxPredict: Error closing session. ", Ort::Status::GetErrorMessage(_status)); } - _session = TF_NewSession(_graph, _sessionOptions, _status); - if (TF_GetCode(_status) != TF_OK) { - throw EssentiaException("OnnxPredict: Error creating new session after reset. ", TF_Message(_status)); + //TF_DeleteSession(_session, _status); + if (Ort::Status::GetErrorCode(_status) != Ort::Status::IsOK) { + throw EssentiaException("OnnxPredict: Error deleting session. ", Ort::Status::GetErrorMessage(_status)); } + + // TODO: do we need this for ORT? + /*_session = TF_NewSession(_graph, _sessionOptions, _status); + if (Ort::Status::GetErrorCode(_status) != Ort::Status::IsOK) { + throw EssentiaException("OnnxPredict: Error creating new session after reset. ", Ort::Status::GetErrorMessage(_status)); + }*/ } void OnnxPredict::compute() { if (!_isConfigured) { throw EssentiaException("OnnxPredict: This algorithm is not configured. To configure this algorithm you " - "should specify a valid `graphFilename` or `savedModel` as input parameter."); + "should specify a valid `graphFilename` as input parameter."); } const Pool& poolIn = _poolIn.get(); Pool& poolOut = _poolOut.get(); + + // TODO: update to ONNX if needed + /*input_tensor_ = Ort::Value::CreateTensor(memory_info, input_image_.data(), input_image_.size(), input_shape_.data(), input_shape_.size()); + output_tensor_ = Ort::Value::CreateTensor(memory_info, results_.data(), results_.size(), output_shape_.data(), output_shape_.size());*/ + // Parse the input tensors from the pool into Tensorflow tensors. for (size_t i = 0; i < _nInputs; i++) { const Tensor& inputData = - poolIn.value >(_inputNames[i]); + poolIn.value >(_inputs[i]); _inputTensors[i] = TensorToTF(inputData); } + // TODO: update to ONNX if needed // Initialize output tensors. for (size_t i = 0; i < _nOutputs; i++) { _outputTensors[i] = NULL; } - // Run the Tensorflow session. + // TODO: okay this is the session run + // TODO: define input_names + _session.Run(run_options, // Run options. + input_names, // Input node names. + &_inputTensors, // Input tensro values. + 1, // Number of inputs. + output_names, // Output node names. + &_outputTensors, // Output tensor values. + 1 // Number of outputs. + ); + + + + /* Run the Tensorflow session. TF_SessionRun(_session, NULL, // Run options. &_inputNodes[0], // Input node names. @@ -259,17 +342,18 @@ void OnnxPredict::compute() { NULL, 0, // Target operations, number of targets. NULL, // Run metadata. _status // Output status. - ); + );*/ - if (TF_GetCode(_status) != TF_OK) { - throw EssentiaException("OnnxPredict: Error running the Tensorflow session. ", TF_Message(_status)); + if (Ort::Status::GetErrorCode(_status) != Ort::Status::IsOK()) { + throw EssentiaException("OnnxPredict: Error running the OnnxRuntime session. ", Ort::Status::GetErrorMessage(_status)); } - + // Copy the desired tensors into the output pool. for (size_t i = 0; i < _nOutputs; i++) { - poolOut.set(_outputNames[i], TFToTensor(_outputTensors[i], _outputNodes[i])); + poolOut.set(_outputs[i], TFToTensor(_outputTensors[i], _outputNodes[i])); } + // TODO: replace with ONNX functionalities // Deallocate tensors. for (size_t i = 0; i < _nInputs; i++) { TF_DeleteTensor(_inputTensors[i]); @@ -280,7 +364,9 @@ void OnnxPredict::compute() { } } +// TODO: do we need this functionalities? +/* TF_Tensor* OnnxPredict::TensorToTF( const Tensor& tensorIn) { int dims = 1; @@ -360,9 +446,9 @@ const Tensor OnnxPredict::TFToTensor( // Return a const reference to the data in Eigen format. return TensorMap(outputData, shape); -} - +}*/ +/* TF_Output OnnxPredict::graphOperationByName(const string nodeName) { int index = 0; const char* name = nodeName.c_str(); @@ -404,8 +490,9 @@ TF_Output OnnxPredict::graphOperationByName(const string nodeName) { } return output; -} +}*/ +// TODO: use ONNX functionalities vector OnnxPredict::nodeNames() { size_t pos = 0; TF_Operation *oper; diff --git a/src/algorithms/machinelearning/onnxpredict.h b/src/algorithms/machinelearning/onnxpredict.h index 39881ec2d..1a85e54e1 100644 --- a/src/algorithms/machinelearning/onnxpredict.h +++ b/src/algorithms/machinelearning/onnxpredict.h @@ -22,7 +22,10 @@ #include "algorithm.h" #include "pool.h" -#include + +// DONE: remove TF include and add ONNX includes +//#include +#include namespace essentia { @@ -31,22 +34,33 @@ namespace standard { class OnnxPredict : public Algorithm { protected: + + // DOUBT: do we need Pool for ONNX? Input _poolIn; Output _poolOut; std::string _graphFilename; - std::vector _inputNames; - std::vector _outputNames; + std::vector _inputs; + std::vector _outputs; + + bool _isTraining; + bool _isTrainingSet; + std::string _isTrainingName; + + bool _squeeze; + + bool _isConfigured; + size_t _nInputs; + size_t _nOutputs; + + /* std::vector _inputTensors; std::vector _outputTensors; std::vector _inputNodes; std::vector _outputNodes; - size_t _nInputs; - size_t _nOutputs; - TF_Graph* _graph; TF_Status* _status; TF_ImportGraphDefOptions* _options; @@ -56,21 +70,34 @@ class OnnxPredict : public Algorithm { std::string _savedModel; std::vector _tags; TF_Buffer* _runOptions; - - bool _isTraining; - bool _isTrainingSet; - std::string _isTrainingName; - - bool _squeeze; - - bool _isConfigured; - - void openGraph(); + */ + + Ort::Value _inputTensors{nullptr}; + Ort::Value _outputTensors{nullptr}; + + Ort::Value _inputNodes; + Ort::Value _outputNodes; + + // DOUBT: NOT sure if we would need that + Ort::Graph* _graph; + Ort::Status* _status; + + Ort::Env _env; + Ort::SessionOptions _sessionOptions{nullptr}; + Ort::Session _session; + + Ort::RunOptions _run_options; + + Ort::MemoryInfo memory_info{ nullptr }; // Used to allocate memory for input + + /*void openGraph(); TF_Tensor* TensorToTF(const Tensor& tensorIn); const Tensor TFToTensor(const TF_Tensor* tensor, TF_Output node); - TF_Output graphOperationByName(const std::string nodeName); + TF_Output graphOperationByName(const std::string nodeName);*/ std::vector nodeNames(); + + // TODO: do we need this one? inline std::string availableNodesInfo() { std::vector nodes = nodeNames(); std::string info = "OnnxPredict: Available node names are:\n"; @@ -79,22 +106,35 @@ class OnnxPredict : public Algorithm { } public: - OnnxPredict() : _graph(TF_NewGraph()), _status(TF_NewStatus()), + /*OnnxPredict() : _graph(TF_NewGraph()), _status(TF_NewStatus()), _options(TF_NewImportGraphDefOptions()), _sessionOptions(TF_NewSessionOptions()), _session(TF_NewSession(_graph, _sessionOptions, _status)), _runOptions(NULL), _isConfigured(false) { declareInput(_poolIn, "poolIn", "the pool where to get the feature tensors"); declareOutput(_poolOut, "poolOut", "the pool where to store the output tensors"); + }*/ + + OnnxPredict() : _env(Ort::Env(ORT_LOGGING_LEVEL_WARNING, "default")), + _sessionOptions(Ort::SessionOptions()), _session(Ort::Session(_env, _graphFilename.c_str(), _sessionOptions)), _isConfigured(false) { + declareInput(_poolIn, "poolIn", "the pool where to get the feature tensors"); + declareOutput(_poolOut, "poolOut", "the pool where to store the output tensors"); } ~OnnxPredict(){ - TF_CloseSession(_session, _status); + // TODO: replace with ONNX functionalities + //Ort::ReleaseSession(_session); + //Ort::ReleaseSessionOptions(_sessionOptions); + //Ort::ReleaseGraph(_graph); // error: no type named 'ReleaseGraph' in namespace 'Ort' + //Ort::ReleaseStatus(_status); + //Ort::ReleaseRunOptions(_runOptions); + + /*TF_CloseSession(_session, _status); TF_DeleteSessionOptions(_sessionOptions); TF_DeleteSession(_session, _status); TF_DeleteImportGraphDefOptions(_options); TF_DeleteStatus(_status); TF_DeleteGraph(_graph); - TF_DeleteBuffer(_runOptions); + TF_DeleteBuffer(_runOptions);*/ } void declareParameters() { @@ -102,12 +142,10 @@ class OnnxPredict : public Algorithm { std::vector defaultTags = arrayToVector(defaultTagsC); declareParameter("graphFilename", "the name of the file from which to load the TensorFlow graph", "", ""); - declareParameter("savedModel", "the name of the TensorFlow SavedModel. Overrides parameter `graphFilename`", "", ""); - declareParameter("tags", "the tags of the savedModel", "", defaultTags); declareParameter("inputs", "will look for these namespaces in poolIn. Should match the names of the input nodes in the Tensorflow graph", "", Parameter::VECTOR_STRING); declareParameter("outputs", "will save the tensors on the graph nodes named after `outputs` to the same namespaces in the output pool. Set the first element of this list as an empty array to print all the available nodes in the graph", "", Parameter::VECTOR_STRING); declareParameter("isTraining", "run the model in training mode (normalized with statistics of the current batch) instead of inference mode (normalized with moving statistics). This only applies to some models", "{true,false}", false); - declareParameter("isTrainingName", "the name of an additional input node indicating whether the model is to be run in a training mode (for models with a training mode, leave it empty otherwise)", "", ""); + //declareParameter("isTrainingName", "the name of an additional input node indicating whether the model is to be run in a training mode (for models with a training mode, leave it empty otherwise)", "", ""); declareParameter("squeeze", "remove singleton dimensions of the inputs tensors. Does not apply to the batch dimension", "{true,false}", true); } From 90da5d4c5ddd73fa4edaa0d95d956623ec547d1a Mon Sep 17 00:00:00 2001 From: xaviliz Date: Tue, 8 Jul 2025 14:05:19 +0200 Subject: [PATCH 08/60] Fix status errors. --- src/algorithms/machinelearning/onnxpredict.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/algorithms/machinelearning/onnxpredict.cpp b/src/algorithms/machinelearning/onnxpredict.cpp index f28d790be..b4a32b61a 100644 --- a/src/algorithms/machinelearning/onnxpredict.cpp +++ b/src/algorithms/machinelearning/onnxpredict.cpp @@ -116,7 +116,6 @@ void OnnxPredict::configure() { _env = Ort::Env(ORT_LOGGING_LEVEL_WARNING, "default"); - //_session = Ort::Session{env, _graphFilename.c_str(), options_ort}; try{ _session = Ort::Session(_env, _graphFilename.c_str(), _sessionOptions); @@ -273,18 +272,18 @@ void OnnxPredict::reset() { // Ort::ReleaseSession(_session); // error: no type named 'ReleaseSession' in namespace 'Ort' - if (Ort::Status::GetErrorCode(_status) != Ort::Status::IsOK()) { - throw EssentiaException("OnnxPredict: Error closing session. ", Ort::Status::GetErrorMessage(_status)); + if (_status->GetErrorCode() != _status->IsOK()) { + throw EssentiaException("OnnxPredict: Error closing session. ", _status->GetErrorMessage()); } //TF_DeleteSession(_session, _status); - if (Ort::Status::GetErrorCode(_status) != Ort::Status::IsOK) { - throw EssentiaException("OnnxPredict: Error deleting session. ", Ort::Status::GetErrorMessage(_status)); + if (_status->GetErrorCode() != _status->IsOK()) { + throw EssentiaException("OnnxPredict: Error deleting session. ", _status->GetErrorMessage()); } // TODO: do we need this for ORT? /*_session = TF_NewSession(_graph, _sessionOptions, _status); - if (Ort::Status::GetErrorCode(_status) != Ort::Status::IsOK) { + if (Ort::Status::GetErrorCode(_status) != Ort::Status::IsOK(_status)) { throw EssentiaException("OnnxPredict: Error creating new session after reset. ", Ort::Status::GetErrorMessage(_status)); }*/ } @@ -304,6 +303,7 @@ void OnnxPredict::compute() { /*input_tensor_ = Ort::Value::CreateTensor(memory_info, input_image_.data(), input_image_.size(), input_shape_.data(), input_shape_.size()); output_tensor_ = Ort::Value::CreateTensor(memory_info, results_.data(), results_.size(), output_shape_.data(), output_shape_.size());*/ + // TODO: implement TensorToOrt() taht converts Pool tensors into Ort tensors // Parse the input tensors from the pool into Tensorflow tensors. for (size_t i = 0; i < _nInputs; i++) { const Tensor& inputData = @@ -344,8 +344,8 @@ void OnnxPredict::compute() { _status // Output status. );*/ - if (Ort::Status::GetErrorCode(_status) != Ort::Status::IsOK()) { - throw EssentiaException("OnnxPredict: Error running the OnnxRuntime session. ", Ort::Status::GetErrorMessage(_status)); + if (_status->GetErrorCode() != _status->IsOK()) { + throw EssentiaException("OnnxPredict: Error running the OnnxRuntime session. ", _status->GetErrorMessage()); } // Copy the desired tensors into the output pool. From 7d357be1d032bbcb70d7a405bc59e5837131f20c Mon Sep 17 00:00:00 2001 From: xaviliz Date: Mon, 1 Sep 2025 13:58:14 +0200 Subject: [PATCH 09/60] Update ONNX Runtime versioning --- packaging/build_config.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packaging/build_config.sh b/packaging/build_config.sh index b44590efb..4c20e2184 100755 --- a/packaging/build_config.sh +++ b/packaging/build_config.sh @@ -2,7 +2,7 @@ HOST=i686-w64-mingw32 if [ -z "${PREFIX}" ]; then - PREFIX=`pwd` + PREFIX=$(pwd) fi echo Installing to: $PREFIX @@ -28,8 +28,7 @@ CHROMAPRINT_VERSION=1.4.3 QT_SOURCE_URL=https://download.qt.io/archive/qt/4.8/4.8.4/qt-everywhere-opensource-src-4.8.4.tar.gz GAIA_VERSION=2.4.6-86-ged433ed TENSORFLOW_VERSION=2.5.0 -LIBONNXRUNTIME_VERSION=1.21.1 - +LIBONNXRUNTIME_VERSION=1.22.1 FFMPEG_AUDIO_FLAGS=" --disable-programs From 56da948a08c74fd4fb643aaa0920e3e75f7f501b Mon Sep 17 00:00:00 2001 From: xaviliz Date: Mon, 1 Sep 2025 14:14:48 +0200 Subject: [PATCH 10/60] Support for MacOS building at arm64 --- packaging/debian_3rdparty/build_onnx.sh | 34 +++++++++++++++++-------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/packaging/debian_3rdparty/build_onnx.sh b/packaging/debian_3rdparty/build_onnx.sh index aa2c60d65..7571cb23b 100644 --- a/packaging/debian_3rdparty/build_onnx.sh +++ b/packaging/debian_3rdparty/build_onnx.sh @@ -2,8 +2,8 @@ set -e . ../build_config.sh -# rm -rf tmp -# mkdir tmp +rm -rf tmp +mkdir tmp cd tmp # Prerequisites: python>=3.10 @@ -11,24 +11,36 @@ cd tmp echo "Building onnxruntime $LIBONNXRUNTIME_VERSION" -#curl -SLO "https://github.com/microsoft/onnxruntime/archive/refs/tags/v$LIBONNXRUNTIME_VERSION.tar.gz" -#! this file has an issue https://github.com/microsoft/onnxruntime/issues/24861 -#! it is fixed manually for testing https://github.com/microsoft/onnxruntime/commit/f57db79743c4d1a3553aa05cf95bcd10966030e6 -#! should be done in three-steps: first, downloading the package, editing the deps.txt file and running the script -#! in the main branch it is fixed, it should be replaced when a release is available. +curl -SLO "https://github.com/microsoft/onnxruntime/archive/refs/tags/v$LIBONNXRUNTIME_VERSION.tar.gz" -#tar -xf v$LIBONNXRUNTIME_VERSION.tar.gz +tar -xf v$LIBONNXRUNTIME_VERSION.tar.gz cd onnxruntime-$LIBONNXRUNTIME_VERSION python3 -m pip install cmake -# compile library with cmake +# Build the dynamic library for Linux +# ./build.sh \ +# --config RelWithDebInfo \ +# --build_shared_lib \ +# --parallel \ +# --compile_no_warning_as_error \ +# --skip_submodule_sync + +# Build the dynamic library for MacOS (build for Intel and Apple silicon CPUs --> "x86_64;arm64") ./build.sh \ --config RelWithDebInfo \ --build_shared_lib \ --parallel \ --compile_no_warning_as_error \ - --skip_submodule_sync + --skip_submodule_sync \ + --cmake_extra_defines CMAKE_OSX_ARCHITECTURES="arm64" FETCHCONTENT_TRY_FIND_PACKAGE_MODE=NEVER CMAKE_INSTALL_PREFIX=$PREFIX + +#! We have found some issues building for cross-platforms, it looks it is much better to build it in a docker +#! In MacOS, we have experienced issues with the brew package. So, it needs to uninstall brew applications first (brew unsnstall onnxruntime) + +# copying .pc file +mkdir -p "${PREFIX}"/lib/pkgconfig/ +cp build/MacOS/RelWithDebInfo/libonnxruntime.pc ${PREFIX}/lib/pkgconfig/ cd ../.. -# rm -fr tmp +rm -fr tmp From d9c8a494a31fd043dcef2d605d187d063ad7f110 Mon Sep 17 00:00:00 2001 From: xaviliz Date: Mon, 1 Sep 2025 14:15:58 +0200 Subject: [PATCH 11/60] Support for multi IO models --- .../machinelearning/onnxpredict.cpp | 654 ++++++++---------- src/algorithms/machinelearning/onnxpredict.h | 126 ++-- 2 files changed, 325 insertions(+), 455 deletions(-) diff --git a/src/algorithms/machinelearning/onnxpredict.cpp b/src/algorithms/machinelearning/onnxpredict.cpp index b4a32b61a..f76f504c4 100644 --- a/src/algorithms/machinelearning/onnxpredict.cpp +++ b/src/algorithms/machinelearning/onnxpredict.cpp @@ -26,53 +26,27 @@ using namespace standard; const char* OnnxPredict::name = "OnnxPredict"; const char* OnnxPredict::category = "Machine Learning"; -// TODO: update description with ONNX const char* OnnxPredict::description = DOC("This algorithm runs a Onnx graph and stores the desired output tensors in a pool.\n" -"The Tensorflow graph should be stored in Protocol Buffer (.pb) binary format [1] or as a SavedModel [2], and should contain both the architecture and the weights of the model.\n" -"The parameter `inputs` should contain a list with the names of the input nodes that feed the model. The input Pool should contain the tensors corresponding to each input node stored using Essetia tensors. " +"The Onnx graph should be stored in Open Neural Network Exchange (.onnx) binary format [1], and should contain both the architecture and the weights of the model.\n" +"The parameter `inputs` should contain a list with the names of the input nodes that feed the model. The input Pool should contain the tensors corresponding to each input node stored using Essentia tensors. " "The pool namespace for each input tensor has to match the input node's name.\n" "In the same way, the `outputs` parameter should contain the names of the tensors to save. These tensors will be stored inside the output pool under a namespace that matches the tensor's name. " -"TensorFlow nodes return tensors identified as `nodeName:n`, where `n` goes from 0 to the number of outputs of the node - 1. " -"The first output tensor of a node can be referred implicitly (e.g., `nodeName`) or explicitly (e.g., `nodeName:0`), and the subsequent tensors require the specific index (e.g., nodeName:1, nodeName:2). " -"In TensorFlow 2 SavedModels all the outputs of the model are contained in the `PartitionedCall` node (e.g., `PartitionedCall:0`, `PartitionedCall:1`). " -"You can use TenorFlow's `saved_model_cli` to find the relation of outputs and `PartitionedCall` indices, for example:\n" -"\n" -">>> saved_model_cli show --dir SavedModel/ --all\n" -">>> ...\n" -">>> The given SavedModel SignatureDef contains the following output(s):\n" -">>> outputs['activations'] tensor_info:\n" -">>> dtype: DT_FLOAT\n" -">>> shape: (1, 400)\n" -">>> name: PartitionedCall:0\n" -">>> outputs['embeddings'] tensor_info:\n" -">>> dtype: DT_FLOAT\n" -">>> shape: (1, 1280)\n" -">>> name: PartitionedCall:1\n" -">>> ...\n" -"\n" "To print a list with all the available nodes in the graph set the first element of `outputs` as an empty string (i.e., \"\")." "\n" -"This algorithm is a wrapper for the Tensorflow C API [3]. The first time it is configured with a non-empty `graphFilename` it will try to load the contained graph and to attach a Tensorflow session to it. " -"The reset method deletes the current session (and the resources attached to it) and creates a new one relying on the available graph. " +"This algorithm is a wrapper for the ONNX Runtime Inferencing API [2]. The first time it is configured with a non-empty `graphFilename` it will try to load the contained graph and to attach a ONNX session to it. " +"The reset method deletes the model inputs and outputs internally stored in a vector. " "By reconfiguring the algorithm the graph is reloaded and the reset method is called.\n" "\n" "References:\n" -" [1] TensorFlow - An open source machine learning library for research and production.\n" -" https://www.tensorflow.org/extend/tool_developers/#protocol_buffers\n\n" -" [2] TensorFlow - An open source machine learning library for research and production.\n" -" https://www.tensorflow.org/guide/saved_model\n\n" -" [3] TensorFlow - An open source machine learning library for research and production.\n" -" https://www.tensorflow.org/api_docs/cc/"); - - -static void DeallocateBuffer(void* data, size_t) { - free(data); -} +" [1] ONNX - The open standard for machine learning interoperability.\n" +" https://onnx.ai/onnx/intro/\n\n" +" [2] ONNX Runtime API - a cross-platform machine-learning model accelerator, with a flexible interface to integrate hardware-specific libraries.\n" +" https://onnxruntime.ai/docs/"); void OnnxPredict::configure() { _graphFilename = parameter("graphFilename").toString(); - + if ((_graphFilename.empty()) and (_isConfigured)) { E_WARNING("OnnxPredict: You are trying to update a valid configuration with invalid parameters. " "If you want to update the configuration specify a valid `graphFilename` parameter."); @@ -80,216 +54,187 @@ void OnnxPredict::configure() { // Do not do anything if we did not get a non-empty model name. if (_graphFilename.empty()) return; - - + + try{ + // Define environment + _env = Ort::Env(ORT_LOGGING_LEVEL_WARNING, "multi_io_inference"); // {"default", "test", "multi_io_inference"} + + // Set graph optimization level - check https://onnxruntime.ai/docs/performance/model-optimizations/graph-optimizations.html + _sessionOptions.SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_EXTENDED); + // To enable model serialization after graph optimization set this + _sessionOptions.SetOptimizedModelFilePath("optimized_file_path"); + _sessionOptions.SetIntraOpNumThreads(1); + + // Initialize session + _session = Ort::Session(_env, _graphFilename.c_str(), _sessionOptions); + } + catch (Ort::Exception oe) { + cout << "ONNX exception caught: " << oe.what() << ". Code: " << oe.GetOrtErrorCode() << ".\n"; + return; + } + + // get input and output info (names, type and shapes) + all_input_infos = setTensorInfos(_session, _allocator, true); + all_output_infos = setTensorInfos(_session, _allocator, false); + + // read inputs and outputs as input parameter _inputs = parameter("inputs").toVectorString(); _outputs = parameter("outputs").toVectorString(); - _isTraining = parameter("isTraining").toBool(); - _isTrainingName = parameter("isTrainingName").toString(); + _squeeze = parameter("squeeze").toBool(); - (_isTrainingName == "") ? _isTrainingSet = false : _isTrainingSet = true; - _nInputs = _inputs.size(); _nOutputs = _outputs.size(); - - // Allocate input and output tensors. - _inputTensors.resize(_nInputs + int(_isTrainingSet)); - _inputNodes.resize(_nInputs + int(_isTrainingSet)); - - _outputTensors.resize(_nOutputs); - _outputNodes.resize(_nOutputs); - - /* Set graph optimization level - // https://onnxruntime.ai/docs/performance/model-optimizations/graph-optimizations.html - session_options.SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_EXTENDED); - - // To enable model serialization after graph optimization set this - session_options.SetOptimizedModelFilePath("optimized_file_path");*/ - - - // DONE: reset graph and create a new empty one - /*Ort::ReleaseGraph(_graph); - _graph = Ort::Graph(); - - openGraph();*/ + cout << "_inputs.size(): " << _nInputs << ".\n"; - _env = Ort::Env(ORT_LOGGING_LEVEL_WARNING, "default"); - - try{ - _session = Ort::Session(_env, _graphFilename.c_str(), _sessionOptions); - } - catch (Ort::Exception oe) { - std::cout << "ONNX exception caught: " << oe.what() << ". Code: " << oe.GetOrtErrorCode() << ".\n"; - //return -1; + // use the first input when no input is defined + if (_nInputs == 0){ + // take the first input + _inputs.push_back(_session.GetInputNames()[0]); + _nInputs = _inputs.size(); + // inform the first model input will be used + E_INFO("OnnxPredict: using the first model input '" + _inputs[0] + "'"); } - _isConfigured = true; - reset(); + // define _outputs with the first model output when no output is provided + if (_nOutputs == 0){ + // take the first output + _outputs.push_back(_session.GetOutputNames()[0]); + _nOutputs = _outputs.size(); + // inform the first model input will be used + E_INFO("OnnxPredict: using the first model output '" + _outputs[0] + "'"); + } // If the first output name is empty just print out the list of nodes and return. if (_outputs[0] == "") { - E_INFO(availableNodesInfo()); - + E_INFO(getTensorInfos(all_input_infos, "Model Inputs")); + E_INFO(getTensorInfos(all_output_infos, "Model Outputs")); return; } - try { - memory_info = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault); + // check model has input and output https://github.com/microsoft/onnxruntime-inference-examples/blob/7a635daae48450ff142e5c0848a564b245f04112/c_cxx/model-explorer/model-explorer.cpp#L99C3-L100C63 + + for (int i = 0; i < _inputs.size(); i++) { + for (int j = 0; j < all_input_infos.size(); j++) { + if (_inputs[i] == all_input_infos[j].name){ + _inputNodes.push_back(all_input_infos[j]); + } + } } - catch (Ort::Exception oe) { - std::cout << "ONNX exception caught: " << oe.what() << ". Code: " << oe.GetOrtErrorCode() << ".\n"; - //return -1; + + if (_inputNodes.size() == 0){ + std::string s; + for (const auto &piece : _inputs) { + s += piece; + s += " "; + } + throw EssentiaException("ONNXRuntimePredict: '" + s + + "' are not valid input name of this model.\n" + + getTensorInfos(all_input_infos, "Model Inputs")); } - - /* Initialize the list of input and output node names. - for (size_t i = 0; i < _nInputs; i++) { - _inputNodes[i] = graphOperationByName(_inputs[i]); + for (int i = 0; i < _outputs.size(); i++) { + for (int j = 0; j < all_output_infos.size(); j++) { + if (_outputs[i] == all_output_infos[j].name){ + _outputNodes.push_back(all_output_infos[j]); + } + } } - - for (size_t i = 0; i < _nOutputs; i++) { - _outputNodes[i] = graphOperationByName(_outputs[i]); - }*/ - - - // print name/shape of inputs - auto input_names = _session.GetInputNames(); - auto input_shapes = _session.GetInputShapes(); - cout << "Input Node Name/Shape (" << input_names.size() << "):" << endl; - for (size_t i = 0; i < input_names.size(); i++) { - cout << "\t" << input_names[i] << " : " << print_shape(input_shapes[i]) << endl; + + if (_outputNodes.size() == 0){ + std::string s; + for (const auto &piece : _outputs) { + s += piece; + s += " "; + } + throw EssentiaException("ONNXRuntimePredict: '" + s + + "' has not a valid output name of this model.\n" + + getTensorInfos(all_output_infos, "Model Outputs")); } - - // print name/shape of outputs - auto output_names = _session.GetOutputNames(); - auto output_shapes = _session.GetOutputShapes(); - cout << "Output Node Name/Shape (" << output_names.size() << "):" << endl; - for (size_t i = 0; i < output_names.size(); i++) { - cout << "\t" << output_names[i] << " : " << print_shape(output_shapes[i]) << endl; + + _isConfigured = true; + reset(); + + for (size_t i = 0; i < _nInputs; i++) { + checkName(_inputs[i], all_input_infos); } - - /* Add isTraining flag if needed. - if (_isTrainingSet) { - const int64_t dims[1] = {}; - TF_Tensor *isTraining = TF_AllocateTensor(TF_BOOL, dims, 0, 1); - void* isTrainingValue = TF_TensorData(isTraining); - - if (isTrainingValue == NULL) { - TF_DeleteTensor(isTraining); - throw EssentiaException("OnnxPredict: Error generating training phase flag"); - } - - memcpy(isTrainingValue, &_isTraining, sizeof(bool)); - - _inputTensors[_nInputs] = isTraining; - _inputNodes[_nInputs] = graphOperationByName(_isTrainingName); + for (size_t i = 0; i < _nOutputs; i++) { + checkName(_outputs[i], all_output_infos); } - */ } -/* -void OnnxPredict::openGraph() { - - if (!_graphFilename.empty()) { - - // Create a ModelProto instance - onnx::ModelProto model; - - // Open file input stream - std::ifstream input(model_path, std::ios::binary); - if (!input.is_open()) { - std::cerr << "Error opening model file: " << model_path << std::endl; - return 1; - } - - // Parse the binary ONNX model into the ModelProto - if (!model.ParseFromIstream(&input)) { - std::cerr << "Failed to parse ONNX model." << std::endl; - return 1; - } - - // Access the graph - onnx::GraphProto* graph = model.mutable_graph(); - - // Print some basic info - std::cout << "Model graph name: " << graph->name() << std::endl; - std::cout << "Number of nodes: " << graph->node_size() << std::endl; +std::vector OnnxPredict::setTensorInfos(const Ort::Session& session, Ort::AllocatorWithDefaultOptions& allocator, bool is_input) { + size_t count = is_input ? session.GetInputCount() : session.GetOutputCount(); + std::vector infos; - // First we load and initialize the model. - const auto f = fopen(_graphFilename.c_str(), "rb"); - if (f == NULL) { - throw EssentiaException( - "OnnxPredict: could not open the Tensorflow graph file."); - } + auto names_raw = is_input + ? session.GetInputNames() + : session.GetOutputNames(); - fseek(f, 0, SEEK_END); - const auto fsize = ftell(f); - fseek(f, 0, SEEK_SET); + for (size_t i = 0; i < count; ++i) { + auto name_raw = names_raw[i]; - // Graph size sanity check. - if (fsize < 1) { - fclose(f); - throw(EssentiaException("OnnxPredict: Graph file is empty.")); - } + std::string name(name_raw); + + Ort::TypeInfo type_info = is_input + ? session.GetInputTypeInfo(i) + : session.GetOutputTypeInfo(i); - // Reserve memory and read the graph. - const auto data = malloc(fsize); - fread(data, fsize, 1, f); - fclose(f); - - // TODO: this should be updated to ONNX. Maybe it is not needed? + auto tensor_info = type_info.GetTensorTypeAndShapeInfo(); - TF_Buffer* buffer = TF_NewBuffer(); - buffer->data = data; - buffer->length = fsize; - buffer->data_deallocator = DeallocateBuffer; + TensorInfo info; + info.name = name; + info.type = tensor_info.GetElementType(); + info.shape = tensor_info.GetShape(); - TF_GraphImportGraphDef(_graph, buffer, _options, _status); + infos.push_back(std::move(info)); + } - TF_DeleteBuffer(buffer); + return infos; +} - if (TF_GetCode(_status) != TF_OK) { - throw EssentiaException("OnnxPredict: Error importing graph. ", TF_Message(_status)); +void OnnxPredict::printTensorInfos(const std::vector& infos, const std::string& label) { + std::cout << "=== " << label << " ===\n"; + for (const auto& info : infos) { + std::cout << "[Name] " << info.name << "\n"; + std::cout << " [Type] " << info.type << "\n"; + std::cout << " [Shape] ["; + for (size_t j = 0; j < info.shape.size(); ++j) { + std::cout << info.shape[j]; + if (j + 1 < info.shape.size()) std::cout << ", "; + } + std::cout << "]\n"; } +} - E_INFO("OnnxPredict: Successfully loaded graph file: `" << _graphFilename << "`"); - - +std::string OnnxPredict::getTensorInfos(const std::vector& infos, const std::string& label) { + std::string out; + out += "=== " + label + " ===\n"; + for (const auto& info : infos) { + out += "[Name] " + info.name + "\n"; + //cout << "info.type: " << typeid(info.type).name() << endl; + std::string type_str = onnxTypeToString(info.type); + out += "\t[Type] " + type_str + "\n"; + out += "\t[Shape] ["; + for (size_t j = 0; j < info.shape.size(); ++j) { + out += info.shape[j]; + if (j + 1 < info.shape.size()) out += ", "; + } + out += "]\n"; } } -*/ - void OnnxPredict::reset() { if (!_isConfigured) return; - // TODO: update this part using ONNX functionalities - - // this should close the session - // Ort::ReleaseSession(_session); // error: no type named 'ReleaseSession' in namespace 'Ort' - - - if (_status->GetErrorCode() != _status->IsOK()) { - throw EssentiaException("OnnxPredict: Error closing session. ", _status->GetErrorMessage()); - } - - //TF_DeleteSession(_session, _status); - if (_status->GetErrorCode() != _status->IsOK()) { - throw EssentiaException("OnnxPredict: Error deleting session. ", _status->GetErrorMessage()); - } - - // TODO: do we need this for ORT? - /*_session = TF_NewSession(_graph, _sessionOptions, _status); - if (Ort::Status::GetErrorCode(_status) != Ort::Status::IsOK(_status)) { - throw EssentiaException("OnnxPredict: Error creating new session after reset. ", Ort::Status::GetErrorMessage(_status)); - }*/ + input_names.clear(); + output_names.clear(); } - void OnnxPredict::compute() { + if (!_isConfigured) { throw EssentiaException("OnnxPredict: This algorithm is not configured. To configure this algorithm you " "should specify a valid `graphFilename` as input parameter."); @@ -297,210 +242,163 @@ void OnnxPredict::compute() { const Pool& poolIn = _poolIn.get(); Pool& poolOut = _poolOut.get(); - - // TODO: update to ONNX if needed - /*input_tensor_ = Ort::Value::CreateTensor(memory_info, input_image_.data(), input_image_.size(), input_shape_.data(), input_shape_.size()); - output_tensor_ = Ort::Value::CreateTensor(memory_info, results_.data(), results_.size(), output_shape_.data(), output_shape_.size());*/ - - // TODO: implement TensorToOrt() taht converts Pool tensors into Ort tensors - // Parse the input tensors from the pool into Tensorflow tensors. - for (size_t i = 0; i < _nInputs; i++) { - const Tensor& inputData = - poolIn.value >(_inputs[i]); - _inputTensors[i] = TensorToTF(inputData); - } - - // TODO: update to ONNX if needed - // Initialize output tensors. - for (size_t i = 0; i < _nOutputs; i++) { - _outputTensors[i] = NULL; - } - - // TODO: okay this is the session run - // TODO: define input_names - _session.Run(run_options, // Run options. - input_names, // Input node names. - &_inputTensors, // Input tensro values. - 1, // Number of inputs. - output_names, // Output node names. - &_outputTensors, // Output tensor values. - 1 // Number of outputs. - ); - - - - /* Run the Tensorflow session. - TF_SessionRun(_session, - NULL, // Run options. - &_inputNodes[0], // Input node names. - &_inputTensors[0], // input tensor values. - _nInputs + (int)_isTrainingSet, // Number of inputs. - &_outputNodes[0], // Output node names. - &_outputTensors[0], // Output tensor values. - _nOutputs, // Number of outputs. - NULL, 0, // Target operations, number of targets. - NULL, // Run metadata. - _status // Output status. - );*/ - - if (_status->GetErrorCode() != _status->IsOK()) { - throw EssentiaException("OnnxPredict: Error running the OnnxRuntime session. ", _status->GetErrorMessage()); - } + std::vector shape; - // Copy the desired tensors into the output pool. - for (size_t i = 0; i < _nOutputs; i++) { - poolOut.set(_outputs[i], TFToTensor(_outputTensors[i], _outputNodes[i])); - } - - // TODO: replace with ONNX functionalities - // Deallocate tensors. + // Parse the input tensors from the pool into ONNX Runtime tensors. for (size_t i = 0; i < _nInputs; i++) { - TF_DeleteTensor(_inputTensors[i]); - } - - for (size_t i = 0; i < _nOutputs; i++) { - TF_DeleteTensor(_outputTensors[i]); - } -} - -// TODO: do we need this functionalities? + + cout << "_inputs[i]: " << _inputs[i] << endl; + const Tensor& inputData = poolIn.value >(_inputs[i]); + + // Convert data to float32 + std::vector float_data(inputData.size()); + for (size_t j = 0; j < inputData.size(); ++j) { + float_data[j] = static_cast(inputData.data()[j]); + } + + // Step 2: Get shape + int dims = 1; -/* -TF_Tensor* OnnxPredict::TensorToTF( - const Tensor& tensorIn) { - int dims = 1; - vector shape; - - // With squeeze, the Batch dimension is the only one allowed to be singleton - shape.push_back((int64_t)tensorIn.dimension(0)); - - if (_squeeze) { - for(int i = 1; i < tensorIn.rank(); i++) { - if (tensorIn.dimension(i) > 1) { - shape.push_back((int64_t)tensorIn.dimension(i)); + shape.push_back((int64_t)inputData.dimension(0)); + + if (_squeeze) { + for(int i = 1; i < inputData.rank(); i++) { + if (inputData.dimension(i) > 1) { + shape.push_back((int64_t)inputData.dimension(i)); + dims++; + } + } + + // There should be at least 2 dimensions (batch, data) + if (dims == 1) { + shape.push_back((int64_t) 1); dims++; } + } else { + dims = inputData.rank(); + for(int j = 1; j < dims; j++) { // HERE we need to jump i = 1 - 4D tensor input + //cout << inputData.dimension(j) << endl; + shape.push_back((int64_t)inputData.dimension(j)); + } } - - // There should be at least 2 dimensions (batch, data) - if (dims == 1) { - shape.push_back((int64_t) 1); - dims++; + + // Step 3: Create ONNX Runtime tensor + _memoryInfo = Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU); + + if (_memoryInfo == NULL) { + throw EssentiaException("OnnxRuntimePredict: Error allocating memory for input tensor."); } - } else { - dims = tensorIn.rank(); - for(int i = 1; i < dims; i++) { - shape.push_back((int64_t)tensorIn.dimension(i)); - } - } - - TF_Tensor* tensorOut = TF_AllocateTensor( - TF_FLOAT, &shape[0], dims, - (size_t)tensorIn.size() * sizeof(Real)); + input_tensors.emplace_back(Ort::Value::CreateTensor(_memoryInfo, float_data.data(), float_data.size(), shape.data(), shape.size())); - if (tensorOut == NULL) { - throw EssentiaException("OnnxPredict: Error generating input tensor."); } - // Get a pointer to the data and fill the tensor. - void* tensorData = TF_TensorData(tensorOut); - - if (tensorData == NULL) { - TF_DeleteTensor(tensorOut); - throw EssentiaException("OnnxPredict: Error generating input tensors data."); + // Define input and output names + for (const auto& tensorInfo : _inputNodes) { + input_names.push_back(tensorInfo.name.c_str()); } - - memcpy(tensorData, tensorIn.data(), - std::min(tensorIn.size() * sizeof(Real), - TF_TensorByteSize(tensorOut))); - - return tensorOut; -} - - -const Tensor OnnxPredict::TFToTensor( - const TF_Tensor* tensor, TF_Output node) { - const Real* outputData = static_cast(TF_TensorData(tensor)); - - // Get the output tensor's shape. - size_t outNDims = TF_GraphGetTensorNumDims(_graph, node, _status); - - if (TF_GetCode(_status) != TF_OK) { - throw EssentiaException("OnnxPredict: Error getting the output tensor's shape. ", TF_Message(_status)); + + for (const auto& tensorInfo : _outputNodes) { + output_names.push_back(tensorInfo.name.c_str()); + } + + // Run the Onnxruntime session. + auto output_tensors = _session.Run(_runOptions, // Run options. + input_names.data(), // Input node names. + input_tensors.data(), // Input tensor values. + _nInputs, // Number of inputs. + output_names.data(), // Output node names. + _nOutputs // Number of outputs. + ); + + // Map output tensors to pool + for (size_t i = 0; i < output_tensors.size(); ++i) { + + const Real* outputData = output_tensors[i].GetTensorData(); + + // Create and array to store the tensor shape. + array _shape {1, 1, 1, 1}; + //_shape[0] = (int)outputShapes[0]; + _shape[0] = (int)shape[0]; + for (size_t j = 1; j < _outputNodes[i].shape.size(); j++){ + _shape[j+1] = (int)_outputNodes[i].shape[j]; + } + + // Store tensor in pool + const Tensor tensorMap = TensorMap(outputData, _shape); + poolOut.set(_outputs[i], tensorMap); } - // Create and array to store the shape of the tensor. - array shape {1, 1, 1, 1}; - shape[0] = (int)TF_Dim(tensor, 0); - - // We are assuming one of the following cases: - // 1 - outNDims = 2 -> Batch + Feats - // 2 - outNDims = 3 -> Batch + Timestamps + Feats - // 3 - outNDims = 4 -> Batch + Channels + Timestamps + Feats - size_t idx = 1; - for (size_t i = shape.size() - outNDims + 1; i < shape.size(); i++, idx++) { - shape[i] = (int)TF_Dim(tensor, idx); + /* Cleanup + for (const auto& tensorInfo : all_input_infos) { + _allocator.Free((void*)tensorInfo.name.c_str()); } + + for (const auto& tensorInfo : all_output_infos) { + _allocator.Free((void*)tensorInfo.name.c_str()); + }*/ + +} - // Return a const reference to the data in Eigen format. - return TensorMap(outputData, shape); -}*/ -/* -TF_Output OnnxPredict::graphOperationByName(const string nodeName) { - int index = 0; - const char* name = nodeName.c_str(); - string newNodeName; - - // TensorFlow operations (or nodes from the graph perspective) return tensors named , where n goes - // from 0 to the number of outputs. The first output tensor of a node can be extracted implicitly (nodeName) - // or explicitly (nodeName:0). To extract the subsequent tensors, the index has to be specified (nodeName:1, - // nodeName:2, ...). - string::size_type n = nodeName.find(':'); - if (n != string::npos) { - try { - newNodeName = nodeName.substr(0, n); - name = newNodeName.c_str(); - index = stoi(nodeName.substr(n + 1, nodeName.size())); - - } catch (const invalid_argument& ) { - throw EssentiaException("OnnxPredict: `" + nodeName + "` is not a valid node name. Make sure that all " - "your inputs and outputs follow the pattern `nodeName:n`, where `n` in an integer that " - "goes from 0 to the number of outputs of the node - 1."); - } +void OnnxPredict::checkName(const string nodeName, std::vector _infos) { + vector _names; + + for(int i = 0; i< _infos.size(); i++) { + _names.push_back(_infos[i].name); } - TF_Operation* oper = TF_GraphOperationByName(_graph, name); + std::unordered_set lookup(_names.begin(), _names.end()); + if (lookup.find(nodeName) == lookup.end()) + throw EssentiaException("OnnxPredict: `" + nodeName + "` is not a valid input node name. Make sure that all " + "your inputs are defined in the node list."); +} - TF_Output output = {oper, index}; - if (output.oper == NULL) { - throw EssentiaException("OnnxPredict: '" + string(nodeName) + - "' is not a valid node name of this graph.\n" + - availableNodesInfo()); +vector OnnxPredict::inputNames() { + + vector inputNames; + + // inputs + for(int i = 0; i< all_input_infos.size(); i++) { + inputNames.push_back(all_input_infos[i].name); } - int n_outputs = TF_OperationNumOutputs(oper); - if (index > n_outputs - 1) { - throw EssentiaException("OnnxPredict: Asked for the output with index `" + to_string(index) + - "`, but the node `" + name + "` only has `" + to_string(n_outputs) + "` outputs."); - } - - return output; -}*/ - -// TODO: use ONNX functionalities -vector OnnxPredict::nodeNames() { - size_t pos = 0; - TF_Operation *oper; - vector nodeNames; + return inputNames; +} - while ((oper = TF_GraphNextOperation(_graph, &pos)) != NULL) { - nodeNames.push_back(string(TF_OperationName(oper))); +vector OnnxPredict::outputNames() { + + vector outputNames; + + // inputs + for(int i = 0; i< all_input_infos.size(); i++) { + outputNames.push_back(all_input_infos[i].name); } - return nodeNames; + return outputNames; +} + +std::string OnnxPredict::onnxTypeToString(ONNXTensorElementDataType type) { + switch (type) { + case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT: return "float32"; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8: return "uint8"; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8: return "int8"; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16: return "uint16"; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16: return "int16"; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32: return "int32"; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64: return "int64"; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING: return "string"; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL: return "bool"; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16: return "float16"; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE: return "float64"; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32: return "uint32"; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64: return "uint64"; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX64: return "complex64"; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX128: return "complex128"; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16: return "bfloat16"; + default: return "unknown"; + } } diff --git a/src/algorithms/machinelearning/onnxpredict.h b/src/algorithms/machinelearning/onnxpredict.h index 1a85e54e1..42cbb4fc1 100644 --- a/src/algorithms/machinelearning/onnxpredict.h +++ b/src/algorithms/machinelearning/onnxpredict.h @@ -23,118 +23,90 @@ #include "algorithm.h" #include "pool.h" -// DONE: remove TF include and add ONNX includes -//#include +#define ORT_ENABLE_EXTENDED_API #include +#include namespace essentia { namespace standard { +struct TensorInfo { + std::string name; + ONNXTensorElementDataType type; + std::vector shape; +}; + class OnnxPredict : public Algorithm { protected: - // DOUBT: do we need Pool for ONNX? Input _poolIn; Output _poolOut; std::string _graphFilename; std::vector _inputs; std::vector _outputs; - - bool _isTraining; - bool _isTrainingSet; - std::string _isTrainingName; bool _squeeze; - bool _isConfigured; size_t _nInputs; size_t _nOutputs; - - /* - std::vector _inputTensors; - std::vector _outputTensors; - - std::vector _inputNodes; - std::vector _outputNodes; - - TF_Graph* _graph; - TF_Status* _status; - TF_ImportGraphDefOptions* _options; - TF_SessionOptions* _sessionOptions; - TF_Session* _session; - - std::string _savedModel; - std::vector _tags; - TF_Buffer* _runOptions; - */ - - Ort::Value _inputTensors{nullptr}; - Ort::Value _outputTensors{nullptr}; - - Ort::Value _inputNodes; - Ort::Value _outputNodes; - // DOUBT: NOT sure if we would need that - Ort::Graph* _graph; - Ort::Status* _status; + Ort::Value _inputTensor{nullptr}; + Ort::Value _outputTensor{nullptr}; + + std::vector input_tensors; + std::vector input_names; + std::vector output_names; - Ort::Env _env; + Ort::Env _env{nullptr}; Ort::SessionOptions _sessionOptions{nullptr}; - Ort::Session _session; + Ort::Session _session{nullptr}; - Ort::RunOptions _run_options; + Ort::RunOptions _runOptions; + Ort::AllocatorWithDefaultOptions _allocator; + Ort::MemoryInfo _memoryInfo{ nullptr }; // Used to allocate memory for input + Ort::Model _model; - Ort::MemoryInfo memory_info{ nullptr }; // Used to allocate memory for input - - /*void openGraph(); - TF_Tensor* TensorToTF(const Tensor& tensorIn); - const Tensor TFToTensor(const TF_Tensor* tensor, TF_Output node); - TF_Output graphOperationByName(const std::string nodeName);*/ - std::vector nodeNames(); - - - // TODO: do we need this one? - inline std::string availableNodesInfo() { - std::vector nodes = nodeNames(); - std::string info = "OnnxPredict: Available node names are:\n"; - for (std::vector::const_iterator i = nodes.begin(); i != nodes.end() - 1; ++i) info += *i + ", "; - return info + nodes.back() + ".\n\nReconfigure this algorithm with valid node names as inputs and outputs before starting the processing."; + std::vector all_input_infos; + std::vector all_output_infos; + + std::vector _inputNodes; + std::vector _outputNodes; + + std::vector inputNames(); + std::vector outputNames(); + std::vector setTensorInfos(const Ort::Session&, Ort::AllocatorWithDefaultOptions&, bool); + void printTensorInfos(const std::vector&, const std::string&); + std::string getTensorInfos(const std::vector&, const std::string&); + void checkName(const std::string, std::vector); + std::string onnxTypeToString(ONNXTensorElementDataType); + + inline std::string availableInputInfo() { + std::vector inputs = inputNames(); + std::string info = "OnnxPredict: Available input names are:\n"; + for (std::vector::const_iterator i = inputs.begin(); i != inputs.end() - 1; ++i) info += *i + ", "; + return info + inputs.back() + ".\n\nReconfigure this algorithm with valid node names as inputs and outputs before starting the processing."; } public: - /*OnnxPredict() : _graph(TF_NewGraph()), _status(TF_NewStatus()), - _options(TF_NewImportGraphDefOptions()), _sessionOptions(TF_NewSessionOptions()), - _session(TF_NewSession(_graph, _sessionOptions, _status)), _runOptions(NULL), - _isConfigured(false) { - declareInput(_poolIn, "poolIn", "the pool where to get the feature tensors"); - declareOutput(_poolOut, "poolOut", "the pool where to store the output tensors"); - }*/ - OnnxPredict() : _env(Ort::Env(ORT_LOGGING_LEVEL_WARNING, "default")), - _sessionOptions(Ort::SessionOptions()), _session(Ort::Session(_env, _graphFilename.c_str(), _sessionOptions)), _isConfigured(false) { + OnnxPredict() : _env(Ort::Env(ORT_LOGGING_LEVEL_WARNING, "test")), + _sessionOptions(Ort::SessionOptions()), _session(Ort::Session(nullptr)), _runOptions(NULL), _isConfigured(false) , _model(Ort::Model(nullptr)){ declareInput(_poolIn, "poolIn", "the pool where to get the feature tensors"); declareOutput(_poolOut, "poolOut", "the pool where to store the output tensors"); } ~OnnxPredict(){ - // TODO: replace with ONNX functionalities - //Ort::ReleaseSession(_session); - //Ort::ReleaseSessionOptions(_sessionOptions); - //Ort::ReleaseGraph(_graph); // error: no type named 'ReleaseGraph' in namespace 'Ort' - //Ort::ReleaseStatus(_status); - //Ort::ReleaseRunOptions(_runOptions); - - /*TF_CloseSession(_session, _status); - TF_DeleteSessionOptions(_sessionOptions); - TF_DeleteSession(_session, _status); - TF_DeleteImportGraphDefOptions(_options); - TF_DeleteStatus(_status); - TF_DeleteGraph(_graph); - TF_DeleteBuffer(_runOptions);*/ + all_input_infos.clear(); + all_output_infos.clear(); + _inputNodes.clear(); + _outputNodes.clear(); + input_tensors.clear(); + input_names.clear(); + output_names.clear(); } void declareParameters() { @@ -143,9 +115,9 @@ class OnnxPredict : public Algorithm { declareParameter("graphFilename", "the name of the file from which to load the TensorFlow graph", "", ""); declareParameter("inputs", "will look for these namespaces in poolIn. Should match the names of the input nodes in the Tensorflow graph", "", Parameter::VECTOR_STRING); + // TODO: fix the empty input and output parameters. declareParameter("outputs", "will save the tensors on the graph nodes named after `outputs` to the same namespaces in the output pool. Set the first element of this list as an empty array to print all the available nodes in the graph", "", Parameter::VECTOR_STRING); declareParameter("isTraining", "run the model in training mode (normalized with statistics of the current batch) instead of inference mode (normalized with moving statistics). This only applies to some models", "{true,false}", false); - //declareParameter("isTrainingName", "the name of an additional input node indicating whether the model is to be run in a training mode (for models with a training mode, leave it empty otherwise)", "", ""); declareParameter("squeeze", "remove singleton dimensions of the inputs tensors. Does not apply to the batch dimension", "{true,false}", true); } From d13638831d8f8382239950de40d3c469966424a9 Mon Sep 17 00:00:00 2001 From: xaviliz Date: Mon, 1 Sep 2025 14:17:06 +0200 Subject: [PATCH 12/60] Except OnnxPredict algorithm to use pool in cpp --- src/python/essentia/standard.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/python/essentia/standard.py b/src/python/essentia/standard.py index ea0e7d059..45d236030 100644 --- a/src/python/essentia/standard.py +++ b/src/python/essentia/standard.py @@ -27,6 +27,7 @@ def _create_essentia_class(name, moduleName = __name__): essentia.log.debug(essentia.EPython, 'Creating essentia.standard class: %s' % name) + # print(f"name: {name}") _algoInstance = _essentia.Algorithm(name) _algoDoc = _algoInstance.getDoc() _algoStruct = _algoInstance.getStruct() @@ -71,7 +72,7 @@ def compute(self, *args): # we have to make some exceptions for YamlOutput and PoolAggregator # because they expect cpp Pools - if name in ('YamlOutput', 'PoolAggregator', 'SvmClassifier', 'PCA', 'GaiaTransform', 'TensorflowPredict'): + if name in ('YamlOutput', 'PoolAggregator', 'SvmClassifier', 'PCA', 'GaiaTransform', 'TensorflowPredict', 'OnnxPredict'): args = (args[0].cppPool,) # verify that all types match and do any necessary conversions @@ -105,7 +106,7 @@ def compute(self, *args): # we have to make an exceptional case for YamlInput, because we need # to wrap the Pool that it outputs w/ our python Pool from common.py - if name in ('YamlInput', 'PoolAggregator', 'SvmClassifier', 'PCA', 'GaiaTransform', 'Extractor', 'TensorflowPredict'): + if name in ('YamlInput', 'PoolAggregator', 'SvmClassifier', 'PCA', 'GaiaTransform', 'Extractor', 'TensorflowPredict', 'OnnxPredict'): return _c.Pool(results) # MusicExtractor and FreesoundExtractor output two pools From f46d48180aa1c37e04d109f3d2608a296712dd12 Mon Sep 17 00:00:00 2001 From: xaviliz Date: Mon, 1 Sep 2025 14:36:13 +0200 Subject: [PATCH 13/60] First onnx runtime unittest with effnet model --- .../machinelearning/test_onnxpredict.py | 383 ++++++++++++++++++ 1 file changed, 383 insertions(+) create mode 100644 test/src/unittests/machinelearning/test_onnxpredict.py diff --git a/test/src/unittests/machinelearning/test_onnxpredict.py b/test/src/unittests/machinelearning/test_onnxpredict.py new file mode 100644 index 000000000..43758cfe4 --- /dev/null +++ b/test/src/unittests/machinelearning/test_onnxpredict.py @@ -0,0 +1,383 @@ +#!/usr/bin/env python + +# Copyright (C) 2006-2021 Music Technology Group - Universitat Pompeu Fabra +# +# This file is part of Essentia +# +# Essentia is free software: you can redistribute it and/or modify it under +# the terms of the GNU Affero General Public License as published by the Free +# Software Foundation (FSF), either version 3 of the License, or (at your +# option) any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the Affero GNU General Public License +# version 3 along with this program. If not, see http://www.gnu.org/licenses/ + + +from essentia_test import * +import sys +import os +from pathlib import Path +import soundfile as sf + + +class TestOnnxPredict(TestCase): + # def testIONameParser(self): + # model = join(testdata.models_dir, "discogs-effnet-bsdynamic-1.onnx") + # print(f"\nmodel: {model}") + # configs = [ + # { + # "graphFilename": model, + # "inputs": ["model/Placeholder"], + # "outputs": ["model/Softmax:"], + # }, # No index. + # { + # "graphFilename": model, + # "inputs": ["model/Placeholder"], + # "outputs": ["model/Softmax:3"], + # }, # Index out of bounds. + # { + # "graphFilename": model, + # "inputs": ["model/Placeholder"], + # "outputs": ["model/Softmax::0"], + # }, # Double colon. + # { + # "graphFilename": model, + # "inputs": ["model/Placeholder"], + # "outputs": ["model/Softmax:s:0"], + # }, # Several colons. + # ] + + # for config in configs[1:]: + # with self.subTest(f"{config} failed"): + # self.assertConfigureFails(OnnxPredict(), config) + + def testInference(self,): + model = join(testdata.models_dir, "discogs-effnet-bsdynamic-1.onnx") + print(f"\nmodel: {model}") + output_layer_name0 = "activations" + output_layer_name1 = "embeddings" + onxx_predict = OnnxPredict( + graphFilename= model, + inputs=[], + outputs=[output_layer_name0, output_layer_name1] + ) + + stem = "387517__deleted_user_7267864__saxophone-going-up" + audio_path = join(testdata.audio_dir, Path("recorded"), f"{stem}.wav") + + audio, sample_rate = sf.read(audio_path, dtype=numpy.float32) + print(f"audio.shape: {audio.shape}") + + frame_size = 512 + hop_size = 256 + patch_size = 128 + number_bands = 96 + + w = Windowing(type="hann", zeroPadding=frame_size) + spectrum = Spectrum(size=frame_size) + mels = MelBands(inputSize=frame_size+1,numberBands=number_bands, type="magnitude") + logNorm = UnaryOperator(type="log") + + # compute mel bands + bands = [] + for frame in FrameGenerator(audio, frameSize=frame_size, hopSize=hop_size): + melFrame = mels(spectrum(w(frame))) + bands.append(logNorm(melFrame)) + bands = array(bands) + + discard = bands.shape[0] % patch_size + bands = numpy.reshape(bands[:-discard, :], [-1, patch_size, number_bands]) + batch = numpy.expand_dims(bands, 1) + print(f"bands.shape: {bands.shape}") + print(f"batch.shape: {batch.shape}") + + pool = Pool() + pool.set("melspectrogram", batch) + + pool_out = onxx_predict(pool) + + # print(f"op: {dir(pool_out)}") + print(f"descriptorNames: {pool_out.descriptorNames()}") + print(f"result['{output_layer_name0}']: {pool_out[output_layer_name0]}") + print(f"result['{output_layer_name0}'].shape(): {pool_out[output_layer_name0].shape}") + + print(f"result['{output_layer_name1}']: {pool_out[output_layer_name1]}") + print(f"result['{output_layer_name0}'].shape(): {pool_out[output_layer_name1].shape}") + + AssertionError() + + """ + def regression(self, parameters): + # Test a simple tensorflow model trained on Essentia features. + # The ground true values were obtained with the following script: + expectedValues = [9.9997020e-01, 5.3647455e-14, 2.9801236e-05, 4.9495230e-12] + + patchSize = 128 + numberBands = 128 + frameSize = 1024 + hopSize = frameSize + + filename = join(testdata.audio_dir, "recorded", "cat_purrrr.wav") + + audio = MonoLoader(filename=filename)() + + w = Windowing(type="hann", zeroPadding=frameSize) + spectrum = Spectrum() + mels = MelBands(numberBands=numberBands, type="magnitude") + logNorm = UnaryOperator(type="log") + + bands = [] + for frame in FrameGenerator(audio, frameSize=frameSize, hopSize=hopSize): + melFrame = mels(spectrum(w(frame))) + bands.append(logNorm(melFrame)) + bands = array(bands) + + discard = bands.shape[0] % patchSize + bands = numpy.reshape(bands[:-discard, :], [-1, patchSize, numberBands]) + batch = numpy.expand_dims(bands, 1) + + pool = Pool() + pool.set("model/Placeholder", batch) + + tfp = TensorflowPredict(**parameters) + poolOut = tfp(pool) + + foundValues = poolOut["model/Softmax"].mean(axis=0).squeeze() + + self.assertAlmostEqualVector(foundValues, expectedValues, 1e-5) + + def testRegressionFrozenModel(self): + parameters = { + "graphFilename": join(testdata.models_dir, "vgg", "vgg4.pb"), + "inputs": ["model/Placeholder"], + "outputs": ["model/Softmax"], + "isTraining": False, + "isTrainingName": "model/Placeholder_1", + } + + self.regression(parameters) + + def testRegressionSavedModel(self): + parameters = { + "savedModel": join(testdata.models_dir, "vgg", "vgg4"), + "inputs": ["model/Placeholder"], + "outputs": ["model/Softmax"], + "isTraining": False, + "isTrainingName": "model/Placeholder_1", + } + + self.regression(parameters) + + def testSavedModelOverridesGraphFilename(self): + # When both are specified, `savedModel` should be preferred. + # Test this by setting an invalid `graphFilename` that should be ignored. + parameters = { + "graphFilename": "wrong_model", + "savedModel": join(testdata.models_dir, "vgg", "vgg4"), + "inputs": ["model/Placeholder"], + "outputs": ["model/Softmax"], + "isTraining": False, + "isTrainingName": "model/Placeholder_1", + } + + self.regression(parameters) + + def testEmptyModelName(self): + # With empty model name the algorithm should skip the configuration without errors. + self.assertConfigureSuccess(TensorflowPredict(), {}) + self.assertConfigureSuccess(TensorflowPredict(), {"graphFilename": ""}) + self.assertConfigureSuccess( + TensorflowPredict(), {"graphFilename": "", "inputs": [""]} + ) + self.assertConfigureSuccess( + TensorflowPredict(), {"graphFilename": "", "inputs": ["wrong_input"]} + ) + self.assertConfigureSuccess(TensorflowPredict(), {"savedModel": ""}) + self.assertConfigureSuccess( + TensorflowPredict(), {"savedModel": "", "inputs": [""]} + ) + self.assertConfigureSuccess( + TensorflowPredict(), {"savedModel": "", "inputs": ["wrong_input"]} + ) + self.assertConfigureSuccess( + TensorflowPredict(), {"graphFilename": "", "savedModel": ""} + ) + self.assertConfigureSuccess( + TensorflowPredict(), {"graphFilename": "", "savedModel": "", "inputs": [""]} + ) + self.assertConfigureSuccess( + TensorflowPredict(), + {"graphFilename": "", "savedModel": "", "inputs": ["wrong_input"]}, + ) + + def testInvalidParam(self): + model = join(testdata.models_dir, "vgg", "vgg4.pb") + self.assertConfigureFails( + TensorflowPredict(), {"graphFilename": model} + ) # inputs and outputs are not defined + self.assertConfigureFails( + TensorflowPredict(), + { + "graphFilename": model, + "inputs": ["model/Placeholder"], + }, + ) # outputs are not defined + self.assertConfigureFails( + TensorflowPredict(), + { + "graphFilename": model, + "inputs": ["wrong_input_name"], + "outputs": ["model/Softmax"], + }, + ) # input does not exist in the model + self.assertConfigureFails( + TensorflowPredict(), + { + "graphFilename": "wrong_model_name", + "inputs": ["model/Placeholder"], + "outputs": ["model/Softmax"], + }, + ) # the model does not exist + + # Repeat tests for savedModel format. + model = join(testdata.models_dir, "vgg", "vgg4/") + self.assertConfigureFails( + TensorflowPredict(), {"savedModel": model} + ) # inputs and outputs are not defined + self.assertConfigureFails( + TensorflowPredict(), + { + "savedModel": model, + "inputs": ["model/Placeholder"], + }, + ) # outputs are not defined + self.assertConfigureFails( + TensorflowPredict(), + { + "savedModel": model, + "inputs": ["wrong_input_name"], + "outputs": ["model/Softmax"], + }, + ) # input does not exist in the model + self.assertConfigureFails( + TensorflowPredict(), + { + "savedModel": "wrong_model_name", + "inputs": ["model/Placeholder"], + "outputs": ["model/Softmax"], + }, + ) # the model does not exist + + def testIdentityModel(self): + # Perform the identity operation in Tensorflow to test if the data is + # being copied correctly backwards and fordwards. + model = join(filedir(), "tensorflowpredict", "identity.pb") + filename = join(testdata.audio_dir, "recorded", "cat_purrrr.wav") + + audio = MonoLoader(filename=filename)() + frames = array([frame for frame in FrameGenerator(audio)]) + batch = frames[numpy.newaxis, numpy.newaxis, :] + + pool = Pool() + pool.set("model/Placeholder", batch) + + poolOut = TensorflowPredict( + graphFilename=model, + inputs=["model/Placeholder"], + outputs=["model/Identity"], + )(pool) + + foundValues = poolOut["model/Identity"] + + self.assertAlmostEqualMatrix(foundValues, batch) + + def testComputeWithoutConfiguration(self): + pool = Pool() + pool.set("model/Placeholder", numpy.zeros((1, 1, 1, 1), dtype="float32")) + + self.assertComputeFails(TensorflowPredict(), pool) + + def testIgnoreInvalidReconfiguration(self): + pool = Pool() + pool.set("model/Placeholder", numpy.ones((1, 1, 1, 1), dtype="float32")) + + model_name = join(filedir(), "tensorflowpredict", "identity.pb") + model = TensorflowPredict( + graphFilename=model_name, + inputs=["model/Placeholder"], + outputs=["model/Identity"], + squeeze=False, + ) + + firstResult = model(pool)["model/Identity"] + + # This attempt to reconfigure the algorithm should be ignored and trigger a Warning. + model.configure() + + secondResult = model(pool)["model/Identity"] + + self.assertEqualMatrix(firstResult, secondResult) + + def testImplicitOutputTensorIndex(self): + model = join(filedir(), "tensorflowpredict", "identity.pb") + batch = numpy.reshape(numpy.arange(4, dtype="float32"), (1, 1, 2, 2)) + + pool = Pool() + pool.set("model/Placeholder", batch) + + implicit_output = "model/Identity" + implicit = TensorflowPredict( + graphFilename=model, + inputs=["model/Placeholder"], + outputs=[implicit_output], + )(pool)[implicit_output].squeeze() + + explicit_output = "model/Identity:0" + explicit = TensorflowPredict( + graphFilename=model, + inputs=["model/Placeholder"], + outputs=[explicit_output], + )(pool)[explicit_output].squeeze() + + self.assertAlmostEqualMatrix(implicit, explicit) + + def testNodeNameParser(self): + model = join(testdata.models_dir, "vgg", "vgg4.pb") + + configs = [ + { + "graphFilename": model, + "inputs": ["model/Placeholder"], + "outputs": ["model/Softmax:"], + }, # No index. + { + "graphFilename": model, + "inputs": ["model/Placeholder"], + "outputs": ["model/Softmax:3"], + }, # Index out of bounds. + { + "graphFilename": model, + "inputs": ["model/Placeholder"], + "outputs": ["model/Softmax::0"], + }, # Double colon. + { + "graphFilename": model, + "inputs": ["model/Placeholder"], + "outputs": ["model/Softmax:s:0"], + }, # Several colons. + ] + + for config in configs[1:]: + with self.subTest(f"{config} failed"): + self.assertConfigureFails(TensorflowPredict(), config) + """ + +suite = allTests(TestOnnxPredict) + +if __name__ == "__main__": + TextTestRunner(verbosity=2).run(suite) From bce71f93df1f6921ec8e3b9ef7faa4a3ca96a524 Mon Sep 17 00:00:00 2001 From: xaviliz Date: Tue, 2 Sep 2025 20:35:30 +0200 Subject: [PATCH 14/60] Add support for building onnxruntime in Linux. --- packaging/debian_3rdparty/build_onnx.sh | 44 ++++++++++++++----------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/packaging/debian_3rdparty/build_onnx.sh b/packaging/debian_3rdparty/build_onnx.sh index 7571cb23b..e3fd7096c 100644 --- a/packaging/debian_3rdparty/build_onnx.sh +++ b/packaging/debian_3rdparty/build_onnx.sh @@ -18,29 +18,33 @@ cd onnxruntime-$LIBONNXRUNTIME_VERSION python3 -m pip install cmake -# Build the dynamic library for Linux -# ./build.sh \ -# --config RelWithDebInfo \ -# --build_shared_lib \ -# --parallel \ -# --compile_no_warning_as_error \ -# --skip_submodule_sync - -# Build the dynamic library for MacOS (build for Intel and Apple silicon CPUs --> "x86_64;arm64") -./build.sh \ - --config RelWithDebInfo \ - --build_shared_lib \ - --parallel \ - --compile_no_warning_as_error \ - --skip_submodule_sync \ - --cmake_extra_defines CMAKE_OSX_ARCHITECTURES="arm64" FETCHCONTENT_TRY_FIND_PACKAGE_MODE=NEVER CMAKE_INSTALL_PREFIX=$PREFIX - -#! We have found some issues building for cross-platforms, it looks it is much better to build it in a docker -#! In MacOS, we have experienced issues with the brew package. So, it needs to uninstall brew applications first (brew unsnstall onnxruntime) +# Build the dynamic library for Linux or MacOS +# build for Intel and Apple silicon CPUs --> "x86_64;arm64" + +CMAKE_EXTRA_DEFINES="FETCHCONTENT_TRY_FIND_PACKAGE_MODE=NEVER CMAKE_INSTALL_PREFIX=${PREFIX}" +OS=$(uname -s) + +if [ "$OS" = "Darwin" ]; then + DIR_OS="MacOS" + CMAKE_EXTRA_DEFINES+=' CMAKE_OSX_ARCHITECTURES="arm64"' +else + DIR_OS="Linux" +fi + +CONFIG="Release" +./build.sh \ + --config $CONFIG \ + --build_shared_lib \ + --parallel \ + --compile_no_warning_as_error \ + --skip_submodule_sync \ + --allow_running_as_root \ + --skip_tests \ + --cmake_extra_defines $CMAKE_EXTRA_DEFINES # copying .pc file mkdir -p "${PREFIX}"/lib/pkgconfig/ -cp build/MacOS/RelWithDebInfo/libonnxruntime.pc ${PREFIX}/lib/pkgconfig/ +cp -r build/$DIR_OS/$CONFIG/libonnxruntime.* ${PREFIX}/lib/pkgconfig/ cd ../.. rm -fr tmp From 28da0a945a06bb69e024f16efbc58c06a2be185f Mon Sep 17 00:00:00 2001 From: xaviliz Date: Wed, 3 Sep 2025 23:55:56 +0200 Subject: [PATCH 15/60] Save onnxruntime files after building --- packaging/debian_3rdparty/build_onnx.sh | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/packaging/debian_3rdparty/build_onnx.sh b/packaging/debian_3rdparty/build_onnx.sh index e3fd7096c..bc0363537 100644 --- a/packaging/debian_3rdparty/build_onnx.sh +++ b/packaging/debian_3rdparty/build_onnx.sh @@ -27,8 +27,10 @@ OS=$(uname -s) if [ "$OS" = "Darwin" ]; then DIR_OS="MacOS" CMAKE_EXTRA_DEFINES+=' CMAKE_OSX_ARCHITECTURES="arm64"' + SUFFIX="${LIBONNXRUNTIME_VERSION}.dylib" else DIR_OS="Linux" + SUFFIX="so.${LIBONNXRUNTIME_VERSION}" fi CONFIG="Release" @@ -42,9 +44,16 @@ CONFIG="Release" --skip_tests \ --cmake_extra_defines $CMAKE_EXTRA_DEFINES -# copying .pc file +# copying onnxruntime files mkdir -p "${PREFIX}"/lib/pkgconfig/ -cp -r build/$DIR_OS/$CONFIG/libonnxruntime.* ${PREFIX}/lib/pkgconfig/ +mkdir -p "${PREFIX}"/include/onnxruntime/ + +cp build/$DIR_OS/$CONFIG/libonnxruntime.pc ${PREFIX}/lib/pkgconfig/ +cp build/$DIR_OS/$CONFIG/libonnxruntime.$SUFFIX ${PREFIX}/lib/ +cp include/onnxruntime/core/session/onnxruntime_cxx_inline.h ${PREFIX}/include/onnxruntime/ +cp include/onnxruntime/core/session/onnxruntime_float16.h ${PREFIX}/include/onnxruntime/ +cp include/onnxruntime/core/session/onnxruntime_c_api.h ${PREFIX}/include/onnxruntime/ +cp include/onnxruntime/core/session/onnxruntime_cxx_api.h ${PREFIX}/include/onnxruntime/ cd ../.. rm -fr tmp From 0da98008dcfa540885188742801ec9fcdb3e3932 Mon Sep 17 00:00:00 2001 From: xaviliz Date: Thu, 4 Sep 2025 12:01:08 +0200 Subject: [PATCH 16/60] Copy all dynamic library files --- packaging/debian_3rdparty/build_onnx.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packaging/debian_3rdparty/build_onnx.sh b/packaging/debian_3rdparty/build_onnx.sh index bc0363537..00945581c 100644 --- a/packaging/debian_3rdparty/build_onnx.sh +++ b/packaging/debian_3rdparty/build_onnx.sh @@ -27,10 +27,10 @@ OS=$(uname -s) if [ "$OS" = "Darwin" ]; then DIR_OS="MacOS" CMAKE_EXTRA_DEFINES+=' CMAKE_OSX_ARCHITECTURES="arm64"' - SUFFIX="${LIBONNXRUNTIME_VERSION}.dylib" + SUFFIX="${LIBONNXRUNTIME_VERSION}.dylib*" else DIR_OS="Linux" - SUFFIX="so.${LIBONNXRUNTIME_VERSION}" + SUFFIX="so*" fi CONFIG="Release" @@ -49,7 +49,8 @@ mkdir -p "${PREFIX}"/lib/pkgconfig/ mkdir -p "${PREFIX}"/include/onnxruntime/ cp build/$DIR_OS/$CONFIG/libonnxruntime.pc ${PREFIX}/lib/pkgconfig/ -cp build/$DIR_OS/$CONFIG/libonnxruntime.$SUFFIX ${PREFIX}/lib/ +cp -r build/$DIR_OS/$CONFIG/libonnxruntime.$SUFFIX ${PREFIX}/lib/ + cp include/onnxruntime/core/session/onnxruntime_cxx_inline.h ${PREFIX}/include/onnxruntime/ cp include/onnxruntime/core/session/onnxruntime_float16.h ${PREFIX}/include/onnxruntime/ cp include/onnxruntime/core/session/onnxruntime_c_api.h ${PREFIX}/include/onnxruntime/ From 88787a225e54825524a5f4ecd838f1ceb4deee58 Mon Sep 17 00:00:00 2001 From: xaviliz Date: Thu, 4 Sep 2025 14:12:26 +0200 Subject: [PATCH 17/60] Fix stem in audio path --- test/src/unittests/machinelearning/test_onnxpredict.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/src/unittests/machinelearning/test_onnxpredict.py b/test/src/unittests/machinelearning/test_onnxpredict.py index 43758cfe4..a13295a5a 100644 --- a/test/src/unittests/machinelearning/test_onnxpredict.py +++ b/test/src/unittests/machinelearning/test_onnxpredict.py @@ -67,7 +67,7 @@ def testInference(self,): outputs=[output_layer_name0, output_layer_name1] ) - stem = "387517__deleted_user_7267864__saxophone-going-up" + stem = "359500__mtg__sax-tenor-e-major" audio_path = join(testdata.audio_dir, Path("recorded"), f"{stem}.wav") audio, sample_rate = sf.read(audio_path, dtype=numpy.float32) From 1ee0fe63abc96696d2e4923d6eeb83a2964169f1 Mon Sep 17 00:00:00 2001 From: xaviliz Date: Fri, 5 Sep 2025 13:53:51 +0200 Subject: [PATCH 18/60] Clean and update the parameter declaration --- src/algorithms/machinelearning/onnxpredict.h | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/algorithms/machinelearning/onnxpredict.h b/src/algorithms/machinelearning/onnxpredict.h index 42cbb4fc1..89e4b80bd 100644 --- a/src/algorithms/machinelearning/onnxpredict.h +++ b/src/algorithms/machinelearning/onnxpredict.h @@ -113,11 +113,9 @@ class OnnxPredict : public Algorithm { const char* defaultTagsC[] = { "serve" }; std::vector defaultTags = arrayToVector(defaultTagsC); - declareParameter("graphFilename", "the name of the file from which to load the TensorFlow graph", "", ""); - declareParameter("inputs", "will look for these namespaces in poolIn. Should match the names of the input nodes in the Tensorflow graph", "", Parameter::VECTOR_STRING); - // TODO: fix the empty input and output parameters. - declareParameter("outputs", "will save the tensors on the graph nodes named after `outputs` to the same namespaces in the output pool. Set the first element of this list as an empty array to print all the available nodes in the graph", "", Parameter::VECTOR_STRING); - declareParameter("isTraining", "run the model in training mode (normalized with statistics of the current batch) instead of inference mode (normalized with moving statistics). This only applies to some models", "{true,false}", false); + declareParameter("graphFilename", "the name of the file from which to load the ONNX model", "", ""); + declareParameter("inputs", "will look for these namespaces in poolIn. Should match the names of the inputs in the ONNX model", "", Parameter::VECTOR_STRING); + declareParameter("outputs", "will save the tensors on the model outputs named after `outputs` to the same namespaces in the output pool. Set the first element of this list as an empty array to print all the available model outputs", "", Parameter::VECTOR_STRING); declareParameter("squeeze", "remove singleton dimensions of the inputs tensors. Does not apply to the batch dimension", "{true,false}", true); } From 83ed01d638c2e94856ddeff62995518df4122080 Mon Sep 17 00:00:00 2001 From: xaviliz Date: Fri, 5 Sep 2025 13:57:28 +0200 Subject: [PATCH 19/60] Small clean --- src/algorithms/machinelearning/onnxpredict.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/algorithms/machinelearning/onnxpredict.h b/src/algorithms/machinelearning/onnxpredict.h index 89e4b80bd..56d403e09 100644 --- a/src/algorithms/machinelearning/onnxpredict.h +++ b/src/algorithms/machinelearning/onnxpredict.h @@ -67,8 +67,7 @@ class OnnxPredict : public Algorithm { Ort::RunOptions _runOptions; Ort::AllocatorWithDefaultOptions _allocator; - Ort::MemoryInfo _memoryInfo{ nullptr }; // Used to allocate memory for input - Ort::Model _model; + Ort::MemoryInfo _memoryInfo{ nullptr }; // Used to allocate memory for inputs std::vector all_input_infos; std::vector all_output_infos; @@ -94,7 +93,7 @@ class OnnxPredict : public Algorithm { public: OnnxPredict() : _env(Ort::Env(ORT_LOGGING_LEVEL_WARNING, "test")), - _sessionOptions(Ort::SessionOptions()), _session(Ort::Session(nullptr)), _runOptions(NULL), _isConfigured(false) , _model(Ort::Model(nullptr)){ + _sessionOptions(Ort::SessionOptions()), _session(Ort::Session(nullptr)), _runOptions(NULL), _isConfigured(false){ declareInput(_poolIn, "poolIn", "the pool where to get the feature tensors"); declareOutput(_poolOut, "poolOut", "the pool where to store the output tensors"); } From 5a300fc5346723a3acd97c1873c5695114d32f16 Mon Sep 17 00:00:00 2001 From: xaviliz Date: Fri, 5 Sep 2025 14:45:15 +0200 Subject: [PATCH 20/60] Fix issue in the EssentiaExceptions for name parsing. --- src/algorithms/machinelearning/onnxpredict.cpp | 17 ++++++++--------- src/algorithms/machinelearning/onnxpredict.h | 9 ++++++++- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/algorithms/machinelearning/onnxpredict.cpp b/src/algorithms/machinelearning/onnxpredict.cpp index f76f504c4..90473d9bf 100644 --- a/src/algorithms/machinelearning/onnxpredict.cpp +++ b/src/algorithms/machinelearning/onnxpredict.cpp @@ -86,7 +86,7 @@ void OnnxPredict::configure() { _nInputs = _inputs.size(); _nOutputs = _outputs.size(); - cout << "_inputs.size(): " << _nInputs << ".\n"; + // cout << "_inputs.size(): " << _nInputs << ".\n"; // use the first input when no input is defined if (_nInputs == 0){ @@ -124,14 +124,14 @@ void OnnxPredict::configure() { } if (_inputNodes.size() == 0){ + std::string s; for (const auto &piece : _inputs) { s += piece; s += " "; } - throw EssentiaException("ONNXRuntimePredict: '" + s + - "' are not valid input name of this model.\n" + - getTensorInfos(all_input_infos, "Model Inputs")); + + throw EssentiaException(availableInputInfo()); } for (int i = 0; i < _outputs.size(); i++) { @@ -148,9 +148,7 @@ void OnnxPredict::configure() { s += piece; s += " "; } - throw EssentiaException("ONNXRuntimePredict: '" + s + - "' has not a valid output name of this model.\n" + - getTensorInfos(all_output_infos, "Model Outputs")); + throw EssentiaException(availableOutputInfo()); } _isConfigured = true; @@ -198,8 +196,8 @@ std::vector OnnxPredict::setTensorInfos(const Ort::Session& session, void OnnxPredict::printTensorInfos(const std::vector& infos, const std::string& label) { std::cout << "=== " << label << " ===\n"; for (const auto& info : infos) { - std::cout << "[Name] " << info.name << "\n"; - std::cout << " [Type] " << info.type << "\n"; + std::cout << "[Name] " << info.name << endl; + std::cout << " [Type] " << info.type << endl; std::cout << " [Shape] ["; for (size_t j = 0; j < info.shape.size(); ++j) { std::cout << info.shape[j]; @@ -224,6 +222,7 @@ std::string OnnxPredict::getTensorInfos(const std::vector& infos, co } out += "]\n"; } + return out; } void OnnxPredict::reset() { diff --git a/src/algorithms/machinelearning/onnxpredict.h b/src/algorithms/machinelearning/onnxpredict.h index 56d403e09..ecbe342ec 100644 --- a/src/algorithms/machinelearning/onnxpredict.h +++ b/src/algorithms/machinelearning/onnxpredict.h @@ -87,7 +87,14 @@ class OnnxPredict : public Algorithm { std::vector inputs = inputNames(); std::string info = "OnnxPredict: Available input names are:\n"; for (std::vector::const_iterator i = inputs.begin(); i != inputs.end() - 1; ++i) info += *i + ", "; - return info + inputs.back() + ".\n\nReconfigure this algorithm with valid node names as inputs and outputs before starting the processing."; + return info + inputs.back() + ".\n\nReconfigure this algorithm with valid input names before starting the processing."; + } + + inline std::string availableOutputInfo() { + std::vector outputs = outputNames(); + std::string info = "OnnxPredict: Available output names are:\n"; + for (std::vector::const_iterator i = outputs.begin(); i != outputs.end() - 1; ++i) info += *i + ", "; + return info + outputs.back() + ".\n\nReconfigure this algorithm with valid output names before starting the processing."; } public: From 4f7955cbb88557dee2fec59b3d5e4bd8940aff3b Mon Sep 17 00:00:00 2001 From: xaviliz Date: Fri, 5 Sep 2025 14:46:27 +0200 Subject: [PATCH 21/60] Add TestIONameParser() --- .../machinelearning/test_onnxpredict.py | 60 +++++++++---------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/test/src/unittests/machinelearning/test_onnxpredict.py b/test/src/unittests/machinelearning/test_onnxpredict.py index a13295a5a..021e6e9e2 100644 --- a/test/src/unittests/machinelearning/test_onnxpredict.py +++ b/test/src/unittests/machinelearning/test_onnxpredict.py @@ -26,35 +26,36 @@ class TestOnnxPredict(TestCase): - # def testIONameParser(self): - # model = join(testdata.models_dir, "discogs-effnet-bsdynamic-1.onnx") - # print(f"\nmodel: {model}") - # configs = [ - # { - # "graphFilename": model, - # "inputs": ["model/Placeholder"], - # "outputs": ["model/Softmax:"], - # }, # No index. - # { - # "graphFilename": model, - # "inputs": ["model/Placeholder"], - # "outputs": ["model/Softmax:3"], - # }, # Index out of bounds. - # { - # "graphFilename": model, - # "inputs": ["model/Placeholder"], - # "outputs": ["model/Softmax::0"], - # }, # Double colon. - # { - # "graphFilename": model, - # "inputs": ["model/Placeholder"], - # "outputs": ["model/Softmax:s:0"], - # }, # Several colons. - # ] - - # for config in configs[1:]: - # with self.subTest(f"{config} failed"): - # self.assertConfigureFails(OnnxPredict(), config) + def testIONameParser(self): + model = join(testdata.models_dir, "discogs-effnet-bsdynamic-1.onnx") + print(f"\nmodel: {model}") + configs = [ + { + "graphFilename": model, + "inputs": ["model/Placeholder"], + "outputs": ["model/Softmax:"], + }, # No index. + { + "graphFilename": model, + "inputs": ["model/Placeholder"], + "outputs": ["model/Softmax:3"], + }, # Index out of bounds. + { + "graphFilename": model, + "inputs": ["model/Placeholder"], + "outputs": ["model/Softmax::0"], + }, # Double colon. + { + "graphFilename": model, + "inputs": ["model/Placeholder"], + "outputs": ["model/Softmax:s:0"], + }, # Several colons. + ] + + for config in configs[1:]: + with self.subTest(f"{config} failed"): + print(config) + self.assertConfigureFails(OnnxPredict(), config) def testInference(self,): model = join(testdata.models_dir, "discogs-effnet-bsdynamic-1.onnx") @@ -101,7 +102,6 @@ def testInference(self,): pool_out = onxx_predict(pool) - # print(f"op: {dir(pool_out)}") print(f"descriptorNames: {pool_out.descriptorNames()}") print(f"result['{output_layer_name0}']: {pool_out[output_layer_name0]}") print(f"result['{output_layer_name0}'].shape(): {pool_out[output_layer_name0].shape}") From 29d0e2bf1eabff1ceefcf64c4d9044e1c73c57a0 Mon Sep 17 00:00:00 2001 From: xaviliz Date: Fri, 5 Sep 2025 14:57:32 +0200 Subject: [PATCH 22/60] Adapt testEmptyModelName() unittest --- .../unittests/machinelearning/test_onnxpredict.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test/src/unittests/machinelearning/test_onnxpredict.py b/test/src/unittests/machinelearning/test_onnxpredict.py index 021e6e9e2..62451f130 100644 --- a/test/src/unittests/machinelearning/test_onnxpredict.py +++ b/test/src/unittests/machinelearning/test_onnxpredict.py @@ -111,6 +111,19 @@ def testInference(self,): AssertionError() + def testEmptyModelName(self): + # With empty model name the algorithm should skip the configuration without errors. + self.assertConfigureSuccess(OnnxPredict(), {}) + self.assertConfigureSuccess(OnnxPredict(), {"graphFilename": ""}) + self.assertConfigureSuccess( + OnnxPredict(), {"graphFilename": "", "inputs": [""]} + ) + self.assertConfigureSuccess( + OnnxPredict(), {"graphFilename": "", "inputs": ["wrong_input"]} + ) + + # TODO: addapt TensorPredict unittests such as testIdentityModel, testComputeWithoutConfiguration, testIgnoreInvalidReconfiguration, testInvalidParameters + """ def regression(self, parameters): # Test a simple tensorflow model trained on Essentia features. From 2655a031c18c1653b353165530047e350ed23fa0 Mon Sep 17 00:00:00 2001 From: xaviliz Date: Tue, 9 Sep 2025 12:35:36 +0200 Subject: [PATCH 23/60] Add testInvalidParams() as unitest --- .../machinelearning/test_onnxpredict.py | 27 ++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/test/src/unittests/machinelearning/test_onnxpredict.py b/test/src/unittests/machinelearning/test_onnxpredict.py index 62451f130..a11f0a7ae 100644 --- a/test/src/unittests/machinelearning/test_onnxpredict.py +++ b/test/src/unittests/machinelearning/test_onnxpredict.py @@ -59,6 +59,8 @@ def testIONameParser(self): def testInference(self,): model = join(testdata.models_dir, "discogs-effnet-bsdynamic-1.onnx") + # TODO: store model card and to extract model output shapes to assert the ouputs + print(f"\nmodel: {model}") output_layer_name0 = "activations" output_layer_name1 = "embeddings" @@ -110,6 +112,7 @@ def testInference(self,): print(f"result['{output_layer_name0}'].shape(): {pool_out[output_layer_name1].shape}") AssertionError() + # TODO: assert the output shapes def testEmptyModelName(self): # With empty model name the algorithm should skip the configuration without errors. @@ -122,7 +125,29 @@ def testEmptyModelName(self): OnnxPredict(), {"graphFilename": "", "inputs": ["wrong_input"]} ) - # TODO: addapt TensorPredict unittests such as testIdentityModel, testComputeWithoutConfiguration, testIgnoreInvalidReconfiguration, testInvalidParameters + def testInvalidParam(self): + model = join(testdata.models_dir, "discogs-effnet-bsdynamic-1.onnx") + self.assertConfigureFails( + OnnxPredict(), + { + "graphFilename": model, + "inputs": ["wrong_input_name"], + "outputs": ["embeddings"], + }, + ) # input does not exist in the model + self.assertConfigureFails( + OnnxPredict(), + { + "graphFilename": "wrong_model_name", #! I suspect the issue is here with OnnxExceptions + "inputs": ["melspectrogram"], + "outputs": ["embeddings"], + }, + ) # the model does not exist + + # TODO: adapt TensorPredict unittests such as testIdentityModel, testComputeWithoutConfiguration, testIgnoreInvalidReconfiguration + # TODO: make a test for squeeze, showing that it fails when it is not applied with 2D models + # TODO: make a test for squeeze, showing that it works well when it is applied for a 2D model + """ def regression(self, parameters): From 2c42e914a49a0a59d3ca2bc4ad9e8c44d05bb8da Mon Sep 17 00:00:00 2001 From: xaviliz Date: Tue, 9 Sep 2025 14:02:49 +0200 Subject: [PATCH 24/60] Update effnetdiscogs-bsdynamic-1.onnx location --- test/src/unittests/machinelearning/test_onnxpredict.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/src/unittests/machinelearning/test_onnxpredict.py b/test/src/unittests/machinelearning/test_onnxpredict.py index a11f0a7ae..a1fc0be41 100644 --- a/test/src/unittests/machinelearning/test_onnxpredict.py +++ b/test/src/unittests/machinelearning/test_onnxpredict.py @@ -27,7 +27,7 @@ class TestOnnxPredict(TestCase): def testIONameParser(self): - model = join(testdata.models_dir, "discogs-effnet-bsdynamic-1.onnx") + model = join(testdata.models_dir, "effnetdiscogs", "effnetdiscogs-bsdynamic-1.onnx") print(f"\nmodel: {model}") configs = [ { @@ -58,7 +58,7 @@ def testIONameParser(self): self.assertConfigureFails(OnnxPredict(), config) def testInference(self,): - model = join(testdata.models_dir, "discogs-effnet-bsdynamic-1.onnx") + model = join(testdata.models_dir, "effnetdiscogs", "effnetdiscogs-bsdynamic-1.onnx") # TODO: store model card and to extract model output shapes to assert the ouputs print(f"\nmodel: {model}") @@ -126,7 +126,7 @@ def testEmptyModelName(self): ) def testInvalidParam(self): - model = join(testdata.models_dir, "discogs-effnet-bsdynamic-1.onnx") + model = join(testdata.models_dir, "effnetdiscogs", "effnetdiscogs-bsdynamic-1.onnx") self.assertConfigureFails( OnnxPredict(), { From c7669524dc64c5255763eaca5ef17ea6b9a2ac8c Mon Sep 17 00:00:00 2001 From: xaviliz Date: Tue, 9 Sep 2025 14:04:47 +0200 Subject: [PATCH 25/60] Throw EssentiaException for empty model names. --- src/algorithms/machinelearning/onnxpredict.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/algorithms/machinelearning/onnxpredict.cpp b/src/algorithms/machinelearning/onnxpredict.cpp index 90473d9bf..71415fe29 100644 --- a/src/algorithms/machinelearning/onnxpredict.cpp +++ b/src/algorithms/machinelearning/onnxpredict.cpp @@ -69,9 +69,9 @@ void OnnxPredict::configure() { _session = Ort::Session(_env, _graphFilename.c_str(), _sessionOptions); } catch (Ort::Exception oe) { - cout << "ONNX exception caught: " << oe.what() << ". Code: " << oe.GetOrtErrorCode() << ".\n"; - return; + throw EssentiaException(string("OnnxPredict:") + oe.what(), oe.GetOrtErrorCode()); } + E_INFO("OnnxPredict: Successfully loaded graph file: `" << _graphFilename << "`"); // get input and output info (names, type and shapes) all_input_infos = setTensorInfos(_session, _allocator, true); From 35b0228c2117102f9f2c95047f7935351744bcab Mon Sep 17 00:00:00 2001 From: xaviliz Date: Mon, 15 Sep 2025 16:14:12 +0200 Subject: [PATCH 26/60] Updated test/models submodule --- test/models | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/models b/test/models index 3ca4130bc..c298db07d 160000 --- a/test/models +++ b/test/models @@ -1 +1 @@ -Subproject commit 3ca4130bcb398a1361867e5d8462d3a7a0c02ccd +Subproject commit c298db07de13e79d9b1ab892b074ccad4cc10c20 From 4302cb989e983ad2d1c9d0df6fca782d76fbc498 Mon Sep 17 00:00:00 2001 From: xaviliz Date: Mon, 15 Sep 2025 16:23:09 +0200 Subject: [PATCH 27/60] Updated test/audio submodule --- test/audio | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/audio b/test/audio index 89df8e4d6..9d181685f 160000 --- a/test/audio +++ b/test/audio @@ -1 +1 @@ -Subproject commit 89df8e4d6fb306db6d2e14b6de6357aacda43b10 +Subproject commit 9d181685fe123624b976baf8918df335432bd2f1 From 9a09d955d2e4c5bb569b2bb0c3fe4c26666ee2d8 Mon Sep 17 00:00:00 2001 From: xaviliz Date: Mon, 15 Sep 2025 16:26:13 +0200 Subject: [PATCH 28/60] Polish ORT building command for MacOS --- packaging/debian_3rdparty/build_onnx.sh | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/packaging/debian_3rdparty/build_onnx.sh b/packaging/debian_3rdparty/build_onnx.sh index 00945581c..fb85f3518 100644 --- a/packaging/debian_3rdparty/build_onnx.sh +++ b/packaging/debian_3rdparty/build_onnx.sh @@ -23,18 +23,25 @@ python3 -m pip install cmake CMAKE_EXTRA_DEFINES="FETCHCONTENT_TRY_FIND_PACKAGE_MODE=NEVER CMAKE_INSTALL_PREFIX=${PREFIX}" OS=$(uname -s) +CONFIG=Release if [ "$OS" = "Darwin" ]; then DIR_OS="MacOS" - CMAKE_EXTRA_DEFINES+=' CMAKE_OSX_ARCHITECTURES="arm64"' + CMAKE_EXTRA_DEFINES+=' CMAKE_OSX_ARCHITECTURES=arm64' SUFFIX="${LIBONNXRUNTIME_VERSION}.dylib*" + + ./build.sh \ + --config $CONFIG \ + --build_shared_lib \ + --parallel \ + --skip_submodule_sync \ + --skip_tests \ + --cmake_extra_defines $CMAKE_EXTRA_DEFINES else DIR_OS="Linux" SUFFIX="so*" -fi -CONFIG="Release" -./build.sh \ + ./build.sh \ --config $CONFIG \ --build_shared_lib \ --parallel \ @@ -43,6 +50,7 @@ CONFIG="Release" --allow_running_as_root \ --skip_tests \ --cmake_extra_defines $CMAKE_EXTRA_DEFINES +fi # copying onnxruntime files mkdir -p "${PREFIX}"/lib/pkgconfig/ From b552601f6019bb7472bc88769463b3c1ad307667 Mon Sep 17 00:00:00 2001 From: xaviliz Date: Mon, 15 Sep 2025 16:27:32 +0200 Subject: [PATCH 29/60] Initial support for multi input models --- .../machinelearning/onnxpredict.cpp | 109 ++++++++++++------ 1 file changed, 73 insertions(+), 36 deletions(-) diff --git a/src/algorithms/machinelearning/onnxpredict.cpp b/src/algorithms/machinelearning/onnxpredict.cpp index 71415fe29..522ce5281 100644 --- a/src/algorithms/machinelearning/onnxpredict.cpp +++ b/src/algorithms/machinelearning/onnxpredict.cpp @@ -242,29 +242,28 @@ void OnnxPredict::compute() { const Pool& poolIn = _poolIn.get(); Pool& poolOut = _poolOut.get(); - std::vector shape; + std:vector> shapes; // Parse the input tensors from the pool into ONNX Runtime tensors. for (size_t i = 0; i < _nInputs; i++) { - cout << "_inputs[i]: " << _inputs[i] << endl; + cout << "_inputs[" << i << "]: " << _inputs[i] << endl; const Tensor& inputData = poolIn.value >(_inputs[i]); - - // Convert data to float32 - std::vector float_data(inputData.size()); - for (size_t j = 0; j < inputData.size(); ++j) { - float_data[j] = static_cast(inputData.data()[j]); - } - - // Step 2: Get shape + cout << "inputData.size(): " << inputData.size() << endl; + + // Step 1: Get tensor shape + std::vector shape; int dims = 1; shape.push_back((int64_t)inputData.dimension(0)); if (_squeeze) { - for(int i = 1; i < inputData.rank(); i++) { - if (inputData.dimension(i) > 1) { - shape.push_back((int64_t)inputData.dimension(i)); + //cout << "Applying squeeze!!!" << endl; + //cout << "inputData.rank(): " << inputData.rank() << endl; + + for(int j = 1; j < inputData.rank(); j++) { + if (inputData.dimension(j) > 1) { + shape.push_back((int64_t)inputData.dimension(j)); dims++; } } @@ -274,14 +273,21 @@ void OnnxPredict::compute() { shape.push_back((int64_t) 1); dims++; } + } else { dims = inputData.rank(); - for(int j = 1; j < dims; j++) { // HERE we need to jump i = 1 - 4D tensor input - //cout << inputData.dimension(j) << endl; + for(int j = 1; j < dims; j++) { + //cout << inputData.dimension(j) << endl; shape.push_back((int64_t)inputData.dimension(j)); } } - + + // Step 2: Convert data to float32 + std::vector float_data(inputData.size()); + for (size_t j = 0; j < inputData.size(); ++j) { + float_data[j] = static_cast(inputData.data()[j]); + } + // Step 3: Create ONNX Runtime tensor _memoryInfo = Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU); @@ -289,8 +295,24 @@ void OnnxPredict::compute() { throw EssentiaException("OnnxRuntimePredict: Error allocating memory for input tensor."); } + // display shape + cout << "shape: ["; + for (size_t j = 0; j < shape.size(); ++j) { + cout << shape[j]; + if (j + 1 < shape.size()) std::cout << ", "; + } + cout << "]" << endl; + + // display float_data + cout << "float_data: ["; + for (size_t j = 0; j < float_data.size(); ++j) { + cout << float_data[j]; + if (j + 1 < float_data.size()) std::cout << ", "; + } + cout << "]" << endl; + input_tensors.emplace_back(Ort::Value::CreateTensor(_memoryInfo, float_data.data(), float_data.size(), shape.data(), shape.size())); - + shapes.push_back(shape); } // Define input and output names @@ -303,12 +325,12 @@ void OnnxPredict::compute() { } // Run the Onnxruntime session. - auto output_tensors = _session.Run(_runOptions, // Run options. - input_names.data(), // Input node names. - input_tensors.data(), // Input tensor values. - _nInputs, // Number of inputs. - output_names.data(), // Output node names. - _nOutputs // Number of outputs. + auto output_tensors = _session.Run(_runOptions, // Run options. + input_names.data(), // Input node names. + input_tensors.data(), // Input tensor values. + _nInputs, // Number of inputs. + output_names.data(), // Output node names. + _nOutputs // Number of outputs. ); // Map output tensors to pool @@ -316,27 +338,42 @@ void OnnxPredict::compute() { const Real* outputData = output_tensors[i].GetTensorData(); - // Create and array to store the tensor shape. + + auto outputInfo = output_tensors[i].GetTensorTypeAndShapeInfo(); + cout << "GetElementType: " << outputInfo.GetElementType() << endl; + cout << "Dimensions of the output: " << outputInfo.GetShape().size() << endl; + cout << "Shape of the output: ["; + auto tensor_size = 1; + for (unsigned int shapeI = 0; shapeI < outputInfo.GetShape().size(); shapeI++){ + tensor_size *= outputInfo.GetShape()[shapeI]; + cout << outputInfo.GetShape()[shapeI]; + if (shapeI + 1 _shape {1, 1, 1, 1}; - //_shape[0] = (int)outputShapes[0]; - _shape[0] = (int)shape[0]; + _shape[0] = (int)shapes[0][0]; + for (size_t j = 1; j < _outputNodes[i].shape.size(); j++){ - _shape[j+1] = (int)_outputNodes[i].shape[j]; + int shape_idx = _shape.size() - j; + _shape[shape_idx] = (int)_outputNodes[i].shape[_outputNodes[i].shape.size() - j]; } // Store tensor in pool const Tensor tensorMap = TensorMap(outputData, _shape); poolOut.set(_outputs[i], tensorMap); } - - /* Cleanup - for (const auto& tensorInfo : all_input_infos) { - _allocator.Free((void*)tensorInfo.name.c_str()); - } - - for (const auto& tensorInfo : all_output_infos) { - _allocator.Free((void*)tensorInfo.name.c_str()); - }*/ } From a5268cc29256eaab4944db3be4a5252b0fffb7b6 Mon Sep 17 00:00:00 2001 From: xaviliz Date: Tue, 16 Sep 2025 11:56:52 +0200 Subject: [PATCH 30/60] Add testIdentityModel() --- .../machinelearning/test_onnxpredict.py | 252 ++++++++++-------- 1 file changed, 146 insertions(+), 106 deletions(-) diff --git a/test/src/unittests/machinelearning/test_onnxpredict.py b/test/src/unittests/machinelearning/test_onnxpredict.py index a1fc0be41..6b7765326 100644 --- a/test/src/unittests/machinelearning/test_onnxpredict.py +++ b/test/src/unittests/machinelearning/test_onnxpredict.py @@ -26,123 +26,163 @@ class TestOnnxPredict(TestCase): - def testIONameParser(self): - model = join(testdata.models_dir, "effnetdiscogs", "effnetdiscogs-bsdynamic-1.onnx") - print(f"\nmodel: {model}") - configs = [ - { - "graphFilename": model, - "inputs": ["model/Placeholder"], - "outputs": ["model/Softmax:"], - }, # No index. - { - "graphFilename": model, - "inputs": ["model/Placeholder"], - "outputs": ["model/Softmax:3"], - }, # Index out of bounds. - { - "graphFilename": model, - "inputs": ["model/Placeholder"], - "outputs": ["model/Softmax::0"], - }, # Double colon. - { - "graphFilename": model, - "inputs": ["model/Placeholder"], - "outputs": ["model/Softmax:s:0"], - }, # Several colons. - ] + # def testIONameParser(self): + # model = join(testdata.models_dir, "effnetdiscogs", "effnetdiscogs-bsdynamic-1.onnx") + # print(f"\nmodel: {model}") + # configs = [ + # { + # "graphFilename": model, + # "inputs": ["model/Placeholder"], + # "outputs": ["model/Softmax:"], + # }, # No index. + # { + # "graphFilename": model, + # "inputs": ["model/Placeholder"], + # "outputs": ["model/Softmax:3"], + # }, # Index out of bounds. + # { + # "graphFilename": model, + # "inputs": ["model/Placeholder"], + # "outputs": ["model/Softmax::0"], + # }, # Double colon. + # { + # "graphFilename": model, + # "inputs": ["model/Placeholder"], + # "outputs": ["model/Softmax:s:0"], + # }, # Several colons. + # ] + + # for config in configs[1:]: + # with self.subTest(f"{config} failed"): + # print(config) + # self.assertConfigureFails(OnnxPredict(), config) + + # def testInference(self,): + # model = join(testdata.models_dir, "effnetdiscogs", "effnetdiscogs-bsdynamic-1.onnx") + # # TODO: store model card and to extract model output shapes to assert the ouputs + + # print(f"\nmodel: {model}") + # output_layer_name0 = "activations" + # output_layer_name1 = "embeddings" + # onxx_predict = OnnxPredict( + # graphFilename= model, + # inputs=[], + # outputs=[output_layer_name0, output_layer_name1] + # ) + + # stem = "359500__mtg__sax-tenor-e-major" + # audio_path = join(testdata.audio_dir, Path("recorded"), f"{stem}.wav") + + # audio, sample_rate = sf.read(audio_path, dtype=numpy.float32) + # print(f"audio.shape: {audio.shape}") + + # frame_size = 512 + # hop_size = 256 + # patch_size = 128 + # number_bands = 96 + + # w = Windowing(type="hann", zeroPadding=frame_size) + # spectrum = Spectrum(size=frame_size) + # mels = MelBands(inputSize=frame_size+1,numberBands=number_bands, type="magnitude") + # logNorm = UnaryOperator(type="log") + + # # compute mel bands + # bands = [] + # for frame in FrameGenerator(audio, frameSize=frame_size, hopSize=hop_size): + # melFrame = mels(spectrum(w(frame))) + # bands.append(logNorm(melFrame)) + # bands = array(bands) + + # discard = bands.shape[0] % patch_size + # bands = numpy.reshape(bands[:-discard, :], [-1, patch_size, number_bands]) + # batch = numpy.expand_dims(bands, 1) + # print(f"bands.shape: {bands.shape}") + # print(f"batch.shape: {batch.shape}") + + # pool = Pool() + # pool.set("melspectrogram", batch) + + # pool_out = onxx_predict(pool) + + # print(f"descriptorNames: {pool_out.descriptorNames()}") + # print(f"result['{output_layer_name0}']: {pool_out[output_layer_name0]}") + # print(f"result['{output_layer_name0}'].shape(): {pool_out[output_layer_name0].shape}") + + # print(f"result['{output_layer_name1}']: {pool_out[output_layer_name1]}") + # print(f"result['{output_layer_name0}'].shape(): {pool_out[output_layer_name1].shape}") + + # AssertionError() + # # TODO: assert the output shapes + + # def testEmptyModelName(self): + # # With empty model name the algorithm should skip the configuration without errors. + # self.assertConfigureSuccess(OnnxPredict(), {}) + # self.assertConfigureSuccess(OnnxPredict(), {"graphFilename": ""}) + # self.assertConfigureSuccess( + # OnnxPredict(), {"graphFilename": "", "inputs": [""]} + # ) + # self.assertConfigureSuccess( + # OnnxPredict(), {"graphFilename": "", "inputs": ["wrong_input"]} + # ) + + # def testInvalidParam(self): + # model = join(testdata.models_dir, "effnetdiscogs", "effnetdiscogs-bsdynamic-1.onnx") + # self.assertConfigureFails( + # OnnxPredict(), + # { + # "graphFilename": model, + # "inputs": ["wrong_input_name"], + # "outputs": ["embeddings"], + # }, + # ) # input does not exist in the model + # self.assertConfigureFails( + # OnnxPredict(), + # { + # "graphFilename": "wrong_model_name", #! I suspect the issue is here with OnnxExceptions + # "inputs": ["melspectrogram"], + # "outputs": ["embeddings"], + # }, + # ) # the model does not exist - for config in configs[1:]: - with self.subTest(f"{config} failed"): - print(config) - self.assertConfigureFails(OnnxPredict(), config) - - def testInference(self,): - model = join(testdata.models_dir, "effnetdiscogs", "effnetdiscogs-bsdynamic-1.onnx") - # TODO: store model card and to extract model output shapes to assert the ouputs - - print(f"\nmodel: {model}") - output_layer_name0 = "activations" - output_layer_name1 = "embeddings" - onxx_predict = OnnxPredict( - graphFilename= model, - inputs=[], - outputs=[output_layer_name0, output_layer_name1] - ) - - stem = "359500__mtg__sax-tenor-e-major" - audio_path = join(testdata.audio_dir, Path("recorded"), f"{stem}.wav") - - audio, sample_rate = sf.read(audio_path, dtype=numpy.float32) - print(f"audio.shape: {audio.shape}") - - frame_size = 512 - hop_size = 256 - patch_size = 128 - number_bands = 96 - - w = Windowing(type="hann", zeroPadding=frame_size) - spectrum = Spectrum(size=frame_size) - mels = MelBands(inputSize=frame_size+1,numberBands=number_bands, type="magnitude") - logNorm = UnaryOperator(type="log") + def testIdentityModel(self): + model = join(testdata.models_dir, "identity", "identity2x2.onnx") - # compute mel bands - bands = [] - for frame in FrameGenerator(audio, frameSize=frame_size, hopSize=hop_size): - melFrame = mels(spectrum(w(frame))) - bands.append(logNorm(melFrame)) - bands = array(bands) + # prepare model inputs and batches + input1, input2 = (numpy.float32(numpy.random.random((3, 3))) for _ in range(2)) - discard = bands.shape[0] % patch_size - bands = numpy.reshape(bands[:-discard, :], [-1, patch_size, number_bands]) - batch = numpy.expand_dims(bands, 1) - print(f"bands.shape: {bands.shape}") - print(f"batch.shape: {batch.shape}") + n, m = input1.shape + + print(f"input1.shape: {(n, m)}") + batch1 = input1.reshape(n, 1, 1, m) + batch2 = input2.reshape(n, 1, 1, m) + + print(f"batch1.shape: {batch1.shape}") + print(f"batch2.shape: {batch2.shape}") + + print(f"batch1: {batch1}") pool = Pool() - pool.set("melspectrogram", batch) + pool.set("input1", batch1) + pool.set("input2", batch2) - pool_out = onxx_predict(pool) + poolOut = OnnxPredict( + graphFilename=model, + inputs=["input1", "input2"], + outputs=["output1", "output2"], + squeeze=True, + )(pool) - print(f"descriptorNames: {pool_out.descriptorNames()}") - print(f"result['{output_layer_name0}']: {pool_out[output_layer_name0]}") - print(f"result['{output_layer_name0}'].shape(): {pool_out[output_layer_name0].shape}") + found_values1 = poolOut["output1"] + found_values2 = poolOut["output2"] - print(f"result['{output_layer_name1}']: {pool_out[output_layer_name1]}") - print(f"result['{output_layer_name0}'].shape(): {pool_out[output_layer_name1].shape}") + print(f"found_values1.shape: {found_values1.shape}") + print(f"found_values2.shape: {found_values2.shape}") - AssertionError() - # TODO: assert the output shapes + print(f"found_values1: {found_values1}") - def testEmptyModelName(self): - # With empty model name the algorithm should skip the configuration without errors. - self.assertConfigureSuccess(OnnxPredict(), {}) - self.assertConfigureSuccess(OnnxPredict(), {"graphFilename": ""}) - self.assertConfigureSuccess( - OnnxPredict(), {"graphFilename": "", "inputs": [""]} - ) - self.assertConfigureSuccess( - OnnxPredict(), {"graphFilename": "", "inputs": ["wrong_input"]} - ) + self.assertAlmostEqualMatrix(found_values1, batch1) + self.assertAlmostEqualMatrix(found_values2, batch2) - def testInvalidParam(self): - model = join(testdata.models_dir, "effnetdiscogs", "effnetdiscogs-bsdynamic-1.onnx") - self.assertConfigureFails( - OnnxPredict(), - { - "graphFilename": model, - "inputs": ["wrong_input_name"], - "outputs": ["embeddings"], - }, - ) # input does not exist in the model - self.assertConfigureFails( - OnnxPredict(), - { - "graphFilename": "wrong_model_name", #! I suspect the issue is here with OnnxExceptions - "inputs": ["melspectrogram"], - "outputs": ["embeddings"], - }, - ) # the model does not exist # TODO: adapt TensorPredict unittests such as testIdentityModel, testComputeWithoutConfiguration, testIgnoreInvalidReconfiguration # TODO: make a test for squeeze, showing that it fails when it is not applied with 2D models From e21cca10a9dbc1e86273f4b702fc4d802d2ebf30 Mon Sep 17 00:00:00 2001 From: xaviliz Date: Tue, 16 Sep 2025 13:08:42 +0200 Subject: [PATCH 31/60] Fix issues with multi input data models --- .../machinelearning/onnxpredict.cpp | 65 +++---------------- 1 file changed, 10 insertions(+), 55 deletions(-) diff --git a/src/algorithms/machinelearning/onnxpredict.cpp b/src/algorithms/machinelearning/onnxpredict.cpp index 522ce5281..95d782d9f 100644 --- a/src/algorithms/machinelearning/onnxpredict.cpp +++ b/src/algorithms/machinelearning/onnxpredict.cpp @@ -85,9 +85,7 @@ void OnnxPredict::configure() { _nInputs = _inputs.size(); _nOutputs = _outputs.size(); - - // cout << "_inputs.size(): " << _nInputs << ".\n"; - + // use the first input when no input is defined if (_nInputs == 0){ // take the first input @@ -212,7 +210,6 @@ std::string OnnxPredict::getTensorInfos(const std::vector& infos, co out += "=== " + label + " ===\n"; for (const auto& info : infos) { out += "[Name] " + info.name + "\n"; - //cout << "info.type: " << typeid(info.type).name() << endl; std::string type_str = onnxTypeToString(info.type); out += "\t[Type] " + type_str + "\n"; out += "\t[Shape] ["; @@ -241,15 +238,14 @@ void OnnxPredict::compute() { const Pool& poolIn = _poolIn.get(); Pool& poolOut = _poolOut.get(); - - std:vector> shapes; + + std::vector> input_datas; // <-- keeps inputs alive + std:vector> shapes; // <-- keeps shapes alive // Parse the input tensors from the pool into ONNX Runtime tensors. for (size_t i = 0; i < _nInputs; i++) { - cout << "_inputs[" << i << "]: " << _inputs[i] << endl; const Tensor& inputData = poolIn.value >(_inputs[i]); - cout << "inputData.size(): " << inputData.size() << endl; // Step 1: Get tensor shape std::vector shape; @@ -258,8 +254,6 @@ void OnnxPredict::compute() { shape.push_back((int64_t)inputData.dimension(0)); if (_squeeze) { - //cout << "Applying squeeze!!!" << endl; - //cout << "inputData.rank(): " << inputData.rank() << endl; for(int j = 1; j < inputData.rank(); j++) { if (inputData.dimension(j) > 1) { @@ -277,41 +271,24 @@ void OnnxPredict::compute() { } else { dims = inputData.rank(); for(int j = 1; j < dims; j++) { - //cout << inputData.dimension(j) << endl; shape.push_back((int64_t)inputData.dimension(j)); } } // Step 2: Convert data to float32 - std::vector float_data(inputData.size()); + input_datas.emplace_back(inputData.size()); for (size_t j = 0; j < inputData.size(); ++j) { - float_data[j] = static_cast(inputData.data()[j]); + input_datas.back()[j] = static_cast(inputData.data()[j]); } // Step 3: Create ONNX Runtime tensor - _memoryInfo = Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU); + _memoryInfo = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault); if (_memoryInfo == NULL) { throw EssentiaException("OnnxRuntimePredict: Error allocating memory for input tensor."); } - - // display shape - cout << "shape: ["; - for (size_t j = 0; j < shape.size(); ++j) { - cout << shape[j]; - if (j + 1 < shape.size()) std::cout << ", "; - } - cout << "]" << endl; - - // display float_data - cout << "float_data: ["; - for (size_t j = 0; j < float_data.size(); ++j) { - cout << float_data[j]; - if (j + 1 < float_data.size()) std::cout << ", "; - } - cout << "]" << endl; - input_tensors.emplace_back(Ort::Value::CreateTensor(_memoryInfo, float_data.data(), float_data.size(), shape.data(), shape.size())); + input_tensors.emplace_back(Ort::Value::CreateTensor(_memoryInfo, input_datas.back().data(), input_datas.back().size(), shape.data(), shape.size())); shapes.push_back(shape); } @@ -332,34 +309,12 @@ void OnnxPredict::compute() { output_names.data(), // Output node names. _nOutputs // Number of outputs. ); - + // Map output tensors to pool for (size_t i = 0; i < output_tensors.size(); ++i) { const Real* outputData = output_tensors[i].GetTensorData(); - - - auto outputInfo = output_tensors[i].GetTensorTypeAndShapeInfo(); - cout << "GetElementType: " << outputInfo.GetElementType() << endl; - cout << "Dimensions of the output: " << outputInfo.GetShape().size() << endl; - cout << "Shape of the output: ["; - auto tensor_size = 1; - for (unsigned int shapeI = 0; shapeI < outputInfo.GetShape().size(); shapeI++){ - tensor_size *= outputInfo.GetShape()[shapeI]; - cout << outputInfo.GetShape()[shapeI]; - if (shapeI + 1 (); // Create and array to store the output tensor shape. array _shape {1, 1, 1, 1}; From 64a86cb4f0ab7af9253299262553ecf8e054233e Mon Sep 17 00:00:00 2001 From: xaviliz Date: Tue, 16 Sep 2025 13:10:58 +0200 Subject: [PATCH 32/60] Small clean --- .../src/unittests/machinelearning/test_onnxpredict.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/test/src/unittests/machinelearning/test_onnxpredict.py b/test/src/unittests/machinelearning/test_onnxpredict.py index 6b7765326..f96bf02eb 100644 --- a/test/src/unittests/machinelearning/test_onnxpredict.py +++ b/test/src/unittests/machinelearning/test_onnxpredict.py @@ -152,15 +152,9 @@ def testIdentityModel(self): n, m = input1.shape - print(f"input1.shape: {(n, m)}") batch1 = input1.reshape(n, 1, 1, m) batch2 = input2.reshape(n, 1, 1, m) - print(f"batch1.shape: {batch1.shape}") - print(f"batch2.shape: {batch2.shape}") - - print(f"batch1: {batch1}") - pool = Pool() pool.set("input1", batch1) pool.set("input2", batch2) @@ -175,11 +169,6 @@ def testIdentityModel(self): found_values1 = poolOut["output1"] found_values2 = poolOut["output2"] - print(f"found_values1.shape: {found_values1.shape}") - print(f"found_values2.shape: {found_values2.shape}") - - print(f"found_values1: {found_values1}") - self.assertAlmostEqualMatrix(found_values1, batch1) self.assertAlmostEqualMatrix(found_values2, batch2) From e2068456b82171d20688c0b7bd2c273fdda3b31a Mon Sep 17 00:00:00 2001 From: xaviliz Date: Wed, 17 Sep 2025 11:10:04 +0200 Subject: [PATCH 33/60] Assert testInference() with input and output shapes --- .../machinelearning/test_onnxpredict.py | 172 +++++++++--------- 1 file changed, 87 insertions(+), 85 deletions(-) diff --git a/test/src/unittests/machinelearning/test_onnxpredict.py b/test/src/unittests/machinelearning/test_onnxpredict.py index f96bf02eb..923167e17 100644 --- a/test/src/unittests/machinelearning/test_onnxpredict.py +++ b/test/src/unittests/machinelearning/test_onnxpredict.py @@ -57,92 +57,94 @@ class TestOnnxPredict(TestCase): # print(config) # self.assertConfigureFails(OnnxPredict(), config) - # def testInference(self,): - # model = join(testdata.models_dir, "effnetdiscogs", "effnetdiscogs-bsdynamic-1.onnx") - # # TODO: store model card and to extract model output shapes to assert the ouputs + def testInference(self,): + model = join(testdata.models_dir, "effnetdiscogs", "effnetdiscogs-bsdynamic-1.onnx") - # print(f"\nmodel: {model}") - # output_layer_name0 = "activations" - # output_layer_name1 = "embeddings" - # onxx_predict = OnnxPredict( - # graphFilename= model, - # inputs=[], - # outputs=[output_layer_name0, output_layer_name1] - # ) - - # stem = "359500__mtg__sax-tenor-e-major" - # audio_path = join(testdata.audio_dir, Path("recorded"), f"{stem}.wav") - - # audio, sample_rate = sf.read(audio_path, dtype=numpy.float32) - # print(f"audio.shape: {audio.shape}") - - # frame_size = 512 - # hop_size = 256 - # patch_size = 128 - # number_bands = 96 - - # w = Windowing(type="hann", zeroPadding=frame_size) - # spectrum = Spectrum(size=frame_size) - # mels = MelBands(inputSize=frame_size+1,numberBands=number_bands, type="magnitude") - # logNorm = UnaryOperator(type="log") - - # # compute mel bands - # bands = [] - # for frame in FrameGenerator(audio, frameSize=frame_size, hopSize=hop_size): - # melFrame = mels(spectrum(w(frame))) - # bands.append(logNorm(melFrame)) - # bands = array(bands) - - # discard = bands.shape[0] % patch_size - # bands = numpy.reshape(bands[:-discard, :], [-1, patch_size, number_bands]) - # batch = numpy.expand_dims(bands, 1) - # print(f"bands.shape: {bands.shape}") - # print(f"batch.shape: {batch.shape}") - - # pool = Pool() - # pool.set("melspectrogram", batch) - - # pool_out = onxx_predict(pool) - - # print(f"descriptorNames: {pool_out.descriptorNames()}") - # print(f"result['{output_layer_name0}']: {pool_out[output_layer_name0]}") - # print(f"result['{output_layer_name0}'].shape(): {pool_out[output_layer_name0].shape}") - - # print(f"result['{output_layer_name1}']: {pool_out[output_layer_name1]}") - # print(f"result['{output_layer_name0}'].shape(): {pool_out[output_layer_name1].shape}") - - # AssertionError() - # # TODO: assert the output shapes - - # def testEmptyModelName(self): - # # With empty model name the algorithm should skip the configuration without errors. - # self.assertConfigureSuccess(OnnxPredict(), {}) - # self.assertConfigureSuccess(OnnxPredict(), {"graphFilename": ""}) - # self.assertConfigureSuccess( - # OnnxPredict(), {"graphFilename": "", "inputs": [""]} - # ) - # self.assertConfigureSuccess( - # OnnxPredict(), {"graphFilename": "", "inputs": ["wrong_input"]} - # ) - - # def testInvalidParam(self): - # model = join(testdata.models_dir, "effnetdiscogs", "effnetdiscogs-bsdynamic-1.onnx") - # self.assertConfigureFails( - # OnnxPredict(), - # { - # "graphFilename": model, - # "inputs": ["wrong_input_name"], - # "outputs": ["embeddings"], - # }, - # ) # input does not exist in the model - # self.assertConfigureFails( - # OnnxPredict(), - # { - # "graphFilename": "wrong_model_name", #! I suspect the issue is here with OnnxExceptions - # "inputs": ["melspectrogram"], - # "outputs": ["embeddings"], - # }, - # ) # the model does not exist + # define input and output groundtruths + input_shape = (1, 128, 96) + outputs = [ + { + "name": "activations", + "shape": (1, 400), + }, + { + "name": "embeddings", + "shape": (1, 1280), + } + ] + + onxx_predict = OnnxPredict( + graphFilename= model, + inputs=[], + outputs=[output["name"] for output in outputs], + ) + + stem = "359500__mtg__sax-tenor-e-major" + audio_path = join(testdata.audio_dir, Path("recorded"), f"{stem}.wav") + + audio, sample_rate = sf.read(audio_path, dtype=numpy.float32) + + frame_size = 512 + hop_size = 256 + patch_size = 128 + number_bands = 96 + + w = Windowing(type="hann", zeroPadding=frame_size) + spectrum = Spectrum(size=frame_size) + mels = MelBands(inputSize=frame_size+1,numberBands=number_bands, type="magnitude") + logNorm = UnaryOperator(type="log") + + # compute mel bands + bands = [] + for frame in FrameGenerator(audio, frameSize=frame_size, hopSize=hop_size): + melFrame = mels(spectrum(w(frame))) + bands.append(logNorm(melFrame)) + bands = array(bands) + + discard = bands.shape[0] % patch_size + bands = numpy.reshape(bands[:-discard, :], [-1, patch_size, number_bands]) + batch = numpy.expand_dims(bands, 1) + + pool = Pool() + pool.set("melspectrogram", batch) + + pool_out = onxx_predict(pool) + + self.assertEqualVector(input_shape, batch.shape[1:]) + self.assertEqualVector(outputs[0]["shape"], pool_out[outputs[0]["name"]].shape[2:]) + self.assertEqualVector(outputs[1]["shape"], pool_out[outputs[1]["name"]].shape[2:]) + self.assertEqual(pool_out.descriptorNames()[0], outputs[0]["name"]) + self.assertEqual(pool_out.descriptorNames()[1], outputs[1]["name"]) + + def testEmptyModelName(self): + # With empty model name the algorithm should skip the configuration without errors. + self.assertConfigureSuccess(OnnxPredict(), {}) + self.assertConfigureSuccess(OnnxPredict(), {"graphFilename": ""}) + self.assertConfigureSuccess( + OnnxPredict(), {"graphFilename": "", "inputs": [""]} + ) + self.assertConfigureSuccess( + OnnxPredict(), {"graphFilename": "", "inputs": ["wrong_input"]} + ) + + def testInvalidParam(self): + model = join(testdata.models_dir, "effnetdiscogs", "effnetdiscogs-bsdynamic-1.onnx") + self.assertConfigureFails( + OnnxPredict(), + { + "graphFilename": model, + "inputs": ["wrong_input_name"], + "outputs": ["embeddings"], + }, + ) # input does not exist in the model + self.assertConfigureFails( + OnnxPredict(), + { + "graphFilename": "wrong_model_name", #! I suspect the issue is here with OnnxExceptions + "inputs": ["melspectrogram"], + "outputs": ["embeddings"], + }, + ) # the model does not exist def testIdentityModel(self): model = join(testdata.models_dir, "identity", "identity2x2.onnx") From d246e2839dc53f0caf72041453e73a371e8e6900 Mon Sep 17 00:00:00 2001 From: xaviliz Date: Wed, 17 Sep 2025 11:13:30 +0200 Subject: [PATCH 34/60] Add testComputeWithoutConfiguration() unittest --- test/src/unittests/machinelearning/test_onnxpredict.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/src/unittests/machinelearning/test_onnxpredict.py b/test/src/unittests/machinelearning/test_onnxpredict.py index 923167e17..1687ea624 100644 --- a/test/src/unittests/machinelearning/test_onnxpredict.py +++ b/test/src/unittests/machinelearning/test_onnxpredict.py @@ -174,9 +174,13 @@ def testIdentityModel(self): self.assertAlmostEqualMatrix(found_values1, batch1) self.assertAlmostEqualMatrix(found_values2, batch2) + def testComputeWithoutConfiguration(self): + pool = Pool() + pool.set("melspectrogram", numpy.zeros((1, 1, 1, 1), dtype="float32")) + + self.assertComputeFails(OnnxPredict(), pool) - # TODO: adapt TensorPredict unittests such as testIdentityModel, testComputeWithoutConfiguration, testIgnoreInvalidReconfiguration - # TODO: make a test for squeeze, showing that it fails when it is not applied with 2D models + # TODO: adapt TensorPredict unittests such as testIgnoreInvalidReconfiguration # TODO: make a test for squeeze, showing that it works well when it is applied for a 2D model From 86c64eb7aae81696bce24fa966a9ecddd38dc9bc Mon Sep 17 00:00:00 2001 From: xaviliz Date: Wed, 17 Sep 2025 11:28:05 +0200 Subject: [PATCH 35/60] Add testIgnoreInvalidReconfiguration() as unittest --- .../machinelearning/test_onnxpredict.py | 26 +++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/test/src/unittests/machinelearning/test_onnxpredict.py b/test/src/unittests/machinelearning/test_onnxpredict.py index 1687ea624..cf2a4d6b8 100644 --- a/test/src/unittests/machinelearning/test_onnxpredict.py +++ b/test/src/unittests/machinelearning/test_onnxpredict.py @@ -180,9 +180,31 @@ def testComputeWithoutConfiguration(self): self.assertComputeFails(OnnxPredict(), pool) - # TODO: adapt TensorPredict unittests such as testIgnoreInvalidReconfiguration - # TODO: make a test for squeeze, showing that it works well when it is applied for a 2D model + def testIgnoreInvalidReconfiguration(self): + pool = Pool() + pool.set("input1", numpy.ones((1, 1, 1, 3), dtype="float32")) + pool.set("input2", numpy.ones((1, 1, 1, 3), dtype="float32")) + + #model_name = join(filedir(), "tensorflowpredict", "identity.pb") + model_name = join(testdata.models_dir, "identity", "identity2x2.onnx") + model = OnnxPredict( + graphFilename=model_name, + inputs=["input1", "input2"], + outputs=["output1"], + squeeze=True, + ) + firstResult = model(pool) + + # This attempt to reconfigure the algorithm should be ignored and trigger a Warning. + model.configure() + + secondResult = model(pool) + + self.assertEqualMatrix(firstResult["output1"], secondResult["output1"]) + + # TODO: make a test for squeeze, showing that it works well when it is applied for a 2D model + # TODO: make a test reusing the algorithm for two models (effnet and identity) """ def regression(self, parameters): From 459652e6b41d55855704dc070febaec658a1b1c9 Mon Sep 17 00:00:00 2001 From: xaviliz Date: Wed, 17 Sep 2025 11:34:32 +0200 Subject: [PATCH 36/60] Add testInvalidSqueezeConfiguration() as unittest --- .../machinelearning/test_onnxpredict.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/test/src/unittests/machinelearning/test_onnxpredict.py b/test/src/unittests/machinelearning/test_onnxpredict.py index cf2a4d6b8..e8caa9606 100644 --- a/test/src/unittests/machinelearning/test_onnxpredict.py +++ b/test/src/unittests/machinelearning/test_onnxpredict.py @@ -204,6 +204,29 @@ def testIgnoreInvalidReconfiguration(self): self.assertEqualMatrix(firstResult["output1"], secondResult["output1"]) # TODO: make a test for squeeze, showing that it works well when it is applied for a 2D model + def testInvalidSqueezeConfiguration(self): + model = join(testdata.models_dir, "identity", "identity2x2.onnx") + + # prepare model inputs and batches + input1, input2 = (numpy.float32(numpy.random.random((3, 3))) for _ in range(2)) + + n, m = input1.shape + + batch1 = input1.reshape(n, 1, 1, m) + batch2 = input2.reshape(n, 1, 1, m) + + pool = Pool() + pool.set("input1", batch1) + pool.set("input2", batch2) + + onnx_predict = OnnxPredict( + graphFilename=model, + inputs=["input1", "input2"], + outputs=["output1", "output2"], + squeeze=False, + ) + self.assertComputeFails(onnx_predict, pool) + # TODO: make a test reusing the algorithm for two models (effnet and identity) """ From 30abf8a775d9193b7a0fee77b416529e7aab2e2b Mon Sep 17 00:00:00 2001 From: xaviliz Date: Wed, 17 Sep 2025 13:14:04 +0200 Subject: [PATCH 37/60] Add testConfigure() as unittest --- .../machinelearning/onnxpredict.cpp | 13 +- .../machinelearning/test_onnxpredict.py | 415 ++++++------------ 2 files changed, 132 insertions(+), 296 deletions(-) diff --git a/src/algorithms/machinelearning/onnxpredict.cpp b/src/algorithms/machinelearning/onnxpredict.cpp index 95d782d9f..8596a9d5d 100644 --- a/src/algorithms/machinelearning/onnxpredict.cpp +++ b/src/algorithms/machinelearning/onnxpredict.cpp @@ -110,9 +110,11 @@ void OnnxPredict::configure() { E_INFO(getTensorInfos(all_output_infos, "Model Outputs")); return; } - - // check model has input and output https://github.com/microsoft/onnxruntime-inference-examples/blob/7a635daae48450ff142e5c0848a564b245f04112/c_cxx/model-explorer/model-explorer.cpp#L99C3-L100C63 + _isConfigured = true; + reset(); + + // check model has input and output https://github.com/microsoft/onnxruntime-inference-examples/blob/7a635daae48450ff142e5c0848a564b245f04112/c_cxx/model-explorer/model-explorer.cpp#L99C3-L100C63 for (int i = 0; i < _inputs.size(); i++) { for (int j = 0; j < all_input_infos.size(); j++) { if (_inputs[i] == all_input_infos[j].name){ @@ -149,9 +151,6 @@ void OnnxPredict::configure() { throw EssentiaException(availableOutputInfo()); } - _isConfigured = true; - reset(); - for (size_t i = 0; i < _nInputs; i++) { checkName(_inputs[i], all_input_infos); } @@ -224,9 +223,11 @@ std::string OnnxPredict::getTensorInfos(const std::vector& infos, co void OnnxPredict::reset() { if (!_isConfigured) return; - input_names.clear(); output_names.clear(); + _inputNodes.clear(); + _outputNodes.clear(); + input_tensors.clear(); } void OnnxPredict::compute() { diff --git a/test/src/unittests/machinelearning/test_onnxpredict.py b/test/src/unittests/machinelearning/test_onnxpredict.py index e8caa9606..f8e8a5ab6 100644 --- a/test/src/unittests/machinelearning/test_onnxpredict.py +++ b/test/src/unittests/machinelearning/test_onnxpredict.py @@ -57,11 +57,68 @@ class TestOnnxPredict(TestCase): # print(config) # self.assertConfigureFails(OnnxPredict(), config) + # def testInference(self,): + # model = join(testdata.models_dir, "effnetdiscogs", "effnetdiscogs-bsdynamic-1.onnx") + + # # define input and output metadata + # input_shape = (1, 128, 96) + # outputs = [ + # { + # "name": "activations", + # "shape": (1, 400), + # }, + # { + # "name": "embeddings", + # "shape": (1, 1280), + # } + # ] + + # onxx_predict = OnnxPredict( + # graphFilename= model, + # inputs=[], + # outputs=[output["name"] for output in outputs], + # ) + + # stem = "359500__mtg__sax-tenor-e-major" + # audio_path = join(testdata.audio_dir, Path("recorded"), f"{stem}.wav") + + # audio, sample_rate = sf.read(audio_path, dtype=numpy.float32) + + # frame_size = 512 + # hop_size = 256 + # patch_size = 128 + # number_bands = 96 + + # w = Windowing(type="hann", zeroPadding=frame_size) + # spectrum = Spectrum(size=frame_size) + # mels = MelBands(inputSize=frame_size+1,numberBands=number_bands, type="magnitude") + # logNorm = UnaryOperator(type="log") + + # # compute mel bands + # bands = [] + # for frame in FrameGenerator(audio, frameSize=frame_size, hopSize=hop_size): + # melFrame = mels(spectrum(w(frame))) + # bands.append(logNorm(melFrame)) + # bands = array(bands) + + # discard = bands.shape[0] % patch_size + # bands = numpy.reshape(bands[:-discard, :], [-1, patch_size, number_bands]) + # batch = numpy.expand_dims(bands, 1) + + # pool = Pool() + # pool.set("melspectrogram", batch) + + # pool_out = onxx_predict(pool) + + # self.assertEqualVector(input_shape, batch.shape[1:]) + # self.assertEqualVector(outputs[0]["shape"], pool_out[outputs[0]["name"]].shape[2:]) + # self.assertEqualVector(outputs[1]["shape"], pool_out[outputs[1]["name"]].shape[2:]) + # self.assertEqual(pool_out.descriptorNames()[0], outputs[0]["name"]) + # self.assertEqual(pool_out.descriptorNames()[1], outputs[1]["name"]) + def testInference(self,): - model = join(testdata.models_dir, "effnetdiscogs", "effnetdiscogs-bsdynamic-1.onnx") - # define input and output groundtruths - input_shape = (1, 128, 96) + # define output metadata outputs = [ { "name": "activations", @@ -73,44 +130,11 @@ def testInference(self,): } ] - onxx_predict = OnnxPredict( - graphFilename= model, - inputs=[], - outputs=[output["name"] for output in outputs], - ) - - stem = "359500__mtg__sax-tenor-e-major" - audio_path = join(testdata.audio_dir, Path("recorded"), f"{stem}.wav") - - audio, sample_rate = sf.read(audio_path, dtype=numpy.float32) - - frame_size = 512 - hop_size = 256 - patch_size = 128 - number_bands = 96 - - w = Windowing(type="hann", zeroPadding=frame_size) - spectrum = Spectrum(size=frame_size) - mels = MelBands(inputSize=frame_size+1,numberBands=number_bands, type="magnitude") - logNorm = UnaryOperator(type="log") - - # compute mel bands - bands = [] - for frame in FrameGenerator(audio, frameSize=frame_size, hopSize=hop_size): - melFrame = mels(spectrum(w(frame))) - bands.append(logNorm(melFrame)) - bands = array(bands) - - discard = bands.shape[0] % patch_size - bands = numpy.reshape(bands[:-discard, :], [-1, patch_size, number_bands]) - batch = numpy.expand_dims(bands, 1) - + onxx_predict = OnnxPredict() pool = Pool() - pool.set("melspectrogram", batch) - pool_out = onxx_predict(pool) + pool_out = runEffnetDiscogsInference(onxx_predict, outputs, pool) - self.assertEqualVector(input_shape, batch.shape[1:]) self.assertEqualVector(outputs[0]["shape"], pool_out[outputs[0]["name"]].shape[2:]) self.assertEqualVector(outputs[1]["shape"], pool_out[outputs[1]["name"]].shape[2:]) self.assertEqual(pool_out.descriptorNames()[0], outputs[0]["name"]) @@ -147,7 +171,6 @@ def testInvalidParam(self): ) # the model does not exist def testIdentityModel(self): - model = join(testdata.models_dir, "identity", "identity2x2.onnx") # prepare model inputs and batches input1, input2 = (numpy.float32(numpy.random.random((3, 3))) for _ in range(2)) @@ -161,15 +184,7 @@ def testIdentityModel(self): pool.set("input1", batch1) pool.set("input2", batch2) - poolOut = OnnxPredict( - graphFilename=model, - inputs=["input1", "input2"], - outputs=["output1", "output2"], - squeeze=True, - )(pool) - - found_values1 = poolOut["output1"] - found_values2 = poolOut["output2"] + found_values1, found_values2 = runIdentityModelInference(OnnxPredict(), pool) self.assertAlmostEqualMatrix(found_values1, batch1) self.assertAlmostEqualMatrix(found_values2, batch2) @@ -185,7 +200,6 @@ def testIgnoreInvalidReconfiguration(self): pool.set("input1", numpy.ones((1, 1, 1, 3), dtype="float32")) pool.set("input2", numpy.ones((1, 1, 1, 3), dtype="float32")) - #model_name = join(filedir(), "tensorflowpredict", "identity.pb") model_name = join(testdata.models_dir, "identity", "identity2x2.onnx") model = OnnxPredict( graphFilename=model_name, @@ -203,7 +217,6 @@ def testIgnoreInvalidReconfiguration(self): self.assertEqualMatrix(firstResult["output1"], secondResult["output1"]) - # TODO: make a test for squeeze, showing that it works well when it is applied for a 2D model def testInvalidSqueezeConfiguration(self): model = join(testdata.models_dir, "identity", "identity2x2.onnx") @@ -228,272 +241,94 @@ def testInvalidSqueezeConfiguration(self): self.assertComputeFails(onnx_predict, pool) # TODO: make a test reusing the algorithm for two models (effnet and identity) - - """ - def regression(self, parameters): - # Test a simple tensorflow model trained on Essentia features. - # The ground true values were obtained with the following script: - expectedValues = [9.9997020e-01, 5.3647455e-14, 2.9801236e-05, 4.9495230e-12] - - patchSize = 128 - numberBands = 128 - frameSize = 1024 - hopSize = frameSize - - filename = join(testdata.audio_dir, "recorded", "cat_purrrr.wav") - - audio = MonoLoader(filename=filename)() - - w = Windowing(type="hann", zeroPadding=frameSize) - spectrum = Spectrum() - mels = MelBands(numberBands=numberBands, type="magnitude") - logNorm = UnaryOperator(type="log") - - bands = [] - for frame in FrameGenerator(audio, frameSize=frameSize, hopSize=hopSize): - melFrame = mels(spectrum(w(frame))) - bands.append(logNorm(melFrame)) - bands = array(bands) - - discard = bands.shape[0] % patchSize - bands = numpy.reshape(bands[:-discard, :], [-1, patchSize, numberBands]) - batch = numpy.expand_dims(bands, 1) - - pool = Pool() - pool.set("model/Placeholder", batch) - - tfp = TensorflowPredict(**parameters) - poolOut = tfp(pool) - - foundValues = poolOut["model/Softmax"].mean(axis=0).squeeze() - - self.assertAlmostEqualVector(foundValues, expectedValues, 1e-5) - - def testRegressionFrozenModel(self): - parameters = { - "graphFilename": join(testdata.models_dir, "vgg", "vgg4.pb"), - "inputs": ["model/Placeholder"], - "outputs": ["model/Softmax"], - "isTraining": False, - "isTrainingName": "model/Placeholder_1", - } - - self.regression(parameters) - - def testRegressionSavedModel(self): - parameters = { - "savedModel": join(testdata.models_dir, "vgg", "vgg4"), - "inputs": ["model/Placeholder"], - "outputs": ["model/Softmax"], - "isTraining": False, - "isTrainingName": "model/Placeholder_1", - } - - self.regression(parameters) - - def testSavedModelOverridesGraphFilename(self): - # When both are specified, `savedModel` should be preferred. - # Test this by setting an invalid `graphFilename` that should be ignored. - parameters = { - "graphFilename": "wrong_model", - "savedModel": join(testdata.models_dir, "vgg", "vgg4"), - "inputs": ["model/Placeholder"], - "outputs": ["model/Softmax"], - "isTraining": False, - "isTrainingName": "model/Placeholder_1", - } - - self.regression(parameters) - - def testEmptyModelName(self): - # With empty model name the algorithm should skip the configuration without errors. - self.assertConfigureSuccess(TensorflowPredict(), {}) - self.assertConfigureSuccess(TensorflowPredict(), {"graphFilename": ""}) - self.assertConfigureSuccess( - TensorflowPredict(), {"graphFilename": "", "inputs": [""]} - ) - self.assertConfigureSuccess( - TensorflowPredict(), {"graphFilename": "", "inputs": ["wrong_input"]} - ) - self.assertConfigureSuccess(TensorflowPredict(), {"savedModel": ""}) - self.assertConfigureSuccess( - TensorflowPredict(), {"savedModel": "", "inputs": [""]} - ) - self.assertConfigureSuccess( - TensorflowPredict(), {"savedModel": "", "inputs": ["wrong_input"]} - ) - self.assertConfigureSuccess( - TensorflowPredict(), {"graphFilename": "", "savedModel": ""} - ) - self.assertConfigureSuccess( - TensorflowPredict(), {"graphFilename": "", "savedModel": "", "inputs": [""]} - ) - self.assertConfigureSuccess( - TensorflowPredict(), - {"graphFilename": "", "savedModel": "", "inputs": ["wrong_input"]}, - ) - - def testInvalidParam(self): - model = join(testdata.models_dir, "vgg", "vgg4.pb") - self.assertConfigureFails( - TensorflowPredict(), {"graphFilename": model} - ) # inputs and outputs are not defined - self.assertConfigureFails( - TensorflowPredict(), - { - "graphFilename": model, - "inputs": ["model/Placeholder"], - }, - ) # outputs are not defined - self.assertConfigureFails( - TensorflowPredict(), + def testConfiguration(self): + # define output metadata + outputs = [ { - "graphFilename": model, - "inputs": ["wrong_input_name"], - "outputs": ["model/Softmax"], + "name": "activations", + "shape": (1, 400), }, - ) # input does not exist in the model - self.assertConfigureFails( - TensorflowPredict(), { - "graphFilename": "wrong_model_name", - "inputs": ["model/Placeholder"], - "outputs": ["model/Softmax"], - }, - ) # the model does not exist + "name": "embeddings", + "shape": (1, 1280), + } + ] - # Repeat tests for savedModel format. - model = join(testdata.models_dir, "vgg", "vgg4/") - self.assertConfigureFails( - TensorflowPredict(), {"savedModel": model} - ) # inputs and outputs are not defined - self.assertConfigureFails( - TensorflowPredict(), - { - "savedModel": model, - "inputs": ["model/Placeholder"], - }, - ) # outputs are not defined - self.assertConfigureFails( - TensorflowPredict(), - { - "savedModel": model, - "inputs": ["wrong_input_name"], - "outputs": ["model/Softmax"], - }, - ) # input does not exist in the model - self.assertConfigureFails( - TensorflowPredict(), - { - "savedModel": "wrong_model_name", - "inputs": ["model/Placeholder"], - "outputs": ["model/Softmax"], - }, - ) # the model does not exist + onxx_predict = OnnxPredict() + pool = Pool() - def testIdentityModel(self): - # Perform the identity operation in Tensorflow to test if the data is - # being copied correctly backwards and fordwards. - model = join(filedir(), "tensorflowpredict", "identity.pb") - filename = join(testdata.audio_dir, "recorded", "cat_purrrr.wav") + _ = runEffnetDiscogsInference(onxx_predict, outputs, pool) + pool.clear() + + # prepare model inputs and batches for identity model + input1, input2 = (numpy.float32(numpy.random.random((3, 3))) for _ in range(2)) - audio = MonoLoader(filename=filename)() - frames = array([frame for frame in FrameGenerator(audio)]) - batch = frames[numpy.newaxis, numpy.newaxis, :] + n, m = input1.shape - pool = Pool() - pool.set("model/Placeholder", batch) + batch1 = input1.reshape(n, 1, 1, m) + batch2 = input2.reshape(n, 1, 1, m) - poolOut = TensorflowPredict( - graphFilename=model, - inputs=["model/Placeholder"], - outputs=["model/Identity"], - )(pool) + pool.set("input1", batch1) + pool.set("input2", batch2) - foundValues = poolOut["model/Identity"] + found_values1, found_values2 = runIdentityModelInference(onxx_predict, pool) - self.assertAlmostEqualMatrix(foundValues, batch) + self.assertAlmostEqualMatrix(found_values1, batch1) + self.assertAlmostEqualMatrix(found_values2, batch2) - def testComputeWithoutConfiguration(self): - pool = Pool() - pool.set("model/Placeholder", numpy.zeros((1, 1, 1, 1), dtype="float32")) +def runEffnetDiscogsInference(onnx_predict, outputs, pool) -> Pool: + model = join(testdata.models_dir, "effnetdiscogs", "effnetdiscogs-bsdynamic-1.onnx") - self.assertComputeFails(TensorflowPredict(), pool) + stem = "359500__mtg__sax-tenor-e-major" + audio_path = join(testdata.audio_dir, Path("recorded"), f"{stem}.wav") - def testIgnoreInvalidReconfiguration(self): - pool = Pool() - pool.set("model/Placeholder", numpy.ones((1, 1, 1, 1), dtype="float32")) + audio, _ = sf.read(audio_path, dtype=numpy.float32) - model_name = join(filedir(), "tensorflowpredict", "identity.pb") - model = TensorflowPredict( - graphFilename=model_name, - inputs=["model/Placeholder"], - outputs=["model/Identity"], - squeeze=False, + onnx_predict.configure( + graphFilename= model, + inputs=[], + outputs=[output["name"] for output in outputs], ) - firstResult = model(pool)["model/Identity"] + frame_size = 512 + hop_size = 256 + patch_size = 128 + number_bands = 96 - # This attempt to reconfigure the algorithm should be ignored and trigger a Warning. - model.configure() + w = Windowing(type="hann", zeroPadding=frame_size) + spectrum = Spectrum(size=frame_size) + mels = MelBands(inputSize=frame_size+1,numberBands=number_bands, type="magnitude") + logNorm = UnaryOperator(type="log") - secondResult = model(pool)["model/Identity"] + # compute mel bands + bands = [] + for frame in FrameGenerator(audio, frameSize=frame_size, hopSize=hop_size): + melFrame = mels(spectrum(w(frame))) + bands.append(logNorm(melFrame)) + bands = array(bands) - self.assertEqualMatrix(firstResult, secondResult) + discard = bands.shape[0] % patch_size + bands = numpy.reshape(bands[:-discard, :], [-1, patch_size, number_bands]) + batch = numpy.expand_dims(bands, 1) - def testImplicitOutputTensorIndex(self): - model = join(filedir(), "tensorflowpredict", "identity.pb") - batch = numpy.reshape(numpy.arange(4, dtype="float32"), (1, 1, 2, 2)) - - pool = Pool() - pool.set("model/Placeholder", batch) + pool.set("melspectrogram", batch) - implicit_output = "model/Identity" - implicit = TensorflowPredict( - graphFilename=model, - inputs=["model/Placeholder"], - outputs=[implicit_output], - )(pool)[implicit_output].squeeze() + return onnx_predict(pool) - explicit_output = "model/Identity:0" - explicit = TensorflowPredict( - graphFilename=model, - inputs=["model/Placeholder"], - outputs=[explicit_output], - )(pool)[explicit_output].squeeze() +def runIdentityModelInference(onnx_predict, pool): + model = join(testdata.models_dir, "identity", "identity2x2.onnx") - self.assertAlmostEqualMatrix(implicit, explicit) + onnx_predict.configure( + graphFilename=model, + inputs=["input1", "input2"], + outputs=["output1", "output2"], + squeeze=True, + ) - def testNodeNameParser(self): - model = join(testdata.models_dir, "vgg", "vgg4.pb") + poolOut = onnx_predict(pool) - configs = [ - { - "graphFilename": model, - "inputs": ["model/Placeholder"], - "outputs": ["model/Softmax:"], - }, # No index. - { - "graphFilename": model, - "inputs": ["model/Placeholder"], - "outputs": ["model/Softmax:3"], - }, # Index out of bounds. - { - "graphFilename": model, - "inputs": ["model/Placeholder"], - "outputs": ["model/Softmax::0"], - }, # Double colon. - { - "graphFilename": model, - "inputs": ["model/Placeholder"], - "outputs": ["model/Softmax:s:0"], - }, # Several colons. - ] + return poolOut["output1"], poolOut["output2"] - for config in configs[1:]: - with self.subTest(f"{config} failed"): - self.assertConfigureFails(TensorflowPredict(), config) - """ suite = allTests(TestOnnxPredict) From 6fa1780cc6a2a53378c7664455be678530950a74 Mon Sep 17 00:00:00 2001 From: xaviliz Date: Wed, 17 Sep 2025 13:14:31 +0200 Subject: [PATCH 38/60] Add testConfigure() as unittest --- test/src/unittests/machinelearning/test_onnxpredict.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/src/unittests/machinelearning/test_onnxpredict.py b/test/src/unittests/machinelearning/test_onnxpredict.py index f8e8a5ab6..4113ba8d6 100644 --- a/test/src/unittests/machinelearning/test_onnxpredict.py +++ b/test/src/unittests/machinelearning/test_onnxpredict.py @@ -240,8 +240,7 @@ def testInvalidSqueezeConfiguration(self): ) self.assertComputeFails(onnx_predict, pool) - # TODO: make a test reusing the algorithm for two models (effnet and identity) - def testConfiguration(self): + def testConfigure(self): # define output metadata outputs = [ { From 46727b7d1a67efeac851bb965d95098d2795be16 Mon Sep 17 00:00:00 2001 From: xaviliz Date: Wed, 17 Sep 2025 16:31:39 +0200 Subject: [PATCH 39/60] Fix issue in testIgnoreInvalidReconfiguration() for Linux --- src/algorithms/machinelearning/onnxpredict.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/algorithms/machinelearning/onnxpredict.cpp b/src/algorithms/machinelearning/onnxpredict.cpp index 8596a9d5d..875fc7f01 100644 --- a/src/algorithms/machinelearning/onnxpredict.cpp +++ b/src/algorithms/machinelearning/onnxpredict.cpp @@ -54,6 +54,7 @@ void OnnxPredict::configure() { // Do not do anything if we did not get a non-empty model name. if (_graphFilename.empty()) return; + cout << "after return" << endl; try{ // Define environment @@ -227,7 +228,6 @@ void OnnxPredict::reset() { output_names.clear(); _inputNodes.clear(); _outputNodes.clear(); - input_tensors.clear(); } void OnnxPredict::compute() { @@ -242,6 +242,10 @@ void OnnxPredict::compute() { std::vector> input_datas; // <-- keeps inputs alive std:vector> shapes; // <-- keeps shapes alive + + if (!input_tensors.empty()) + input_tensors.clear(); // <-- destroy input tensors + // Parse the input tensors from the pool into ONNX Runtime tensors. for (size_t i = 0; i < _nInputs; i++) { @@ -315,7 +319,6 @@ void OnnxPredict::compute() { for (size_t i = 0; i < output_tensors.size(); ++i) { const Real* outputData = output_tensors[i].GetTensorData(); - const float* outputFloat = output_tensors[i].GetTensorData(); // Create and array to store the output tensor shape. array _shape {1, 1, 1, 1}; From 7193415074e2bc151454df4772fda66e7117a2e0 Mon Sep 17 00:00:00 2001 From: xaviliz Date: Thu, 18 Sep 2025 18:37:03 +0200 Subject: [PATCH 40/60] Clean and small change --- .../machinelearning/test_onnxpredict.py | 120 +++++------------- 1 file changed, 31 insertions(+), 89 deletions(-) diff --git a/test/src/unittests/machinelearning/test_onnxpredict.py b/test/src/unittests/machinelearning/test_onnxpredict.py index 4113ba8d6..bd7bbae84 100644 --- a/test/src/unittests/machinelearning/test_onnxpredict.py +++ b/test/src/unittests/machinelearning/test_onnxpredict.py @@ -26,95 +26,37 @@ class TestOnnxPredict(TestCase): - # def testIONameParser(self): - # model = join(testdata.models_dir, "effnetdiscogs", "effnetdiscogs-bsdynamic-1.onnx") - # print(f"\nmodel: {model}") - # configs = [ - # { - # "graphFilename": model, - # "inputs": ["model/Placeholder"], - # "outputs": ["model/Softmax:"], - # }, # No index. - # { - # "graphFilename": model, - # "inputs": ["model/Placeholder"], - # "outputs": ["model/Softmax:3"], - # }, # Index out of bounds. - # { - # "graphFilename": model, - # "inputs": ["model/Placeholder"], - # "outputs": ["model/Softmax::0"], - # }, # Double colon. - # { - # "graphFilename": model, - # "inputs": ["model/Placeholder"], - # "outputs": ["model/Softmax:s:0"], - # }, # Several colons. - # ] - - # for config in configs[1:]: - # with self.subTest(f"{config} failed"): - # print(config) - # self.assertConfigureFails(OnnxPredict(), config) - - # def testInference(self,): - # model = join(testdata.models_dir, "effnetdiscogs", "effnetdiscogs-bsdynamic-1.onnx") - - # # define input and output metadata - # input_shape = (1, 128, 96) - # outputs = [ - # { - # "name": "activations", - # "shape": (1, 400), - # }, - # { - # "name": "embeddings", - # "shape": (1, 1280), - # } - # ] - - # onxx_predict = OnnxPredict( - # graphFilename= model, - # inputs=[], - # outputs=[output["name"] for output in outputs], - # ) - - # stem = "359500__mtg__sax-tenor-e-major" - # audio_path = join(testdata.audio_dir, Path("recorded"), f"{stem}.wav") - - # audio, sample_rate = sf.read(audio_path, dtype=numpy.float32) - - # frame_size = 512 - # hop_size = 256 - # patch_size = 128 - # number_bands = 96 - - # w = Windowing(type="hann", zeroPadding=frame_size) - # spectrum = Spectrum(size=frame_size) - # mels = MelBands(inputSize=frame_size+1,numberBands=number_bands, type="magnitude") - # logNorm = UnaryOperator(type="log") - - # # compute mel bands - # bands = [] - # for frame in FrameGenerator(audio, frameSize=frame_size, hopSize=hop_size): - # melFrame = mels(spectrum(w(frame))) - # bands.append(logNorm(melFrame)) - # bands = array(bands) - - # discard = bands.shape[0] % patch_size - # bands = numpy.reshape(bands[:-discard, :], [-1, patch_size, number_bands]) - # batch = numpy.expand_dims(bands, 1) - - # pool = Pool() - # pool.set("melspectrogram", batch) - - # pool_out = onxx_predict(pool) - - # self.assertEqualVector(input_shape, batch.shape[1:]) - # self.assertEqualVector(outputs[0]["shape"], pool_out[outputs[0]["name"]].shape[2:]) - # self.assertEqualVector(outputs[1]["shape"], pool_out[outputs[1]["name"]].shape[2:]) - # self.assertEqual(pool_out.descriptorNames()[0], outputs[0]["name"]) - # self.assertEqual(pool_out.descriptorNames()[1], outputs[1]["name"]) + + def testIONameParser(self): + model = join(testdata.models_dir, "effnetdiscogs", "effnetdiscogs-bsdynamic-1.onnx") + print(f"\nmodel: {model}") + configs = [ + { + "graphFilename": model, + "inputs": ["model/Placeholder"], + "outputs": ["model/Softmax:"], + }, # No index. + { + "graphFilename": model, + "inputs": ["model/Placeholder"], + "outputs": ["model/Softmax:3"], + }, # Index out of bounds. + { + "graphFilename": model, + "inputs": ["model/Placeholder"], + "outputs": ["model/Softmax::0"], + }, # Double colon. + { + "graphFilename": model, + "inputs": ["model/Placeholder"], + "outputs": ["model/Softmax:s:0"], + }, # Several colons. + ] + + for config in configs[1:]: + with self.subTest(f"{config} failed"): + print(config) + self.assertConfigureFails(OnnxPredict(), config) def testInference(self,): From 509b99146c71c47ab2d3e0e824dbcdce9f4f05f9 Mon Sep 17 00:00:00 2001 From: xaviliz Date: Tue, 23 Sep 2025 11:31:27 +0200 Subject: [PATCH 41/60] Remove MacOS support --- packaging/debian_3rdparty/build_onnx.sh | 47 +++++++------------------ 1 file changed, 12 insertions(+), 35 deletions(-) diff --git a/packaging/debian_3rdparty/build_onnx.sh b/packaging/debian_3rdparty/build_onnx.sh index fb85f3518..766eb1d0c 100644 --- a/packaging/debian_3rdparty/build_onnx.sh +++ b/packaging/debian_3rdparty/build_onnx.sh @@ -18,46 +18,23 @@ cd onnxruntime-$LIBONNXRUNTIME_VERSION python3 -m pip install cmake -# Build the dynamic library for Linux or MacOS -# build for Intel and Apple silicon CPUs --> "x86_64;arm64" - -CMAKE_EXTRA_DEFINES="FETCHCONTENT_TRY_FIND_PACKAGE_MODE=NEVER CMAKE_INSTALL_PREFIX=${PREFIX}" -OS=$(uname -s) -CONFIG=Release - -if [ "$OS" = "Darwin" ]; then - DIR_OS="MacOS" - CMAKE_EXTRA_DEFINES+=' CMAKE_OSX_ARCHITECTURES=arm64' - SUFFIX="${LIBONNXRUNTIME_VERSION}.dylib*" - - ./build.sh \ - --config $CONFIG \ - --build_shared_lib \ - --parallel \ - --skip_submodule_sync \ - --skip_tests \ - --cmake_extra_defines $CMAKE_EXTRA_DEFINES -else - DIR_OS="Linux" - SUFFIX="so*" - - ./build.sh \ - --config $CONFIG \ - --build_shared_lib \ - --parallel \ - --compile_no_warning_as_error \ - --skip_submodule_sync \ - --allow_running_as_root \ - --skip_tests \ - --cmake_extra_defines $CMAKE_EXTRA_DEFINES -fi +# Build the dynamic library +./build.sh \ + --config Release \ + --build_shared_lib \ + --parallel \ + --compile_no_warning_as_error \ + --skip_submodule_sync \ + --allow_running_as_root \ + --skip_tests \ + --cmake_extra_defines FETCHCONTENT_TRY_FIND_PACKAGE_MODE=NEVER CMAKE_INSTALL_PREFIX=${PREFIX} # copying onnxruntime files mkdir -p "${PREFIX}"/lib/pkgconfig/ mkdir -p "${PREFIX}"/include/onnxruntime/ -cp build/$DIR_OS/$CONFIG/libonnxruntime.pc ${PREFIX}/lib/pkgconfig/ -cp -r build/$DIR_OS/$CONFIG/libonnxruntime.$SUFFIX ${PREFIX}/lib/ +cp build/Linux/Release/libonnxruntime.pc ${PREFIX}/lib/pkgconfig/ +cp -r build/Linux/Release/libonnxruntime.so* ${PREFIX}/lib/ cp include/onnxruntime/core/session/onnxruntime_cxx_inline.h ${PREFIX}/include/onnxruntime/ cp include/onnxruntime/core/session/onnxruntime_float16.h ${PREFIX}/include/onnxruntime/ From 00c09ee26a89f8f5ece24c0b22003e59ae67120b Mon Sep 17 00:00:00 2001 From: xaviliz Date: Tue, 23 Sep 2025 12:08:39 +0200 Subject: [PATCH 42/60] Clean debug output --- src/algorithms/machinelearning/onnxpredict.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/algorithms/machinelearning/onnxpredict.cpp b/src/algorithms/machinelearning/onnxpredict.cpp index 875fc7f01..058158d09 100644 --- a/src/algorithms/machinelearning/onnxpredict.cpp +++ b/src/algorithms/machinelearning/onnxpredict.cpp @@ -26,8 +26,8 @@ using namespace standard; const char* OnnxPredict::name = "OnnxPredict"; const char* OnnxPredict::category = "Machine Learning"; -const char* OnnxPredict::description = DOC("This algorithm runs a Onnx graph and stores the desired output tensors in a pool.\n" -"The Onnx graph should be stored in Open Neural Network Exchange (.onnx) binary format [1], and should contain both the architecture and the weights of the model.\n" +const char* OnnxPredict::description = DOC("This algorithm runs a Onnx model and stores the desired output tensors in a pool.\n" +"The Onnx model should be saved in Open Neural Network Exchange (.onnx) binary format [1], and should contain both the architecture and the weights of the model.\n" "The parameter `inputs` should contain a list with the names of the input nodes that feed the model. The input Pool should contain the tensors corresponding to each input node stored using Essentia tensors. " "The pool namespace for each input tensor has to match the input node's name.\n" "In the same way, the `outputs` parameter should contain the names of the tensors to save. These tensors will be stored inside the output pool under a namespace that matches the tensor's name. " @@ -54,7 +54,6 @@ void OnnxPredict::configure() { // Do not do anything if we did not get a non-empty model name. if (_graphFilename.empty()) return; - cout << "after return" << endl; try{ // Define environment From 6dc8f55750385113d00b61e40a610908ea9ac4f9 Mon Sep 17 00:00:00 2001 From: xaviliz Date: Tue, 23 Sep 2025 14:47:56 +0200 Subject: [PATCH 43/60] Skip saving the optimized graph --- src/algorithms/machinelearning/onnxpredict.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/algorithms/machinelearning/onnxpredict.cpp b/src/algorithms/machinelearning/onnxpredict.cpp index 058158d09..c4f4b0f24 100644 --- a/src/algorithms/machinelearning/onnxpredict.cpp +++ b/src/algorithms/machinelearning/onnxpredict.cpp @@ -61,8 +61,6 @@ void OnnxPredict::configure() { // Set graph optimization level - check https://onnxruntime.ai/docs/performance/model-optimizations/graph-optimizations.html _sessionOptions.SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_EXTENDED); - // To enable model serialization after graph optimization set this - _sessionOptions.SetOptimizedModelFilePath("optimized_file_path"); _sessionOptions.SetIntraOpNumThreads(1); // Initialize session From 881fe48d7e1d9a18ba66e9e37683ee9e1d8f4064 Mon Sep 17 00:00:00 2001 From: xaviliz Date: Tue, 23 Sep 2025 14:52:29 +0200 Subject: [PATCH 44/60] Set to 0 the intraop number of threads --- src/algorithms/machinelearning/onnxpredict.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/algorithms/machinelearning/onnxpredict.cpp b/src/algorithms/machinelearning/onnxpredict.cpp index c4f4b0f24..57669bbac 100644 --- a/src/algorithms/machinelearning/onnxpredict.cpp +++ b/src/algorithms/machinelearning/onnxpredict.cpp @@ -61,7 +61,7 @@ void OnnxPredict::configure() { // Set graph optimization level - check https://onnxruntime.ai/docs/performance/model-optimizations/graph-optimizations.html _sessionOptions.SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_EXTENDED); - _sessionOptions.SetIntraOpNumThreads(1); + _sessionOptions.SetIntraOpNumThreads(0); // Initialize session _session = Ort::Session(_env, _graphFilename.c_str(), _sessionOptions); From cac1cb4e3417d8ae4b5f312db930025a20a546c0 Mon Sep 17 00:00:00 2001 From: xaviliz Date: Tue, 23 Sep 2025 15:33:48 +0200 Subject: [PATCH 45/60] Improve setTensorInfos() for readablity --- src/algorithms/machinelearning/onnxpredict.cpp | 18 +++++++----------- src/algorithms/machinelearning/onnxpredict.h | 2 +- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/src/algorithms/machinelearning/onnxpredict.cpp b/src/algorithms/machinelearning/onnxpredict.cpp index 57669bbac..7d7ed80ca 100644 --- a/src/algorithms/machinelearning/onnxpredict.cpp +++ b/src/algorithms/machinelearning/onnxpredict.cpp @@ -72,8 +72,8 @@ void OnnxPredict::configure() { E_INFO("OnnxPredict: Successfully loaded graph file: `" << _graphFilename << "`"); // get input and output info (names, type and shapes) - all_input_infos = setTensorInfos(_session, _allocator, true); - all_output_infos = setTensorInfos(_session, _allocator, false); + all_input_infos = setTensorInfos(_session, _allocator, "inputs"); + all_output_infos = setTensorInfos(_session, _allocator, "outputs"); // read inputs and outputs as input parameter _inputs = parameter("inputs").toVectorString(); @@ -158,22 +158,18 @@ void OnnxPredict::configure() { } } -std::vector OnnxPredict::setTensorInfos(const Ort::Session& session, Ort::AllocatorWithDefaultOptions& allocator, bool is_input) { - size_t count = is_input ? session.GetInputCount() : session.GetOutputCount(); +std::vector OnnxPredict::setTensorInfos(const Ort::Session& session, Ort::AllocatorWithDefaultOptions& allocator, const std::string& port) { + std::vector infos; - auto names_raw = is_input - ? session.GetInputNames() - : session.GetOutputNames(); + size_t count = (port == "inputs") ? session.GetInputCount() : session.GetOutputCount(); + auto names_raw = (port == "inputs") ? session.GetInputNames() : session.GetOutputNames(); for (size_t i = 0; i < count; ++i) { auto name_raw = names_raw[i]; std::string name(name_raw); - - Ort::TypeInfo type_info = is_input - ? session.GetInputTypeInfo(i) - : session.GetOutputTypeInfo(i); + Ort::TypeInfo type_info = (port == "inputs") ? session.GetInputTypeInfo(i) : session.GetOutputTypeInfo(i); auto tensor_info = type_info.GetTensorTypeAndShapeInfo(); diff --git a/src/algorithms/machinelearning/onnxpredict.h b/src/algorithms/machinelearning/onnxpredict.h index ecbe342ec..e0637cd44 100644 --- a/src/algorithms/machinelearning/onnxpredict.h +++ b/src/algorithms/machinelearning/onnxpredict.h @@ -77,7 +77,7 @@ class OnnxPredict : public Algorithm { std::vector inputNames(); std::vector outputNames(); - std::vector setTensorInfos(const Ort::Session&, Ort::AllocatorWithDefaultOptions&, bool); + std::vector setTensorInfos(const Ort::Session&, Ort::AllocatorWithDefaultOptions&, const std::string&); void printTensorInfos(const std::vector&, const std::string&); std::string getTensorInfos(const std::vector&, const std::string&); void checkName(const std::string, std::vector); From 6b971f6e2c8a815bbe909f2a1f88b713ec9fd4eb Mon Sep 17 00:00:00 2001 From: xaviliz Date: Tue, 23 Sep 2025 15:59:41 +0200 Subject: [PATCH 46/60] Raise an exception if no input is provided --- src/algorithms/machinelearning/onnxpredict.cpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/algorithms/machinelearning/onnxpredict.cpp b/src/algorithms/machinelearning/onnxpredict.cpp index 7d7ed80ca..5f288b485 100644 --- a/src/algorithms/machinelearning/onnxpredict.cpp +++ b/src/algorithms/machinelearning/onnxpredict.cpp @@ -86,11 +86,7 @@ void OnnxPredict::configure() { // use the first input when no input is defined if (_nInputs == 0){ - // take the first input - _inputs.push_back(_session.GetInputNames()[0]); - _nInputs = _inputs.size(); - // inform the first model input will be used - E_INFO("OnnxPredict: using the first model input '" + _inputs[0] + "'"); + throw EssentiaException("No model input was defined.\n" + availableInputInfo()); } // define _outputs with the first model output when no output is provided From 1f8f0fb943a4030aa025ce2697e57f7e5ef8f110 Mon Sep 17 00:00:00 2001 From: xaviliz Date: Fri, 26 Sep 2025 12:24:21 +0200 Subject: [PATCH 47/60] Small clean --- src/algorithms/machinelearning/onnxpredict.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/algorithms/machinelearning/onnxpredict.h b/src/algorithms/machinelearning/onnxpredict.h index e0637cd44..d7f778f39 100644 --- a/src/algorithms/machinelearning/onnxpredict.h +++ b/src/algorithms/machinelearning/onnxpredict.h @@ -85,16 +85,16 @@ class OnnxPredict : public Algorithm { inline std::string availableInputInfo() { std::vector inputs = inputNames(); - std::string info = "OnnxPredict: Available input names are:\n"; + std::string info = "Available input names are:\n"; for (std::vector::const_iterator i = inputs.begin(); i != inputs.end() - 1; ++i) info += *i + ", "; - return info + inputs.back() + ".\n\nReconfigure this algorithm with valid input names before starting the processing."; + return info + inputs.back() + "\n\nReconfigure this algorithm with valid input names before starting the processing."; } inline std::string availableOutputInfo() { std::vector outputs = outputNames(); std::string info = "OnnxPredict: Available output names are:\n"; for (std::vector::const_iterator i = outputs.begin(); i != outputs.end() - 1; ++i) info += *i + ", "; - return info + outputs.back() + ".\n\nReconfigure this algorithm with valid output names before starting the processing."; + return info + outputs.back() + "\n\nReconfigure this algorithm with valid output names before starting the processing."; } public: From 679b2fb18c2b47dac83d9666118acae1e4a0fbf9 Mon Sep 17 00:00:00 2001 From: xaviliz Date: Fri, 26 Sep 2025 12:26:42 +0200 Subject: [PATCH 48/60] Fix testInference() unitest after declaring inputs as mandatory parameter --- test/src/unittests/machinelearning/test_onnxpredict.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/src/unittests/machinelearning/test_onnxpredict.py b/test/src/unittests/machinelearning/test_onnxpredict.py index bd7bbae84..ee43d6aba 100644 --- a/test/src/unittests/machinelearning/test_onnxpredict.py +++ b/test/src/unittests/machinelearning/test_onnxpredict.py @@ -227,7 +227,7 @@ def runEffnetDiscogsInference(onnx_predict, outputs, pool) -> Pool: onnx_predict.configure( graphFilename= model, - inputs=[], + inputs=["melspectrogram"], outputs=[output["name"] for output in outputs], ) From 31b7964150a6e415f174eab229b7ec96eaa44582 Mon Sep 17 00:00:00 2001 From: xaviliz Date: Fri, 26 Sep 2025 12:30:15 +0200 Subject: [PATCH 49/60] Small change --- src/algorithms/machinelearning/onnxpredict.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/algorithms/machinelearning/onnxpredict.cpp b/src/algorithms/machinelearning/onnxpredict.cpp index 5f288b485..629d7c849 100644 --- a/src/algorithms/machinelearning/onnxpredict.cpp +++ b/src/algorithms/machinelearning/onnxpredict.cpp @@ -84,7 +84,7 @@ void OnnxPredict::configure() { _nInputs = _inputs.size(); _nOutputs = _outputs.size(); - // use the first input when no input is defined + // excepts if no input is provided if (_nInputs == 0){ throw EssentiaException("No model input was defined.\n" + availableInputInfo()); } From e2c6c2aa04ab7e5168647332e55405b879aed9dd Mon Sep 17 00:00:00 2001 From: xaviliz Date: Fri, 26 Sep 2025 12:33:00 +0200 Subject: [PATCH 50/60] Excepts when no outputs are defined --- src/algorithms/machinelearning/onnxpredict.cpp | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/algorithms/machinelearning/onnxpredict.cpp b/src/algorithms/machinelearning/onnxpredict.cpp index 629d7c849..a14927242 100644 --- a/src/algorithms/machinelearning/onnxpredict.cpp +++ b/src/algorithms/machinelearning/onnxpredict.cpp @@ -84,18 +84,14 @@ void OnnxPredict::configure() { _nInputs = _inputs.size(); _nOutputs = _outputs.size(); - // excepts if no input is provided + // excepts if no inputs are defined if (_nInputs == 0){ throw EssentiaException("No model input was defined.\n" + availableInputInfo()); } - // define _outputs with the first model output when no output is provided + // excepts if no outputs are defined if (_nOutputs == 0){ - // take the first output - _outputs.push_back(_session.GetOutputNames()[0]); - _nOutputs = _outputs.size(); - // inform the first model input will be used - E_INFO("OnnxPredict: using the first model output '" + _outputs[0] + "'"); + throw EssentiaException("No model output was defined.\n" + availableOutputInfo()); } // If the first output name is empty just print out the list of nodes and return. From f86fd5220bb757c16e2991b0fd5749794dae587d Mon Sep 17 00:00:00 2001 From: xaviliz Date: Fri, 26 Sep 2025 12:37:01 +0200 Subject: [PATCH 51/60] Clean unused checkers for inputs and outputs --- .../machinelearning/onnxpredict.cpp | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/src/algorithms/machinelearning/onnxpredict.cpp b/src/algorithms/machinelearning/onnxpredict.cpp index a14927242..e666dc661 100644 --- a/src/algorithms/machinelearning/onnxpredict.cpp +++ b/src/algorithms/machinelearning/onnxpredict.cpp @@ -112,17 +112,6 @@ void OnnxPredict::configure() { } } } - - if (_inputNodes.size() == 0){ - - std::string s; - for (const auto &piece : _inputs) { - s += piece; - s += " "; - } - - throw EssentiaException(availableInputInfo()); - } for (int i = 0; i < _outputs.size(); i++) { for (int j = 0; j < all_output_infos.size(); j++) { @@ -131,15 +120,6 @@ void OnnxPredict::configure() { } } } - - if (_outputNodes.size() == 0){ - std::string s; - for (const auto &piece : _outputs) { - s += piece; - s += " "; - } - throw EssentiaException(availableOutputInfo()); - } for (size_t i = 0; i < _nInputs; i++) { checkName(_inputs[i], all_input_infos); From 59ae04e5c37a234248820a9c9eae3221803fe37e Mon Sep 17 00:00:00 2001 From: xaviliz Date: Fri, 26 Sep 2025 12:41:35 +0200 Subject: [PATCH 52/60] Replace std::cout by E_INFO() --- src/algorithms/machinelearning/onnxpredict.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/algorithms/machinelearning/onnxpredict.cpp b/src/algorithms/machinelearning/onnxpredict.cpp index e666dc661..08760ff1f 100644 --- a/src/algorithms/machinelearning/onnxpredict.cpp +++ b/src/algorithms/machinelearning/onnxpredict.cpp @@ -157,16 +157,16 @@ std::vector OnnxPredict::setTensorInfos(const Ort::Session& session, } void OnnxPredict::printTensorInfos(const std::vector& infos, const std::string& label) { - std::cout << "=== " << label << " ===\n"; + E_INFO("=== " << label << " ===\n"); for (const auto& info : infos) { - std::cout << "[Name] " << info.name << endl; - std::cout << " [Type] " << info.type << endl; - std::cout << " [Shape] ["; + E_INFO("[Name] " << info.name); + E_INFO(" [Type] " << info.type); + E_INFO(" [Shape] ["); for (size_t j = 0; j < info.shape.size(); ++j) { - std::cout << info.shape[j]; - if (j + 1 < info.shape.size()) std::cout << ", "; + E_INFO(info.shape[j]); + if (j + 1 < info.shape.size()) E_INFO(", "); } - std::cout << "]\n"; + E_INFO("]\n"); } } From 108ee4c14f47fdd4adfa4dcf32742de51fecbcc9 Mon Sep 17 00:00:00 2001 From: xaviliz Date: Thu, 2 Oct 2025 11:46:13 +0200 Subject: [PATCH 53/60] Add onnxruntime library in the Debian building pipeline. --- packaging/build_3rdparty_static_debian.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/packaging/build_3rdparty_static_debian.sh b/packaging/build_3rdparty_static_debian.sh index 7fed688fc..d2d61d4b5 100755 --- a/packaging/build_3rdparty_static_debian.sh +++ b/packaging/build_3rdparty_static_debian.sh @@ -11,6 +11,7 @@ cd $BASEDIR/debian_3rdparty ./build_taglib.sh ./build_yaml.sh ./build_chromaprint.sh +./build_onnx.sh #!/usr/bin/env bash if [[ "$*" == *--with-gaia* ]] From 442bae553703474e04031c1156fbe49cac1c6135 Mon Sep 17 00:00:00 2001 From: xaviliz Date: Thu, 2 Oct 2025 13:10:26 +0200 Subject: [PATCH 54/60] Configure GPU providers dynamically --- .../machinelearning/onnxpredict.cpp | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/src/algorithms/machinelearning/onnxpredict.cpp b/src/algorithms/machinelearning/onnxpredict.cpp index 08760ff1f..3bfcd67a3 100644 --- a/src/algorithms/machinelearning/onnxpredict.cpp +++ b/src/algorithms/machinelearning/onnxpredict.cpp @@ -58,6 +58,21 @@ void OnnxPredict::configure() { try{ // Define environment _env = Ort::Env(ORT_LOGGING_LEVEL_WARNING, "multi_io_inference"); // {"default", "test", "multi_io_inference"} + + // Auto-detect EPs + auto providers = Ort::GetAvailableProviders(); + if (std::find(providers.begin(), providers.end(), "CUDAExecutionProvider") != providers.end()) { + OrtSessionOptionsAppendExecutionProvider_CUDA(session_options, 0); // device_id = 0 + E_INFO("✅ Using CUDA Execution Provider"); + } else if (std::find(providers.begin(), providers.end(), "MetalExecutionProvider") != providers.end()) { + OrtSessionOptionsAppendExecutionProvider_Metal(session_options, 0); // device_id = 0 + E_INFO("✅ Using Metal Execution Provider"); + } else if (std::find(providers.begin(), providers.end(), "CoreMLExecutionProvider") != providers.end()) { + OrtSessionOptionsAppendExecutionProvider_CoreML(session_options, 0); // device_id = 0 + E_INFO("✅ Using Core ML Execution Provider"); + } else { + // Default = CPU - CPU is always available, no need to append explicitly + } // Set graph optimization level - check https://onnxruntime.ai/docs/performance/model-optimizations/graph-optimizations.html _sessionOptions.SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_EXTENDED); @@ -112,6 +127,7 @@ void OnnxPredict::configure() { } } } +// TODO: check if _inputNodes is empty - release an error instead for (int i = 0; i < _outputs.size(); i++) { for (int j = 0; j < all_output_infos.size(); j++) { @@ -120,7 +136,8 @@ void OnnxPredict::configure() { } } } - + // TODO: check if _outputNodes is empty - release an error instead + // TODO: remove for (size_t i = 0; i < _nInputs; i++) { checkName(_inputs[i], all_input_infos); } @@ -193,13 +210,15 @@ void OnnxPredict::reset() { output_names.clear(); _inputNodes.clear(); _outputNodes.clear(); + // TODO: execute the reset() at the beginning in configure() + // TODO: include here the model and session creation } void OnnxPredict::compute() { if (!_isConfigured) { throw EssentiaException("OnnxPredict: This algorithm is not configured. To configure this algorithm you " - "should specify a valid `graphFilename` as input parameter."); + "should specify a valid `graphFilename`, `inputs` and `outputs` as input parameter."); } const Pool& poolIn = _poolIn.get(); From 580a62ce4c2540267ac702ea0f0fef752399d35f Mon Sep 17 00:00:00 2001 From: xaviliz Date: Fri, 3 Oct 2025 12:57:27 +0200 Subject: [PATCH 55/60] Handle parallel computational providers with macros to avoid errors when cuda, metal or open_ml are not compiled in onnxruntime library --- .../machinelearning/onnxpredict.cpp | 31 ++++++++++++++++--- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/src/algorithms/machinelearning/onnxpredict.cpp b/src/algorithms/machinelearning/onnxpredict.cpp index 3bfcd67a3..2d1167f7f 100644 --- a/src/algorithms/machinelearning/onnxpredict.cpp +++ b/src/algorithms/machinelearning/onnxpredict.cpp @@ -59,20 +59,41 @@ void OnnxPredict::configure() { // Define environment _env = Ort::Env(ORT_LOGGING_LEVEL_WARNING, "multi_io_inference"); // {"default", "test", "multi_io_inference"} - // Auto-detect EPs + /* Auto-detect EPs auto providers = Ort::GetAvailableProviders(); if (std::find(providers.begin(), providers.end(), "CUDAExecutionProvider") != providers.end()) { - OrtSessionOptionsAppendExecutionProvider_CUDA(session_options, 0); // device_id = 0 + OrtSessionOptionsAppendExecutionProvider_CUDA(_sessionOptions, 0); // device_id = 0 E_INFO("✅ Using CUDA Execution Provider"); } else if (std::find(providers.begin(), providers.end(), "MetalExecutionProvider") != providers.end()) { - OrtSessionOptionsAppendExecutionProvider_Metal(session_options, 0); // device_id = 0 + OrtSessionOptionsAppendExecutionProvider_Metal(_sessionOptions, 0); // device_id = 0 E_INFO("✅ Using Metal Execution Provider"); } else if (std::find(providers.begin(), providers.end(), "CoreMLExecutionProvider") != providers.end()) { - OrtSessionOptionsAppendExecutionProvider_CoreML(session_options, 0); // device_id = 0 + OrtSessionOptionsAppendExecutionProvider_CoreML(_sessionOptions, 0); // device_id = 0 E_INFO("✅ Using Core ML Execution Provider"); - } else { + }else { // Default = CPU - CPU is always available, no need to append explicitly + }*/ + + #ifdef USE_CUDA + if (std::find(providers.begin(), providers.end(), "CUDAExecutionProvider") != providers.end()) { + OrtSessionOptionsAppendExecutionProvider_CUDA(_sessionOptions, 0); + E_INFO("✅ Using CUDA Execution Provider"); + } + #endif + + #ifdef USE_METAL + if (std::find(providers.begin(), providers.end(), "MetalExecutionProvider") != providers.end()) { + OrtSessionOptionsAppendExecutionProvider_Metal(_sessionOptions, 0); + E_INFO("✅ Using Metal Execution Provider"); + } + #endif + + #ifdef USE_COREML + if (std::find(providers.begin(), providers.end(), "CoreMLExecutionProvider") != providers.end()) { + OrtSessionOptionsAppendExecutionProvider_CoreML(_sessionOptions, 0); + E_INFO("✅ Using Core ML Execution Provider"); } + #endif // Set graph optimization level - check https://onnxruntime.ai/docs/performance/model-optimizations/graph-optimizations.html _sessionOptions.SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_EXTENDED); From 382dd172b0b4eaef06d48beb53080716a0c5f179 Mon Sep 17 00:00:00 2001 From: xaviliz Date: Fri, 3 Oct 2025 14:59:19 +0200 Subject: [PATCH 56/60] Support for resetting Ort::Session and Ort::SessionOption --- .../machinelearning/onnxpredict.cpp | 43 ++++++++----------- src/algorithms/machinelearning/onnxpredict.h | 6 ++- 2 files changed, 22 insertions(+), 27 deletions(-) diff --git a/src/algorithms/machinelearning/onnxpredict.cpp b/src/algorithms/machinelearning/onnxpredict.cpp index 2d1167f7f..107c2aea7 100644 --- a/src/algorithms/machinelearning/onnxpredict.cpp +++ b/src/algorithms/machinelearning/onnxpredict.cpp @@ -58,22 +58,14 @@ void OnnxPredict::configure() { try{ // Define environment _env = Ort::Env(ORT_LOGGING_LEVEL_WARNING, "multi_io_inference"); // {"default", "test", "multi_io_inference"} - - /* Auto-detect EPs - auto providers = Ort::GetAvailableProviders(); - if (std::find(providers.begin(), providers.end(), "CUDAExecutionProvider") != providers.end()) { - OrtSessionOptionsAppendExecutionProvider_CUDA(_sessionOptions, 0); // device_id = 0 - E_INFO("✅ Using CUDA Execution Provider"); - } else if (std::find(providers.begin(), providers.end(), "MetalExecutionProvider") != providers.end()) { - OrtSessionOptionsAppendExecutionProvider_Metal(_sessionOptions, 0); // device_id = 0 - E_INFO("✅ Using Metal Execution Provider"); - } else if (std::find(providers.begin(), providers.end(), "CoreMLExecutionProvider") != providers.end()) { - OrtSessionOptionsAppendExecutionProvider_CoreML(_sessionOptions, 0); // device_id = 0 - E_INFO("✅ Using Core ML Execution Provider"); - }else { - // Default = CPU - CPU is always available, no need to append explicitly - }*/ + // Reset session + _session.reset(); + + // Reset SessionOptions by constructing a fresh object + _sessionOptions = Ort::SessionOptions{}; + + // Auto-detect EPs #ifdef USE_CUDA if (std::find(providers.begin(), providers.end(), "CUDAExecutionProvider") != providers.end()) { OrtSessionOptionsAppendExecutionProvider_CUDA(_sessionOptions, 0); @@ -100,7 +92,8 @@ void OnnxPredict::configure() { _sessionOptions.SetIntraOpNumThreads(0); // Initialize session - _session = Ort::Session(_env, _graphFilename.c_str(), _sessionOptions); + _session = std::make_unique(_env, _graphFilename.c_str(), _sessionOptions); + } catch (Ort::Exception oe) { throw EssentiaException(string("OnnxPredict:") + oe.what(), oe.GetOrtErrorCode()); @@ -108,8 +101,8 @@ void OnnxPredict::configure() { E_INFO("OnnxPredict: Successfully loaded graph file: `" << _graphFilename << "`"); // get input and output info (names, type and shapes) - all_input_infos = setTensorInfos(_session, _allocator, "inputs"); - all_output_infos = setTensorInfos(_session, _allocator, "outputs"); + all_input_infos = setTensorInfos(*_session, _allocator, "inputs"); + all_output_infos = setTensorInfos(*_session, _allocator, "outputs"); // read inputs and outputs as input parameter _inputs = parameter("inputs").toVectorString(); @@ -312,13 +305,13 @@ void OnnxPredict::compute() { } // Run the Onnxruntime session. - auto output_tensors = _session.Run(_runOptions, // Run options. - input_names.data(), // Input node names. - input_tensors.data(), // Input tensor values. - _nInputs, // Number of inputs. - output_names.data(), // Output node names. - _nOutputs // Number of outputs. - ); + auto output_tensors = _session->Run(_runOptions, // Run options. + input_names.data(), // Input node names. + input_tensors.data(), // Input tensor values. + _nInputs, // Number of inputs. + output_names.data(), // Output node names. + _nOutputs // Number of outputs. + ); // Map output tensors to pool for (size_t i = 0; i < output_tensors.size(); ++i) { diff --git a/src/algorithms/machinelearning/onnxpredict.h b/src/algorithms/machinelearning/onnxpredict.h index d7f778f39..f9f8c9bfa 100644 --- a/src/algorithms/machinelearning/onnxpredict.h +++ b/src/algorithms/machinelearning/onnxpredict.h @@ -63,7 +63,9 @@ class OnnxPredict : public Algorithm { Ort::Env _env{nullptr}; Ort::SessionOptions _sessionOptions{nullptr}; - Ort::Session _session{nullptr}; + //Ort::Session _session{nullptr}; + std::unique_ptr _session; + Ort::RunOptions _runOptions; Ort::AllocatorWithDefaultOptions _allocator; @@ -100,7 +102,7 @@ class OnnxPredict : public Algorithm { public: OnnxPredict() : _env(Ort::Env(ORT_LOGGING_LEVEL_WARNING, "test")), - _sessionOptions(Ort::SessionOptions()), _session(Ort::Session(nullptr)), _runOptions(NULL), _isConfigured(false){ + _sessionOptions(Ort::SessionOptions()), _session(nullptr), _runOptions(NULL), _isConfigured(false){ declareInput(_poolIn, "poolIn", "the pool where to get the feature tensors"); declareOutput(_poolOut, "poolOut", "the pool where to store the output tensors"); } From 8e137b32f429a066b577cd2f9b5fb989f76dda6f Mon Sep 17 00:00:00 2001 From: xaviliz Date: Fri, 3 Oct 2025 15:17:57 +0200 Subject: [PATCH 57/60] Check input and output nodes before any computating --- src/algorithms/machinelearning/onnxpredict.cpp | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/algorithms/machinelearning/onnxpredict.cpp b/src/algorithms/machinelearning/onnxpredict.cpp index 107c2aea7..86dcb0375 100644 --- a/src/algorithms/machinelearning/onnxpredict.cpp +++ b/src/algorithms/machinelearning/onnxpredict.cpp @@ -141,8 +141,11 @@ void OnnxPredict::configure() { } } } -// TODO: check if _inputNodes is empty - release an error instead - + + // Check if _inputNodes is empty - release an exception instead + if (!_inputNodes.size) + throw EssentiaException("No input node was found.\n" + availableInputInfo()); + for (int i = 0; i < _outputs.size(); i++) { for (int j = 0; j < all_output_infos.size(); j++) { if (_outputs[i] == all_output_infos[j].name){ @@ -150,8 +153,11 @@ void OnnxPredict::configure() { } } } - // TODO: check if _outputNodes is empty - release an error instead - // TODO: remove + + // Check if _outputNodes is empty - release an exception instead + if (!_outputNodes.size) + throw EssentiaException("No output node was found.\n" + availableOutputInfo()); + for (size_t i = 0; i < _nInputs; i++) { checkName(_inputs[i], all_input_infos); } From 37e69125457f68c150a61a82618a56213bffa62a Mon Sep 17 00:00:00 2001 From: xaviliz Date: Fri, 3 Oct 2025 15:34:56 +0200 Subject: [PATCH 58/60] Reset Ort::Session and Ort::SessionOptions for each configure() --- .../machinelearning/onnxpredict.cpp | 103 +++++++++--------- 1 file changed, 53 insertions(+), 50 deletions(-) diff --git a/src/algorithms/machinelearning/onnxpredict.cpp b/src/algorithms/machinelearning/onnxpredict.cpp index 86dcb0375..6c9aad845 100644 --- a/src/algorithms/machinelearning/onnxpredict.cpp +++ b/src/algorithms/machinelearning/onnxpredict.cpp @@ -55,50 +55,7 @@ void OnnxPredict::configure() { // Do not do anything if we did not get a non-empty model name. if (_graphFilename.empty()) return; - try{ - // Define environment - _env = Ort::Env(ORT_LOGGING_LEVEL_WARNING, "multi_io_inference"); // {"default", "test", "multi_io_inference"} - - // Reset session - _session.reset(); - - // Reset SessionOptions by constructing a fresh object - _sessionOptions = Ort::SessionOptions{}; - - // Auto-detect EPs - #ifdef USE_CUDA - if (std::find(providers.begin(), providers.end(), "CUDAExecutionProvider") != providers.end()) { - OrtSessionOptionsAppendExecutionProvider_CUDA(_sessionOptions, 0); - E_INFO("✅ Using CUDA Execution Provider"); - } - #endif - - #ifdef USE_METAL - if (std::find(providers.begin(), providers.end(), "MetalExecutionProvider") != providers.end()) { - OrtSessionOptionsAppendExecutionProvider_Metal(_sessionOptions, 0); - E_INFO("✅ Using Metal Execution Provider"); - } - #endif - - #ifdef USE_COREML - if (std::find(providers.begin(), providers.end(), "CoreMLExecutionProvider") != providers.end()) { - OrtSessionOptionsAppendExecutionProvider_CoreML(_sessionOptions, 0); - E_INFO("✅ Using Core ML Execution Provider"); - } - #endif - - // Set graph optimization level - check https://onnxruntime.ai/docs/performance/model-optimizations/graph-optimizations.html - _sessionOptions.SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_EXTENDED); - _sessionOptions.SetIntraOpNumThreads(0); - - // Initialize session - _session = std::make_unique(_env, _graphFilename.c_str(), _sessionOptions); - - } - catch (Ort::Exception oe) { - throw EssentiaException(string("OnnxPredict:") + oe.what(), oe.GetOrtErrorCode()); - } - E_INFO("OnnxPredict: Successfully loaded graph file: `" << _graphFilename << "`"); + reset(); // get input and output info (names, type and shapes) all_input_infos = setTensorInfos(*_session, _allocator, "inputs"); @@ -131,7 +88,6 @@ void OnnxPredict::configure() { } _isConfigured = true; - reset(); // check model has input and output https://github.com/microsoft/onnxruntime-inference-examples/blob/7a635daae48450ff142e5c0848a564b245f04112/c_cxx/model-explorer/model-explorer.cpp#L99C3-L100C63 for (int i = 0; i < _inputs.size(); i++) { @@ -143,7 +99,7 @@ void OnnxPredict::configure() { } // Check if _inputNodes is empty - release an exception instead - if (!_inputNodes.size) + if (!_inputNodes.size()) throw EssentiaException("No input node was found.\n" + availableInputInfo()); for (int i = 0; i < _outputs.size(); i++) { @@ -155,7 +111,7 @@ void OnnxPredict::configure() { } // Check if _outputNodes is empty - release an exception instead - if (!_outputNodes.size) + if (!_outputNodes.size()) throw EssentiaException("No output node was found.\n" + availableOutputInfo()); for (size_t i = 0; i < _nInputs; i++) { @@ -225,13 +181,60 @@ std::string OnnxPredict::getTensorInfos(const std::vector& infos, co } void OnnxPredict::reset() { - if (!_isConfigured) return; + //if (!_isConfigured) return; input_names.clear(); output_names.clear(); _inputNodes.clear(); _outputNodes.clear(); - // TODO: execute the reset() at the beginning in configure() - // TODO: include here the model and session creation + + // TODO: execute the reset() at the beginning in configure() + // TODO: include here the model and session creation + + try{ + // Define environment + _env = Ort::Env(ORT_LOGGING_LEVEL_WARNING, "multi_io_inference"); // {"default", "test", "multi_io_inference"} + + // Reset session + _session.reset(); + + // Reset SessionOptions by constructing a fresh object + _sessionOptions = Ort::SessionOptions{}; + + // Auto-detect EPs + #ifdef USE_CUDA + if (std::find(providers.begin(), providers.end(), "CUDAExecutionProvider") != providers.end()) { + OrtSessionOptionsAppendExecutionProvider_CUDA(_sessionOptions, 0); + E_INFO("✅ Using CUDA Execution Provider"); + } + #endif + + #ifdef USE_METAL + if (std::find(providers.begin(), providers.end(), "MetalExecutionProvider") != providers.end()) { + OrtSessionOptionsAppendExecutionProvider_Metal(_sessionOptions, 0); + E_INFO("✅ Using Metal Execution Provider"); + } + #endif + + #ifdef USE_COREML + if (std::find(providers.begin(), providers.end(), "CoreMLExecutionProvider") != providers.end()) { + OrtSessionOptionsAppendExecutionProvider_CoreML(_sessionOptions, 0); + E_INFO("✅ Using Core ML Execution Provider"); + } + #endif + + // Set graph optimization level - check https://onnxruntime.ai/docs/performance/model-optimizations/graph-optimizations.html + _sessionOptions.SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_EXTENDED); + _sessionOptions.SetIntraOpNumThreads(0); + + // Initialize session + _session = std::make_unique(_env, _graphFilename.c_str(), _sessionOptions); + + } + catch (Ort::Exception oe) { + throw EssentiaException(string("OnnxPredict:") + oe.what(), oe.GetOrtErrorCode()); + } + + E_INFO("OnnxPredict: Successfully loaded graph file: `" << _graphFilename << "`"); } void OnnxPredict::compute() { From 1cdcb2b6150ac668f2b73ced3f646fd83e2ba57e Mon Sep 17 00:00:00 2001 From: xaviliz Date: Fri, 3 Oct 2025 15:37:01 +0200 Subject: [PATCH 59/60] Create Ort::Env once per application and reuse it for all sessions --- src/algorithms/machinelearning/onnxpredict.cpp | 9 ++------- src/algorithms/machinelearning/onnxpredict.h | 2 +- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/src/algorithms/machinelearning/onnxpredict.cpp b/src/algorithms/machinelearning/onnxpredict.cpp index 6c9aad845..a66fc4257 100644 --- a/src/algorithms/machinelearning/onnxpredict.cpp +++ b/src/algorithms/machinelearning/onnxpredict.cpp @@ -181,19 +181,14 @@ std::string OnnxPredict::getTensorInfos(const std::vector& infos, co } void OnnxPredict::reset() { - //if (!_isConfigured) return; + input_names.clear(); output_names.clear(); _inputNodes.clear(); _outputNodes.clear(); - - // TODO: execute the reset() at the beginning in configure() - // TODO: include here the model and session creation try{ - // Define environment - _env = Ort::Env(ORT_LOGGING_LEVEL_WARNING, "multi_io_inference"); // {"default", "test", "multi_io_inference"} - + // Reset session _session.reset(); diff --git a/src/algorithms/machinelearning/onnxpredict.h b/src/algorithms/machinelearning/onnxpredict.h index f9f8c9bfa..3647dbbff 100644 --- a/src/algorithms/machinelearning/onnxpredict.h +++ b/src/algorithms/machinelearning/onnxpredict.h @@ -101,7 +101,7 @@ class OnnxPredict : public Algorithm { public: - OnnxPredict() : _env(Ort::Env(ORT_LOGGING_LEVEL_WARNING, "test")), + OnnxPredict() : _env(Ort::Env(ORT_LOGGING_LEVEL_WARNING, "multi_io_inference")), // {"default", "test", "multi_io_inference"} - reuse it for all sessions _sessionOptions(Ort::SessionOptions()), _session(nullptr), _runOptions(NULL), _isConfigured(false){ declareInput(_poolIn, "poolIn", "the pool where to get the feature tensors"); declareOutput(_poolOut, "poolOut", "the pool where to store the output tensors"); From 74a585b3eb2b206535adb1aea9a8973f013bd92a Mon Sep 17 00:00:00 2001 From: xaviliz Date: Fri, 3 Oct 2025 16:02:35 +0200 Subject: [PATCH 60/60] Add deviceId parameter to specify gpu id for inferencing when CUDA, METAL or OPEN_ML providers are found --- src/algorithms/machinelearning/onnxpredict.cpp | 7 ++++--- src/algorithms/machinelearning/onnxpredict.h | 2 ++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/algorithms/machinelearning/onnxpredict.cpp b/src/algorithms/machinelearning/onnxpredict.cpp index a66fc4257..45c5855f8 100644 --- a/src/algorithms/machinelearning/onnxpredict.cpp +++ b/src/algorithms/machinelearning/onnxpredict.cpp @@ -46,6 +46,7 @@ const char* OnnxPredict::description = DOC("This algorithm runs a Onnx model and void OnnxPredict::configure() { _graphFilename = parameter("graphFilename").toString(); + _deviceId = parameter("deviceId").toInt(); if ((_graphFilename.empty()) and (_isConfigured)) { E_WARNING("OnnxPredict: You are trying to update a valid configuration with invalid parameters. " @@ -198,21 +199,21 @@ void OnnxPredict::reset() { // Auto-detect EPs #ifdef USE_CUDA if (std::find(providers.begin(), providers.end(), "CUDAExecutionProvider") != providers.end()) { - OrtSessionOptionsAppendExecutionProvider_CUDA(_sessionOptions, 0); + OrtSessionOptionsAppendExecutionProvider_CUDA(_sessionOptions, _deviceId); E_INFO("✅ Using CUDA Execution Provider"); } #endif #ifdef USE_METAL if (std::find(providers.begin(), providers.end(), "MetalExecutionProvider") != providers.end()) { - OrtSessionOptionsAppendExecutionProvider_Metal(_sessionOptions, 0); + OrtSessionOptionsAppendExecutionProvider_Metal(_sessionOptions, _deviceId); E_INFO("✅ Using Metal Execution Provider"); } #endif #ifdef USE_COREML if (std::find(providers.begin(), providers.end(), "CoreMLExecutionProvider") != providers.end()) { - OrtSessionOptionsAppendExecutionProvider_CoreML(_sessionOptions, 0); + OrtSessionOptionsAppendExecutionProvider_CoreML(_sessionOptions, _deviceId); E_INFO("✅ Using Core ML Execution Provider"); } #endif diff --git a/src/algorithms/machinelearning/onnxpredict.h b/src/algorithms/machinelearning/onnxpredict.h index 3647dbbff..d4878fe71 100644 --- a/src/algorithms/machinelearning/onnxpredict.h +++ b/src/algorithms/machinelearning/onnxpredict.h @@ -47,6 +47,7 @@ class OnnxPredict : public Algorithm { std::string _graphFilename; std::vector _inputs; std::vector _outputs; + size_t _deviceId; bool _squeeze; bool _isConfigured; @@ -125,6 +126,7 @@ class OnnxPredict : public Algorithm { declareParameter("inputs", "will look for these namespaces in poolIn. Should match the names of the inputs in the ONNX model", "", Parameter::VECTOR_STRING); declareParameter("outputs", "will save the tensors on the model outputs named after `outputs` to the same namespaces in the output pool. Set the first element of this list as an empty array to print all the available model outputs", "", Parameter::VECTOR_STRING); declareParameter("squeeze", "remove singleton dimensions of the inputs tensors. Does not apply to the batch dimension", "{true,false}", true); + declareParameter("deviceId", "the gpu device id when CUDA support is available", "[0,inf)", 0); } void configure();