From 5a454626b36c50e7764e73358caab31f8fb809dd Mon Sep 17 00:00:00 2001 From: Matteo Nicoli Date: Tue, 12 Aug 2025 10:32:28 +0200 Subject: [PATCH 1/2] sparse tensor data structures COOTensor init CSRTensor init added SparseTensorAllocator moved SparseTensor to arm_compute/core fixed to_dense method for COOTensor fixed to_sparse and to_dense methods for CSRTensor changed CSR indices; print function moved to the bae class implemented get_value to CSR and COO tensors added SparseIterator Change-Id: Id5ed095587662b750b89cbabd600a286ab9bf53b --- arm_compute/core/Coordinates.h | 10 + arm_compute/core/Dimensions.h | 9 + arm_compute/core/Helpers.h | 37 ++ arm_compute/core/Helpers.inl | 40 ++ arm_compute/core/IReducibleTensor.h | 76 ++++ arm_compute/core/ITensorInfo.h | 18 + arm_compute/core/SparseTensor.h | 102 +++++ arm_compute/core/SubTensorInfo.h | 12 + arm_compute/core/TensorFormat.h | 34 ++ arm_compute/core/TensorInfo.h | 15 + arm_compute/runtime/COOTensor.h | 86 ++++ arm_compute/runtime/CSRTensor.h | 92 ++++ arm_compute/runtime/SparseTensorAllocator.h | 153 +++++++ arm_compute/runtime/Tensor.h | 18 +- filelist.json | 8 +- src/BUILD.bazel | 4 + src/CMakeLists.txt | 4 + src/core/SparseTensor.cpp | 164 +++++++ src/core/SubTensorInfo.cpp | 6 + src/core/TensorInfo.cpp | 6 +- src/runtime/COOTensor.cpp | 264 +++++++++++ src/runtime/CSRTensor.cpp | 272 ++++++++++++ src/runtime/SparseTensorAllocator.cpp | 177 ++++++++ src/runtime/Tensor.cpp | 25 ++ tests/AssetsLibrary.h | 64 +++ tests/validation/cpu/unit/SparseTensor.cpp | 467 ++++++++++++++++++++ 26 files changed, 2155 insertions(+), 8 deletions(-) create mode 100644 arm_compute/core/IReducibleTensor.h create mode 100644 arm_compute/core/SparseTensor.h create mode 100644 arm_compute/core/TensorFormat.h create mode 100644 arm_compute/runtime/COOTensor.h create mode 100644 arm_compute/runtime/CSRTensor.h create mode 100644 arm_compute/runtime/SparseTensorAllocator.h create mode 100644 src/core/SparseTensor.cpp create mode 100644 src/runtime/COOTensor.cpp create mode 100644 src/runtime/CSRTensor.cpp create mode 100644 src/runtime/SparseTensorAllocator.cpp create mode 100644 tests/validation/cpu/unit/SparseTensor.cpp diff --git a/arm_compute/core/Coordinates.h b/arm_compute/core/Coordinates.h index 71576b749a0..3dcd656f9dc 100644 --- a/arm_compute/core/Coordinates.h +++ b/arm_compute/core/Coordinates.h @@ -49,6 +49,16 @@ class Coordinates : public Dimensions constexpr Coordinates(Ts... coords) : Dimensions{coords...} { } + + /** Constructor to initialize the coordinates from a vector. + * + * @param[in] coords Vector containing the values to initialize the dimensions. + */ + template + constexpr Coordinates(std::vector coords) : Dimensions(coords) + { + } + /** Allow instances of this class to be copy constructed */ constexpr Coordinates(const Coordinates &) = default; /** Allow instances of this class to be copied */ diff --git a/arm_compute/core/Dimensions.h b/arm_compute/core/Dimensions.h index ba4a1b1d64a..afcb61f3a77 100644 --- a/arm_compute/core/Dimensions.h +++ b/arm_compute/core/Dimensions.h @@ -58,6 +58,15 @@ class Dimensions { } + /** Constructor to initialize the tensor shape with a list of dim values. + * + * @param[in] dims Vector of values to initialize the dimensions. + */ + explicit Dimensions(std::vector dims) : _id(), _num_dimensions{dims.size()} + { + std::copy_n(dims.begin(), std::min(num_max_dimensions, dims.size()), _id.begin()); + } + /** Allow instances of this class to be copy constructed */ Dimensions(const Dimensions &) = default; diff --git a/arm_compute/core/Helpers.h b/arm_compute/core/Helpers.h index 73728ed5a3a..979c1699498 100644 --- a/arm_compute/core/Helpers.h +++ b/arm_compute/core/Helpers.h @@ -34,6 +34,7 @@ #include "arm_compute/core/Types.h" #include "arm_compute/core/Validate.h" #include "arm_compute/core/Window.h" +#include "arm_compute/core/SparseTensor.h" #include #include @@ -126,6 +127,42 @@ class Iterator std::array _dims; }; +class SparseIterator +{ +public: + /** Create an iterator for the metadata and allocation contained in the SparseTensor + * + * @param[in] tensor A reference to the tensor to associate to the iterator. + */ + SparseIterator(const SparseTensor &tensor); + + /** Returns true if there is at least one more element to iterate */ + bool has_next() const; + + /** Move to the next non-zero element */ + void next(); + + /** Get the coordinates of the current non-zero element */ + Coordinates coordinates() const; + + /** Get the value of the current non-zero element */ + const uint8_t *value() const; + + /** Reset iterator to the beginning */ + void reset(); + + /** Get current index (nth non-zero) */ + size_t index() const; + + /** Get the number of non-zero elements that are left to iterate */ + size_t num_left() const; + +private: + const SparseTensor &_tensor; + size_t _index; + size_t _nnz; +}; + /** Iterate through the passed window, automatically adjusting the iterators and calling the lambda_functino for each element. * It passes the x and y positions to the lambda_function for each iteration * diff --git a/arm_compute/core/Helpers.inl b/arm_compute/core/Helpers.inl index d18809fc6b5..5ce0fc60472 100644 --- a/arm_compute/core/Helpers.inl +++ b/arm_compute/core/Helpers.inl @@ -101,6 +101,7 @@ inline Iterator::Iterator(const ITensor *tensor, const Window &win) : Iterator() { ARM_COMPUTE_ERROR_ON(tensor == nullptr); ARM_COMPUTE_ERROR_ON(tensor->info() == nullptr); + ARM_COMPUTE_ERROR_ON_MSG(tensor->info()->is_sparse(), "Sparse tensors are not supported by Iterators. Use a SparseIterator instead"); initialize(tensor->info()->num_dimensions(), tensor->info()->strides_in_bytes(), tensor->buffer(), tensor->info()->offset_first_element_in_bytes(), win); @@ -169,6 +170,45 @@ inline void Iterator::reset(const size_t dimension) } } +inline SparseIterator::SparseIterator(const SparseTensor &tensor) : _tensor(tensor), _index(0), _nnz(tensor.nnz()) +{ +} + +inline bool SparseIterator::has_next() const +{ + return _index < _nnz; +} + +inline void SparseIterator::next() +{ + ++_index; +} + +inline Coordinates SparseIterator::coordinates() const +{ + return _tensor.get_coordinates(_index); +} + +inline const uint8_t *SparseIterator::value() const +{ + return _tensor.get_value(coordinates()); +} + +inline void SparseIterator::reset() +{ + _index = 0; +} + +inline size_t SparseIterator::index() const +{ + return _index; +} + +inline size_t SparseIterator::num_left() const +{ + return _nnz - _index; +} + inline Coordinates index2coords(const TensorShape &shape, int index) { int num_elements = shape.total_size(); diff --git a/arm_compute/core/IReducibleTensor.h b/arm_compute/core/IReducibleTensor.h new file mode 100644 index 00000000000..09a8ab73b3f --- /dev/null +++ b/arm_compute/core/IReducibleTensor.h @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2025 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef ACL_ARM_COMPUTE_CORE_IREDUCIBLETENSOR_H +#define ACL_ARM_COMPUTE_CORE_IREDUCIBLETENSOR_H + +#include "arm_compute/core/SparseTensor.h" +#include "arm_compute/runtime/COOTensor.h" +#include "arm_compute/runtime/CSRTensor.h" + +namespace arm_compute +{ +/** */ +class IReducibleTensor +{ +public: + virtual ~IReducibleTensor() = default; + /** Convert a dense tensor to sparse tensor with specified sparse dimensions using the default + * sparse tensor representation: COO format. + * + * @param[in] dim sparse dimension + * + * @return A unique pointer to a SparseTensor object. + */ + virtual std::unique_ptr to_sparse(size_t dim) const = 0; + /** Convert a dense tensor to COO sparse tensor with specified sparse dimensions. + * + * @param[in] dim sparse dimension + * + * @return A unique pointer to a COOTensor object. + */ + virtual std::unique_ptr to_coo_sparse(size_t dim) const = 0; + /** Convert a dense tensor to COO sparse tensor with the default sparse dimension. + * For COO format, the number of sparse dimensions is equal to the total number of dimensions. + * + * @return A unique pointer to a COOTensor object. + */ + virtual std::unique_ptr to_coo_sparse() const = 0; + /** Convert a dense tensor to CSR sparse tensor with specified sparse dimensions. + * + * @param[in] dim sparse dimension + * + * @return A unique pointer to a CSRTensor object. + */ + virtual std::unique_ptr to_csr_sparse(size_t dim) const = 0; + /** Convert a dense tensor to CSR sparse tensor with the default sparse dimension. + * For CSR format, the number of sparse dimensions is equal to 2. + * + * @return A unique pointer to a CSRTensor object. + */ + virtual std::unique_ptr to_csr_sparse() const = 0; +}; +} + +#endif // ACL_ARM_COMPUTE_CORE_IREDUCIBLETENSOR_H diff --git a/arm_compute/core/ITensorInfo.h b/arm_compute/core/ITensorInfo.h index 239f200f7f2..8bd32b073c6 100644 --- a/arm_compute/core/ITensorInfo.h +++ b/arm_compute/core/ITensorInfo.h @@ -31,6 +31,7 @@ #include "arm_compute/core/Coordinates.h" #include "arm_compute/core/Strides.h" #include "arm_compute/core/TensorShape.h" +#include "arm_compute/core/TensorFormat.h" #include "arm_compute/core/Types.h" #include "arm_compute/core/utils/misc/Utility.h" @@ -105,6 +106,13 @@ class ITensorInfo : public misc::ICloneable * @return Reference to this ITensorInfo object */ virtual ITensorInfo &set_format(Format format) = 0; + /** Set the tensor format of an already initialized tensor. + * + * @param[in] tensor format to distinguish between dense and sparse tensors. + * + * @return Reference to this ITensorInfo object + */ + virtual ITensorInfo &set_tensor_format(TensorFormat tf) = 0; /** Set the shape of an already initialized tensor. * * @warning Changing the shape requires to recompute the strides and is @@ -261,6 +269,16 @@ class ITensorInfo : public misc::ICloneable * @return True if the tensor size can be changed. */ virtual bool is_resizable() const = 0; + /** Flag indicating whether the tensor is sparse. + * + * @return True if the tensor format different from TensorFormat::Dense. + */ + virtual bool is_sparse() const = 0; + /** Returns the format of the Tensor + * + * @return True if the tensor is an instance of SparseTensor. + */ + virtual TensorFormat tensor_format() const = 0; /** Set the tensor as dynamic/static * * @param[in] dynamic True if tensor is dynamic diff --git a/arm_compute/core/SparseTensor.h b/arm_compute/core/SparseTensor.h new file mode 100644 index 00000000000..36b1431beed --- /dev/null +++ b/arm_compute/core/SparseTensor.h @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2025 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef ACL_ARM_COMPUTE_RUNTIME_SPARSETENSOR_H +#define ACL_ARM_COMPUTE_RUNTIME_SPARSETENSOR_H + +#include "arm_compute/core/ITensor.h" +#include "arm_compute/core/Types.h" + +#include +#include + +typedef std::function predicate_t; + +namespace arm_compute +{ +/** Common base class for all sparse tensors */ +class SparseTensor : public ITensor +{ +public: + /** Prevent instances of this class to be constructed by the default constructor */ + SparseTensor() = delete; + /** Prevent instances of this class to be copy constructed */ + SparseTensor(const SparseTensor&) = delete; + /** Prevent instances of this class to be copied */ + SparseTensor& operator=(const SparseTensor&) = delete; + ~SparseTensor() = default; + + /** Returns the number of sparse dimensions */ + size_t sparse_dim() const; + /** Returns the number of dense dimensions */ + size_t dense_dim() const; + /** Returns the (total) number of dimensions */ + size_t dim() const; + /** Returns true if the tensor is hybrid (contains both sparse and dense dimensions) + * + * @note A sparse tensor is hybrid if it has at least one dense dimension. + */ + bool is_hybrid() const; + /** Returns the ratio of zero-valued elements to the total number of elements */ + float sparsity() const; + /** Returns the ratio of non-zero elements to the total number of elements */ + float density() const; + /** Returns the dense volume */ + uint32_t dense_volume(size_t sparse_dim) const; + /** Returns the number of non zero elements */ + virtual size_t nnz() const = 0; + /** Converts the sparse tensor to a dense tensor */ + virtual std::unique_ptr to_dense() = 0; + /** Returns the coordinates of the n-th (non-zero) element. + * + * @param nth The *zero-base* index of the element + * + * @return The coordinates of the element + */ + virtual Coordinates get_coordinates(size_t nth) const = 0; + /** Returns a pointer to the n-th (non-zero) element. If the element specified by + * the coordinates is zero, nullptr is returned. + * + * @param nth The *zero-base* index of the element + * + * @return The value of the element + * + * @note The value has size dense_volume(sparse_dim()). + */ + virtual const uint8_t *get_value(Coordinates coords) const = 0; + +protected: + SparseTensor(size_t dim, size_t sparse_dim); + + std::function make_is_nonzero_predicate(DataType dt) const; + bool has_non_zero_elements(uint8_t *arr, size_t len, size_t element_size, predicate_t is_non_zero) const; + void print_values(std::ostream &os, const uint8_t *data, size_t offset, size_t count) const; + +private: + size_t _total_dim; + size_t _sparse_dim; +}; +} + +#endif // ACL_ARM_COMPUTE_RUNTIME_SPARSETENSOR_H diff --git a/arm_compute/core/SubTensorInfo.h b/arm_compute/core/SubTensorInfo.h index df15c9dbc96..7a67788ae4f 100644 --- a/arm_compute/core/SubTensorInfo.h +++ b/arm_compute/core/SubTensorInfo.h @@ -34,6 +34,7 @@ #include "arm_compute/core/Strides.h" #include "arm_compute/core/TensorInfo.h" #include "arm_compute/core/TensorShape.h" +#include "arm_compute/core/TensorFormat.h" #include #include @@ -200,6 +201,17 @@ class SubTensorInfo final : public ITensorInfo ARM_COMPUTE_ERROR_ON(_parent == nullptr); return _parent->is_resizable(); } + bool is_sparse() const override + { + ARM_COMPUTE_ERROR_ON(_parent == nullptr); + return _parent->is_sparse(); + } + ITensorInfo &set_tensor_format(TensorFormat tf) override; + TensorFormat tensor_format() const override + { + ARM_COMPUTE_ERROR_ON(_parent == nullptr); + return _parent->tensor_format(); + } ITensorInfo &set_dynamic(bool dynamic) override; bool is_dynamic() const override { diff --git a/arm_compute/core/TensorFormat.h b/arm_compute/core/TensorFormat.h new file mode 100644 index 00000000000..db6dec05d7b --- /dev/null +++ b/arm_compute/core/TensorFormat.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2025 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ACL_ARM_COMPUTE_RUNTIME_TENSOR_FORMAT_H +#define ACL_ARM_COMPUTE_RUNTIME_TENSOR_FORMAT_H + +enum class TensorFormat +{ + Dense = 0, + COO = 1, + CSR = 2, +}; + +#endif // ACL_ARM_COMPUTE_RUNTIME_TENSOR_FORMAT_H diff --git a/arm_compute/core/TensorInfo.h b/arm_compute/core/TensorInfo.h index 629139fddd6..9ebe97c0c51 100644 --- a/arm_compute/core/TensorInfo.h +++ b/arm_compute/core/TensorInfo.h @@ -34,6 +34,7 @@ #include "arm_compute/core/Strides.h" #include "arm_compute/core/TensorShape.h" #include "arm_compute/core/Types.h" +#include "arm_compute/core/TensorFormat.h" #include "ITensorInfo.h" #include @@ -278,6 +279,19 @@ class TensorInfo final : public ITensorInfo { return _is_resizable; } + bool is_sparse() const override + { + return _tensor_format != TensorFormat::Dense; + } + TensorFormat tensor_format() const override + { + return _tensor_format; + } + ITensorInfo &set_tensor_format(TensorFormat tf) override + { + _tensor_format = tf; + return *this; + } ITensorInfo &set_dynamic(bool dynamic) override { std::fill(std::begin(_dims_state), std::end(_dims_state), @@ -362,6 +376,7 @@ class TensorInfo final : public ITensorInfo bool _are_values_constant; ITensorInfo::Id _id; bool _lock_paddings; + TensorFormat _tensor_format; }; /** Check whether two tensor info are equal. diff --git a/arm_compute/runtime/COOTensor.h b/arm_compute/runtime/COOTensor.h new file mode 100644 index 00000000000..66aec0d27a6 --- /dev/null +++ b/arm_compute/runtime/COOTensor.h @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2025 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef ACL_ARM_COMPUTE_RUNTIME_COOTENSOR_H +#define ACL_ARM_COMPUTE_RUNTIME_COOTENSOR_H + +#include "arm_compute/core/SparseTensor.h" +#include "arm_compute/runtime/SparseTensorAllocator.h" + +#include + +namespace arm_compute +{ +class COOTensor final : public SparseTensor, public IMemoryManageable +{ +public: + /** Prevent instances of this class to be move constructed */ + COOTensor(COOTensor &&) = delete; + /** Prevent instances of this class to be moved */ + COOTensor &operator=(COOTensor &&) = delete; + + /** Print the internal state of the COOTensor instance + * + * @param[in] os the output stream; std::cout set as default. + * + * @note It prints (on os stream) the vector of the indices with the format + * index: [idx_0, idx_1, ...], and the corresponding values with the format + * value: [val_0, val_1, ...]. + * @note This print function should overlap the one defined for ITensor. + */ + void print(std::ostream &os = std::cout) const; + + // Inherited methods overridden: + ITensorInfo *info() const override; + ITensorInfo *info() override; + uint8_t *buffer() const override; + size_t nnz() const override; + std::unique_ptr to_dense() override; + Coordinates get_coordinates(size_t nth) const override; + const uint8_t *get_value(Coordinates coords) const override; + void associate_memory_group(IMemoryGroup *memory_group) override; + +private: + /** Convert a dense tensor to sparse tensor with specified sparse dimensions using COO format. + * + * @param[in] tensor + * @param[in] sparse_dim Belongs to [1, tensor->info->num_dimensions()] + */ + COOTensor(const ITensor *tensor, size_t sparse_dim); + /** Convert a dense tensor to a *fully* sparse COOTensor. + * + * @param[in] tensor + * + * @note sparse_dim = tensor->info->num_dimensions() + */ + COOTensor(const ITensor *tensor); + + std::vector _indices; + mutable SparseTensorAllocator _allocator; /**< Instance of the basic CPU allocator.*/ + +friend class Tensor; +}; +} + +#endif // ACL_ARM_COMPUTE_RUNTIME_COOTENSOR_H diff --git a/arm_compute/runtime/CSRTensor.h b/arm_compute/runtime/CSRTensor.h new file mode 100644 index 00000000000..44e536718fd --- /dev/null +++ b/arm_compute/runtime/CSRTensor.h @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2025 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef ACL_ARM_COMPUTE_RUNTIME_CSRTENSOR_H +#define ACL_ARM_COMPUTE_RUNTIME_CSRTENSOR_H + +#include "arm_compute/core/SparseTensor.h" +#include "arm_compute/runtime/SparseTensorAllocator.h" + +#include + +namespace arm_compute +{ +class CSRTensor final : public SparseTensor, public IMemoryManageable +{ +public: + /** Prevent instances of this class to be move constructed */ + CSRTensor(CSRTensor &&) = delete; + /** Prevent instances of this class to be moved */ + CSRTensor &operator=(CSRTensor &&) = delete; + + /** Print the internal state of the CSRTensor instance + * + * @param[in] os the output stream; std::cout set as default. + * + * @note It prints (on os stream) the two vectors of the indices with the format + * [row_idx_0, row_idx_1, ...] and [col_idx_0, col_idx_1, ...] + * @note This print function should overlap the one defined for ITensor. + */ + void print(std::ostream &os = std::cout) const; + + // Inherited methods overridden: + ITensorInfo *info() const override; + ITensorInfo *info() override; + uint8_t *buffer() const override; + size_t nnz() const override; + std::unique_ptr to_dense() override; + Coordinates get_coordinates(size_t nth) const override; + const uint8_t *get_value(Coordinates coords) const override; + void associate_memory_group(IMemoryGroup *memory_group) override; + +private: + /** The size of each index element */ + static constexpr size_t index_size = sizeof(int32_t); + + /** Convert a dense tensor to sparse tensor with specified sparse dimensions using COO format. + * + * @param[in] tensor + * @param[in] sparse_dim It should belong to [1, tensor->info->num_dimensions()] + */ + CSRTensor(const ITensor *tensor, size_t sparse_dim); + /** Convert a dense tensor to a *fully* sparse tensor. + * + * @note sparse_dim = tensor->info->num_dimensions(). + * If tensor->info->num_dimensions() > 2 an error is raised. + */ + CSRTensor(const ITensor *tensor); + + size_t _crow_bytes; /**< Row index size in bytes */ + size_t _col_bytes; /**< Column index size in bytes */ + // In the SparseTensorAllocator buffer, the memory is stored that way + // +---------------+---------------+----------... + // | Row offsets | Col indices | Values ... + // +---------------+---------------+----------... + mutable SparseTensorAllocator _allocator; /**< Instance of the basic CPU allocator.*/ + +friend class Tensor; +}; +} + +#endif // ACL_ARM_COMPUTE_RUNTIME_CSRTENSOR_H diff --git a/arm_compute/runtime/SparseTensorAllocator.h b/arm_compute/runtime/SparseTensorAllocator.h new file mode 100644 index 00000000000..a435c8c20d7 --- /dev/null +++ b/arm_compute/runtime/SparseTensorAllocator.h @@ -0,0 +1,153 @@ +/* + * Copyright (c) 2025 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ACL_ARM_COMPUTE_RUNTIME_SPARSETENSORALLOCATOR_H +#define ACL_ARM_COMPUTE_RUNTIME_SPARSETENSORALLOCATOR_H + +/** @file + * @publicapi + */ + +#include "arm_compute/runtime/ITensorAllocator.h" +#include "arm_compute/runtime/Memory.h" +#include "arm_compute/runtime/MemoryGroup.h" + +#include +#include +#include + +namespace arm_compute +{ +// Forward declaration +class Coordinates; +class TensorInfo; + +/** Basic implementation of a CPU memory sparse tensor allocator. */ +class SparseTensorAllocator : public ITensorAllocator +{ +public: + /** Default constructor. + * + * @param[in] owner Memory manageable owner + */ + SparseTensorAllocator(IMemoryManageable *owner); + /** Default destructor */ + ~SparseTensorAllocator(); + /** Prevent instances of this class from being copied (As this class contains pointers) */ + SparseTensorAllocator(const SparseTensorAllocator &) = delete; + /** Prevent instances of this class from being copy assigned (As this class contains pointers) */ + SparseTensorAllocator &operator=(const SparseTensorAllocator &) = delete; + /** Allow instances of this class to be moved */ + SparseTensorAllocator(SparseTensorAllocator &&) noexcept; + /** Allow instances of this class to be moved */ + SparseTensorAllocator &operator=(SparseTensorAllocator &&) noexcept; + + /** Initializes the sparse tensor allocator with the number of non-zero elements. + * + * @param[in] input The input tensor information + * @param[in] values_bytes The bytes to be allocated for the values + * @param[in] indices_bytes The bytes to be allocated for the indices + * @param[in] alignment The alignment requirement + * + * @note ITensorAllocator's init methods should be disabled. Sparse tensors' memory allocator + * should be initialized with the number of non zero elements. + */ + void init(const TensorInfo &input, size_t values_bytes, size_t indices_bytes, size_t alignment = 0); + + /** Shares the same backing memory with another tensor allocator, while the tensor info might be different. + * In other words this can be used to create a sub-tensor from another tensor while sharing the same memory. + * + * @note SparseTensorAllocator have to be of the same specialized type. + * + * @param[in] allocator The allocator that owns the backing memory to be shared. Ownership becomes shared afterwards. + * @param[in] coords The starting coordinates of the new tensor inside the parent tensor. + * @param[in] sub_info The new tensor information (e.g. shape etc) + */ + void init(const SparseTensorAllocator &allocator, const Coordinates &coords, TensorInfo &sub_info); + + /** Returns the pointer to the allocated data. + * + * @return a pointer to the allocated data. + */ + uint8_t *data() const; + + /** Allocate size specified by TensorInfo of CPU memory. + * + * @note The tensor must not already be allocated when calling this function. + * + */ + void allocate() override; + + bool is_allocated() const override; + + /** Free allocated CPU memory. + * + * @note The sparse tensor must have been allocated when calling this function. + * + */ + void free() override; + /** Import an existing memory as a tensor's backing memory + * + * @warning size is expected to be compliant with total_size reported by ITensorInfo. + * @warning ownership of memory is not transferred. + * @warning tensor shouldn't be memory managed. + * @warning padding should be accounted by the client code. + * @warning memory must be writable in case of in-place operations + * @note buffer alignment will be checked to be compliant with alignment reported by ITensorInfo. + * + * @param[in] memory Raw memory pointer to be used as backing memory + * + * @return An error status + */ + Status import_memory(void *memory); + /** Associates the tensor with a memory group + * + * @param[in] associated_memory_group Memory group to associate the tensor with + */ + void set_associated_memory_group(IMemoryGroup *associated_memory_group); + /** Returns the size in bytes of the allocated memory + * + * @return The size in bytes of the allocated memory + */ + size_t size_bytes() const; + +protected: + /** No-op for CPU memory + * + * @return A pointer to the beginning of the tensor's allocation. + */ + uint8_t *lock() override; + + /** No-op for CPU memory. */ + void unlock() override; + +private: + IMemoryManageable *_owner; /**< Memory manageable object that owns the allocator */ + IMemoryGroup *_associated_memory_group; /**< Registered memory manager */ + Memory _memory; /**< CPU memory */ + size_t _values_bytes; /**< Size in bytes of the allocated values */ + size_t _indices_bytes; /**< Size in bytes of the allocated indices */ +}; +} // namespace arm_compute + +#endif // ACL_ARM_COMPUTE_RUNTIME_SPARSETENSORALLOCATOR_H diff --git a/arm_compute/runtime/Tensor.h b/arm_compute/runtime/Tensor.h index a28dd859fde..e10284c7013 100644 --- a/arm_compute/runtime/Tensor.h +++ b/arm_compute/runtime/Tensor.h @@ -29,7 +29,10 @@ */ #include "arm_compute/core/ITensor.h" +#include "arm_compute/runtime/COOTensor.h" +#include "arm_compute/runtime/CSRTensor.h" #include "arm_compute/runtime/TensorAllocator.h" +#include "arm_compute/core/IReducibleTensor.h" #include @@ -38,7 +41,7 @@ namespace arm_compute class ITensorInfo; class IRuntimeContext; /** Basic implementation of the tensor interface */ -class Tensor : public ITensor, public IMemoryManageable +class Tensor : public ITensor, public IMemoryManageable, public IReducibleTensor { public: /** Constructor @@ -60,10 +63,15 @@ class Tensor : public ITensor, public IMemoryManageable TensorAllocator *allocator(); // Inherited methods overridden: - ITensorInfo *info() const override; - ITensorInfo *info() override; - uint8_t *buffer() const override; - void associate_memory_group(IMemoryGroup *memory_group) override; + ITensorInfo *info() const override; + ITensorInfo *info() override; + uint8_t *buffer() const override; + void associate_memory_group(IMemoryGroup *memory_group) override; + std::unique_ptr to_sparse(size_t dim) const override; + std::unique_ptr to_coo_sparse(size_t dim) const override; + std::unique_ptr to_coo_sparse() const override; + std::unique_ptr to_csr_sparse(size_t dim) const override; + std::unique_ptr to_csr_sparse() const override; private: mutable TensorAllocator _allocator; /**< Instance of the basic CPU allocator.*/ diff --git a/filelist.json b/filelist.json index 81bd5ef9c8f..109fe14b341 100644 --- a/filelist.json +++ b/filelist.json @@ -21,6 +21,7 @@ "src/core/ITensorPack.cpp", "src/core/Rounding.cpp", "src/core/Size2D.cpp", + "src/core/SparseTensor.cpp", "src/core/SubTensorInfo.cpp", "src/core/TensorInfo.cpp", "src/core/Utils.cpp", @@ -67,8 +68,11 @@ "src/runtime/SchedulerFactory.cpp", "src/runtime/SchedulerUtils.cpp", "src/runtime/SubTensor.cpp", + "src/runtime/COOTensor.cpp", + "src/runtime/CSRTensor.cpp", "src/runtime/Tensor.cpp", "src/runtime/TensorAllocator.cpp", + "src/runtime/SparseTensorAllocator.cpp", "src/runtime/Utils.cpp", "src/runtime/CPP/ICPPSimpleFunction.cpp", "src/runtime/CPP/functions/CPPBoxWithNonMaximaSuppressionLimit.cpp", @@ -1631,7 +1635,7 @@ "src/runtime/experimental/operators/CpuMul.cpp", "src/runtime/experimental/operators/CpuQuantize.cpp", "src/runtime/experimental/operators/CpuSoftmax.cpp", - "src/runtime/experimental/operators/CpuPool2d.cpp", + "src/runtime/experimental/operators/CpuPool2d.cpp", "src/runtime/experimental/operators/CpuSub.cpp", "src/runtime/experimental/operators/CpuTranspose.cpp", "src/runtime/experimental/operators/CpuWinogradConv2d.cpp" @@ -1869,7 +1873,7 @@ "src/core/NEON/kernels/arm_gemm/kernels/sve_ffinterleaved_fp16_mla_8x3VL/generic.cpp", "src/core/NEON/kernels/arm_gemm/kernels/sve_ffinterleaved_fp32_mla_8x3VL/a64fx.cpp", "src/core/NEON/kernels/arm_gemm/kernels/sve_ffinterleaved_fp32_mla_8x3VL/generic.cpp" - ] + ] } } }, diff --git a/src/BUILD.bazel b/src/BUILD.bazel index 89e632ddd41..2dc36b5bed6 100644 --- a/src/BUILD.bazel +++ b/src/BUILD.bazel @@ -634,6 +634,7 @@ filegroup( "core/NEON/kernels/convolution/winograd/winograd_fp32.cpp", "core/Rounding.cpp", "core/Size2D.cpp", + "core/SparseTensor.cpp", "core/SubTensorInfo.cpp", "core/TensorInfo.cpp", "core/Utils.cpp", @@ -980,8 +981,11 @@ filegroup( "runtime/SchedulerFactory.cpp", "runtime/SchedulerUtils.cpp", "runtime/SubTensor.cpp", + "runtime/COOTensor.cpp", + "runtime/CSRTensor.cpp", "runtime/Tensor.cpp", "runtime/TensorAllocator.cpp", + "runtime/SparseTensorAllocator.cpp", "runtime/Utils.cpp", "runtime/experimental/low_level/CpuGemmAssemblyDispatch.cpp", "runtime/experimental/operators/CpuActivation.cpp", diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 3a8815d836b..90042703247 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -627,6 +627,7 @@ target_sources( core/NEON/kernels/convolution/winograd/winograd_fp32.cpp core/Rounding.cpp core/Size2D.cpp + core/SparseTensor.cpp core/SubTensorInfo.cpp core/TensorInfo.cpp core/Utils.cpp @@ -974,7 +975,10 @@ target_sources( runtime/SchedulerUtils.cpp runtime/SubTensor.cpp runtime/Tensor.cpp + runtime/COOTensor.cpp + runtime/CSRTensor.cpp runtime/TensorAllocator.cpp + runtime/SparseTensorAllocator.cpp runtime/Utils.cpp runtime/experimental/low_level/CpuGemmAssemblyDispatch.cpp runtime/experimental/operators/CpuActivation.cpp diff --git a/src/core/SparseTensor.cpp b/src/core/SparseTensor.cpp new file mode 100644 index 00000000000..a616c19f2de --- /dev/null +++ b/src/core/SparseTensor.cpp @@ -0,0 +1,164 @@ +/* + * Copyright (c) 2025 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "arm_compute/core/SparseTensor.h" + +#include "arm_compute/core/Error.h" + +namespace arm_compute +{ +SparseTensor::SparseTensor(size_t dim, size_t sparse_dim) : _total_dim(dim), _sparse_dim(sparse_dim) +{ +} + +size_t SparseTensor::sparse_dim() const +{ + return _sparse_dim; +} + +size_t SparseTensor::dense_dim() const +{ + return _total_dim - _sparse_dim; +} + +float SparseTensor::sparsity() const +{ + return 1.0f - density(); +} + +float SparseTensor::density() const +{ + return static_cast(nnz()) / static_cast(info()->total_size()); +} + +size_t SparseTensor::dim() const +{ + return _total_dim; +} + +bool SparseTensor::is_hybrid() const +{ + return dense_dim() > 0; +} + +uint32_t SparseTensor::dense_volume(size_t sparse_dim) const +{ + const auto &ts = info()->tensor_shape(); + return std::accumulate(ts.begin() + sparse_dim, ts.end(), 1, std::multiplies()); +} + +bool SparseTensor::has_non_zero_elements(uint8_t *arr, size_t len, size_t element_size, predicate_t is_non_zero) const +{ + for(size_t i = 0; i < len; i += element_size) + { + if(is_non_zero(arr + i)) + { + return true; + } + } + return false; +} + +std::function SparseTensor::make_is_nonzero_predicate(DataType dt) const +{ + switch (dt) + { + case DataType::F32: + return [](const void *ptr) { + return *(static_cast(ptr)) != 0.0f; + }; + case DataType::F16: // raw bitwise comparison + case DataType::U16: + case DataType::QSYMM16: + case DataType::QASYMM16: + return [](const void *ptr) { + return *(static_cast(ptr)) != 0; + }; + case DataType::S32: + return [](const void *ptr) { + return *(static_cast(ptr)) != 0; + }; + case DataType::S16: + return [](const void *ptr) { + return *(static_cast(ptr)) != 0; + }; + case DataType::U32: + return [](const void *ptr) { + return *(static_cast(ptr)) != 0; + }; + case DataType::U8: + case DataType::QSYMM8: + case DataType::QASYMM8: + case DataType::QSYMM8_PER_CHANNEL: + return [](const void *ptr) { + return *(static_cast(ptr)) != 0; + }; + case DataType::S8: + case DataType::QASYMM8_SIGNED: + return [](const void *ptr) { + return *(static_cast(ptr)) != 0; + }; + default: + throw std::runtime_error("Unsupported DataType in make_is_nonzero_predicate()"); + } +} + +void SparseTensor::print_values(std::ostream &os, const uint8_t *data, size_t offset, size_t count) const +{ + const size_t element_size = info()->element_size(); + const uint8_t *block_ptr = data + offset * count * element_size; + + os << "["; + for(size_t j = 0; j < count; ++j) + { + const void *value_ptr = block_ptr + j * element_size; + + switch(info()->data_type()) + { + case DataType::U8: + os << static_cast(*reinterpret_cast(value_ptr)); + break; + case DataType::S8: + os << static_cast(*reinterpret_cast(value_ptr)); + break; + case DataType::U32: + os << *reinterpret_cast(value_ptr); + break; + case DataType::S32: + os << *reinterpret_cast(value_ptr); + break; + case DataType::F16: + os << static_cast(*reinterpret_cast(value_ptr)); + break; + case DataType::F32: + os << *reinterpret_cast(value_ptr); + break; + default: + os << ""; + } + + if(j < count - 1) os << ", "; + } + os << "]" << std::endl; +} +} // namespace arm_compute diff --git a/src/core/SubTensorInfo.cpp b/src/core/SubTensorInfo.cpp index ac25d725ed9..8e0d23f77f5 100644 --- a/src/core/SubTensorInfo.cpp +++ b/src/core/SubTensorInfo.cpp @@ -133,6 +133,12 @@ ITensorInfo &SubTensorInfo::set_tensor_dims_state(const TensorDimsState &state) return *this; } +ITensorInfo &SubTensorInfo::set_tensor_format(TensorFormat tf) +{ + ARM_COMPUTE_ERROR_ON(_parent == nullptr); + return _parent->set_tensor_format(tf); +} + ITensorInfo &SubTensorInfo::set_dynamic(bool dynamic) { if (dynamic) diff --git a/src/core/TensorInfo.cpp b/src/core/TensorInfo.cpp index 36e2772514b..2e17c4bd7d9 100644 --- a/src/core/TensorInfo.cpp +++ b/src/core/TensorInfo.cpp @@ -25,6 +25,7 @@ #include "arm_compute/core/Error.h" #include "arm_compute/core/Helpers.h" +#include "arm_compute/core/TensorFormat.h" #include "arm_compute/core/TensorInfo.h" #include "arm_compute/core/Validate.h" @@ -50,7 +51,8 @@ TensorInfo::TensorInfo() _data_layout(DataLayout::NCHW), _are_values_constant(true), _id(invalid_tensor_id), - _lock_paddings(false) + _lock_paddings(false), + _tensor_format(TensorFormat::Dense) { _dims_state.fill(ITensorInfo::get_static_state_value()); } @@ -73,6 +75,7 @@ TensorInfo::TensorInfo(const ITensorInfo &info) : TensorInfo() _are_values_constant = info.are_values_constant(); _id = info.id(); _lock_paddings = info.lock_paddings(); + _tensor_format = info.tensor_format(); } TensorInfo::TensorInfo(const TensorInfo &info) : TensorInfo() @@ -93,6 +96,7 @@ TensorInfo::TensorInfo(const TensorInfo &info) : TensorInfo() _are_values_constant = info.are_values_constant(); _id = info.id(); _lock_paddings = false; + _tensor_format = info.tensor_format(); } TensorInfo::TensorInfo(Format format) : TensorInfo(TensorShape(), format) { diff --git a/src/runtime/COOTensor.cpp b/src/runtime/COOTensor.cpp new file mode 100644 index 00000000000..bf03a4068d8 --- /dev/null +++ b/src/runtime/COOTensor.cpp @@ -0,0 +1,264 @@ +/* + * Copyright (c) 2025 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "arm_compute/runtime/COOTensor.h" + +#include "arm_compute/core/CoreTypes.h" +#include "arm_compute/core/Error.h" +#include "arm_compute/core/TensorFormat.h" +#include "arm_compute/runtime/Tensor.h" + +#include "src/core/helpers/Utils.h" + + +namespace arm_compute +{ +namespace +{ +TensorInfo coo_tensor_info(const ITensorInfo *src_info) +{ + return src_info->clone()->set_tensor_format(TensorFormat::COO); +} +} // namespace + +COOTensor::COOTensor(const ITensor *tensor, size_t sparse_dim) : SparseTensor(tensor->info()->num_dimensions(), sparse_dim), _indices(), _allocator(this) +{ + ARM_COMPUTE_ERROR_ON_NULLPTR(tensor); + const ITensorInfo *info = tensor->info(); + + ARM_COMPUTE_ERROR_ON_MSG(info->data_layout() != DataLayout::NCHW, "COOTensor only supports NCHW layout at the moment"); + ARM_COMPUTE_ERROR_ON_MSG(info->is_sparse(), "cannot create a COOTensor from a sparse tensor"); + ARM_COMPUTE_ERROR_ON_MSG_VAR(sparse_dim < 1 || sparse_dim > dim(), + "argument must be in [1,%zu] range. %zu is given", + dim(), sparse_dim); + + const uint8_t *data = tensor->buffer(); + const size_t dense_dims = dense_dim(); + const auto is_nonzero = make_is_nonzero_predicate(info->data_type()); + + std::vector sparse_shape(sparse_dim); + std::vector dense_shape(dense_dims); + for(size_t i = 0; i < sparse_dim; i++) + { + sparse_shape[i] = info->dimension(i); + } + for(size_t i = 0; i < dense_dims; i++) + { + dense_shape[i] = info->dimension(sparse_dim + i); + } + + std::vector temp_values; + + const size_t step = std::accumulate(dense_shape.begin(), dense_shape.end(), size_t(1), std::multiplies()); + const size_t max_iter = std::accumulate(sparse_shape.begin(), sparse_shape.end(), size_t(1), std::multiplies()); + const size_t element_size = info->element_size(); + const size_t slice_size = step * element_size; + + size_t value_byte_size = 0; + size_t indices_bytes = 0; + for(size_t i = 0; i < max_iter; i++) + { + const size_t offset = i * slice_size; + if(has_non_zero_elements(const_cast(data + offset), slice_size, element_size, is_nonzero)) + { + value_byte_size += slice_size; + indices_bytes += dim() * sizeof(int32_t); + } + } + + _allocator.init(coo_tensor_info(info), value_byte_size, indices_bytes); + _allocator.allocate(); + + for(size_t i = 0; i < max_iter; i++) + { + const size_t offset = i * slice_size; + if(!has_non_zero_elements(const_cast(data + offset), slice_size, element_size, is_nonzero)) + { + continue; + } + + // ----------------- + // TODO: I built this function in a similar way to numpy's unravel_index. + // Do you think it's worth creating a util function somewhere else? + // Also, it stores the coordinates in a reverse way, with the fastest + // changing dimension last. Probably it would be better to store them + // in a more natural order. + std::vector multi_index(dim(), std::int32_t{0}); + size_t remainder = i; + for(size_t sd = sparse_dim; sd-- > 0;) + { + multi_index[sd] = remainder % sparse_shape[sd]; + remainder /= sparse_shape[sd]; + } + // ----------------- + + _indices.push_back(Coordinates(multi_index)); + temp_values.insert(temp_values.end(), data + offset, data + offset + slice_size); + } + + if(!temp_values.empty()) + { + std::memcpy(_allocator.data(), temp_values.data(), temp_values.size()); + } +} + +COOTensor::COOTensor(const ITensor *tensor) : COOTensor(tensor, tensor->info()->num_dimensions()) +{ +} + +size_t COOTensor::nnz() const +{ + return _indices.size(); +} + +ITensorInfo *COOTensor::info() const +{ + return &_allocator.info(); +} + +ITensorInfo *COOTensor::info() +{ + return &_allocator.info(); +} + +uint8_t *COOTensor::buffer() const +{ + return _allocator.data(); +} + +std::unique_ptr COOTensor::to_dense() +{ + ARM_COMPUTE_ERROR_ON_MSG(info()->data_layout() != DataLayout::NCHW, "COOTensor only supports NCHW layout at the moment"); + + std::unique_ptr tensor = std::make_unique(); + tensor->allocator()->init(info()->clone()->set_tensor_format(TensorFormat::Dense)); + tensor->allocator()->allocate(); + + const size_t element_size = info()->element_size(); + const size_t total_size = info()->total_size(); + const size_t dense_vol = dense_volume(sparse_dim()); + const size_t first_elem_offset = info()->offset_first_element_in_bytes(); + + std::memset(tensor->buffer() + first_elem_offset, 0, total_size); + + if(nnz() == 0) + { + return tensor; + } + + for(size_t i = 0; i < _indices.size(); ++i) + { + const Coordinates &c = _indices[i]; + const uint8_t *block_ptr = buffer() + i * dense_vol * element_size; + + size_t final_offset = 0; + for(size_t d = 0; d < sparse_dim(); ++d) + { + final_offset += c[d] * dense_volume(d+1); + } + final_offset *= element_size; + + for(size_t j = 0; j < dense_vol; ++j) + { + const void *value_ptr = block_ptr + j * element_size; + uint8_t *base_ptr = tensor->buffer() + final_offset + j * element_size; + + std::memcpy(base_ptr, value_ptr, element_size); + } + } + + return tensor; +} + +Coordinates COOTensor::get_coordinates(size_t nth) const +{ + ARM_COMPUTE_ERROR_ON_MSG(nth >= nnz(), "Invalid index"); + + return _indices[nth]; +} + +const uint8_t *COOTensor::get_value(Coordinates coords) const +{ + ARM_COMPUTE_ERROR_ON_MSG(coords.num_dimensions() != info()->num_dimensions(), "Invalid coordinate dimension"); + for(size_t i = 0; i < coords.num_dimensions(); ++i) + { + ARM_COMPUTE_ERROR_ON_MSG(static_cast(coords[i]) >= info()->tensor_shape()[i], "Invalid coordinates shape"); + } + + const uint8_t *data = static_cast(buffer()); + const size_t dense_vol = dense_volume(sparse_dim()); + + for(size_t i = 0; i < _indices.size(); ++i) + { + const Coordinates &c = _indices[i]; + bool match = false; + + for(size_t d = 0; d < coords.num_dimensions(); ++d) + { + if(c[d] == coords[d]) + { + match = true; + } + } + if(match) + { + return data + i * dense_vol * info()->element_size(); + } + } + + // This coordinates point to a zero value + return nullptr; +} + +void COOTensor::associate_memory_group(IMemoryGroup *memory_group) +{ + _allocator.set_associated_memory_group(memory_group); +} + +#ifdef ARM_COMPUTE_ASSERTS_ENABLED +void COOTensor::print(std::ostream &os) const +{ + const uint8_t *data = static_cast(buffer()); + + if(_indices.empty()) + { + os << "index: [] values: []" << std::endl; + return; + } + + for(size_t i = 0; i < _indices.size(); ++i) + { + const Coordinates &coord = _indices[i]; + os << "index: ["; + for (size_t j = 0; j < coord.num_dimensions(); ++j) + { + os << coord[j]; + if (j < coord.num_dimensions() - 1) os << ", "; + } + os << "] values: "; + print_values(os, data, i, dense_volume(sparse_dim())); + } +} +#endif // ARM_COMPUTE_ASSERTS_ENABLED + +} // namespace arm_compute diff --git a/src/runtime/CSRTensor.cpp b/src/runtime/CSRTensor.cpp new file mode 100644 index 00000000000..035022a61a8 --- /dev/null +++ b/src/runtime/CSRTensor.cpp @@ -0,0 +1,272 @@ +/* + * Copyright (c) 2025 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "arm_compute/runtime/COOTensor.h" + +#include "arm_compute/core/CoreTypes.h" +#include "arm_compute/core/Error.h" +#include "arm_compute/core/TensorFormat.h" +#include "arm_compute/runtime/Tensor.h" + +#include "src/core/helpers/Utils.h" + +#include + +namespace arm_compute +{ + +namespace +{ +TensorInfo csr_tensor_info(const ITensorInfo *src_info) +{ + return src_info->clone()->set_tensor_format(TensorFormat::CSR); +} +} // namespace + +CSRTensor::CSRTensor(const ITensor *tensor, size_t sparse_dim) : SparseTensor(tensor->info()->num_dimensions(), 2), _crow_bytes(), _col_bytes(), _allocator(this) +{ + ARM_COMPUTE_UNUSED(sparse_dim); + ARM_COMPUTE_ERROR_ON_NULLPTR(tensor); + const auto *info = tensor->info(); + + // As of now, CSRTensor only supports 2D tensors with NCHW layout. + ARM_COMPUTE_ERROR_ON_MSG(info->data_layout() != DataLayout::NCHW, "CSRTensor only supports NCHW layout at the moment"); + ARM_COMPUTE_ERROR_ON_MSG(info->is_sparse(), "cannot create a CSRTensor from a sparse tensor"); + ARM_COMPUTE_ERROR_ON_MSG(dim() != 2, "CSRTensor only supports 2D tensors at the moment"); + + const int32_t rows = info->dimension(0); + const int32_t cols = info->dimension(1); + const int32_t element_size = info->element_size(); + const int32_t row_size_bytes = cols * element_size; + const auto is_nonzero = make_is_nonzero_predicate(info->data_type()); + const uint8_t *data = tensor->buffer() + info->offset_first_element_in_bytes(); + size_t value_byte_size = 0; + + _crow_bytes = index_size; // The first row index is always a 0 + _col_bytes = 0; + + for(int32_t row = 0; row < rows; ++row) + { + const int32_t row_offset = row * row_size_bytes; + _crow_bytes += index_size; + for(int32_t col = 0; col < cols; ++col) + { + const int32_t element_offset = row_offset + col * element_size; + if(is_nonzero(data + element_offset)) + { + _col_bytes += index_size; + value_byte_size += element_size * dense_volume(sparse_dim); + } + } + } + + _allocator.init(csr_tensor_info(info), value_byte_size, _crow_bytes + _col_bytes); + _allocator.allocate(); + + uint8_t *_row_offsets = _allocator.data(); + uint8_t *_col_indices = _allocator.data() + _crow_bytes; + uint8_t *_values = _allocator.data() + _crow_bytes + _col_bytes; + size_t num_non_zero = 0; + int32_t last_row = 0; + int32_t col_index = 0; + + std::memcpy(_row_offsets, &last_row, index_size); + + for(int32_t row = 0; row < rows; ++row) + { + const int32_t row_offset = row * row_size_bytes; + int32_t non_zero_row_elements = 0; + for(int32_t col = 0; col < cols; ++col) + { + const size_t element_offset = row_offset + col * element_size; + if(is_nonzero(data + element_offset)) + { + std::memcpy(_col_indices + col_index * index_size, &col, index_size); + std::memcpy(_values + num_non_zero * element_size, data + element_offset, element_size); + non_zero_row_elements++; + num_non_zero++; + col_index++; + } + } + + last_row += non_zero_row_elements; + std::memcpy(_row_offsets + (row + 1) * index_size, &last_row, index_size); + } +} + +CSRTensor::CSRTensor(const ITensor *tensor) : CSRTensor(tensor, 2) +{ +} + +size_t CSRTensor::nnz() const +{ + return static_cast(_col_bytes / index_size); +} + +ITensorInfo *CSRTensor::info() const +{ + return &_allocator.info(); +} + +ITensorInfo *CSRTensor::info() +{ + return &_allocator.info(); +} + +uint8_t *CSRTensor::buffer() const +{ + return _allocator.data(); +} + +Coordinates CSRTensor::get_coordinates(size_t nth) const +{ + ARM_COMPUTE_ERROR_ON_MSG(nth >= nnz(), "Invalid index"); + + const uint8_t *row_indices = _allocator.data(); + const uint8_t *col_indices = _allocator.data() + _crow_bytes; + size_t low = 0; + size_t high = (_crow_bytes / index_size) - 1; + + while(low < high) + { + const size_t mid = (low + high) / 2; + if(*reinterpret_cast(row_indices + (mid + 1) * index_size) <= static_cast(nth)) + { + low = mid + 1; + } + else + { + high = mid; + } + } + + return Coordinates{ low , *reinterpret_cast(col_indices + (nth * index_size)) }; +} + +const uint8_t *CSRTensor::get_value(Coordinates coords) const +{ + ARM_COMPUTE_ERROR_ON_MSG(coords.num_dimensions() != info()->num_dimensions(), "Invalid coordinate dimension"); + for(size_t i = 0; i < coords.num_dimensions(); ++i) + { + ARM_COMPUTE_ERROR_ON_MSG(static_cast(coords[i]) >= info()->tensor_shape()[i], "Invalid coordinates shape"); + } + + const uint8_t *row_indices = _allocator.data(); + const uint8_t *col_indices = _allocator.data() + _crow_bytes; + const uint8_t *values = _allocator.data() + _crow_bytes + _col_bytes; + const int32_t row = coords[0]; + const int32_t col = coords[1]; + + const int32_t start = *reinterpret_cast(row_indices + row * index_size); + const int32_t end = *reinterpret_cast(row_indices + (row + 1) * index_size); + + for(int32_t i = start; i < end; ++i) + { + if(*reinterpret_cast(col_indices + (i * index_size)) == col) + { + return values + i * info()->element_size(); + } + } + + // This coordinates point to a zero value + return nullptr; +} + +std::unique_ptr CSRTensor::to_dense() +{ + ARM_COMPUTE_ERROR_ON_MSG(info()->data_layout() != DataLayout::NCHW, "CSRTensor only supports NCHW layout at the moment"); + + auto tensor = std::make_unique(); + tensor->allocator()->init(info()->clone()->set_tensor_format(TensorFormat::Dense)); + tensor->allocator()->allocate(); + + const size_t first_elem_offset = info()->offset_first_element_in_bytes(); + uint8_t *data = tensor->buffer() + first_elem_offset; + const size_t element_size = info()->element_size(); + const size_t total_size = info()->total_size(); + const uint8_t *row_offsets = _allocator.data(); + const uint8_t *col_indices = _allocator.data() + _crow_bytes; + const uint8_t *values = _allocator.data() + _crow_bytes + _col_bytes; + size_t element = 0; + + std::memset(data, 0, total_size); + + for(size_t i = 0, j = 1, row = 0; j < _crow_bytes / index_size; ++i, ++j, ++row) + { + const int32_t current_col = *reinterpret_cast(row_offsets + (i * index_size)); + const int32_t next_col = *reinterpret_cast(row_offsets + (j * index_size)); + + if(current_col == next_col) + { + continue; + } + + for(size_t current = static_cast(current_col); current < static_cast(next_col); ++current) + { + const size_t col = *reinterpret_cast(col_indices + (current * index_size)); + const uint8_t *value_ptr = values + (element * element_size); + + std::memcpy(data + (row * info()->dimension(1) + col) * element_size, value_ptr, element_size); + element++; + } + } + + return tensor; +} + +void CSRTensor::associate_memory_group(IMemoryGroup *memory_group) +{ + _allocator.set_associated_memory_group(memory_group); +} + +#ifdef ARM_COMPUTE_ASSERTS_ENABLED +void CSRTensor::print(std::ostream &os) const +{ + const uint8_t *_row_offsets = _allocator.data(); + const uint8_t *_col_indices = _allocator.data() + _crow_bytes; + const uint8_t *values = _allocator.data() + _crow_bytes + _col_bytes; + + os << "r_offsets: ["; + for(size_t i = 0; i < _crow_bytes / index_size; ++i) + { + os << *reinterpret_cast(_row_offsets + (i * index_size)); + if (i < _crow_bytes / index_size - 1) + { + os << ", "; + } + } + os << "] cols: ["; + for(size_t i = 0; i < _col_bytes / index_size; ++i) + { + os << *reinterpret_cast(_col_indices + (i * index_size)); + if (i < _col_bytes / index_size - 1) + { + os << ", "; + } + } + os << "] values: "; + print_values(os, values, 0, nnz()); +} +#endif // ARM_COMPUTE_ASSERTS_ENABLED + +} // namespace arm_compute diff --git a/src/runtime/SparseTensorAllocator.cpp b/src/runtime/SparseTensorAllocator.cpp new file mode 100644 index 00000000000..4c404c5edc6 --- /dev/null +++ b/src/runtime/SparseTensorAllocator.cpp @@ -0,0 +1,177 @@ +/* + * Copyright (c) 2025 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "arm_compute/runtime/SparseTensorAllocator.h" + +#include "arm_compute/core/Coordinates.h" +#include "arm_compute/core/Error.h" +#include "arm_compute/core/TensorInfo.h" +#include "arm_compute/runtime/MemoryGroup.h" +#include "arm_compute/runtime/MemoryRegion.h" + +#include + +namespace arm_compute +{ +SparseTensorAllocator::SparseTensorAllocator(IMemoryManageable *owner) : _owner(owner), _associated_memory_group(nullptr), _memory(), _values_bytes(0), _indices_bytes(0) +{ +} + +SparseTensorAllocator::~SparseTensorAllocator() +{ + info().set_is_resizable(true); +} + +SparseTensorAllocator::SparseTensorAllocator(SparseTensorAllocator &&o) noexcept + : ITensorAllocator(std::move(o)), + _owner(o._owner), + _associated_memory_group(o._associated_memory_group), + _memory(std::move(o._memory)), + _values_bytes(o._values_bytes), + _indices_bytes(o._indices_bytes) +{ + o._owner = nullptr; + o._associated_memory_group = nullptr; + o._memory = Memory(); + o._values_bytes = 0; + o._indices_bytes = 0; +} + +SparseTensorAllocator &SparseTensorAllocator::operator=(SparseTensorAllocator &&o) noexcept +{ + if (&o != this) + { + _owner = o._owner; + o._owner = nullptr; + + _associated_memory_group = o._associated_memory_group; + o._associated_memory_group = nullptr; + + _memory = std::move(o._memory); + o._memory = Memory(); + + _values_bytes = o._values_bytes; + _values_bytes = 0; + + _indices_bytes = o._indices_bytes; + _indices_bytes = 0; + + ITensorAllocator::operator=(std::move(o)); + } + return *this; +} + +void SparseTensorAllocator::init(const TensorInfo &input, size_t values_bytes, size_t indices_bytes, size_t alignment) +{ + ITensorAllocator::init(input, alignment); + _values_bytes = values_bytes; + _indices_bytes = indices_bytes; +} + +void SparseTensorAllocator::init(const SparseTensorAllocator &allocator, const Coordinates &coords, TensorInfo &sub_info) +{ + // Get parent info + const TensorInfo parent_info = allocator.info(); + + // Copy pointer to buffer + _memory = Memory(allocator._memory.region()); + + // Init tensor info with new dimensions + size_t total_size = + parent_info.offset_element_in_bytes(coords) + sub_info.total_size() - sub_info.offset_first_element_in_bytes(); + sub_info.init(sub_info.tensor_shape(), sub_info.format(), parent_info.strides_in_bytes(), + parent_info.offset_element_in_bytes(coords), total_size); + + // Set TensorInfo + ITensorAllocator::init(sub_info); +} + +uint8_t *SparseTensorAllocator::data() const +{ + return (_memory.region() == nullptr) ? nullptr : reinterpret_cast(_memory.region()->buffer()); +} + +void SparseTensorAllocator::allocate() +{ + // Align to 64-byte boundaries by default if alignment is not specified + const size_t alignment_to_use = (alignment() != 0) ? alignment() : 64; + const size_t size = size_bytes(); + + if (_associated_memory_group == nullptr) + { + _memory.set_owned_region(std::make_unique(size, alignment_to_use)); + } + else + { + _associated_memory_group->finalize_memory(_owner, _memory, size, alignment_to_use); + } + info().set_is_resizable(false); +} + +void SparseTensorAllocator::free() +{ + _memory.set_region(nullptr); + info().set_is_resizable(true); +} + +bool SparseTensorAllocator::is_allocated() const +{ + return _memory.region() != nullptr; +} + +Status SparseTensorAllocator::import_memory(void *memory) +{ + ARM_COMPUTE_RETURN_ERROR_ON(memory == nullptr); + ARM_COMPUTE_RETURN_ERROR_ON(_associated_memory_group != nullptr); + ARM_COMPUTE_RETURN_ERROR_ON(alignment() != 0 && !arm_compute::utility::check_aligned(memory, alignment())); + + _memory.set_owned_region(std::make_unique(memory, info().total_size())); + info().set_is_resizable(false); + + return Status{}; +} + +void SparseTensorAllocator::set_associated_memory_group(IMemoryGroup *associated_memory_group) +{ + ARM_COMPUTE_ERROR_ON(associated_memory_group == nullptr); + ARM_COMPUTE_ERROR_ON(_associated_memory_group != nullptr && _associated_memory_group != associated_memory_group); + ARM_COMPUTE_ERROR_ON(_memory.region() != nullptr && _memory.region()->buffer() != nullptr); + + _associated_memory_group = associated_memory_group; +} + +size_t SparseTensorAllocator::size_bytes() const +{ + return _values_bytes + _indices_bytes; +} + +uint8_t *SparseTensorAllocator::lock() +{ + ARM_COMPUTE_ERROR_ON(_memory.region() == nullptr); + return reinterpret_cast(_memory.region()->buffer()); +} + +void SparseTensorAllocator::unlock() +{ +} +} // namespace arm_compute diff --git a/src/runtime/Tensor.cpp b/src/runtime/Tensor.cpp index f17e3236945..b4c5e262798 100644 --- a/src/runtime/Tensor.cpp +++ b/src/runtime/Tensor.cpp @@ -49,6 +49,31 @@ TensorAllocator *Tensor::allocator() return &_allocator; } +std::unique_ptr Tensor::to_sparse(size_t dim) const +{ + return to_coo_sparse(dim); +} + +std::unique_ptr Tensor::to_coo_sparse(size_t dim) const +{ + return std::unique_ptr(new COOTensor(this, dim)); +} + +std::unique_ptr Tensor::to_coo_sparse() const +{ + return std::unique_ptr(new COOTensor(this)); +} + +std::unique_ptr Tensor::to_csr_sparse(size_t dim) const +{ + return std::unique_ptr(new CSRTensor(this, dim)); +} + +std::unique_ptr Tensor::to_csr_sparse() const +{ + return std::unique_ptr(new CSRTensor(this)); +} + void Tensor::associate_memory_group(IMemoryGroup *memory_group) { _allocator.set_associated_memory_group(memory_group); diff --git a/tests/AssetsLibrary.h b/tests/AssetsLibrary.h index dedad5227f9..41e330d9ef9 100644 --- a/tests/AssetsLibrary.h +++ b/tests/AssetsLibrary.h @@ -41,6 +41,7 @@ #include #include #include +#include #include #include #include @@ -358,6 +359,27 @@ class AssetsLibrary final template void fill_tensor_uniform(T &&tensor, std::random_device::result_type seed_offset, D low, D high) const; + /** Fill a tensor with zeros and randomly modify a portion of its elements + * + * This function sets all elements of the tensor to zero and then modifies + * approximately @p modification_ratio fraction of them with random values + * uniformly sampled from the range [1, 255]. + * + * The modified elements are selected randomly across the entire tensor. + * + * @tparam T Type of the tensor (e.g., Tensor, ITensor). + * + * @param[in, out] tensor Tensor to be filled and modified. Must be allocated. + * @param[in] modification_ratio Fraction [0,1] of elements to randomly assign a non-zero value. + * + * @note The tensor will be completely filled with zeros, and then @p modification_ratio * total_elements + * positions will be overwritten with random values. The rest remain zero. + * For floating point types, random values are drawn from a uniform real distribution. + * For integral types, a uniform integer distribution is used. + */ + template + void fill_tensor_sparse_random(T &&tensor, float modification_ratio) const; + /** Fill a tensor with uniform distribution across the specified range * * @param[in, out] tensor To be filled tensor. @@ -1018,6 +1040,48 @@ void AssetsLibrary::fill_tensor_uniform(T &&tensor, std::random_device::result_t } } +template +void AssetsLibrary::fill_tensor_sparse_random(T &&tensor, float modification_ratio) const +{ + if(modification_ratio < 0.0f) modification_ratio = 0.0f; + if(modification_ratio > 1.0f) modification_ratio = 1.0f; + + const unsigned min = 1; + const unsigned max = 255; + const size_t num_elements = tensor.shape().total_size(); + const size_t num_mod = static_cast(modification_ratio * num_elements); + + std::vector indices(num_elements); + std::iota(indices.begin(), indices.end(), 0); + std::shuffle(indices.begin(), indices.end(), std::mt19937{std::random_device{}()}); + indices.resize(num_mod); + + std::mt19937 rng{std::random_device{}()}; + + if(tensor.data_type() == DataType::F32) + { + std::vector zero_filled(num_elements, float{0}); + std::uniform_real_distribution dist_real(static_cast(min), static_cast(max)); + for(size_t idx : indices) + { + zero_filled[idx] = static_cast(dist_real(rng)); + } + fill_static_values(std::forward(tensor), zero_filled); + } + else + { + // Signed types will interpret numbers >= 128 as negative; + // for our purposes, it doesn't matter, as long as they are different from 0. + std::vector zero_filled(num_elements, unsigned{0}); + std::uniform_int_distribution dist_int(min, max); + for(size_t idx : indices) + { + zero_filled[idx] = static_cast(dist_int(rng)); + } + fill_static_values(std::forward(tensor), zero_filled); + } +} + template void AssetsLibrary::fill_layer_data(T &&tensor, std::string name) const { diff --git a/tests/validation/cpu/unit/SparseTensor.cpp b/tests/validation/cpu/unit/SparseTensor.cpp new file mode 100644 index 00000000000..49145d7b642 --- /dev/null +++ b/tests/validation/cpu/unit/SparseTensor.cpp @@ -0,0 +1,467 @@ +/* + * Copyright (c) 2025 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "arm_compute/core/TensorFormat.h" +#include "arm_compute/core/Types.h" +#include "arm_compute/runtime/Tensor.h" +#include "arm_compute/runtime/COOTensor.h" +#include "tests/framework/Asserts.h" +#include "tests/framework/Macros.h" +#include "tests/framework/datasets/Datasets.h" +#include "tests/validation/Validation.h" +#include "tests/validation/Helpers.h" + +#include "tests/NEON/Accessor.h" +#include "tests/NEON/Helper.h" + +#include + +namespace arm_compute +{ +namespace +{ +std::vector layouts = { + DataLayout::NCHW, + DataLayout::NHWC +}; +std::vector types = { + DataType::U8, + DataType::S8, + DataType::U32, + DataType::S32, + DataType::F16, + DataType::F32 +}; +std::vector shapes = { + TensorShape(8U), + TensorShape(8U), + TensorShape(3U, 3U), + TensorShape(3U, 3U), + TensorShape(2U, 5U, 5U), + TensorShape(2U, 5U, 5U), + TensorShape(4U, 2U, 2U, 9U), + TensorShape(4U, 2U, 2U, 9U) +}; +// For any shape, we have check the zero tensor, to be sure that `to_sparse` +// correctly returns an empty vector for each dimension. +std::vector> sparse_data = { + { 0, 0, 0, 0, 0, 0, 0, 0 }, + { 4, 0, 0, 0, 9, 0, 0, 0 }, + { + 0, 0, 0, + 0, 0, 0, + 0, 0, 0 + }, + { + 2, 0, 3, + 0, 1, 0, + 0, 0, 0 + }, + { + 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0 + }, + { + 0, 0, 0, 0, 0, + 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, + 0, 0, 0, 10, 0, + 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, + 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, + 0, 0, 0, 155, 0, + 0, 0, 0, 0, 0 + }, + { + 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0 + }, + { + 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 10, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 2, 0, 0, + 3, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 5, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 55, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0 + } +}; +std::vector>> coo_indices = { + { + { } + }, + { + { + Coordinates({ 0 }), + Coordinates({ 4 }) + } + }, + { + { }, + { } + }, + { + { + Coordinates({ 0, 0 }), + Coordinates({ 1, 0 }) + }, + { + Coordinates({ 0, 0 }), + Coordinates({ 0, 2 }), + Coordinates({ 1, 1 }) + } + }, + { + { }, + { }, + { } + }, + { + { + Coordinates({ 0, 0, 0 }), + Coordinates({ 1, 0, 0 }) + }, + { + Coordinates({ 0, 1, 0 }), + Coordinates({ 0, 3, 0 }), + Coordinates({ 1, 1, 0}), + Coordinates({ 1, 3, 0}) + }, + { + Coordinates({ 0, 1, 1 }), + Coordinates({ 0, 3, 3 }), + Coordinates({ 1, 1, 1 }), + Coordinates({ 1, 3, 3 }) + } + }, + { + { }, + { }, + { }, + { } + }, + { + { + Coordinates({ 0, 0, 0, 0 }), + Coordinates({ 1, 0, 0, 0 }), + Coordinates({ 2, 0, 0, 0 }), + Coordinates({ 3, 0, 0, 0 }) + }, + { + Coordinates({ 0, 0, 0, 0 }), + Coordinates({ 1, 0, 0, 0 }), + Coordinates({ 2, 0, 0, 0 }), + Coordinates({ 2, 1, 0, 0 }), + Coordinates({ 3, 0, 0, 0 }), + Coordinates({ 3, 1, 0, 0 }) + }, + { + Coordinates({ 0, 0, 1, 0 }), + Coordinates({ 1, 0, 0, 0 }), + Coordinates({ 1, 0, 1, 0 }), + Coordinates({ 2, 0, 1, 0 }), + Coordinates({ 2, 1, 0, 0 }), + Coordinates({ 3, 0, 1, 0 }), + Coordinates({ 3, 1, 0, 0 }) + }, + { + Coordinates({ 0, 0, 1, 2 }), + Coordinates({ 1, 0, 0, 6 }), + Coordinates({ 1, 0, 1, 0 }), + Coordinates({ 2, 0, 1, 4 }), + Coordinates({ 2, 1, 0, 8 }), + Coordinates({ 3, 0, 1, 1 }), + Coordinates({ 3, 1, 0, 7 }) + } + } +}; +std::vector> csr_nnz_coordinates = { + // The first two shapes has a dimension < 2, so they are not represented in CSR format + { + }, + { + }, + { + }, + { + Coordinates({ 0, 0 }), + Coordinates({ 0, 2 }), + Coordinates({ 1, 1 }) + }, + { + }, + { + Coordinates({ 1, 1 }) + }, + { + }, + { + } +}; + +bool are_values_equal(const uint8_t *a, const uint8_t *b, DataType dt, size_t element_size) +{ + if(dt == DataType::F32) + { + float va = *reinterpret_cast(a); + float vb = *reinterpret_cast(b); + if(std::fabs(va - vb) > 0e-5f) + { + return false; + } + } else + { + if(std::memcmp(a, b, element_size) != 0) + { + return false; + } + } + + return true; +} + +bool tensors_are_equal(const test::Accessor &a, const test::Accessor &b) +{ + if(a.shape() != b.shape() || a.data_type() != b.data_type()) + return false; + + const size_t element_size = a.element_size(); + Window window; + window.use_tensor_dimensions(a.shape()); + + bool equal = true; + + execute_window_loop(window, [&](const Coordinates &id) + { + const uint8_t *a_value = static_cast(a(id)); + const uint8_t *b_value = static_cast(b(id)); + + equal = are_values_equal(a_value, b_value, a.data_type(), element_size); + }); + + return equal; +} +} // namespace + +namespace test +{ +namespace validation +{ +TEST_SUITE(CPU) +TEST_SUITE(UNIT) +TEST_SUITE(SparseTensor) + +TEST_CASE(ConvertDenseTensorToCOOTensor, framework::DatasetMode::ALL) +{ + for(size_t i = 0; i < shapes.size(); ++i) + { + for(auto type : types) + { + const auto src_info = TensorInfo(shapes[i], 1, type, layouts[0]); + auto src = create_tensor(src_info); + bool is_src_dense = src.info()->tensor_format() == TensorFormat::Dense; + + src.allocator()->allocate(); + library->fill_static_values(Accessor(src), sparse_data[i]); + + ARM_COMPUTE_EXPECT(is_src_dense, framework::LogLevel::ERRORS); + + for(size_t sparse_dim = 1; sparse_dim <= shapes[i].num_dimensions(); sparse_dim++) + { + auto st = src.to_coo_sparse(sparse_dim); + +#ifdef ARM_COMPUTE_ASSERTS_ENABLED + st->print(std::cout); +#endif // ARM_COMPUTE_ASSERTS_ENABLED + + bool is_sparse = st->info()->is_sparse(); + bool is_coo = st->info()->tensor_format() == TensorFormat::COO; + size_t dense_dim = shapes[i].num_dimensions() - sparse_dim; + size_t is_hybrid = dense_dim > 0; + size_t nnz = coo_indices[i][sparse_dim - 1].size(); + + ARM_COMPUTE_EXPECT(is_sparse, framework::LogLevel::ERRORS); + ARM_COMPUTE_EXPECT(is_coo, framework::LogLevel::ERRORS); + + ARM_COMPUTE_EXPECT(st->sparse_dim() == sparse_dim, framework::LogLevel::ERRORS); + ARM_COMPUTE_EXPECT(st->dense_dim() == dense_dim, framework::LogLevel::ERRORS); + ARM_COMPUTE_EXPECT(st->is_hybrid() == is_hybrid, framework::LogLevel::ERRORS); + ARM_COMPUTE_EXPECT(st->nnz() == nnz, framework::LogLevel::ERRORS); + + for(size_t j = 0; j < nnz; ++j) + { + const Coordinates coord = st->get_coordinates(j); + + for(size_t k = 0; k < coord.num_dimensions(); ++k) + { + ARM_COMPUTE_EXPECT(coord[k] == coo_indices[i][sparse_dim - 1][j][k], framework::LogLevel::ERRORS); + } + + const uint8_t *value = st->get_value(coord); + ARM_COMPUTE_EXPECT(value != nullptr, framework::LogLevel::ERRORS); + } + } + } + } +} + +TEST_CASE(ConvertCOOTensorToDense, framework::DatasetMode::ALL) +{ + for(size_t i = 0; i < shapes.size(); ++i) + { + for(auto type : types) + { + const auto t_info = TensorInfo(shapes[i], 1, type, layouts[0]); + auto t = create_tensor(t_info); + + t.allocator()->allocate(); + library->fill_tensor_sparse_random(Accessor(t), 0.2); + + for(size_t sparse_dim = 1; sparse_dim <= shapes[i].num_dimensions(); sparse_dim++) + { + auto st = t.to_coo_sparse(sparse_dim); + auto td = st->to_dense(); + ARM_COMPUTE_EXPECT(tensors_are_equal(Accessor(t), Accessor(*td)), framework::LogLevel::ERRORS); + } + } + } +} + +TEST_CASE(ConvertDenseTensorToCSRTensor, framework::DatasetMode::ALL) +{ + for(size_t i = 0; i < shapes.size(); ++i) + { + for(auto type : types) + { + // Currently, CSRTensor only supports 2D tensors + if(shapes[i].num_dimensions() < 2) + { + continue; + } + const TensorShape shape(shapes[i][0], shapes[i][1]); + std::vector data(sparse_data[i].begin(), sparse_data[i].begin() + shapes[i][0] * shapes[i][1]); + + const auto src_info = TensorInfo(shape, 1, type, layouts[0]); + auto src = create_tensor(src_info); + bool is_src_dense = src.info()->tensor_format() == TensorFormat::Dense; + + src.allocator()->allocate(); + library->fill_static_values(Accessor(src), data); + + ARM_COMPUTE_EXPECT(is_src_dense, framework::LogLevel::ERRORS); + + auto st = src.to_csr_sparse(); + +#ifdef ARM_COMPUTE_ASSERTS_ENABLED + st->print(std::cout); +#endif // ARM_COMPUTE_ASSERTS_ENABLED + + bool is_sparse = st->info()->is_sparse(); + bool is_csr = st->info()->tensor_format() == TensorFormat::CSR; + size_t nnz = csr_nnz_coordinates[i].empty() ? 0 : csr_nnz_coordinates[i].size(); + + ARM_COMPUTE_EXPECT(is_sparse, framework::LogLevel::ERRORS); + ARM_COMPUTE_EXPECT(is_csr, framework::LogLevel::ERRORS); + ARM_COMPUTE_EXPECT(nnz == st->nnz(), framework::LogLevel::ERRORS); + + for(size_t j = 0; j < nnz; ++j) + { + const Coordinates coord = st->get_coordinates(j); + for(size_t k = 0; k < coord.num_dimensions(); ++k) + { + ARM_COMPUTE_EXPECT(csr_nnz_coordinates[i][j][k] == coord[k], framework::LogLevel::ERRORS); + } + + const uint8_t *value = st->get_value(coord); + ARM_COMPUTE_EXPECT(value != nullptr, framework::LogLevel::ERRORS); + } + } + } +} + +TEST_CASE(ConvertCSRTensorToDense, framework::DatasetMode::ALL) +{ + for(size_t i = 0; i < shapes.size(); ++i) + { + for(auto type : types) + { + // Currently, CSRTensor only supports 2D tensors + if(shapes[i].num_dimensions() < 2) + { + continue; + } + const TensorShape shape(shapes[i][0], shapes[i][1]); + + const auto t_info = TensorInfo(shape, 1, type, layouts[0]); + auto t = create_tensor(t_info); + + t.allocator()->allocate(); + library->fill_tensor_sparse_random(Accessor(t), 0.2); + + auto st = t.to_csr_sparse(); + auto td = st->to_dense(); + + ARM_COMPUTE_EXPECT(tensors_are_equal(Accessor(t), Accessor(*td)), framework::LogLevel::ERRORS); + } + } +} + +TEST_SUITE_END() // SparseTensor +TEST_SUITE_END() // UNIT +TEST_SUITE_END() // CPU + +} // namespace validation +} // namespace test +} // namespace arm_compute From 6d993d146dbd4d1f1c55263b3c76fe4abfc3044f Mon Sep 17 00:00:00 2001 From: Matteo Nicoli Date: Tue, 7 Oct 2025 00:53:51 +0200 Subject: [PATCH 2/2] cr fixes Change-Id: I3a3fca2b842702672c915e1357602bf0fe47bd47 --- arm_compute/core/IReducibleTensor.h | 10 +- tests/validation/UNIT/SparseTensor.cpp | 206 +++++++++ tests/validation/cpu/unit/SparseTensor.cpp | 467 --------------------- 3 files changed, 213 insertions(+), 470 deletions(-) create mode 100644 tests/validation/UNIT/SparseTensor.cpp delete mode 100644 tests/validation/cpu/unit/SparseTensor.cpp diff --git a/arm_compute/core/IReducibleTensor.h b/arm_compute/core/IReducibleTensor.h index 09a8ab73b3f..b99c5311ed2 100644 --- a/arm_compute/core/IReducibleTensor.h +++ b/arm_compute/core/IReducibleTensor.h @@ -26,12 +26,16 @@ #define ACL_ARM_COMPUTE_CORE_IREDUCIBLETENSOR_H #include "arm_compute/core/SparseTensor.h" -#include "arm_compute/runtime/COOTensor.h" -#include "arm_compute/runtime/CSRTensor.h" namespace arm_compute { -/** */ +/** Forward declaration of COOTensor and CSRTensor class */ +class COOTensor; +class CSRTensor; + +/** Interface for all reducible tensors, i.e. all tensors that can be + * converted to a sparse representation. + */ class IReducibleTensor { public: diff --git a/tests/validation/UNIT/SparseTensor.cpp b/tests/validation/UNIT/SparseTensor.cpp new file mode 100644 index 00000000000..e26e8d4c0a0 --- /dev/null +++ b/tests/validation/UNIT/SparseTensor.cpp @@ -0,0 +1,206 @@ +/* + * Copyright (c) 2025 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "arm_compute/core/TensorFormat.h" +#include "arm_compute/core/Types.h" +#include "arm_compute/runtime/Tensor.h" +#include "arm_compute/runtime/COOTensor.h" +#include "tests/framework/Asserts.h" +#include "tests/framework/Macros.h" +#include "tests/framework/datasets/Datasets.h" +#include "tests/validation/Validation.h" +#include "tests/validation/Helpers.h" + +#include "tests/NEON/Accessor.h" +#include "tests/NEON/Helper.h" + +#include + +namespace arm_compute +{ +namespace +{ +bool are_values_equal(const uint8_t *a, const uint8_t *b, DataType dt, size_t element_size) +{ + if(dt == DataType::F32) + { + float va = *reinterpret_cast(a); + float vb = *reinterpret_cast(b); + if(std::fabs(va - vb) > 0e-5f) + { + return false; + } + } else + { + if(std::memcmp(a, b, element_size) != 0) + { + return false; + } + } + + return true; +} + +bool tensors_are_equal(const test::Accessor &a, const test::Accessor &b) +{ + if(a.shape() != b.shape() || a.data_type() != b.data_type()) + return false; + + const size_t element_size = a.element_size(); + Window window; + window.use_tensor_dimensions(a.shape()); + + bool equal = true; + + execute_window_loop(window, [&](const Coordinates &id) + { + const uint8_t *a_value = static_cast(a(id)); + const uint8_t *b_value = static_cast(b(id)); + + equal = are_values_equal(a_value, b_value, a.data_type(), element_size); + }); + + return equal; +} +} // namespace + +namespace test +{ +namespace validation +{ +TEST_SUITE(UNIT) +TEST_SUITE(SparseTensor) + +// clang-format off +/** Validates TensorInfo Autopadding */ +DATA_TEST_CASE(ConvertCOOTensorToDense, framework::DatasetMode::ALL, combine( + framework::dataset::make("TensorShape", { + TensorShape(8U), + TensorShape(3U, 3U), + TensorShape(2U, 5U, 5U), + TensorShape(4U, 2U, 2U, 9U)}), + framework::dataset::make("TensorType", { + DataType::U8, + DataType::S8, + DataType::U32, + DataType::S32, + DataType::F16, + DataType::F32}) + ), shape, type) +{ + const auto t_info = TensorInfo(shape, 1, type, DataLayout::NCHW); + auto t = create_tensor(t_info); + auto t_zero = create_tensor(t_info); + + t.allocator()->allocate(); + library->fill_tensor_sparse_random(Accessor(t), 0.2); + + t_zero.allocator()->allocate(); + library->fill_static_values(Accessor(t_zero), std::vector(shape.total_size(), 0)); + + for(size_t sparse_dim = 1; sparse_dim <= shape.num_dimensions(); sparse_dim++) + { + auto st = t.to_coo_sparse(sparse_dim); + bool is_sparse = st->info()->is_sparse(); + bool is_coo = st->info()->tensor_format() == TensorFormat::COO; + size_t dense_dim = shape.num_dimensions() - sparse_dim; + size_t is_hybrid = dense_dim > 0; + auto td = st->to_dense(); + + ARM_COMPUTE_EXPECT(is_sparse, framework::LogLevel::ERRORS); + ARM_COMPUTE_EXPECT(is_coo, framework::LogLevel::ERRORS); + ARM_COMPUTE_EXPECT(st->sparse_dim() == sparse_dim, framework::LogLevel::ERRORS); + ARM_COMPUTE_EXPECT(st->dense_dim() == dense_dim, framework::LogLevel::ERRORS); + ARM_COMPUTE_EXPECT(st->is_hybrid() == is_hybrid, framework::LogLevel::ERRORS); + ARM_COMPUTE_EXPECT(tensors_are_equal(Accessor(t), Accessor(*td)), framework::LogLevel::ERRORS); + + auto st_zero = t_zero.to_coo_sparse(sparse_dim); + auto td_zero = st_zero->to_dense(); + ARM_COMPUTE_EXPECT(tensors_are_equal(Accessor(t_zero), Accessor(*td_zero)), framework::LogLevel::ERRORS); + } +} +// clang-format on +// *INDENT-ON* + +// clang-format off +/** Validates TensorInfo Autopadding */ +DATA_TEST_CASE(ConvertCSRTensorToDense, framework::DatasetMode::ALL, combine( + framework::dataset::make("TensorShape", { + TensorShape(8U), + TensorShape(3U, 3U), + TensorShape(2U, 5U, 5U), + TensorShape(4U, 2U, 2U, 9U)}), + framework::dataset::make("TensorType", { + DataType::U8, + DataType::S8, + DataType::U32, + DataType::S32, + DataType::F16, + DataType::F32}) + ), shape, type) +{ + // Currently, CSRTensor only supports 2D tensors + if(shape.num_dimensions() < 2) + { + return; + } + const TensorShape tensor_shape(shape[0], shape[1]); + + const auto t_info = TensorInfo(tensor_shape, 1, type, DataLayout::NCHW); + auto t = create_tensor(t_info); + auto t_zero = create_tensor(t_info); + + t.allocator()->allocate(); + library->fill_tensor_sparse_random(Accessor(t), 0.2); + + t_zero.allocator()->allocate(); + library->fill_static_values(Accessor(t_zero), std::vector(tensor_shape.total_size(), 0)); + + auto st = t.to_csr_sparse(); + auto td = st->to_dense(); + bool is_sparse = st->info()->is_sparse(); + bool is_csr = st->info()->tensor_format() == TensorFormat::CSR; + size_t sparse_dim = tensor_shape.num_dimensions(); + size_t dense_dim = tensor_shape.num_dimensions() - sparse_dim; + size_t is_hybrid = dense_dim > 0; + + ARM_COMPUTE_EXPECT(is_sparse, framework::LogLevel::ERRORS); + ARM_COMPUTE_EXPECT(is_csr, framework::LogLevel::ERRORS); + ARM_COMPUTE_EXPECT(st->sparse_dim() == sparse_dim, framework::LogLevel::ERRORS); + ARM_COMPUTE_EXPECT(st->dense_dim() == dense_dim, framework::LogLevel::ERRORS); + ARM_COMPUTE_EXPECT(st->is_hybrid() == is_hybrid, framework::LogLevel::ERRORS); + ARM_COMPUTE_EXPECT(tensors_are_equal(Accessor(t), Accessor(*td)), framework::LogLevel::ERRORS); + + auto st_zero = t_zero.to_coo_sparse(sparse_dim); + auto td_zero = st_zero->to_dense(); + ARM_COMPUTE_EXPECT(tensors_are_equal(Accessor(t_zero), Accessor(*td_zero)), framework::LogLevel::ERRORS); +} +// clang-format on +// *INDENT-ON* + +TEST_SUITE_END() // SparseTensor +TEST_SUITE_END() // UNIT + +} // namespace validation +} // namespace test +} // namespace arm_compute diff --git a/tests/validation/cpu/unit/SparseTensor.cpp b/tests/validation/cpu/unit/SparseTensor.cpp deleted file mode 100644 index 49145d7b642..00000000000 --- a/tests/validation/cpu/unit/SparseTensor.cpp +++ /dev/null @@ -1,467 +0,0 @@ -/* - * Copyright (c) 2025 Arm Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#include "arm_compute/core/TensorFormat.h" -#include "arm_compute/core/Types.h" -#include "arm_compute/runtime/Tensor.h" -#include "arm_compute/runtime/COOTensor.h" -#include "tests/framework/Asserts.h" -#include "tests/framework/Macros.h" -#include "tests/framework/datasets/Datasets.h" -#include "tests/validation/Validation.h" -#include "tests/validation/Helpers.h" - -#include "tests/NEON/Accessor.h" -#include "tests/NEON/Helper.h" - -#include - -namespace arm_compute -{ -namespace -{ -std::vector layouts = { - DataLayout::NCHW, - DataLayout::NHWC -}; -std::vector types = { - DataType::U8, - DataType::S8, - DataType::U32, - DataType::S32, - DataType::F16, - DataType::F32 -}; -std::vector shapes = { - TensorShape(8U), - TensorShape(8U), - TensorShape(3U, 3U), - TensorShape(3U, 3U), - TensorShape(2U, 5U, 5U), - TensorShape(2U, 5U, 5U), - TensorShape(4U, 2U, 2U, 9U), - TensorShape(4U, 2U, 2U, 9U) -}; -// For any shape, we have check the zero tensor, to be sure that `to_sparse` -// correctly returns an empty vector for each dimension. -std::vector> sparse_data = { - { 0, 0, 0, 0, 0, 0, 0, 0 }, - { 4, 0, 0, 0, 9, 0, 0, 0 }, - { - 0, 0, 0, - 0, 0, 0, - 0, 0, 0 - }, - { - 2, 0, 3, - 0, 1, 0, - 0, 0, 0 - }, - { - 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0 - }, - { - 0, 0, 0, 0, 0, - 0, 2, 0, 0, 0, - 0, 0, 0, 0, 0, - 0, 0, 0, 10, 0, - 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, - 0, 3, 0, 0, 0, - 0, 0, 0, 0, 0, - 0, 0, 0, 155, 0, - 0, 0, 0, 0, 0 - }, - { - 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0 - }, - { - 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 10, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 2, 0, 0, - 3, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 5, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 1, - 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 1, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 55, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0 - } -}; -std::vector>> coo_indices = { - { - { } - }, - { - { - Coordinates({ 0 }), - Coordinates({ 4 }) - } - }, - { - { }, - { } - }, - { - { - Coordinates({ 0, 0 }), - Coordinates({ 1, 0 }) - }, - { - Coordinates({ 0, 0 }), - Coordinates({ 0, 2 }), - Coordinates({ 1, 1 }) - } - }, - { - { }, - { }, - { } - }, - { - { - Coordinates({ 0, 0, 0 }), - Coordinates({ 1, 0, 0 }) - }, - { - Coordinates({ 0, 1, 0 }), - Coordinates({ 0, 3, 0 }), - Coordinates({ 1, 1, 0}), - Coordinates({ 1, 3, 0}) - }, - { - Coordinates({ 0, 1, 1 }), - Coordinates({ 0, 3, 3 }), - Coordinates({ 1, 1, 1 }), - Coordinates({ 1, 3, 3 }) - } - }, - { - { }, - { }, - { }, - { } - }, - { - { - Coordinates({ 0, 0, 0, 0 }), - Coordinates({ 1, 0, 0, 0 }), - Coordinates({ 2, 0, 0, 0 }), - Coordinates({ 3, 0, 0, 0 }) - }, - { - Coordinates({ 0, 0, 0, 0 }), - Coordinates({ 1, 0, 0, 0 }), - Coordinates({ 2, 0, 0, 0 }), - Coordinates({ 2, 1, 0, 0 }), - Coordinates({ 3, 0, 0, 0 }), - Coordinates({ 3, 1, 0, 0 }) - }, - { - Coordinates({ 0, 0, 1, 0 }), - Coordinates({ 1, 0, 0, 0 }), - Coordinates({ 1, 0, 1, 0 }), - Coordinates({ 2, 0, 1, 0 }), - Coordinates({ 2, 1, 0, 0 }), - Coordinates({ 3, 0, 1, 0 }), - Coordinates({ 3, 1, 0, 0 }) - }, - { - Coordinates({ 0, 0, 1, 2 }), - Coordinates({ 1, 0, 0, 6 }), - Coordinates({ 1, 0, 1, 0 }), - Coordinates({ 2, 0, 1, 4 }), - Coordinates({ 2, 1, 0, 8 }), - Coordinates({ 3, 0, 1, 1 }), - Coordinates({ 3, 1, 0, 7 }) - } - } -}; -std::vector> csr_nnz_coordinates = { - // The first two shapes has a dimension < 2, so they are not represented in CSR format - { - }, - { - }, - { - }, - { - Coordinates({ 0, 0 }), - Coordinates({ 0, 2 }), - Coordinates({ 1, 1 }) - }, - { - }, - { - Coordinates({ 1, 1 }) - }, - { - }, - { - } -}; - -bool are_values_equal(const uint8_t *a, const uint8_t *b, DataType dt, size_t element_size) -{ - if(dt == DataType::F32) - { - float va = *reinterpret_cast(a); - float vb = *reinterpret_cast(b); - if(std::fabs(va - vb) > 0e-5f) - { - return false; - } - } else - { - if(std::memcmp(a, b, element_size) != 0) - { - return false; - } - } - - return true; -} - -bool tensors_are_equal(const test::Accessor &a, const test::Accessor &b) -{ - if(a.shape() != b.shape() || a.data_type() != b.data_type()) - return false; - - const size_t element_size = a.element_size(); - Window window; - window.use_tensor_dimensions(a.shape()); - - bool equal = true; - - execute_window_loop(window, [&](const Coordinates &id) - { - const uint8_t *a_value = static_cast(a(id)); - const uint8_t *b_value = static_cast(b(id)); - - equal = are_values_equal(a_value, b_value, a.data_type(), element_size); - }); - - return equal; -} -} // namespace - -namespace test -{ -namespace validation -{ -TEST_SUITE(CPU) -TEST_SUITE(UNIT) -TEST_SUITE(SparseTensor) - -TEST_CASE(ConvertDenseTensorToCOOTensor, framework::DatasetMode::ALL) -{ - for(size_t i = 0; i < shapes.size(); ++i) - { - for(auto type : types) - { - const auto src_info = TensorInfo(shapes[i], 1, type, layouts[0]); - auto src = create_tensor(src_info); - bool is_src_dense = src.info()->tensor_format() == TensorFormat::Dense; - - src.allocator()->allocate(); - library->fill_static_values(Accessor(src), sparse_data[i]); - - ARM_COMPUTE_EXPECT(is_src_dense, framework::LogLevel::ERRORS); - - for(size_t sparse_dim = 1; sparse_dim <= shapes[i].num_dimensions(); sparse_dim++) - { - auto st = src.to_coo_sparse(sparse_dim); - -#ifdef ARM_COMPUTE_ASSERTS_ENABLED - st->print(std::cout); -#endif // ARM_COMPUTE_ASSERTS_ENABLED - - bool is_sparse = st->info()->is_sparse(); - bool is_coo = st->info()->tensor_format() == TensorFormat::COO; - size_t dense_dim = shapes[i].num_dimensions() - sparse_dim; - size_t is_hybrid = dense_dim > 0; - size_t nnz = coo_indices[i][sparse_dim - 1].size(); - - ARM_COMPUTE_EXPECT(is_sparse, framework::LogLevel::ERRORS); - ARM_COMPUTE_EXPECT(is_coo, framework::LogLevel::ERRORS); - - ARM_COMPUTE_EXPECT(st->sparse_dim() == sparse_dim, framework::LogLevel::ERRORS); - ARM_COMPUTE_EXPECT(st->dense_dim() == dense_dim, framework::LogLevel::ERRORS); - ARM_COMPUTE_EXPECT(st->is_hybrid() == is_hybrid, framework::LogLevel::ERRORS); - ARM_COMPUTE_EXPECT(st->nnz() == nnz, framework::LogLevel::ERRORS); - - for(size_t j = 0; j < nnz; ++j) - { - const Coordinates coord = st->get_coordinates(j); - - for(size_t k = 0; k < coord.num_dimensions(); ++k) - { - ARM_COMPUTE_EXPECT(coord[k] == coo_indices[i][sparse_dim - 1][j][k], framework::LogLevel::ERRORS); - } - - const uint8_t *value = st->get_value(coord); - ARM_COMPUTE_EXPECT(value != nullptr, framework::LogLevel::ERRORS); - } - } - } - } -} - -TEST_CASE(ConvertCOOTensorToDense, framework::DatasetMode::ALL) -{ - for(size_t i = 0; i < shapes.size(); ++i) - { - for(auto type : types) - { - const auto t_info = TensorInfo(shapes[i], 1, type, layouts[0]); - auto t = create_tensor(t_info); - - t.allocator()->allocate(); - library->fill_tensor_sparse_random(Accessor(t), 0.2); - - for(size_t sparse_dim = 1; sparse_dim <= shapes[i].num_dimensions(); sparse_dim++) - { - auto st = t.to_coo_sparse(sparse_dim); - auto td = st->to_dense(); - ARM_COMPUTE_EXPECT(tensors_are_equal(Accessor(t), Accessor(*td)), framework::LogLevel::ERRORS); - } - } - } -} - -TEST_CASE(ConvertDenseTensorToCSRTensor, framework::DatasetMode::ALL) -{ - for(size_t i = 0; i < shapes.size(); ++i) - { - for(auto type : types) - { - // Currently, CSRTensor only supports 2D tensors - if(shapes[i].num_dimensions() < 2) - { - continue; - } - const TensorShape shape(shapes[i][0], shapes[i][1]); - std::vector data(sparse_data[i].begin(), sparse_data[i].begin() + shapes[i][0] * shapes[i][1]); - - const auto src_info = TensorInfo(shape, 1, type, layouts[0]); - auto src = create_tensor(src_info); - bool is_src_dense = src.info()->tensor_format() == TensorFormat::Dense; - - src.allocator()->allocate(); - library->fill_static_values(Accessor(src), data); - - ARM_COMPUTE_EXPECT(is_src_dense, framework::LogLevel::ERRORS); - - auto st = src.to_csr_sparse(); - -#ifdef ARM_COMPUTE_ASSERTS_ENABLED - st->print(std::cout); -#endif // ARM_COMPUTE_ASSERTS_ENABLED - - bool is_sparse = st->info()->is_sparse(); - bool is_csr = st->info()->tensor_format() == TensorFormat::CSR; - size_t nnz = csr_nnz_coordinates[i].empty() ? 0 : csr_nnz_coordinates[i].size(); - - ARM_COMPUTE_EXPECT(is_sparse, framework::LogLevel::ERRORS); - ARM_COMPUTE_EXPECT(is_csr, framework::LogLevel::ERRORS); - ARM_COMPUTE_EXPECT(nnz == st->nnz(), framework::LogLevel::ERRORS); - - for(size_t j = 0; j < nnz; ++j) - { - const Coordinates coord = st->get_coordinates(j); - for(size_t k = 0; k < coord.num_dimensions(); ++k) - { - ARM_COMPUTE_EXPECT(csr_nnz_coordinates[i][j][k] == coord[k], framework::LogLevel::ERRORS); - } - - const uint8_t *value = st->get_value(coord); - ARM_COMPUTE_EXPECT(value != nullptr, framework::LogLevel::ERRORS); - } - } - } -} - -TEST_CASE(ConvertCSRTensorToDense, framework::DatasetMode::ALL) -{ - for(size_t i = 0; i < shapes.size(); ++i) - { - for(auto type : types) - { - // Currently, CSRTensor only supports 2D tensors - if(shapes[i].num_dimensions() < 2) - { - continue; - } - const TensorShape shape(shapes[i][0], shapes[i][1]); - - const auto t_info = TensorInfo(shape, 1, type, layouts[0]); - auto t = create_tensor(t_info); - - t.allocator()->allocate(); - library->fill_tensor_sparse_random(Accessor(t), 0.2); - - auto st = t.to_csr_sparse(); - auto td = st->to_dense(); - - ARM_COMPUTE_EXPECT(tensors_are_equal(Accessor(t), Accessor(*td)), framework::LogLevel::ERRORS); - } - } -} - -TEST_SUITE_END() // SparseTensor -TEST_SUITE_END() // UNIT -TEST_SUITE_END() // CPU - -} // namespace validation -} // namespace test -} // namespace arm_compute