[TensorV2] Feature Scaling Functions

This pull request adds two new feature scaling functions - standardization and normalization - to the Tensor class. These functions help users preprocess input data before feeding it into models, improving model performance and accuracy.

**Changes proposed in this PR:**
* Added normalization() function to rescale values to a range between 0 and 1
* Added standardization() function to center data around the mean and scales to a standard deviation of 1

**Self-evaluation:**
1. Build test: [X]Passed [ ]Failed [ ]Skipped
2. Run test:   [X]Passed [ ]Failed [ ]Skipped

Signed-off-by: Donghyeon Jeong <dhyeon.jeong@samsung.com>

Author: djeong20

nntrainer/tensor/float_tensor.cpp

@@ -681,6 +681,10 @@ TensorV2 &FloatTensor::sum(unsigned int axis, TensorV2 &output, float alpha,
   return output;
 }
 
+float FloatTensor::l2norm() const {
+  return snrm2(size(), (float *)getData(), 1);
+}
+
 TensorV2 &FloatTensor::pow(float exponent, TensorV2 &output) const {
   auto f = [exponent](float in) { return powf(in, exponent); };
   apply(f, output);
@@ -783,6 +787,16 @@ void FloatTensor::copyData(const TensorV2 &from) {
   }
 }
 
+float FloatTensor::maxValue() const {
+  const float *data = (float *)getData();
+  return *std::max_element(data, data + size());
+}
+
+float FloatTensor::minValue() const {
+  const float *data = (float *)getData();
+  return *std::min_element(data, data + size());
+}
+
 TensorV2 &FloatTensor::transpose(const std::string &direction,
                                  TensorV2 &output) const {
   unsigned int SL, SI, SJ, SK;

nntrainer/tensor/float_tensor.h

@@ -270,6 +270,11 @@ class FloatTensor : public TensorBase {
   TensorV2 &sum(unsigned int axis, TensorV2 &output, float alpha,
                 float beta) const override;
 
+  /**
+   * @copydoc TensorV2::l2norm
+   */
+  float l2norm() const override;
+
   /**
    * @copydoc TensorV2::pow(float exponent, TensorV2 &output)
    */
@@ -312,6 +317,16 @@ class FloatTensor : public TensorBase {
    */
   void copyData(const TensorV2 &from);
 
+  /**
+   * @copydoc Tensor::maxValue()
+   */
+  float maxValue() const override;
+
+  /**
+   * @copydoc Tensor::minValue()
+   */
+  float minValue() const override;
+
   /**
    * @copydoc TensorV2::transpose(const std::string &direction, TensorV2 &out)
    */

nntrainer/tensor/half_tensor.cpp

@@ -631,6 +631,10 @@ TensorV2 &HalfTensor::sum(unsigned int axis, TensorV2 &output, float alpha,
   return output;
 }
 
+float HalfTensor::l2norm() const {
+  return snrm2(size(), (_FP16 *)getData(), 1);
+}
+
 TensorV2 &HalfTensor::pow(float exponent, TensorV2 &output) const {
   auto f = [exponent](float in) {
     return static_cast<_FP16>(powf(in, exponent));
@@ -858,6 +862,16 @@ void HalfTensor::copyData(const TensorV2 &from) {
   }
 }
 
+float HalfTensor::maxValue() const {
+  const _FP16 *data = (_FP16 *)getData();
+  return (float)*std::max_element(data, data + size());
+}
+
+float HalfTensor::minValue() const {
+  const _FP16 *data = (_FP16 *)getData();
+  return (float)*std::min_element(data, data + size());
+}
+
 TensorV2 &HalfTensor::transpose(const std::string &direction,
                                 TensorV2 &output) const {
   unsigned int SL, SI, SJ, SK;

nntrainer/tensor/half_tensor.h

@@ -269,6 +269,11 @@ class HalfTensor : public TensorBase {
   TensorV2 &sum(unsigned int axis, TensorV2 &output, float alpha,
                 float beta) const override;
 
+  /**
+   * @copydoc TensorV2::l2norm
+   */
+  float l2norm() const override;
+
   /**
    * @copydoc TensorV2::pow(float exponent, TensorV2 &output)
    */
@@ -311,6 +316,16 @@ class HalfTensor : public TensorBase {
    */
   void copyData(const TensorV2 &from);
 
+  /**
+   * @copydoc Tensor::maxValue()
+   */
+  float maxValue() const override;
+
+  /**
+   * @copydoc Tensor::minValue()
+   */
+  float minValue() const override;
+
   /**
    * @copydoc TensorV2::transpose(const std::string &direction, TensorV2 &out)
    */

nntrainer/tensor/tensor_base.h

@@ -288,6 +288,11 @@ class TensorBase {
   virtual TensorV2 &sum(unsigned int axis, TensorV2 &output, float alpha,
                         float beta) const = 0;
 
+  /**
+   * @copydoc TensorV2::l2norm
+   */
+  virtual float l2norm() const = 0;
+
   /**
    * @copydoc TensorV2::pow(float exponent, TensorV2 &output)
    */
@@ -364,6 +369,16 @@ class TensorBase {
    */
   virtual void copyData(const TensorV2 &from) = 0;
 
+  /**
+   * @copydoc TensorV2::maxValue()
+   */
+  virtual float maxValue() const = 0;
+
+  /**
+   * @copydoc TensorV2::minValue()
+   */
+  virtual float minValue() const = 0;
+
   /**
    * @copydoc TensorV2::transpose(const std::string &direction, TensorV2 &out)
    */

nntrainer/tensor/tensor_v2.cpp

@@ -463,6 +463,57 @@ TensorV2 &TensorV2::erf(TensorV2 &output) const {
   return output;
 }
 
+float TensorV2::l2norm() const { return itensor->l2norm(); }
+
+void TensorV2::normalization_i() {
+  NNTR_THROW_IF(!getContiguous(), std::invalid_argument)
+    << getName() << " is not contiguous, cannot do normalization.";
+
+  const float min = minValue();
+  const float max = maxValue();
+
+  if (max == min) {
+    TensorV2 tmp = *this;
+    this->subtract_i(tmp);
+  } else {
+    this->subtract_i(min);
+    this->divide_i(max - min);
+  }
+}
+
+void TensorV2::standardization_i() {
+  TensorV2 mean_by_batch = this->sum_by_batch();
+  mean_by_batch.divide_i(getDim().getFeatureLen());
+
+  this->subtract_i(mean_by_batch);
+  TensorV2 std_dev_by_batch(batch(), 1, 1, 1, getFormat(), getDataType());
+  std_dev_by_batch.setZero();
+
+  /// @todo remove conditional statement
+  if (getDataType() == ml::train::TensorDim::DataType::FP32) {
+    float *std_dev = std_dev_by_batch.getData<float>();
+
+    for (unsigned int k = 0; k < batch(); ++k) {
+      TensorV2 sub_this = this->getBatchSlice(k, 1);
+      std_dev[k] = sub_this.l2norm();
+    }
+  } else if (getDataType() == ml::train::TensorDim::DataType::FP16) {
+#ifdef ENABLE_FP16
+    _FP16 *std_dev = std_dev_by_batch.getData<_FP16>();
+
+    for (unsigned int k = 0; k < batch(); ++k) {
+      TensorV2 sub_this = this->getBatchSlice(k, 1);
+      std_dev[k] = static_cast<_FP16>(sub_this.l2norm());
+    }
+#else
+    throw std::invalid_argument("Error: enable-fp16 is not enabled");
+#endif
+  }
+
+  std_dev_by_batch.divide_i(getDim().getFeatureLen());
+  this->divide_i(std_dev_by_batch);
+}
+
 TensorV2 TensorV2::dot(TensorV2 const &input, bool trans, bool trans_in) const {
   TensorV2 output("", this->getFormat(), this->getDataType());
   dot(input, output, trans, trans_in);
@@ -684,6 +735,10 @@ TensorV2 TensorV2::clone() const {
   return output;
 }
 
+float TensorV2::maxValue() const { return itensor->maxValue(); }
+
+float TensorV2::minValue() const { return itensor->minValue(); }
+
 TensorV2 TensorV2::transpose(const std::string &direction) const {
   TensorV2 output(getDim());
   transpose(direction, output);

nntrainer/tensor/tensor_v2.h

@@ -844,6 +844,36 @@ class TensorV2 {
    */
   TensorV2 &erf(TensorV2 &output) const;
 
+  /**
+   * @brief     l2norm the Tensor elements
+   * @retval    Calculated l2norm
+   */
+  float l2norm() const;
+
+  /**
+   * @brief     Normalize the Tensor elements
+   * @retval    Calculated Tensor
+   */
+  TensorV2 &normalization(TensorV2 &output) const;
+
+  /**
+   * @brief     Standardize the Tensor elements
+   * @retval    Calculated Tensor
+   */
+  TensorV2 &standardization(TensorV2 &output) const;
+
+  /**
+   * @brief     Normalize the Tensor elements in-place
+   * @retval    Calculated Tensor
+   */
+  void normalization_i();
+
+  /**
+   * @brief     Standardize the Tensor elements in-place
+   * @retval    Calculated Tensor
+   */
+  void standardization_i();
+
   /**
    * @brief     Dot Product of Tensor ( equal MxM )
    * @details   This applies dot of the last dimension of this and second-last
@@ -1017,6 +1047,18 @@ class TensorV2 {
    */
   TensorV2 clone() const;
 
+  /**
+   * @brief  return maximum value
+   * @retval Maximum value of the tensor data
+   */
+  float maxValue() const;
+
+  /**
+   * @brief  return minimum value
+   * @retval Minimum value of the tensor data
+   */
+  float minValue() const;
+
   /**
    * @brief  Transpose Tensor
    * @param  direction to transpose ex) 0:2:1