[Layer] Modify Layer for mixed type

fc, conv2d, softmax, pooling layers are modified for mxied type.
It supports In/Out tensors as 32/16 float type, also for weights.

Signed-off-by: Jiho Chu <jiho.chu@samsung.com>

Author: jihochu

nntrainer/layers/common_properties.h

@@ -1398,10 +1398,12 @@ class PropsUserData final : public Property<void *> {
 /**
  * @brief LossScale property, loss is scaled by this value
  *
+ * @note It is uesed to enable/disable loss scale,
+ *       and '0.0f' means disable.
  */
 class LossScale : public nntrainer::Property<float> {
 public:
-  LossScale(float value = 1.0f);
+  LossScale(float value = 0.0f);
   static constexpr const char *key = "loss_scale"; /**< unique key to access */
   using prop_tag = float_prop_tag;               /**< property type */
 };

nntrainer/layers/conv2d_layer.cpp

@@ -117,10 +117,16 @@ static void col2im(const Tensor &col_matrix, const TensorDim &kdim,
   if (image.getDataType() == nntrainer::Tdatatype::FP32) {
     float val;
     apply_data(&val);
-  } else if (image.getDataType() == nntrainer::Tdatatype::FP16) {
+  }
+  #ifdef ENABLE_FP16
+  else if (image.getDataType() == nntrainer::Tdatatype::FP16) {
     _FP16 val;
     apply_data(&val);
   }
+  #endif
+  else {
+    throw std::runtime_error("Not supported datatype");
+  }
 }
 
 /**
@@ -254,10 +260,16 @@ static void im2col(const Tensor &in, const TensorDim &kdim,
   if (out.getDataType() == nntrainer::Tdatatype::FP32) {
     float *out_data = out.getData<float>();
     apply_data(out_data);
-  } else if (out.getDataType() == nntrainer::Tdatatype::FP16) {
+  }
+  #ifdef ENABLE_FP16 
+  else if (out.getDataType() == nntrainer::Tdatatype::FP16) {
     _FP16 *out_data = out.getData<_FP16>();
     apply_data(out_data);
   }
+  #endif
+  else {
+    throw std::runtime_error("Not supported datatype");
+  }
 }
 
 } // namespace
@@ -355,7 +367,14 @@ void Conv2DLayer::forwarding(RunLayerContext &context, bool training) {
   Tensor &input_ = context.getInput(SINGLE_INOUT_IDX);
   Tensor &hidden_ = context.getOutput(SINGLE_INOUT_IDX);
 
-  Tensor &filter_kernel = context.getWeight(wt_idx[ConvParams::weight]);
+  Tensor &filter_kernel_ = context.getWeight(wt_idx[ConvParams::weight]);
+
+  Tensor filter_kernel;
+  if (filter_kernel_.getDataType() != input_.getDataType()) {
+    filter_kernel = filter_kernel_.clone(input_.getDataType());
+  } else {
+    filter_kernel = filter_kernel_;
+  }
 
   /** Calculate Convolution 2D
    *
@@ -431,12 +450,21 @@ void Conv2DLayer::forwarding(RunLayerContext &context, bool training) {
   filter_kernel.reshape(filter_dim);
   if (auto &disable_bias = std::get<props::DisableBias>(*layer_impl_props);
       disable_bias.empty() || disable_bias.get() == false) {
-    Tensor &bias_kernel = context.getWeight(wt_idx[ConvParams::bias]);
+    Tensor &bias_kernel_ = context.getWeight(wt_idx[ConvParams::bias]);
+    Tensor bias_kernel;
+    if (bias_kernel_.getDataType() != hidden_.getDataType()) {
+      bias_kernel = bias_kernel_.clone(hidden_.getDataType());
+    } else {
+      bias_kernel = bias_kernel_;
+    }
+
     status = hidden_.add_i(bias_kernel);
     if (status != ML_ERROR_NONE) {
       throw std::invalid_argument("[Conv2D] adding bias failed");
     }
   }
+  //export_to_file(input_, std::string("debug/var/") + input_.getName() + ".var");
+  //export_to_file(hidden_, std::string("debug/var/") + hidden_.getName() + ".var");
 }
 
 void Conv2DLayer::calcDerivative(RunLayerContext &context) {
@@ -447,7 +475,14 @@ void Conv2DLayer::calcDerivative(RunLayerContext &context) {
 
   const Tensor &derivative = context.getIncomingDerivative(SINGLE_INOUT_IDX);
   Tensor &input_derivative = context.getOutgoingDerivative(SINGLE_INOUT_IDX);
-  Tensor &filter_kernel = context.getWeight(wt_idx[ConvParams::weight]);
+  Tensor &filter_kernel_ = context.getWeight(wt_idx[ConvParams::weight]);
+
+  Tensor filter_kernel;
+  if (filter_kernel_.getDataType() != input_derivative.getDataType()) {
+    filter_kernel = filter_kernel_.clone(input_derivative.getDataType());
+  } else {
+    filter_kernel = filter_kernel_;
+  }
 
   TensorDim filter_dim = filter_kernel.getDim();
   TensorDim filter_dim_squeezed{filter_kernel.batch(),
@@ -495,7 +530,14 @@ void Conv2DLayer::calcGradient(RunLayerContext &context) {
   const Tensor &derivative = context.getIncomingDerivative(SINGLE_INOUT_IDX);
   Tensor &input_ = context.getInput(SINGLE_INOUT_IDX);
 
-  Tensor &delK = context.getWeightGrad(wt_idx[ConvParams::weight]);
+  Tensor &delK_ = context.getWeightGrad(wt_idx[ConvParams::weight]);
+  Tensor delK;
+  if (delK.getDataType() != input_.getDataType()) {
+    delK = delK_.clone(input_.getDataType());
+  } else {
+    delK = delK_;
+  }
+
   delK.setZero();
 
   TensorDim filter_dim = delK.getDim();
@@ -577,9 +619,17 @@ void Conv2DLayer::calcGradient(RunLayerContext &context) {
   delK.reshape(filter_dim);
   if (auto &disable_bias = std::get<props::DisableBias>(*layer_impl_props);
       disable_bias.empty() || disable_bias.get() == false) {
-    Tensor &delBias = context.getWeightGrad(wt_idx[ConvParams::bias]);
+    Tensor &delBias_ = context.getWeightGrad(wt_idx[ConvParams::bias]);
+    Tensor delBias;
+    if (delBias_.getDataType() != derivative.getDataType()) {
+      delBias = delBias_.clone(derivative.getDataType());
+    } else {
+      delBias = delBias_;
+    }
     derivative.sum({0, 2, 3}, delBias);
+    delBias_.copyData(delBias);
   }
+  delK_.copyData(delK);
 }
 
 void Conv2DLayer::exportTo(Exporter &exporter,

nntrainer/layers/fc_layer.cpp

@@ -116,10 +116,17 @@ void FullyConnectedLayer::setProperty(const std::vector<std::string> &values) {
 }
 
 void FullyConnectedLayer::forwarding(RunLayerContext &context, bool training) {
-  Tensor &weight = context.getWeight(weight_idx[FCParams::weight]);
+  Tensor &weight_ = context.getWeight(weight_idx[FCParams::weight]);
   Tensor &hidden_ = context.getOutput(SINGLE_INOUT_IDX);
   Tensor &input_ = context.getInput(SINGLE_INOUT_IDX);
 
+  Tensor weight;
+  if (weight.getDataType() != input_.getDataType()) {
+    weight = weight_.clone(input_.getDataType());
+  } else {
+    weight = weight_;
+  }
+
   if (weight.getDataType() == nntrainer::Tdatatype::QINT4 ||
       weight.getDataType() == nntrainer::Tdatatype::QINT8) {
     Tdatatype dtype = input_.getDataType();
@@ -141,7 +148,13 @@ void FullyConnectedLayer::forwarding(RunLayerContext &context, bool training) {
   if (auto &disable_bias = std::get<props::DisableBias>(*layer_impl_props);
       disable_bias.empty() || disable_bias.get() == false) {
     Tensor &bias = context.getWeight(weight_idx[FCParams::bias]);
-    hidden_.add_i(bias);
+    Tensor b;
+    if (bias.getDataType() != input_.getDataType()) {
+      b = bias.clone(input_.getDataType());
+    } else {
+      b = bias;
+    }
+    hidden_.add_i(b);
   }
 }
 
@@ -187,23 +200,44 @@ void FullyConnectedLayer::incremental_forwarding(RunLayerContext &context,
 }
 
 void FullyConnectedLayer::calcDerivative(RunLayerContext &context) {
-  Tensor &weight = context.getWeight(weight_idx[FCParams::weight]);
+  Tensor &weight_ = context.getWeight(weight_idx[FCParams::weight]);
 
   const Tensor &derivative_ = context.getIncomingDerivative(SINGLE_INOUT_IDX);
   Tensor &ret_ = context.getOutgoingDerivative(SINGLE_INOUT_IDX);
 
+  Tensor weight;
+  if (weight_.getDataType() != derivative_.getDataType()) {
+    weight  = weight_.clone(derivative_.getDataType());
+  } else {
+    weight = weight_;
+  }
+
   ret_.dot_deriv_wrt_1(weight, derivative_, false, false);
 }
 
 void FullyConnectedLayer::calcGradient(RunLayerContext &context) {
-  Tensor &djdw = context.getWeightGrad(weight_idx[FCParams::weight]);
+  Tensor &djdw_ = context.getWeightGrad(weight_idx[FCParams::weight]);
 
   const Tensor &derivative_ = context.getIncomingDerivative(SINGLE_INOUT_IDX);
   Tensor &input_ = context.getInput(SINGLE_INOUT_IDX);
 
+  Tensor djdw;
+  if (djdw_.getDataType() != derivative_.getDataType()) {
+    djdw = djdw_.clone(derivative_.getDataType());
+  } else {
+    djdw = djdw_;
+  }
+
   if (auto &disable_bias = std::get<props::DisableBias>(*layer_impl_props);
       disable_bias.empty() || disable_bias.get() == false) {
-    Tensor &djdb = context.getWeightGrad(weight_idx[FCParams::bias]);
+    Tensor &djdb_ = context.getWeightGrad(weight_idx[FCParams::bias]);
+
+    Tensor djdb;
+    if (djdb_.getDataType() != input_.getDataType()) {
+      djdb = djdb_.clone(input_.getDataType());
+    } else {
+      djdb = djdb_;
+    }
 
     if (context.isGradientFirstAccess(weight_idx[FCParams::bias])) {
       derivative_.sum({0, 1, 2}, djdb);
@@ -212,11 +246,13 @@ void FullyConnectedLayer::calcGradient(RunLayerContext &context) {
       Tensor t = derivative_.sum({0, 1, 2});
       djdb.add_i(t);
     }
+    djdb_.copyData(djdb);
   }
 
   input_.dot_deriv_wrt_2(
     djdw, derivative_, false, false,
     !context.isGradientFirstAccess(weight_idx[FCParams::weight]));
+  djdw_.copyData(djdw);
 }
 
 } /* namespace nntrainer */

nntrainer/layers/loss/cross_entropy_softmax_loss_layer.cpp

@@ -29,6 +29,8 @@ void CrossEntropySoftmaxLossLayer::forwarding(RunLayerContext &context,
   Tensor &hidden_ = context.getOutput(SINGLE_INOUT_IDX);
   Tensor &y = context.getInput(SINGLE_INOUT_IDX);
 
+  //export_to_file(y, std::string("debug/var/") + y.getName() + ".var");
+
   // fill the output
   auto out_type = hidden_.getDataType();
   if (out_type == ml::train::TensorDim::DataType::FP32) {
@@ -93,7 +95,8 @@ void CrossEntropySoftmaxLossLayer::calcDerivative(RunLayerContext &context) {
                              "Error when calculating loss");
   }
 
-  ret_derivative.multiply_i(loss_scale);
+  if (loss_scale != 0.0f)
+    ret_derivative.multiply_i(loss_scale);
 }
 
 } // namespace nntrainer

nntrainer/layers/pooling2d_layer.cpp

@@ -155,6 +155,9 @@ void Pooling2DLayer::forwarding(RunLayerContext &context, bool training) {
   } else {
     forwarding_job(0, in_dim.batch(), 0, nullptr);
   }
+
+  //export_to_file(input_, std::string("debug/var/") + input_.getName() + ".var");
+  //export_to_file(hidden_, std::string("debug/var/") + hidden_.getName() + ".var");
 }
 
 void Pooling2DLayer::calcDerivative(RunLayerContext &context) {
@@ -258,21 +261,33 @@ void Pooling2DLayer::calcDerivative(RunLayerContext &context) {
   case props::PoolingTypeInfo::Enum::max:
     if (in_dim.getDataType() == ml::train::TensorDim::DataType::FP32)
       apply_max(result.getData<float>());
+#ifdef ENABLE_FP16
     else if (in_dim.getDataType() == ml::train::TensorDim::DataType::FP16)
       apply_max(result.getData<_FP16>());
+#endif
+    else
+      throw std::runtime_error("Not supported datatype");
     break;
   case props::PoolingTypeInfo::Enum::global_average:
   case props::PoolingTypeInfo::Enum::average:
     if (in_dim.getDataType() == ml::train::TensorDim::DataType::FP32)
       apply_average(result.getData<float>());
+#ifdef ENABLE_FP16
     else if (in_dim.getDataType() == ml::train::TensorDim::DataType::FP16)
       apply_average(result.getData<_FP16>());
+#endif
+    else
+      throw std::runtime_error("Not supported datatype");
     break;
   case props::PoolingTypeInfo::Enum::global_max:
     if (in_dim.getDataType() == ml::train::TensorDim::DataType::FP32)
       apply_global_max(result.getData<float>());
+#ifdef ENABLE_FP16
     else if (in_dim.getDataType() == ml::train::TensorDim::DataType::FP16)
       apply_global_max(result.getData<_FP16>());
+#endif
+    else
+      throw std::runtime_error("Not supported datatype");
     break;
   default:
     throw std::runtime_error("Error: Unknown Pooling Type");
@@ -320,7 +335,9 @@ void Pooling2DLayer::pooling2d(Tensor &in, bool training, Tensor &output,
    * @return result value of pooling
    */
   PoolFunc<float>::Type pool_fn_fp32;
+#ifdef ENABLE_FP16
   PoolFunc<_FP16>::Type pool_fn_fp16;
+#endif
 
   unsigned int max_idx_count = 0;
 
@@ -411,16 +428,22 @@ void Pooling2DLayer::pooling2d(Tensor &in, bool training, Tensor &output,
   switch (pooling_type) {
   case props::PoolingTypeInfo::Enum::max:
     pool_fn_fp32 = pool_fn_max;
+#ifdef ENABLE_FP16
     pool_fn_fp16 = pool_fn_max;
+#endif
     break;
   case props::PoolingTypeInfo::Enum::global_max:
     pool_fn_fp32 = pool_fn_global_max;
+#ifdef ENABLE_FP16
     pool_fn_fp16 = pool_fn_global_max;
+#endif
     break;
   case props::PoolingTypeInfo::Enum::global_average:
   case props::PoolingTypeInfo::Enum::average:
     pool_fn_fp32 = pool_fn_average;
+#ifdef ENABLE_FP16
     pool_fn_fp16 = pool_fn_average;
+#endif
     break;
   case props::PoolingTypeInfo::Enum::unknown:
   default:
@@ -446,7 +469,9 @@ void Pooling2DLayer::pooling2d(Tensor &in, bool training, Tensor &output,
         }
       }
     }
-  } else if (in.getDataType() == ml::train::TensorDim::DataType::FP16) {
+  }
+#ifdef ENABLE_FP16
+  else if (in.getDataType() == ml::train::TensorDim::DataType::FP16) {
     const _FP16 *in_data = in.getData<_FP16>();
     _FP16 *out_data = output.getData<_FP16>();
 
@@ -465,6 +490,10 @@ void Pooling2DLayer::pooling2d(Tensor &in, bool training, Tensor &output,
       }
     }
   }
+#endif
+  else {
+    throw std::runtime_error("Not supported datatype");
+  }
 }
 
 void Pooling2DLayer::setBatch(RunLayerContext &context, unsigned int batch) {