Merge pull request #7 from fszewczyk/dropout

Dropout Layer and Moduliarizing Activations

examples/xor_nn.cpp

@@ -14,9 +14,12 @@ int main() {
     xs.push_back(Vec32::of({0, 0})); ys.push_back(Vec32::of({0}));
 
     auto mlp = SequentialBuilder<Type::float32>::begin()
-                .add(Layer32::create(2, 15, Activation::relu<Type::float32>))
-                .add(Layer32::create(15, 5, Activation::relu<Type::float32>))
-                .add(Layer32::create(5, 1, Activation::sigmoid<Type::float32>))
+                .add(Linear32::create(2, 15))
+                .add(ReLU32::create())
+                .add(Dropout32::create(15, 5, 0.2))
+                .add(ReLU32::create())
+                .add(Linear32::create(5, 1))
+                .add(Sigmoid32::create())
                 .build();
     // clang-format on
 

include/ShkyeraGrad.hpp

@@ -7,13 +7,21 @@
 
 #pragma once
 
-#include "src/core/Type.hpp"
-#include "src/core/Value.hpp"
-#include "src/core/Vector.hpp"
-#include "src/nn/Activation.hpp"
-#include "src/nn/Layer.hpp"
-#include "src/nn/Loss.hpp"
-#include "src/nn/Module.hpp"
-#include "src/nn/Neuron.hpp"
-#include "src/nn/Optimizer.hpp"
-#include "src/nn/Sequential.hpp"
+#include "core/Type.hpp"
+#include "core/Value.hpp"
+#include "core/Vector.hpp"
+
+#include "nn/Loss.hpp"
+#include "nn/Module.hpp"
+#include "nn/Neuron.hpp"
+#include "nn/Optimizer.hpp"
+#include "nn/Sequential.hpp"
+
+#include "nn/activation/Activation.hpp"
+#include "nn/activation/Exp.hpp"
+#include "nn/activation/ReLU.hpp"
+#include "nn/activation/Sigmoid.hpp"
+#include "nn/activation/Tanh.hpp"
+
+#include "nn/layers/Dropout.hpp"
+#include "nn/layers/Linear.hpp"

include/src/core/Utils.hpp → include/core/Utils.hpp

@@ -7,7 +7,9 @@
 
 #pragma once
 
+#include <algorithm>
 #include <random>
+#include <vector>
 
 namespace shkyera::utils {
 
@@ -35,12 +37,16 @@ template <typename T> std::enable_if_t<std::is_integral_v<T>, T> sample(T from,
     return distribution(generator);
 }
 
-template <typename T> std::enable_if_t<std::is_integral_v<T>, std::vector<T>> sample(T from, T to, size_t size) {
+template <typename T>
+std::enable_if_t<std::is_integral_v<T>, std::vector<T>> sample(T from, T to, size_t size, bool withReplacement = true) {
     std::uniform_int_distribution<T> distribution(from, to);
 
     std::vector<T> sampled(size);
     for (size_t i = 0; i < size; i++) {
-        sampled[i] = distribution(generator);
+        T candidate = distribution(generator);
+        while (!withReplacement && std::find(sampled.begin(), sampled.end(), candidate) != sampled.end())
+            candidate = distribution(generator);
+        sampled[i] = candidate;
     }
 
     return sampled;

include/src/nn/Neuron.hpp → include/nn/Neuron.hpp

@@ -23,11 +23,9 @@ template <typename T> class Neuron {
   private:
     ValuePtr<T> _bias;
     Vector<T> _weights;
-    std::function<ValuePtr<T>(ValuePtr<T>)> _activation = [](ValuePtr<T> a) { return a; };
 
   public:
     Neuron(size_t input);
-    Neuron(size_t input, std::function<ValuePtr<T>(ValuePtr<T>)> activation);
 
     Vector<T> operator()(const Vector<T> &x) const;
     std::vector<ValuePtr<T>> parameters() const;
@@ -40,13 +38,8 @@ template <typename T> Neuron<T>::Neuron(size_t input) {
     _bias = Value<T>::create(utils::sample<T>(-1, 1));
 }
 
-template <typename T>
-Neuron<T>::Neuron(size_t input, std::function<ValuePtr<T>(ValuePtr<T>)> activation) : Neuron<T>(input) {
-    _activation = activation;
-}
-
 template <typename T> Vector<T> Neuron<T>::operator()(const Vector<T> &x) const {
-    return Vector<T>({_activation(_bias + _weights.dot(x))});
+    return Vector<T>({_bias + _weights.dot(x)});
 }
 
 template <typename T> std::vector<ValuePtr<T>> Neuron<T>::parameters() const {

include/src/nn/Sequential.hpp → include/nn/Sequential.hpp

@@ -8,7 +8,6 @@
 #pragma once
 
 #include "../core/Type.hpp"
-#include "Activation.hpp"
 #include "Module.hpp"
 
 namespace shkyera {

include/nn/activation/Activation.hpp

@@ -0,0 +1,26 @@
+/**
+ * Copyright © 2023 Franciszek Szewczyk. None of the rights reserved.
+ * This code is released under the Beerware License. If you find this code useful or you appreciate the work, you are
+ * encouraged to buy the author a beer in return.
+ * Contact the author at szewczyk.franciszek02@gmail.com for inquiries and support.
+ */
+
+#pragma once
+
+#include "../../core/Type.hpp"
+#include "../Module.hpp"
+
+namespace shkyera {
+
+template <typename T> class Activation;
+
+template <typename T> class Activation : public Module<T> {
+  protected:
+    Activation() = default;
+
+  public:
+    virtual Vector<T> operator()(const Vector<T> &x) const override { return x; }
+    virtual std::vector<ValuePtr<T>> parameters() const override { return {}; }
+};
+
+} // namespace shkyera

include/nn/activation/Exp.hpp

@@ -0,0 +1,38 @@
+/**
+ * Copyright © 2023 Franciszek Szewczyk. None of the rights reserved.
+ * This code is released under the Beerware License. If you find this code useful or you appreciate the work, you are
+ * encouraged to buy the author a beer in return.
+ * Contact the author at szewczyk.franciszek02@gmail.com for inquiries and support.
+ */
+
+#pragma once
+
+#include "Activation.hpp"
+
+namespace shkyera {
+
+template <typename T> class Exp;
+using Exp32 = Exp<Type::float32>;
+using Exp64 = Exp<Type::float64>;
+
+template <typename T> class Exp : public Activation<T> {
+  public:
+    static std::shared_ptr<Exp<T>> create();
+
+    virtual Vector<T> operator()(const Vector<T> &x) const override;
+};
+
+template <typename T> std::shared_ptr<Exp<T>> Exp<T>::create() { return std::shared_ptr<Exp<T>>(new Exp<T>()); }
+
+template <typename T> Vector<T> Exp<T>::operator()(const Vector<T> &x) const {
+    std::vector<ValuePtr<T>> out;
+    out.reserve(x.size());
+
+    for (size_t i = 0; i < x.size(); ++i) {
+        out.emplace_back(x[i]->exp());
+    }
+
+    return Vector<T>(out);
+}
+
+} // namespace shkyera

include/nn/activation/ReLU.hpp

@@ -0,0 +1,38 @@
+/**
+ * Copyright © 2023 Franciszek Szewczyk. None of the rights reserved.
+ * This code is released under the Beerware License. If you find this code useful or you appreciate the work, you are
+ * encouraged to buy the author a beer in return.
+ * Contact the author at szewczyk.franciszek02@gmail.com for inquiries and support.
+ */
+
+#pragma once
+
+#include "Activation.hpp"
+
+namespace shkyera {
+
+template <typename T> class ReLU;
+using ReLU32 = ReLU<Type::float32>;
+using ReLU64 = ReLU<Type::float64>;
+
+template <typename T> class ReLU : public Activation<T> {
+  public:
+    static std::shared_ptr<ReLU<T>> create();
+
+    virtual Vector<T> operator()(const Vector<T> &x) const override;
+};
+
+template <typename T> std::shared_ptr<ReLU<T>> ReLU<T>::create() { return std::shared_ptr<ReLU<T>>(new ReLU<T>()); }
+
+template <typename T> Vector<T> ReLU<T>::operator()(const Vector<T> &x) const {
+    std::vector<ValuePtr<T>> out;
+    out.reserve(x.size());
+
+    for (size_t i = 0; i < x.size(); ++i) {
+        out.emplace_back(x[i]->relu());
+    }
+
+    return Vector<T>(out);
+}
+
+} // namespace shkyera

include/nn/activation/Sigmoid.hpp

@@ -0,0 +1,40 @@
+/**
+ * Copyright © 2023 Franciszek Szewczyk. None of the rights reserved.
+ * This code is released under the Beerware License. If you find this code useful or you appreciate the work, you are
+ * encouraged to buy the author a beer in return.
+ * Contact the author at szewczyk.franciszek02@gmail.com for inquiries and support.
+ */
+
+#pragma once
+
+#include "Activation.hpp"
+
+namespace shkyera {
+
+template <typename T> class Sigmoid;
+using Sigmoid32 = Sigmoid<Type::float32>;
+using Sigmoid64 = Sigmoid<Type::float64>;
+
+template <typename T> class Sigmoid : public Activation<T> {
+  public:
+    static std::shared_ptr<Sigmoid<T>> create();
+
+    virtual Vector<T> operator()(const Vector<T> &x) const override;
+};
+
+template <typename T> std::shared_ptr<Sigmoid<T>> Sigmoid<T>::create() {
+    return std::shared_ptr<Sigmoid<T>>(new Sigmoid<T>());
+}
+
+template <typename T> Vector<T> Sigmoid<T>::operator()(const Vector<T> &x) const {
+    std::vector<ValuePtr<T>> out;
+    out.reserve(x.size());
+
+    for (size_t i = 0; i < x.size(); ++i) {
+        out.emplace_back(x[i]->sigmoid());
+    }
+
+    return Vector<T>(out);
+}
+
+} // namespace shkyera

include/nn/activation/Tanh.hpp

@@ -0,0 +1,38 @@
+/**
+ * Copyright © 2023 Franciszek Szewczyk. None of the rights reserved.
+ * This code is released under the Beerware License. If you find this code useful or you appreciate the work, you are
+ * encouraged to buy the author a beer in return.
+ * Contact the author at szewczyk.franciszek02@gmail.com for inquiries and support.
+ */
+
+#pragma once
+
+#include "Activation.hpp"
+
+namespace shkyera {
+
+template <typename T> class Tanh;
+using Tanh32 = Tanh<Type::float32>;
+using Tanh64 = Tanh<Type::float64>;
+
+template <typename T> class Tanh : public Activation<T> {
+  public:
+    static std::shared_ptr<Tanh<T>> create();
+
+    virtual Vector<T> operator()(const Vector<T> &x) const override;
+};
+
+template <typename T> std::shared_ptr<Tanh<T>> Tanh<T>::create() { return std::shared_ptr<Tanh<T>>(new Tanh<T>()); }
+
+template <typename T> Vector<T> Tanh<T>::operator()(const Vector<T> &x) const {
+    std::vector<ValuePtr<T>> out;
+    out.reserve(x.size());
+
+    for (size_t i = 0; i < x.size(); ++i) {
+        out.emplace_back(x[i]->tanh());
+    }
+
+    return Vector<T>(out);
+}
+
+} // namespace shkyera

include/nn/layers/Dropout.hpp

@@ -0,0 +1,59 @@
+/**
+ * Copyright © 2023 Franciszek Szewczyk. None of the rights reserved.
+ * This code is released under the Beerware License. If you find this code useful or you appreciate the work, you are
+ * encouraged to buy the author a beer in return.
+ * Contact the author at szewczyk.franciszek02@gmail.com for inquiries and support.
+ */
+
+#pragma once
+
+#include "../../core/Utils.hpp"
+#include "Linear.hpp"
+
+namespace shkyera {
+
+template <typename T> class Dropout;
+template <typename T> using DropoutPtr = std::shared_ptr<Dropout<T>>;
+
+using Dropout32 = Dropout<Type::float32>;
+using Dropout64 = Dropout<Type::float64>;
+
+template <typename T> class Dropout : public Linear<T> {
+  private:
+    double _dropout;
+
+    Dropout(size_t input, size_t size, double dropout);
+
+  public:
+    static DropoutPtr<T> create(size_t input, size_t size, double dropout);
+
+    virtual Vector<T> operator()(const Vector<T> &x) const override;
+};
+
+template <typename T> Dropout<T>::Dropout(size_t input, size_t size, double dropout) : Linear<T>(input, size) {
+    if (_dropout < 0 || _dropout >= 1) {
+        throw std::invalid_argument("Droput rate must be in the range [0,1). You set it to " + std::to_string(dropout) +
+                                    ".");
+    }
+    _dropout = dropout;
+}
+
+template <typename T> DropoutPtr<T> Dropout<T>::create(size_t input, size_t size, double dropout) {
+    return std::shared_ptr<Dropout<T>>(new Dropout(input, size, dropout));
+}
+
+template <typename T> Vector<T> Dropout<T>::operator()(const Vector<T> &x) const {
+    std::vector<ValuePtr<T>> alteredInput;
+    alteredInput.reserve(x.size());
+    auto scaling = Value<T>::create(1.0 / (1 - _dropout));
+    for (size_t i = 0; i < x.size(); ++i)
+        alteredInput.push_back(x[i] * scaling);
+
+    std::vector<size_t> indicesToRemove = utils::sample<size_t>(0, x.size() - 1, _dropout * x.size(), false);
+    for (size_t idxToRemove : indicesToRemove)
+        alteredInput[idxToRemove] = Value<T>::create(0);
+
+    return Linear<T>::operator()(Vector<T>(alteredInput));
+}
+
+} // namespace shkyera