From f428f710780a4b7dbb33255097d77991944f44df Mon Sep 17 00:00:00 2001
From: Gaurav Singh <gs8763076@gmail.com>
Date: Sat, 4 Apr 2020 17:04:26 +0530
Subject: [PATCH 01/17] Initial commit.

---
 include/ensmallen_bits/cmaes/cmaes.hpp      |  30 ++++--
 include/ensmallen_bits/cmaes/cmaes_impl.hpp | 107 ++++++++++++++++++--
 tests/cmaes_test.cpp                        |  11 +-
 3 files changed, 128 insertions(+), 20 deletions(-)

diff --git a/include/ensmallen_bits/cmaes/cmaes.hpp b/include/ensmallen_bits/cmaes/cmaes.hpp
index 9ee2efc0c..9fbe0b83f 100644
--- a/include/ensmallen_bits/cmaes/cmaes.hpp
+++ b/include/ensmallen_bits/cmaes/cmaes.hpp
@@ -66,6 +66,7 @@ class CMAES
    * @param maxIterations Maximum number of iterations allowed (0 means no
    *     limit).
    * @param tolerance Maximum absolute tolerance to terminate algorithm.
+   * @param initialSigma The initial step size
    * @param selectionPolicy Instantiated selection policy used to calculate the
    *     objective.
    */
@@ -75,7 +76,8 @@ class CMAES
         const size_t batchSize = 32,
         const size_t maxIterations = 1000,
         const double tolerance = 1e-5,
-        const SelectionPolicyType& selectionPolicy = SelectionPolicyType());
+        const SelectionPolicyType& selectionPolicy = SelectionPolicyType(),
+        const double initialSigma = 0.6);
 
   /**
    * Optimize the given function using CMA-ES. The given starting point will be
@@ -97,9 +99,9 @@ class CMAES
                                        MatType& iterate,
                                        CallbackTypes&&... callbacks);
 
-  //! Get the step size.
+  //! Get the population size.
   size_t PopulationSize() const { return lambda; }
-  //! Modify the step size.
+  //! Modify the population size.
   size_t& PopulationSize() { return lambda; }
 
   //! Get the lower bound of decision variables.
@@ -107,9 +109,9 @@ class CMAES
   //! Modify the lower bound of decision variables.
   double& LowerBound() { return lowerBound; }
 
-  //! Get the upper bound of decision variables
+  //! Get the upper bound of decision variables.
   double UpperBound() const { return upperBound; }
-  //! Modify the upper bound of decision variables
+  //! Modify the upper bound of decision variables.
   double& UpperBound() { return upperBound; }
 
   //! Get the batch size.
@@ -132,6 +134,11 @@ class CMAES
   //! Modify the selection policy.
   SelectionPolicyType& SelectionPolicy() { return selectionPolicy; }
 
+  //! Get the initial step size.
+  double InitialSigma() const { return initialSigma; }
+  //! Modify the initial step size.
+  double& InitialSigma() { return initialSigma; }
+
  private:
   //! Population size.
   size_t lambda;
@@ -139,7 +146,7 @@ class CMAES
   //! Lower bound of decision variables.
   double lowerBound;
 
-  //! Upper bound of decision variables
+  //! Upper bound of decision variables.
   double upperBound;
 
   //! The batch size for processing.
@@ -153,6 +160,15 @@ class CMAES
 
   //! The selection policy used to calculate the objective.
   SelectionPolicyType selectionPolicy;
+
+  //! Initial step size.
+  double initialSigma;
+
+  //! Methods used to transform the candidates into the constraints.
+  template<typename ElemType, typename BaseMatType>
+  void BoundaryTransform(BaseMatType& matrix);
+  template<typename ElemType, typename BaseMatType>
+  void BoundaryTransformInverse(BaseMatType& matrix);
 };
 
 /**
@@ -166,4 +182,4 @@ using ApproxCMAES = CMAES<SelectionPolicyType>;
 // Include implementation.
 #include "cmaes_impl.hpp"
 
-#endif
+#endif
\ No newline at end of file
diff --git a/include/ensmallen_bits/cmaes/cmaes_impl.hpp b/include/ensmallen_bits/cmaes/cmaes_impl.hpp
index 32a05dde9..cbb051d2b 100644
--- a/include/ensmallen_bits/cmaes/cmaes_impl.hpp
+++ b/include/ensmallen_bits/cmaes/cmaes_impl.hpp
@@ -29,15 +29,22 @@ CMAES<SelectionPolicyType>::CMAES(const size_t lambda,
                                   const size_t batchSize,
                                   const size_t maxIterations,
                                   const double tolerance,
-                                  const SelectionPolicyType& selectionPolicy) :
+                                  const SelectionPolicyType& selectionPolicy,
+                                  const double initialSigma) :
     lambda(lambda),
     lowerBound(lowerBound),
     upperBound(upperBound),
     batchSize(batchSize),
     maxIterations(maxIterations),
     tolerance(tolerance),
-    selectionPolicy(selectionPolicy)
-{ /* Nothing to do. */ }
+    selectionPolicy(selectionPolicy),
+    initialSigma(initialSigma)
+{
+  if (lowerBound == upperBound)
+  {
+    Warn << "Lower and upper bounds must be different." << std::endl;
+  }
+}
 
 //! Optimize the function (minimize).
 template<typename SelectionPolicyType>
@@ -78,7 +85,7 @@ typename MatType::elem_type CMAES<SelectionPolicyType>::Optimize(
 
   // Step size control parameters.
   BaseMatType sigma(2, 1); // sigma is vector-shaped.
-  sigma(0) = 0.3 * (upperBound - lowerBound);
+  sigma(0) = initialSigma;
   const double cs = (muEffective + 2) / (iterate.n_elem + muEffective + 5);
   const double ds = 1 + cs + 2 * std::max(std::sqrt((muEffective - 1) /
       (iterate.n_elem + 1)) - 1, 0.0);
@@ -99,13 +106,16 @@ typename MatType::elem_type CMAES<SelectionPolicyType>::Optimize(
 
   std::vector<BaseMatType> mPosition(2, BaseMatType(iterate.n_rows,
       iterate.n_cols));
-  mPosition[0] = lowerBound + arma::randu<BaseMatType>(
-      iterate.n_rows, iterate.n_cols) * (upperBound - lowerBound);
+  BaseMatType initialVal;
+  initialVal.randu(iterate.n_rows, iterate.n_cols);
+  initialVal += (BaseMatType)(iterateIn);
+  mPosition[0] = initialVal;
 
   BaseMatType step(iterate.n_rows, iterate.n_cols);
   step.zeros();
 
   // Calculate the first objective function.
+  BoundaryTransform<ElemType, BaseMatType>(mPosition[0]);
   ElemType currentObjective = 0;
   for (size_t f = 0; f < numFunctions; f += batchSize)
   {
@@ -177,6 +187,7 @@ typename MatType::elem_type CMAES<SelectionPolicyType>::Optimize(
       pPosition[idx(j)] = mPosition[idx0] + sigma(idx0) * pStep[idx(j)];
 
       // Calculate the objective function.
+      BoundaryTransform<ElemType, BaseMatType>(pPosition[idx(j)]);
       pObjective(idx(j)) = selectionPolicy.Select(function, batchSize,
           pPosition[idx(j)], callbacks...);
     }
@@ -191,6 +202,7 @@ typename MatType::elem_type CMAES<SelectionPolicyType>::Optimize(
     mPosition[idx1] = mPosition[idx0] + sigma(idx0) * step;
 
     // Calculate the objective function.
+    BoundaryTransform<ElemType, BaseMatType>(mPosition[idx1]);
     currentObjective = selectionPolicy.Select(function, batchSize,
         mPosition[idx1], callbacks...);
 
@@ -313,6 +325,89 @@ typename MatType::elem_type CMAES<SelectionPolicyType>::Optimize(
   return overallObjective;
 }
 
+// Transforms the candidate into the given bounds.
+template<typename SelectionPolicyType>
+template<typename ElemType, typename BaseMatType>
+void CMAES<SelectionPolicyType>::BoundaryTransform(BaseMatType& matrix)
+{
+  double diff = (upperBound - lowerBound) / 2.0;
+  double al = std::min(diff, (1 + std::abs(lowerBound)) / 20.0);
+  double au = std::min(diff, (1 + std::abs(upperBound)) / 20.0);
+  double xlow = lowerBound - 2 * al - diff;
+  double xup = upperBound + 2 * au + diff;
+  double r = 2 * (upperBound - lowerBound + al + au);
+
+  for (size_t col=0; col < matrix.n_cols; col++)
+  {
+    for (size_t row=0; row < matrix.n_rows; row++)
+    {
+      ElemType y = matrix(row, col);
+      //boundary transformation shift into feasible preimage.
+      if (y < xlow)
+      {
+        y += (ElemType)(r * (1 + (xlow - y) / r));
+      }
+      if (y > xup)
+      {
+        y -= (ElemType)(r * (1 + (y - xup) / r));
+      }
+      if (y < lowerBound - al)
+      {
+        y += (ElemType)(2 * (lowerBound - al - y));
+      }
+      if (y > upperBound + au)
+      {
+        y -= (ElemType)(2 * (y - upperBound - au));
+      }
+      //boundary transformation
+      if (y < lowerBound + al)
+      {
+        y = (ElemType)(lowerBound + (y - (lowerBound - al)) *
+            (y - (lowerBound - al)) / 4.0 / al);
+      }
+      else if (y > upperBound - au)
+      {
+        y = (ElemType)(upperBound - (y - (upperBound + au)) *
+            (y - (upperBound + au)) / 4.0 / au);
+      }
+
+      matrix(row, col) = y;
+    }
+  }
+  
+}
+
+// Computes the inverse of the transformation.
+template<typename SelectionPolicyType>
+template<typename ElemType, typename BaseMatType>
+void CMAES<SelectionPolicyType>::BoundaryTransformInverse(BaseMatType& matrix)
+{
+  double diff = (upperBound - lowerBound) / 2.0;
+  double al = std::min(diff, (1 + std::abs(lowerBound)) / 20.0);
+  double au = std::min(diff, (1 + std::abs(upperBound)) / 20.0);
+
+  for (size_t col=0; col < matrix.n_cols; col++)
+  {
+    for (size_t row=0; row < matrix.n_rows; row++)
+    {
+      ElemType y = matrix(row, col);
+      
+      if (y < lowerBound + al)
+      {
+        y =  (ElemType)(lowerBound - al) + 2 *
+              std::pow(std::abs(al * (y - lowerBound)),0.5);
+      }
+      else if (y > upperBound - au)
+      {
+        y = (ElemType)(upperBound + au) - 2 *
+             std::pow(std::abs(au * (upperBound - y)),0.5);
+      }
+
+      matrix(row, col) = y;
+    }
+  }
+}
+
 } // namespace ens
 
 #endif
diff --git a/tests/cmaes_test.cpp b/tests/cmaes_test.cpp
index 45c7f1722..054c31c47 100644
--- a/tests/cmaes_test.cpp
+++ b/tests/cmaes_test.cpp
@@ -50,7 +50,7 @@ TEST_CASE("CMAESLogisticRegressionTest", "[CMAESTest]")
         responses, testResponses, shuffledResponses);
     LogisticRegression<> lr(shuffledData, shuffledResponses, 0.5);
 
-    CMAES<> cmaes(0, -1, 1, 32, 200, 1e-3);
+    CMAES<> cmaes(0, -20, 20, 32, 200, 1e-3);
     arma::mat coordinates = lr.GetInitialPoint();
     cmaes.Optimize(lr, coordinates);
 
@@ -85,7 +85,7 @@ TEST_CASE("ApproxCMAESLogisticRegressionTest", "[CMAESTest]")
         responses, testResponses, shuffledResponses);
     LogisticRegression<> lr(shuffledData, shuffledResponses, 0.5);
 
-    ApproxCMAES<> cmaes(0, -1, 1, 32, 200, 1e-3);
+    ApproxCMAES<> cmaes(0, -20, 20, 32, 200, 1e-3);
     arma::mat coordinates = lr.GetInitialPoint();
     cmaes.Optimize(lr, coordinates);
 
@@ -120,7 +120,7 @@ TEST_CASE("CMAESLogisticRegressionFMatTest", "[CMAESTest]")
         responses, testResponses, shuffledResponses);
     LogisticRegression<arma::fmat> lr(shuffledData, shuffledResponses, 0.5);
 
-    CMAES<> cmaes(0, -1, 1, 32, 200, 1e-3);
+    CMAES<> cmaes(0, -20, 20, 32, 200, 1e-3);
     arma::fmat coordinates = lr.GetInitialPoint();
     cmaes.Optimize(lr, coordinates);
 
@@ -155,7 +155,7 @@ TEST_CASE("ApproxCMAESLogisticRegressionFMatTest", "[CMAESTest]")
         responses, testResponses, shuffledResponses);
     LogisticRegression<arma::fmat> lr(shuffledData, shuffledResponses, 0.5);
 
-    ApproxCMAES<> cmaes(0, -1, 1, 32, 200, 1e-3);
+    ApproxCMAES<> cmaes(0, -20, 20, 32, 200, 1e-3);
     arma::fmat coordinates = lr.GetInitialPoint();
     cmaes.Optimize(lr, coordinates);
 
@@ -169,6 +169,3 @@ TEST_CASE("ApproxCMAESLogisticRegressionFMatTest", "[CMAESTest]")
       break;
     }
   }
-
-  REQUIRE(success == true);
-}

From c0a5f28c652174068967f85a830f7547a30d3c8c Mon Sep 17 00:00:00 2001
From: Gaurav Singh <gs8763076@gmail.com>
Date: Sat, 4 Apr 2020 17:21:02 +0530
Subject: [PATCH 02/17] Changes in cmaes test.

---
 tests/cmaes_test.cpp | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/cmaes_test.cpp b/tests/cmaes_test.cpp
index 054c31c47..a5c5f2a60 100644
--- a/tests/cmaes_test.cpp
+++ b/tests/cmaes_test.cpp
@@ -169,3 +169,6 @@ TEST_CASE("ApproxCMAESLogisticRegressionFMatTest", "[CMAESTest]")
       break;
     }
   }
+
+  REQUIRE(success == true);
+}

From 1e848fd585f88b4670ff6f985354acfa5f3189d8 Mon Sep 17 00:00:00 2001
From: Gaurav Singh <gs8763076@gmail.com>
Date: Wed, 8 Apr 2020 12:25:43 +0530
Subject: [PATCH 03/17] Update include/ensmallen_bits/cmaes/cmaes_impl.hpp

Co-Authored-By: Marcus Edel <marcus.edel@fu-berlin.de>
---
 include/ensmallen_bits/cmaes/cmaes_impl.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/include/ensmallen_bits/cmaes/cmaes_impl.hpp b/include/ensmallen_bits/cmaes/cmaes_impl.hpp
index cbb051d2b..91b87a915 100644
--- a/include/ensmallen_bits/cmaes/cmaes_impl.hpp
+++ b/include/ensmallen_bits/cmaes/cmaes_impl.hpp
@@ -359,7 +359,7 @@ void CMAES<SelectionPolicyType>::BoundaryTransform(BaseMatType& matrix)
       {
         y -= (ElemType)(2 * (y - upperBound - au));
       }
-      //boundary transformation
+      // Boundary transformation.
       if (y < lowerBound + al)
       {
         y = (ElemType)(lowerBound + (y - (lowerBound - al)) *

From 2e503f16d8c5445375f6905b2294c993bbb79ba1 Mon Sep 17 00:00:00 2001
From: Gaurav Singh <gs8763076@gmail.com>
Date: Wed, 8 Apr 2020 12:25:57 +0530
Subject: [PATCH 04/17] Update include/ensmallen_bits/cmaes/cmaes_impl.hpp

Co-Authored-By: Marcus Edel <marcus.edel@fu-berlin.de>
---
 include/ensmallen_bits/cmaes/cmaes_impl.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/include/ensmallen_bits/cmaes/cmaes_impl.hpp b/include/ensmallen_bits/cmaes/cmaes_impl.hpp
index 91b87a915..24289d61e 100644
--- a/include/ensmallen_bits/cmaes/cmaes_impl.hpp
+++ b/include/ensmallen_bits/cmaes/cmaes_impl.hpp
@@ -339,7 +339,7 @@ void CMAES<SelectionPolicyType>::BoundaryTransform(BaseMatType& matrix)
 
   for (size_t col=0; col < matrix.n_cols; col++)
   {
-    for (size_t row=0; row < matrix.n_rows; row++)
+    for (size_t row = 0; row < matrix.n_rows; row++)
     {
       ElemType y = matrix(row, col);
       //boundary transformation shift into feasible preimage.

From 93146c50db2cbcdef318693fa7abbaf7fc1facdf Mon Sep 17 00:00:00 2001
From: Gaurav Singh <gs8763076@gmail.com>
Date: Wed, 8 Apr 2020 12:26:07 +0530
Subject: [PATCH 05/17] Update include/ensmallen_bits/cmaes/cmaes_impl.hpp

Co-Authored-By: Marcus Edel <marcus.edel@fu-berlin.de>
---
 include/ensmallen_bits/cmaes/cmaes_impl.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/include/ensmallen_bits/cmaes/cmaes_impl.hpp b/include/ensmallen_bits/cmaes/cmaes_impl.hpp
index 24289d61e..ac87f54b3 100644
--- a/include/ensmallen_bits/cmaes/cmaes_impl.hpp
+++ b/include/ensmallen_bits/cmaes/cmaes_impl.hpp
@@ -337,7 +337,7 @@ void CMAES<SelectionPolicyType>::BoundaryTransform(BaseMatType& matrix)
   double xup = upperBound + 2 * au + diff;
   double r = 2 * (upperBound - lowerBound + al + au);
 
-  for (size_t col=0; col < matrix.n_cols; col++)
+  for (size_t col = 0; col < matrix.n_cols; col++)
   {
     for (size_t row = 0; row < matrix.n_rows; row++)
     {

From 1edbfe78c771505390b65f1a509da27bd298f870 Mon Sep 17 00:00:00 2001
From: Gaurav Singh <gs8763076@gmail.com>
Date: Wed, 8 Apr 2020 12:26:24 +0530
Subject: [PATCH 06/17] Update include/ensmallen_bits/cmaes/cmaes_impl.hpp

Co-Authored-By: Marcus Edel <marcus.edel@fu-berlin.de>
---
 include/ensmallen_bits/cmaes/cmaes_impl.hpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/include/ensmallen_bits/cmaes/cmaes_impl.hpp b/include/ensmallen_bits/cmaes/cmaes_impl.hpp
index ac87f54b3..b7d78367d 100644
--- a/include/ensmallen_bits/cmaes/cmaes_impl.hpp
+++ b/include/ensmallen_bits/cmaes/cmaes_impl.hpp
@@ -374,7 +374,6 @@ void CMAES<SelectionPolicyType>::BoundaryTransform(BaseMatType& matrix)
       matrix(row, col) = y;
     }
   }
-  
 }
 
 // Computes the inverse of the transformation.

From 7de865bb8f3f6b3a6771461490ee3c952392ce2f Mon Sep 17 00:00:00 2001
From: Gaurav Singh <gs8763076@gmail.com>
Date: Wed, 8 Apr 2020 12:26:45 +0530
Subject: [PATCH 07/17] Update include/ensmallen_bits/cmaes/cmaes_impl.hpp

Co-Authored-By: Marcus Edel <marcus.edel@fu-berlin.de>
---
 include/ensmallen_bits/cmaes/cmaes_impl.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/include/ensmallen_bits/cmaes/cmaes_impl.hpp b/include/ensmallen_bits/cmaes/cmaes_impl.hpp
index b7d78367d..624a2c70d 100644
--- a/include/ensmallen_bits/cmaes/cmaes_impl.hpp
+++ b/include/ensmallen_bits/cmaes/cmaes_impl.hpp
@@ -342,7 +342,7 @@ void CMAES<SelectionPolicyType>::BoundaryTransform(BaseMatType& matrix)
     for (size_t row = 0; row < matrix.n_rows; row++)
     {
       ElemType y = matrix(row, col);
-      //boundary transformation shift into feasible preimage.
+      // Boundary transformation shift into feasible pre-image.
       if (y < xlow)
       {
         y += (ElemType)(r * (1 + (xlow - y) / r));

From 659d0c68545e2f297b651053133a42cc8c07c5d2 Mon Sep 17 00:00:00 2001
From: Gaurav Singh <gs8763076@gmail.com>
Date: Wed, 8 Apr 2020 13:17:19 +0530
Subject: [PATCH 08/17] Changes in implementation.

---
 include/ensmallen_bits/cmaes/cmaes_impl.hpp | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/include/ensmallen_bits/cmaes/cmaes_impl.hpp b/include/ensmallen_bits/cmaes/cmaes_impl.hpp
index 624a2c70d..e6262779b 100644
--- a/include/ensmallen_bits/cmaes/cmaes_impl.hpp
+++ b/include/ensmallen_bits/cmaes/cmaes_impl.hpp
@@ -330,12 +330,12 @@ template<typename SelectionPolicyType>
 template<typename ElemType, typename BaseMatType>
 void CMAES<SelectionPolicyType>::BoundaryTransform(BaseMatType& matrix)
 {
-  double diff = (upperBound - lowerBound) / 2.0;
-  double al = std::min(diff, (1 + std::abs(lowerBound)) / 20.0);
-  double au = std::min(diff, (1 + std::abs(upperBound)) / 20.0);
-  double xlow = lowerBound - 2 * al - diff;
-  double xup = upperBound + 2 * au + diff;
-  double r = 2 * (upperBound - lowerBound + al + au);
+  const double diff = (upperBound - lowerBound) / 2.0;
+  const double al = std::min(diff, (1 + std::abs(lowerBound)) / 20.0);
+  const double au = std::min(diff, (1 + std::abs(upperBound)) / 20.0);
+  const double xlow = lowerBound - 2 * al - diff;
+  const double xup = upperBound + 2 * au + diff;
+  const double r = 2 * (upperBound - lowerBound + al + au);
 
   for (size_t col = 0; col < matrix.n_cols; col++)
   {
@@ -347,15 +347,15 @@ void CMAES<SelectionPolicyType>::BoundaryTransform(BaseMatType& matrix)
       {
         y += (ElemType)(r * (1 + (xlow - y) / r));
       }
-      if (y > xup)
+      else if (y > xup)
       {
         y -= (ElemType)(r * (1 + (y - xup) / r));
       }
-      if (y < lowerBound - al)
+      else if (y < lowerBound - al)
       {
         y += (ElemType)(2 * (lowerBound - al - y));
       }
-      if (y > upperBound + au)
+      else if (y > upperBound + au)
       {
         y -= (ElemType)(2 * (y - upperBound - au));
       }

From 748bbd421cdd03f1b301967498d8ba9cf39eabb1 Mon Sep 17 00:00:00 2001
From: Gaurav Singh <gs8763076@gmail.com>
Date: Wed, 8 Apr 2020 14:06:25 +0530
Subject: [PATCH 09/17] Minor style fixes.

---
 include/ensmallen_bits/cmaes/cmaes_impl.hpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/include/ensmallen_bits/cmaes/cmaes_impl.hpp b/include/ensmallen_bits/cmaes/cmaes_impl.hpp
index e6262779b..2f13c7507 100644
--- a/include/ensmallen_bits/cmaes/cmaes_impl.hpp
+++ b/include/ensmallen_bits/cmaes/cmaes_impl.hpp
@@ -385,9 +385,9 @@ void CMAES<SelectionPolicyType>::BoundaryTransformInverse(BaseMatType& matrix)
   double al = std::min(diff, (1 + std::abs(lowerBound)) / 20.0);
   double au = std::min(diff, (1 + std::abs(upperBound)) / 20.0);
 
-  for (size_t col=0; col < matrix.n_cols; col++)
+  for (size_t col = 0; col < matrix.n_cols; col++)
   {
-    for (size_t row=0; row < matrix.n_rows; row++)
+    for (size_t row = 0; row < matrix.n_rows; row++)
     {
       ElemType y = matrix(row, col);
       

From 9ad285213ca77feeed17820f4d2ad8b429a2d096 Mon Sep 17 00:00:00 2001
From: Gaurav Singh <gs8763076@gmail.com>
Date: Wed, 8 Apr 2020 14:13:58 +0530
Subject: [PATCH 10/17] Changes in inverse implementation.

---
 include/ensmallen_bits/cmaes/cmaes_impl.hpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/include/ensmallen_bits/cmaes/cmaes_impl.hpp b/include/ensmallen_bits/cmaes/cmaes_impl.hpp
index 2f13c7507..d20b9a8d9 100644
--- a/include/ensmallen_bits/cmaes/cmaes_impl.hpp
+++ b/include/ensmallen_bits/cmaes/cmaes_impl.hpp
@@ -381,9 +381,9 @@ template<typename SelectionPolicyType>
 template<typename ElemType, typename BaseMatType>
 void CMAES<SelectionPolicyType>::BoundaryTransformInverse(BaseMatType& matrix)
 {
-  double diff = (upperBound - lowerBound) / 2.0;
-  double al = std::min(diff, (1 + std::abs(lowerBound)) / 20.0);
-  double au = std::min(diff, (1 + std::abs(upperBound)) / 20.0);
+  const double diff = (upperBound - lowerBound) / 2.0;
+  const double al = std::min(diff, (1 + std::abs(lowerBound)) / 20.0);
+  const double au = std::min(diff, (1 + std::abs(upperBound)) / 20.0);
 
   for (size_t col = 0; col < matrix.n_cols; col++)
   {

From 63aed56f8ca674b598e6872caaa56c45693ed456 Mon Sep 17 00:00:00 2001
From: Gaurav Singh <gs8763076@gmail.com>
Date: Sun, 10 May 2020 00:55:55 +0530
Subject: [PATCH 11/17] Changes in Documentation and some other minor changes.

---
 doc/optimizers.md                           |  4 +++-
 include/ensmallen_bits/cmaes/cmaes.hpp      |  4 ++--
 include/ensmallen_bits/cmaes/cmaes_impl.hpp | 12 +++++++-----
 3 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/doc/optimizers.md b/doc/optimizers.md
index fa5a2007f..351bf21eb 100644
--- a/doc/optimizers.md
+++ b/doc/optimizers.md
@@ -584,6 +584,7 @@ matrix within an iterative procedure using the covariance matrix.
  * `CMAES<`_`SelectionPolicyType`_`>(`_`lambda, lowerBound, upperBound`_`)`
  * `CMAES<`_`SelectionPolicyType`_`>(`_`lambda, lowerBound, upperBound, batchSize`_`)`
  * `CMAES<`_`SelectionPolicyType`_`>(`_`lambda, lowerBound, upperBound, batchSize, maxIterations, tolerance, selectionPolicy`_`)`
+ * `CMAES<`_`SelectionPolicyType`_`>(`_`lambda, lowerBound, upperBound, batchSize, maxIterations, tolerance, selectionPolicy, initialSigma`_`)`
 
 The _`SelectionPolicyType`_ template parameter refers to the strategy used to
 compute the (approximate) objective function.  The `FullSelection` and
@@ -606,10 +607,11 @@ For convenience the following types can be used:
 | `size_t` | **`maxIterations`** | Maximum number of iterations. | `1000` |
 | `double` | **`tolerance`** | Maximum absolute tolerance to terminate algorithm. | `1e-5` |
 | `SelectionPolicyType` | **`selectionPolicy`** | Instantiated selection policy used to calculate the objective. | `SelectionPolicyType()` |
+| `double` | **`initialSigma`** | The initial step size. | `0.6` |
 
 Attributes of the optimizer may also be changed via the member methods
 `Lambda()`, `LowerBound()`, `UpperBound()`, `BatchSize()`, `MaxIterations()`,
-`Tolerance()`, and `SelectionPolicy()`.
+`Tolerance()`, `SelectionPolicy()` and `InitialSigma()`.
 
 The `selectionPolicy` attribute allows an instantiated `SelectionPolicyType` to
 be given.  The `FullSelection` policy has no need to be instantiated and thus
diff --git a/include/ensmallen_bits/cmaes/cmaes.hpp b/include/ensmallen_bits/cmaes/cmaes.hpp
index 9fbe0b83f..c182c958a 100644
--- a/include/ensmallen_bits/cmaes/cmaes.hpp
+++ b/include/ensmallen_bits/cmaes/cmaes.hpp
@@ -165,9 +165,9 @@ class CMAES
   double initialSigma;
 
   //! Methods used to transform the candidates into the constraints.
-  template<typename ElemType, typename BaseMatType>
+  template<typename BaseMatType>
   void BoundaryTransform(BaseMatType& matrix);
-  template<typename ElemType, typename BaseMatType>
+  template<typename BaseMatType>
   void BoundaryTransformInverse(BaseMatType& matrix);
 };
 
diff --git a/include/ensmallen_bits/cmaes/cmaes_impl.hpp b/include/ensmallen_bits/cmaes/cmaes_impl.hpp
index d20b9a8d9..d7251fbc9 100644
--- a/include/ensmallen_bits/cmaes/cmaes_impl.hpp
+++ b/include/ensmallen_bits/cmaes/cmaes_impl.hpp
@@ -115,7 +115,7 @@ typename MatType::elem_type CMAES<SelectionPolicyType>::Optimize(
   step.zeros();
 
   // Calculate the first objective function.
-  BoundaryTransform<ElemType, BaseMatType>(mPosition[0]);
+  BoundaryTransform<BaseMatType>(mPosition[0]);
   ElemType currentObjective = 0;
   for (size_t f = 0; f < numFunctions; f += batchSize)
   {
@@ -187,7 +187,7 @@ typename MatType::elem_type CMAES<SelectionPolicyType>::Optimize(
       pPosition[idx(j)] = mPosition[idx0] + sigma(idx0) * pStep[idx(j)];
 
       // Calculate the objective function.
-      BoundaryTransform<ElemType, BaseMatType>(pPosition[idx(j)]);
+      BoundaryTransform<BaseMatType>(pPosition[idx(j)]);
       pObjective(idx(j)) = selectionPolicy.Select(function, batchSize,
           pPosition[idx(j)], callbacks...);
     }
@@ -202,7 +202,7 @@ typename MatType::elem_type CMAES<SelectionPolicyType>::Optimize(
     mPosition[idx1] = mPosition[idx0] + sigma(idx0) * step;
 
     // Calculate the objective function.
-    BoundaryTransform<ElemType, BaseMatType>(mPosition[idx1]);
+    BoundaryTransform<BaseMatType>(mPosition[idx1]);
     currentObjective = selectionPolicy.Select(function, batchSize,
         mPosition[idx1], callbacks...);
 
@@ -327,9 +327,10 @@ typename MatType::elem_type CMAES<SelectionPolicyType>::Optimize(
 
 // Transforms the candidate into the given bounds.
 template<typename SelectionPolicyType>
-template<typename ElemType, typename BaseMatType>
+template<typename BaseMatType>
 void CMAES<SelectionPolicyType>::BoundaryTransform(BaseMatType& matrix)
 {
+  typedef typename BaseMatType::elem_type ElemType;
   const double diff = (upperBound - lowerBound) / 2.0;
   const double al = std::min(diff, (1 + std::abs(lowerBound)) / 20.0);
   const double au = std::min(diff, (1 + std::abs(upperBound)) / 20.0);
@@ -378,9 +379,10 @@ void CMAES<SelectionPolicyType>::BoundaryTransform(BaseMatType& matrix)
 
 // Computes the inverse of the transformation.
 template<typename SelectionPolicyType>
-template<typename ElemType, typename BaseMatType>
+template<typename BaseMatType>
 void CMAES<SelectionPolicyType>::BoundaryTransformInverse(BaseMatType& matrix)
 {
+  typedef typename BaseMatType::elem_type ElemType;
   const double diff = (upperBound - lowerBound) / 2.0;
   const double al = std::min(diff, (1 + std::abs(lowerBound)) / 20.0);
   const double au = std::min(diff, (1 + std::abs(upperBound)) / 20.0);

From 881de8a308dad2013e443a79a2cba79e30d6159e Mon Sep 17 00:00:00 2001
From: Gaurav Singh <gs8763076@gmail.com>
Date: Sun, 10 May 2020 01:10:08 +0530
Subject: [PATCH 12/17] Changes in HISTORY.md

---
 HISTORY.md | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/HISTORY.md b/HISTORY.md
index 45a1931c1..4aa0fc5cc 100644
--- a/HISTORY.md
+++ b/HISTORY.md
@@ -3,6 +3,9 @@
  * Fix CMake package export
     ([#198](https://github.com/mlpack/ensmallen/pull/198)).
 
+ * Fix CMA-ES inconsistencies
+    ([#193](https://github.com/mlpack/ensmallen/pull/193)).
+
 ### ensmallen 2.12.1: "Stir Crazy"
 ###### 2020-04-20
  * Fix total number of epochs and time estimation for ProgressBar callback
@@ -20,7 +23,7 @@
     ([#183](https://github.com/mlpack/ensmallen/pull/183)).
 
  * Remove deprecated methods from PrimalDualSolver implementation
-    ([#185](https://github.com/mlpack/ensmallen/pull/185).
+    ([#185](https://github.com/mlpack/ensmallen/pull/185)).
 
  * Update logo ([#186](https://github.com/mlpack/ensmallen/pull/186)).
 

From 2ec0daa9056d1b48de89f25274cf58abb1ac3756 Mon Sep 17 00:00:00 2001
From: Gaurav Singh <gs8763076@gmail.com>
Date: Thu, 14 May 2020 23:08:29 +0530
Subject: [PATCH 13/17] Changing pow to sqrt.

---
 include/ensmallen_bits/cmaes/cmaes_impl.hpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/include/ensmallen_bits/cmaes/cmaes_impl.hpp b/include/ensmallen_bits/cmaes/cmaes_impl.hpp
index d7251fbc9..f8592ffe3 100644
--- a/include/ensmallen_bits/cmaes/cmaes_impl.hpp
+++ b/include/ensmallen_bits/cmaes/cmaes_impl.hpp
@@ -396,12 +396,12 @@ void CMAES<SelectionPolicyType>::BoundaryTransformInverse(BaseMatType& matrix)
       if (y < lowerBound + al)
       {
         y =  (ElemType)(lowerBound - al) + 2 *
-              std::pow(std::abs(al * (y - lowerBound)),0.5);
+              std::sqrt(std::abs(al * (y - lowerBound));
       }
       else if (y > upperBound - au)
       {
         y = (ElemType)(upperBound + au) - 2 *
-             std::pow(std::abs(au * (upperBound - y)),0.5);
+             std::sqrt(std::abs(au * (upperBound - y));
       }
 
       matrix(row, col) = y;

From c5bebd63bda6c011887b997cdc98a33d3fafa215 Mon Sep 17 00:00:00 2001
From: Gaurav Singh <gs8763076@gmail.com>
Date: Fri, 15 May 2020 13:56:16 +0530
Subject: [PATCH 14/17] Changing ens::Warn to assert.

---
 include/ensmallen_bits/cmaes/cmaes_impl.hpp | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/include/ensmallen_bits/cmaes/cmaes_impl.hpp b/include/ensmallen_bits/cmaes/cmaes_impl.hpp
index f8592ffe3..72ea59181 100644
--- a/include/ensmallen_bits/cmaes/cmaes_impl.hpp
+++ b/include/ensmallen_bits/cmaes/cmaes_impl.hpp
@@ -17,6 +17,7 @@
 
 // In case it hasn't been included yet.
 #include "cmaes.hpp"
+#include <assert.h>
 
 #include <ensmallen_bits/function.hpp>
 
@@ -40,10 +41,8 @@ CMAES<SelectionPolicyType>::CMAES(const size_t lambda,
     selectionPolicy(selectionPolicy),
     initialSigma(initialSigma)
 {
-  if (lowerBound == upperBound)
-  {
-    Warn << "Lower and upper bounds must be different." << std::endl;
-  }
+  assert(this->lowerBound != this->upperBound && "The values of "
+        "lowerbound and upperbound must be different.");
 }
 
 //! Optimize the function (minimize).
@@ -396,12 +395,12 @@ void CMAES<SelectionPolicyType>::BoundaryTransformInverse(BaseMatType& matrix)
       if (y < lowerBound + al)
       {
         y =  (ElemType)(lowerBound - al) + 2 *
-              std::sqrt(std::abs(al * (y - lowerBound));
+              std::sqrt(std::abs(al * (y - lowerBound)));
       }
       else if (y > upperBound - au)
       {
         y = (ElemType)(upperBound + au) - 2 *
-             std::sqrt(std::abs(au * (upperBound - y));
+             std::sqrt(std::abs(au * (upperBound - y)));
       }
 
       matrix(row, col) = y;

From 89fe19c8f7a737cfa1e8e00f8c481cd756f4f050 Mon Sep 17 00:00:00 2001
From: Gaurav Singh <gs8763076@gmail.com>
Date: Fri, 15 May 2020 17:04:32 +0530
Subject: [PATCH 15/17] Update include/ensmallen_bits/cmaes/cmaes.hpp

Co-authored-by: favre49 <40389657+favre49@users.noreply.github.com>
---
 include/ensmallen_bits/cmaes/cmaes.hpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/include/ensmallen_bits/cmaes/cmaes.hpp b/include/ensmallen_bits/cmaes/cmaes.hpp
index c182c958a..a0e059089 100644
--- a/include/ensmallen_bits/cmaes/cmaes.hpp
+++ b/include/ensmallen_bits/cmaes/cmaes.hpp
@@ -66,7 +66,7 @@ class CMAES
    * @param maxIterations Maximum number of iterations allowed (0 means no
    *     limit).
    * @param tolerance Maximum absolute tolerance to terminate algorithm.
-   * @param initialSigma The initial step size
+   * @param initialSigma The initial step size.
    * @param selectionPolicy Instantiated selection policy used to calculate the
    *     objective.
    */
@@ -182,4 +182,4 @@ using ApproxCMAES = CMAES<SelectionPolicyType>;
 // Include implementation.
 #include "cmaes_impl.hpp"
 
-#endif
\ No newline at end of file
+#endif

From 2b86192343369fac3a32c072f2be03f26d072125 Mon Sep 17 00:00:00 2001
From: Gaurav Singh <gs8763076@gmail.com>
Date: Sun, 24 May 2020 17:19:30 +0530
Subject: [PATCH 16/17] Adding boundary transformation test.

---
 include/ensmallen_bits/cmaes/cmaes_impl.hpp |  2 +-
 tests/cmaes_test.cpp                        | 23 +++++++++++++++++++++
 2 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/include/ensmallen_bits/cmaes/cmaes_impl.hpp b/include/ensmallen_bits/cmaes/cmaes_impl.hpp
index 72ea59181..88db35623 100644
--- a/include/ensmallen_bits/cmaes/cmaes_impl.hpp
+++ b/include/ensmallen_bits/cmaes/cmaes_impl.hpp
@@ -42,7 +42,7 @@ CMAES<SelectionPolicyType>::CMAES(const size_t lambda,
     initialSigma(initialSigma)
 {
   assert(this->lowerBound != this->upperBound && "The values of "
-        "lowerbound and upperbound must be different.");
+        "lower bound and upper bound must be different.");
 }
 
 //! Optimize the function (minimize).
diff --git a/tests/cmaes_test.cpp b/tests/cmaes_test.cpp
index a5c5f2a60..2a307cffb 100644
--- a/tests/cmaes_test.cpp
+++ b/tests/cmaes_test.cpp
@@ -172,3 +172,26 @@ TEST_CASE("ApproxCMAESLogisticRegressionFMatTest", "[CMAESTest]")
 
   REQUIRE(success == true);
 }
+
+/**
+ * Test to check if the boundary transformation
+ * works as expected.
+ */
+TEST_CASE("BoundaryConditionTestFunction", "[CMAESTest]")
+{
+  SGDTestFunction f;
+  CMAES<> optimizer(0, -10, 10, 32, 200, -1);
+
+  arma::mat coordinates = f.GetInitialPoint();
+  optimizer.Optimize(f, coordinates);
+
+  for (size_t col = 0; col < coordinates.n_cols; col++)
+  {
+    for (size_t row = 0; row < coordinates.n_rows; row++)
+    {
+      bool success = coordinates(row, col) <= 10 && 
+                     coordinates(row, col) >= -10;
+      REQUIRE(success == true);
+    }
+  }
+}

From 486930ae44ccf238de7b118cc38d1a0efd6ebec4 Mon Sep 17 00:00:00 2001
From: Gaurav Singh <gs8763076@gmail.com>
Date: Sun, 4 Oct 2020 14:25:26 +0530
Subject: [PATCH 17/17] Changed the test function from SGD to Logistic
 Regression.

---
 tests/cmaes_test.cpp | 31 ++++++++++++++++++++-----------
 1 file changed, 20 insertions(+), 11 deletions(-)

diff --git a/tests/cmaes_test.cpp b/tests/cmaes_test.cpp
index 2a307cffb..bbbf193ba 100644
--- a/tests/cmaes_test.cpp
+++ b/tests/cmaes_test.cpp
@@ -179,19 +179,28 @@ TEST_CASE("ApproxCMAESLogisticRegressionFMatTest", "[CMAESTest]")
  */
 TEST_CASE("BoundaryConditionTestFunction", "[CMAESTest]")
 {
-  SGDTestFunction f;
-  CMAES<> optimizer(0, -10, 10, 32, 200, -1);
+  const size_t trials = 3;
+  for(size_t trail=0; trail < trials; ++trail)
+  {
+    arma::mat data, testData, shuffledData;
+    arma::Row<size_t> responses, testResponses, shuffledResponses;
 
-  arma::mat coordinates = f.GetInitialPoint();
-  optimizer.Optimize(f, coordinates);
+    LogisticRegressionTestData(data, testData, shuffledData, 
+         responses, testResponses, shuffledResponses);
+    LogisticRegression<> lr(shuffledData, shuffledResponses, 0.5);
 
-  for (size_t col = 0; col < coordinates.n_cols; col++)
-  {
-    for (size_t row = 0; row < coordinates.n_rows; row++)
+    CMAES<> optimizer(0, -10, 10, 32, 200, -1);
+    arma::mat coordinates = lr.GetInitialPoint();
+    optimizer.Optimize(lr, coordinates);
+
+    for (size_t col = 0; col < coordinates.n_cols; col++)
     {
-      bool success = coordinates(row, col) <= 10 && 
-                     coordinates(row, col) >= -10;
-      REQUIRE(success == true);
+      for (size_t row = 0; row < coordinates.n_rows; row++)
+      {
+        bool success = coordinates(row, col) <= 10 && 
+                       coordinates(row, col) >= -10;
+        REQUIRE(success == true);
+      }
     }
-  }
+  }  
 }