implement evaluateBestParameter(): evaluate the best parameters found…

… during optimization on the test set, code refactoring

- change type of observed values
- restrict objectives to error and accuracy
- returns the best parameter after the optimization run has finished
- implement evaluateBestParameter(): evaluate the best parameters that were found during optimization on the test set
- fix path in runExampleAutotuner (use relative path)
- remove some TODOs
- update README
- some code refactoring

README.md

@@ -10,9 +10,9 @@ import { TensorflowlModelAutotuner } from '@piximi/autotuner';
 
 ### Getting Started
 
-Initialize the autotuner by specifying metrics and a dataset.
+Initialize the autotuner by specifying metrics, a dataset and the number of categories.
 ```javascript
-var autotuner = new TensorflowlModelAutotuner(['accuracy'], dataset.trainData, dataset.testData, dataset.validationData);
+var autotuner = new TensorflowlModelAutotuner(['accuracy'], dataset, numberOfCategories);
 ```
 ### Adding a model to the autotuner
 ```javascript
@@ -24,27 +24,32 @@ autotuner.addModel('testModel', testModel, parameters);
 ```
 
 ### Tuning the hyperparameters
-Specify the optimization algorith. The hyperparameters can be tuned by either doing bayesian optimization or by doing a simple grid search.
+Specify the optimization algorithm: the hyperparameters can be tuned by either doing bayesian optimization or by doing a simple grid search.
 ```javascript
 autotuner.bayesianOptimization();
 ```
 ```javascript
 autotuner.gridSearchOptimizytion();
 ```
-#### Optinal parameters
-The ojective function of the optimization can be specified (either 'error' or 'accuracy').
+The ojective function of the optimization can be specified (either 'error' or 'accuracy'):
 ```javascript
 autotuner.gridSearchOptimizytion('accuracy');
 ```
-Also one can enable cross validation when evaluating a model.
+Evaluating a model can be done using cross validation:
 ```javascript
 autotuner.gridSearchOptimizytion('accuracy', true);
 ```
-When doing bayesian optimization the maximum number of domain points to be evaluated can be specified as an optional parameter.
+When doing bayesian optimization the maximum number of domain points to be evaluated can be specified as an optional parameter:
 ```javascript
 autotuner.bayesianOptimization('accuracy', true, 0.8);
 ```
-In the example above the optimizytion search stops after 80% of the domain ponits have been evaluated. By default this value is set to 0.75.  
+In the example above the optimizytion stops after 80% of the domain ponits have been evaluated. By default this value is set to 0.75.  
+### Evaluate the best hyperparameters
+The best hyperparameters found in the optimization run can be evaluated on the test set. Specify the objective and wheter or not to use cross validation.
+```javascript
+autotuner.evaluateBestParameter('error', true);
+```
+
 ### Example
 An example usage can be found here:
 ```bash
@@ -54,17 +59,18 @@ tets/runExampleAutotuner.ts
 
 Pull and initialize:
 ```bash
-git pull https://github.com/piximi/autotuner.git
+git clone https://github.com/piximi/autotuner.git
 cd autotuner
 npm install
 ```
 
 To run tests:
 ```bash
-npm test
+npm run test
+npm run runExampleAutotuner
 ```
 
 To compile the code and check for type errors:
 ```bash
-npm build
+npm run build
 ```

src/autotuner.ts

@@ -1,14 +1,15 @@
 import * as tensorflow from '@tensorflow/tfjs';
-import { ModelDict, SequentialModelParameters, DataPoint, BaysianOptimisationStep, LossFunction, DomainPointValue } from '../types/types';
+import { ModelDict, SequentialModelParameters, DataPoint, BaysianOptimisationStep, LossFunction, ObservedValues, DomainPointValue } from '../types/types';
 import * as bayesianOptimizer from './bayesianOptimizer';
 import * as gridSearchOptimizer from './gridSearchOptimizer';
 import * as paramspace from './paramspace';
 import * as priors from './priors';
 import * as modelEvaluator from './modelEvaluater';
+import { argmax } from './util';
 
 class AutotunerBaseClass {
     metrics: string[] = [];
-    observedValues: DomainPointValue[] = [];
+    observedValues: ObservedValues = {};
     /**
      * Fraction of domain indices that should be evaluated at most
      */
@@ -18,7 +19,7 @@ class AutotunerBaseClass {
      * 
      * @return {boolean} false if tuning the hyperparameters should be stopped, true otherwise
      */
-    metricsStopingCriteria: (observedValues: DomainPointValue[]) => boolean;
+    metricsStopingCriteria: (observedValues: ObservedValues) => boolean;
     modelOptimizersDict: { [model: string]: tensorflow.Optimizer[] } = {};
     paramspace: any;
     optimizer: any;
@@ -31,7 +32,29 @@ class AutotunerBaseClass {
      * @param {number} domainIndex Index of the domain point to be evaluated.
      * @return {Promise<number>} Value of the domain point
      */
-    evaluateModel: (domainIndex: number, objective: string, useCrossValidation: boolean) => Promise<number>;
+    evaluateModel: (domainIndex: number, objective: string, useCrossValidation: boolean, useTestData: boolean) => Promise<number>;
+
+    bestParameter(objective: string) {
+        if (this.checkObjective(objective)) {
+            return
+        }
+        const observedValues = this.observedValues[objective];
+        const bestScoreIndex = argmax(observedValues);
+        const bestScoreDomainIndex = this.observedValues['domainIndices'][bestScoreIndex];
+        return bestScoreDomainIndex;
+    }
+
+    /**
+     * Tests the best parameters that were found during optimization on the test set.
+     * 
+     * @param {string} objective Define the metric that should be evaluated. Either 'error' or 'accuracy'
+     * @param {boolean} useCrossValidation Indicate wheter or not to use cross validation to evaluate the model. Set to 'false' by default.
+     * @return {number} Returns the score of.
+     */
+    async evaluateBestParameter(objective: string, useCrossValidation: boolean = false) {
+        var bestScoreDomainIndex = this.bestParameter(objective) as number;
+        return await this.evaluateModel(bestScoreDomainIndex, objective, useCrossValidation, true);
+    }
 
     /**
      * Decide whether to continue tuning the hyperparameters.
@@ -41,7 +64,7 @@ class AutotunerBaseClass {
      */
     stopingCriteria() {
         const domainSize = this.paramspace.domainIndices.length;
-        const numberOfObservedValues = this.observedValues.length;
+        const numberOfObservedValues = this.observedValues['domainIndices'].length;
         var fractionOfEvaluatedPoints = numberOfObservedValues / domainSize;
         var maxIterationsReached: boolean = fractionOfEvaluatedPoints <= this.maxIterations;
 
@@ -58,7 +81,7 @@ class AutotunerBaseClass {
     }
 
     checkObjective (objective: string): boolean {
-        const allowedObjectives: string[] = ['error'].concat(this.metrics);
+        const allowedObjectives = ['error', 'accuracy'];
         if (!allowedObjectives.includes(objective)) {
             console.log("Invalid objective function selected!");
             console.log("Objective function must be one of the following: " + allowedObjectives.join());
@@ -71,6 +94,9 @@ class AutotunerBaseClass {
         this.paramspace = new paramspace.Paramspace();
         this.modelEvaluator = new modelEvaluator.ModelEvaluater(dataSet, numberOfCategories, validationSetRatio, testSetRatio);
         this.metrics = metrics;
+        this.observedValues['domainIndices'] = [];
+        this.observedValues['error'] = [];
+        this.observedValues['accuracy'] = [];
     }
 
     /**
@@ -80,8 +106,9 @@ class AutotunerBaseClass {
      * @param {boolean} [useCrossValidation=false] Indicate wheter or not to use cross validation to evaluate the model. Set to 'false' by default.
      * @param {number} [maxIteration=0.75] Fraction of domain points that should be evaluated at most. (e.g. for 'maxIteration=0.75' the optimization stops if 75% of the domain has been evaluated)
      * @param {boolean} [stopingCriteria] Predicate on the observed values when to stop the optimization
+     * @return Returns the best parameters found in the optimization run.
      */
-    async bayesianOptimization(objective: string = 'error', useCrossValidation: boolean = false, maxIteration: number = 0.75, stopingCriteria?: ((observedValues: DomainPointValue[]) => boolean)) {
+    async bayesianOptimization(objective: string = 'error', useCrossValidation: boolean = false, maxIteration: number = 0.75, stopingCriteria?: ((observedValues: ObservedValues) => boolean)) {
         if (this.checkObjective(objective)) {
             return;
         }
@@ -92,14 +119,15 @@ class AutotunerBaseClass {
             this.metricsStopingCriteria = stopingCriteria;
         }
 
-        this.tuneHyperparameters(objective, useCrossValidation);
+        return await this.tuneHyperparameters(objective, useCrossValidation);
     }
 
     /**
      * Search the best Parameters using grid search.
      * 
      * @param {string} [objective='error'] Define the objective of the optimization. Set to 'error' by default.
      * @param {boolean} [useCrossValidation=false] Indicate wheter or not to use cross validation to evaluate the model. Set to 'false' by default.
+     * @return Returns the best parameters found in the optimization run.
      */
     async gridSearchOptimizytion(objective: string = 'error', useCrossValidation: boolean = false) {
         if (this.checkObjective(objective)) {
@@ -109,7 +137,7 @@ class AutotunerBaseClass {
         this.optimizer = new gridSearchOptimizer.Optimizer(this.paramspace.domainIndices, this.paramspace.modelsDomains);
         this.maxIterations = 1;
 
-        this.tuneHyperparameters(objective, useCrossValidation);
+        return await this.tuneHyperparameters(objective, useCrossValidation);
     }
 
 
@@ -123,7 +151,7 @@ class AutotunerBaseClass {
             var nextOptimizationPoint: BaysianOptimisationStep = this.optimizer.getNextPoint();
 
             // Train a model given the params and obtain a quality metric value.
-            var value = await this.evaluateModel(nextOptimizationPoint.nextPoint, objective, useCrossValidation);
+            var value = await this.evaluateModel(nextOptimizationPoint.nextPoint, objective, useCrossValidation, false);
 
             // Report the obtained quality metric value.
             this.optimizer.addSample(nextOptimizationPoint.nextPoint, value);
@@ -135,6 +163,12 @@ class AutotunerBaseClass {
 
         console.log("============================");
         console.log("finished tuning the hyperparameters");
+        console.log();
+        var bestScoreDomainIndex = this.bestParameter(objective) as number;
+        var bestParameters = this.paramspace.domain[bestScoreDomainIndex]['params'];
+        console.log("The best parameters found are:");
+        console.log(bestParameters);
+        return bestParameters;
     }
 }
 
@@ -153,7 +187,7 @@ class TensorflowlModelAutotuner extends AutotunerBaseClass {
     constructor(metrics: string[], dataSet: DataPoint[], numberOfCategories: number, validationSetRatio: number = 0.25, testSetRatio: number = 0.25) {
         super(metrics, dataSet, numberOfCategories, validationSetRatio, testSetRatio);
 
-        this.evaluateModel = async (point: number, objective: string, useCrossValidation: boolean) => {
+        this.evaluateModel = async (point: number, objective: string, useCrossValidation: boolean, useTestData: boolean = false) => {
             const modelIdentifier = this.paramspace.domain[point]['model'];
             const model = this.modelDict[modelIdentifier];
             const params = this.paramspace.domain[point]['params'];
@@ -169,13 +203,16 @@ class TensorflowlModelAutotuner extends AutotunerBaseClass {
                 metrics: metrics,
                 optimizer: optimizerFunction
             });
-
-            let dataPointValue: DomainPointValue = useCrossValidation 
-                ? await this.modelEvaluator.EvaluateSequentialTensorflowModelCV(model, args)
-                : await this.modelEvaluator.EvaluateSequentialTensorflowModel(model, args);
-            this.observedValues.push(dataPointValue);
-            return objective === 'error' ? dataPointValue.error : dataPointValue.metricScores[0];
-        }
+
+            let domainPointValue: DomainPointValue = useCrossValidation 
+                ? await this.modelEvaluator.EvaluateSequentialTensorflowModelCV(model, args, useTestData)
+                : await this.modelEvaluator.EvaluateSequentialTensorflowModel(model, args, useTestData);
+
+            this.observedValues['domainIndices'].push(point);
+            this.observedValues['error'].push(domainPointValue.error);
+            this.observedValues['accuracy'].push(domainPointValue.accuracy);
+            return objective === 'error' ? domainPointValue.error : 1 - domainPointValue.accuracy;
+        } 
     }
 
     /**

src/modelEvaluater.ts

@@ -30,40 +30,27 @@ export class ModelEvaluater{
         this.trainData = dataSet;
     }
 
-    ConcatenateTensorData(data: DataPoint[]) {
-        const trainData: tensorflow.Tensor<tensorflow.Rank>[] = [];
-        const trainLables: number[] = [];
-        for (let i = 0; i < data.length; i++) {
-            trainData.push(data[i].data);
-            trainLables.push(data[i].lables);
-        }
-
-        let concatenatedTensorData = tensorflow.tidy(() => tensorflow.concat(trainData));
-        let concatenatedLables = tensorflow.tidy(() => tensorflow.oneHot(trainLables, this.numberOfCategories));
-        return { concatenatedTensorData, concatenatedLables };
-    }
-
-    EvaluateSequentialTensorflowModel = async (model: tensorflow.Sequential, args: any): Promise<DomainPointValue> => {
+    EvaluateSequentialTensorflowModel = async (model: tensorflow.Sequential, args: any, useTestData: boolean): Promise<DomainPointValue> => {
         var trainData = this.ConcatenateTensorData(this.trainData);
         await model.fit(trainData.concatenatedTensorData, trainData.concatenatedLables, args);
 
-        var validationData = this.ConcatenateTensorData(this.validationData);
+        var validationData = useTestData ? this.ConcatenateTensorData(this.testData) : this.ConcatenateTensorData(this.validationData);
         const evaluationResult = model.evaluate(validationData.concatenatedTensorData, validationData.concatenatedLables) as tensorflow.Tensor[];
 
         const error = evaluationResult[0].dataSync()[0];
-        const score = evaluationResult[1].dataSync()[0];
-        return {error: error, metricScores: [score]} as DomainPointValue;
+        const accuracy = evaluationResult[1].dataSync()[0];
+        return {error: error, accuracy: accuracy} as DomainPointValue;
     }
 
-    EvaluateSequentialTensorflowModelCV = async (model: tensorflow.Sequential, args: any): Promise<DomainPointValue> => {
-        const dataSet = this.trainData.concat(this.validationData);
+    EvaluateSequentialTensorflowModelCV = async (model: tensorflow.Sequential, args: any, useTestData: boolean): Promise<DomainPointValue> => {
+        const dataSet = useTestData ? this.testData : this.trainData.concat(this.validationData);
         const dataSize = dataSet.length;
         const k = math.min(10, math.floor(math.nthRoot(dataSize) as number));
 
         const dataFolds: DataPoint[][] = Array.from(Array(math.ceil(dataSet.length/k)), (_,i) => dataSet.slice(i*k,i*k+k));
 
         var error = 0;
-        var score = 0;
+        var accuracy = 0;
         for (let i = 0; i < k; i++) {
             var validationData = dataFolds[i];
             var trainData: DataPoint[] = [];
@@ -77,16 +64,34 @@ export class ModelEvaluater{
             var concatenatedTrainData = this.ConcatenateTensorData(trainData);
             await model.fit(concatenatedTrainData.concatenatedTensorData, concatenatedTrainData.concatenatedLables, args);
 
-            var concatenatedValidationData = this.ConcatenateTensorData(validationData);
-            const evaluationResult = model.evaluate(concatenatedValidationData.concatenatedTensorData, concatenatedValidationData.concatenatedLables) as tensorflow.Tensor[];
-
-            const foldError = evaluationResult[0].dataSync()[0];
-            const foldScore = evaluationResult[1].dataSync()[0];
-            error += foldError;
-            score += foldScore;
+            var evaluationResult = await this.EvaluateTensorflowModel(model, validationData);
+            error += evaluationResult.error;
+            accuracy += evaluationResult.accuracy;
         }
-        return {error: error/dataSize, metricScores: [score/k]} as DomainPointValue;
+        return {error: error/dataSize, accuracy: accuracy/k} as DomainPointValue;
 
     }
 
+    ConcatenateTensorData = (data: DataPoint[]) => {
+        const trainData: tensorflow.Tensor<tensorflow.Rank>[] = [];
+        const trainLables: number[] = [];
+        for (let i = 0; i < data.length; i++) {
+            trainData.push(data[i].data);
+            trainLables.push(data[i].lables);
+        }
+
+        let concatenatedTensorData = tensorflow.tidy(() => tensorflow.concat(trainData));
+        let concatenatedLables = tensorflow.tidy(() => tensorflow.oneHot(trainLables, this.numberOfCategories));
+        return { concatenatedTensorData, concatenatedLables };
+    }
+
+    EvaluateTensorflowModel = async (model: tensorflow.Sequential, evaluationData: DataPoint[]) => {
+        var concatenatedEvaluationData = this.ConcatenateTensorData(evaluationData);
+        const evaluationResult = model.evaluate(concatenatedEvaluationData.concatenatedTensorData, concatenatedEvaluationData.concatenatedLables) as tensorflow.Tensor[];
+
+        const error = evaluationResult[0].dataSync()[0];
+        const accuracy = evaluationResult[1].dataSync()[0];
+        return {error: error, accuracy: accuracy} as DomainPointValue;
+    }
+
 }

src/priors.ts

@@ -62,7 +62,6 @@ class Priors {
                             }
                         }
                     }
-                    // TODO: test matrix operations and casts to matrix
                     cov /= (this.observedValues[point].length * this.observedValues[point2].length)
                     this.kernel = math.subset(this.kernel, math.index(idx, idx2), cov) as math.Matrix;
                     this.kernel = math.subset(this.kernel, math.index(idx2, idx), cov) as math.Matrix;

src/util.ts

@@ -18,7 +18,6 @@ const argmax = (array: number[]) => {
  * @param {Array.<number>} std Standard deviation values of the probability distribution over a given domain.
  * @return {Array.<number>} Values of the expected improvement for all points of the mean and std.
  */
-// TODO: change type of parameters, use math.matrix, adjust usages
 const expectedImprovement = (bestObjective: number, mean: math.Matrix, std: math.Matrix) => {
     var mean: math.Matrix;
     var std: math.Matrix;

tests/runExampleAutotuner.ts

@@ -5,20 +5,26 @@ import * as tensorflow from '@tensorflow/tfjs';
 import { Classifier } from '@piximi/types';
 
 const runExampleAutotuner = async () => {
-        var fs = require("fs");
-        var stringContent = fs.readFileSync("C:/Users/m_lev/Projects/BA/piximi/autotuner/tests/data/smallMNISTTest.piximi");
-        var classifier = JSON.parse(stringContent) as Classifier;
+    var path = require("path");
+    var fs = require("fs");
+    var testFilePath = path.resolve('tests', 'data', 'smallMNISTTest.piximi');
+    var stringContent = fs.readFileSync(testFilePath);
+    var classifier = JSON.parse(stringContent) as Classifier;
 
-        const dataset = await createDataset(classifier.categories, classifier.images);
+    const dataset = await createDataset(classifier.categories, classifier.images);
 
-        var autotuner = new TensorflowlModelAutotuner(['accuracy'], dataset.dataSet as DataPoint[], dataset.numberOfCategories);
-        
-        const testModel = await createModel();
+    var autotuner = new TensorflowlModelAutotuner(['accuracy'], dataset.dataSet as DataPoint[], dataset.numberOfCategories);
+
+    const testModel = await createModel();
 
-        const parameters = { lossfunction: [LossFunction.categoricalCrossentropy], optimizerAlgorithm: [tensorflow.train.adadelta()], batchSize: [10], epochs: [5,10] };
-        autotuner.addModel('testModel', testModel, parameters);
+    const parameters = { lossfunction: [LossFunction.categoricalCrossentropy], optimizerAlgorithm: [tensorflow.train.adadelta()], batchSize: [10], epochs: [5,10] };
+    autotuner.addModel('testModel', testModel, parameters);
 
-        autotuner.bayesianOptimization();
+    // tune the hyperparameters
+    await autotuner.bayesianOptimization();
+
+    // evaluate the best parameters found on the test set
+    autotuner.evaluateBestParameter('error', true)
 };
 
 runExampleAutotuner();