From 90eb64d62b8783c7cf141be5d83811051427baaf Mon Sep 17 00:00:00 2001 From: moserle Date: Fri, 30 Aug 2019 12:25:06 +0200 Subject: [PATCH] implement evaluateBestParameter(): evaluate the best parameters found during optimization on the test set, code refactoring - change type of observed values - restrict objectives to error and accuracy - returns the best parameter after the optimization run has finished - implement evaluateBestParameter(): evaluate the best parameters that were found during optimization on the test set - fix path in runExampleAutotuner (use relative path) - remove some TODOs - update README - some code refactoring --- README.md | 28 ++++++++------ src/autotuner.ts | 73 +++++++++++++++++++++++++++--------- src/modelEvaluater.ts | 61 ++++++++++++++++-------------- src/priors.ts | 1 - src/util.ts | 1 - tests/runExampleAutotuner.ts | 26 ++++++++----- types/types.ts | 4 +- 7 files changed, 124 insertions(+), 70 deletions(-) diff --git a/README.md b/README.md index e1f71ff..174a4c5 100644 --- a/README.md +++ b/README.md @@ -10,9 +10,9 @@ import { TensorflowlModelAutotuner } from '@piximi/autotuner'; ### Getting Started -Initialize the autotuner by specifying metrics and a dataset. +Initialize the autotuner by specifying metrics, a dataset and the number of categories. ```javascript -var autotuner = new TensorflowlModelAutotuner(['accuracy'], dataset.trainData, dataset.testData, dataset.validationData); +var autotuner = new TensorflowlModelAutotuner(['accuracy'], dataset, numberOfCategories); ``` ### Adding a model to the autotuner ```javascript @@ -24,27 +24,32 @@ autotuner.addModel('testModel', testModel, parameters); ``` ### Tuning the hyperparameters -Specify the optimization algorith. The hyperparameters can be tuned by either doing bayesian optimization or by doing a simple grid search. +Specify the optimization algorithm: the hyperparameters can be tuned by either doing bayesian optimization or by doing a simple grid search. ```javascript autotuner.bayesianOptimization(); ``` ```javascript autotuner.gridSearchOptimizytion(); ``` -#### Optinal parameters -The ojective function of the optimization can be specified (either 'error' or 'accuracy'). +The ojective function of the optimization can be specified (either 'error' or 'accuracy'): ```javascript autotuner.gridSearchOptimizytion('accuracy'); ``` -Also one can enable cross validation when evaluating a model. +Evaluating a model can be done using cross validation: ```javascript autotuner.gridSearchOptimizytion('accuracy', true); ``` -When doing bayesian optimization the maximum number of domain points to be evaluated can be specified as an optional parameter. +When doing bayesian optimization the maximum number of domain points to be evaluated can be specified as an optional parameter: ```javascript autotuner.bayesianOptimization('accuracy', true, 0.8); ``` -In the example above the optimizytion search stops after 80% of the domain ponits have been evaluated. By default this value is set to 0.75. +In the example above the optimizytion stops after 80% of the domain ponits have been evaluated. By default this value is set to 0.75. +### Evaluate the best hyperparameters +The best hyperparameters found in the optimization run can be evaluated on the test set. Specify the objective and wheter or not to use cross validation. +```javascript +autotuner.evaluateBestParameter('error', true); +``` + ### Example An example usage can be found here: ```bash @@ -54,17 +59,18 @@ tets/runExampleAutotuner.ts Pull and initialize: ```bash -git pull https://github.com/piximi/autotuner.git +git clone https://github.com/piximi/autotuner.git cd autotuner npm install ``` To run tests: ```bash -npm test +npm run test +npm run runExampleAutotuner ``` To compile the code and check for type errors: ```bash -npm build +npm run build ``` \ No newline at end of file diff --git a/src/autotuner.ts b/src/autotuner.ts index dd852f9..826d71f 100644 --- a/src/autotuner.ts +++ b/src/autotuner.ts @@ -1,14 +1,15 @@ import * as tensorflow from '@tensorflow/tfjs'; -import { ModelDict, SequentialModelParameters, DataPoint, BaysianOptimisationStep, LossFunction, DomainPointValue } from '../types/types'; +import { ModelDict, SequentialModelParameters, DataPoint, BaysianOptimisationStep, LossFunction, ObservedValues, DomainPointValue } from '../types/types'; import * as bayesianOptimizer from './bayesianOptimizer'; import * as gridSearchOptimizer from './gridSearchOptimizer'; import * as paramspace from './paramspace'; import * as priors from './priors'; import * as modelEvaluator from './modelEvaluater'; +import { argmax } from './util'; class AutotunerBaseClass { metrics: string[] = []; - observedValues: DomainPointValue[] = []; + observedValues: ObservedValues = {}; /** * Fraction of domain indices that should be evaluated at most */ @@ -18,7 +19,7 @@ class AutotunerBaseClass { * * @return {boolean} false if tuning the hyperparameters should be stopped, true otherwise */ - metricsStopingCriteria: (observedValues: DomainPointValue[]) => boolean; + metricsStopingCriteria: (observedValues: ObservedValues) => boolean; modelOptimizersDict: { [model: string]: tensorflow.Optimizer[] } = {}; paramspace: any; optimizer: any; @@ -31,7 +32,29 @@ class AutotunerBaseClass { * @param {number} domainIndex Index of the domain point to be evaluated. * @return {Promise} Value of the domain point */ - evaluateModel: (domainIndex: number, objective: string, useCrossValidation: boolean) => Promise; + evaluateModel: (domainIndex: number, objective: string, useCrossValidation: boolean, useTestData: boolean) => Promise; + + bestParameter(objective: string) { + if (this.checkObjective(objective)) { + return + } + const observedValues = this.observedValues[objective]; + const bestScoreIndex = argmax(observedValues); + const bestScoreDomainIndex = this.observedValues['domainIndices'][bestScoreIndex]; + return bestScoreDomainIndex; + } + + /** + * Tests the best parameters that were found during optimization on the test set. + * + * @param {string} objective Define the metric that should be evaluated. Either 'error' or 'accuracy' + * @param {boolean} useCrossValidation Indicate wheter or not to use cross validation to evaluate the model. Set to 'false' by default. + * @return {number} Returns the score of. + */ + async evaluateBestParameter(objective: string, useCrossValidation: boolean = false) { + var bestScoreDomainIndex = this.bestParameter(objective) as number; + return await this.evaluateModel(bestScoreDomainIndex, objective, useCrossValidation, true); + } /** * Decide whether to continue tuning the hyperparameters. @@ -41,7 +64,7 @@ class AutotunerBaseClass { */ stopingCriteria() { const domainSize = this.paramspace.domainIndices.length; - const numberOfObservedValues = this.observedValues.length; + const numberOfObservedValues = this.observedValues['domainIndices'].length; var fractionOfEvaluatedPoints = numberOfObservedValues / domainSize; var maxIterationsReached: boolean = fractionOfEvaluatedPoints <= this.maxIterations; @@ -58,7 +81,7 @@ class AutotunerBaseClass { } checkObjective (objective: string): boolean { - const allowedObjectives: string[] = ['error'].concat(this.metrics); + const allowedObjectives = ['error', 'accuracy']; if (!allowedObjectives.includes(objective)) { console.log("Invalid objective function selected!"); console.log("Objective function must be one of the following: " + allowedObjectives.join()); @@ -71,6 +94,9 @@ class AutotunerBaseClass { this.paramspace = new paramspace.Paramspace(); this.modelEvaluator = new modelEvaluator.ModelEvaluater(dataSet, numberOfCategories, validationSetRatio, testSetRatio); this.metrics = metrics; + this.observedValues['domainIndices'] = []; + this.observedValues['error'] = []; + this.observedValues['accuracy'] = []; } /** @@ -80,8 +106,9 @@ class AutotunerBaseClass { * @param {boolean} [useCrossValidation=false] Indicate wheter or not to use cross validation to evaluate the model. Set to 'false' by default. * @param {number} [maxIteration=0.75] Fraction of domain points that should be evaluated at most. (e.g. for 'maxIteration=0.75' the optimization stops if 75% of the domain has been evaluated) * @param {boolean} [stopingCriteria] Predicate on the observed values when to stop the optimization + * @return Returns the best parameters found in the optimization run. */ - async bayesianOptimization(objective: string = 'error', useCrossValidation: boolean = false, maxIteration: number = 0.75, stopingCriteria?: ((observedValues: DomainPointValue[]) => boolean)) { + async bayesianOptimization(objective: string = 'error', useCrossValidation: boolean = false, maxIteration: number = 0.75, stopingCriteria?: ((observedValues: ObservedValues) => boolean)) { if (this.checkObjective(objective)) { return; } @@ -92,7 +119,7 @@ class AutotunerBaseClass { this.metricsStopingCriteria = stopingCriteria; } - this.tuneHyperparameters(objective, useCrossValidation); + return await this.tuneHyperparameters(objective, useCrossValidation); } /** @@ -100,6 +127,7 @@ class AutotunerBaseClass { * * @param {string} [objective='error'] Define the objective of the optimization. Set to 'error' by default. * @param {boolean} [useCrossValidation=false] Indicate wheter or not to use cross validation to evaluate the model. Set to 'false' by default. + * @return Returns the best parameters found in the optimization run. */ async gridSearchOptimizytion(objective: string = 'error', useCrossValidation: boolean = false) { if (this.checkObjective(objective)) { @@ -109,7 +137,7 @@ class AutotunerBaseClass { this.optimizer = new gridSearchOptimizer.Optimizer(this.paramspace.domainIndices, this.paramspace.modelsDomains); this.maxIterations = 1; - this.tuneHyperparameters(objective, useCrossValidation); + return await this.tuneHyperparameters(objective, useCrossValidation); } @@ -123,7 +151,7 @@ class AutotunerBaseClass { var nextOptimizationPoint: BaysianOptimisationStep = this.optimizer.getNextPoint(); // Train a model given the params and obtain a quality metric value. - var value = await this.evaluateModel(nextOptimizationPoint.nextPoint, objective, useCrossValidation); + var value = await this.evaluateModel(nextOptimizationPoint.nextPoint, objective, useCrossValidation, false); // Report the obtained quality metric value. this.optimizer.addSample(nextOptimizationPoint.nextPoint, value); @@ -135,6 +163,12 @@ class AutotunerBaseClass { console.log("============================"); console.log("finished tuning the hyperparameters"); + console.log(); + var bestScoreDomainIndex = this.bestParameter(objective) as number; + var bestParameters = this.paramspace.domain[bestScoreDomainIndex]['params']; + console.log("The best parameters found are:"); + console.log(bestParameters); + return bestParameters; } } @@ -153,7 +187,7 @@ class TensorflowlModelAutotuner extends AutotunerBaseClass { constructor(metrics: string[], dataSet: DataPoint[], numberOfCategories: number, validationSetRatio: number = 0.25, testSetRatio: number = 0.25) { super(metrics, dataSet, numberOfCategories, validationSetRatio, testSetRatio); - this.evaluateModel = async (point: number, objective: string, useCrossValidation: boolean) => { + this.evaluateModel = async (point: number, objective: string, useCrossValidation: boolean, useTestData: boolean = false) => { const modelIdentifier = this.paramspace.domain[point]['model']; const model = this.modelDict[modelIdentifier]; const params = this.paramspace.domain[point]['params']; @@ -169,13 +203,16 @@ class TensorflowlModelAutotuner extends AutotunerBaseClass { metrics: metrics, optimizer: optimizerFunction }); - - let dataPointValue: DomainPointValue = useCrossValidation - ? await this.modelEvaluator.EvaluateSequentialTensorflowModelCV(model, args) - : await this.modelEvaluator.EvaluateSequentialTensorflowModel(model, args); - this.observedValues.push(dataPointValue); - return objective === 'error' ? dataPointValue.error : dataPointValue.metricScores[0]; - } + + let domainPointValue: DomainPointValue = useCrossValidation + ? await this.modelEvaluator.EvaluateSequentialTensorflowModelCV(model, args, useTestData) + : await this.modelEvaluator.EvaluateSequentialTensorflowModel(model, args, useTestData); + + this.observedValues['domainIndices'].push(point); + this.observedValues['error'].push(domainPointValue.error); + this.observedValues['accuracy'].push(domainPointValue.accuracy); + return objective === 'error' ? domainPointValue.error : 1 - domainPointValue.accuracy; + } } /** diff --git a/src/modelEvaluater.ts b/src/modelEvaluater.ts index dc84035..ba40fcd 100644 --- a/src/modelEvaluater.ts +++ b/src/modelEvaluater.ts @@ -30,40 +30,27 @@ export class ModelEvaluater{ this.trainData = dataSet; } - ConcatenateTensorData(data: DataPoint[]) { - const trainData: tensorflow.Tensor[] = []; - const trainLables: number[] = []; - for (let i = 0; i < data.length; i++) { - trainData.push(data[i].data); - trainLables.push(data[i].lables); - } - - let concatenatedTensorData = tensorflow.tidy(() => tensorflow.concat(trainData)); - let concatenatedLables = tensorflow.tidy(() => tensorflow.oneHot(trainLables, this.numberOfCategories)); - return { concatenatedTensorData, concatenatedLables }; - } - - EvaluateSequentialTensorflowModel = async (model: tensorflow.Sequential, args: any): Promise => { + EvaluateSequentialTensorflowModel = async (model: tensorflow.Sequential, args: any, useTestData: boolean): Promise => { var trainData = this.ConcatenateTensorData(this.trainData); await model.fit(trainData.concatenatedTensorData, trainData.concatenatedLables, args); - var validationData = this.ConcatenateTensorData(this.validationData); + var validationData = useTestData ? this.ConcatenateTensorData(this.testData) : this.ConcatenateTensorData(this.validationData); const evaluationResult = model.evaluate(validationData.concatenatedTensorData, validationData.concatenatedLables) as tensorflow.Tensor[]; const error = evaluationResult[0].dataSync()[0]; - const score = evaluationResult[1].dataSync()[0]; - return {error: error, metricScores: [score]} as DomainPointValue; + const accuracy = evaluationResult[1].dataSync()[0]; + return {error: error, accuracy: accuracy} as DomainPointValue; } - EvaluateSequentialTensorflowModelCV = async (model: tensorflow.Sequential, args: any): Promise => { - const dataSet = this.trainData.concat(this.validationData); + EvaluateSequentialTensorflowModelCV = async (model: tensorflow.Sequential, args: any, useTestData: boolean): Promise => { + const dataSet = useTestData ? this.testData : this.trainData.concat(this.validationData); const dataSize = dataSet.length; const k = math.min(10, math.floor(math.nthRoot(dataSize) as number)); const dataFolds: DataPoint[][] = Array.from(Array(math.ceil(dataSet.length/k)), (_,i) => dataSet.slice(i*k,i*k+k)); var error = 0; - var score = 0; + var accuracy = 0; for (let i = 0; i < k; i++) { var validationData = dataFolds[i]; var trainData: DataPoint[] = []; @@ -77,16 +64,34 @@ export class ModelEvaluater{ var concatenatedTrainData = this.ConcatenateTensorData(trainData); await model.fit(concatenatedTrainData.concatenatedTensorData, concatenatedTrainData.concatenatedLables, args); - var concatenatedValidationData = this.ConcatenateTensorData(validationData); - const evaluationResult = model.evaluate(concatenatedValidationData.concatenatedTensorData, concatenatedValidationData.concatenatedLables) as tensorflow.Tensor[]; - - const foldError = evaluationResult[0].dataSync()[0]; - const foldScore = evaluationResult[1].dataSync()[0]; - error += foldError; - score += foldScore; + var evaluationResult = await this.EvaluateTensorflowModel(model, validationData); + error += evaluationResult.error; + accuracy += evaluationResult.accuracy; } - return {error: error/dataSize, metricScores: [score/k]} as DomainPointValue; + return {error: error/dataSize, accuracy: accuracy/k} as DomainPointValue; } + ConcatenateTensorData = (data: DataPoint[]) => { + const trainData: tensorflow.Tensor[] = []; + const trainLables: number[] = []; + for (let i = 0; i < data.length; i++) { + trainData.push(data[i].data); + trainLables.push(data[i].lables); + } + + let concatenatedTensorData = tensorflow.tidy(() => tensorflow.concat(trainData)); + let concatenatedLables = tensorflow.tidy(() => tensorflow.oneHot(trainLables, this.numberOfCategories)); + return { concatenatedTensorData, concatenatedLables }; + } + + EvaluateTensorflowModel = async (model: tensorflow.Sequential, evaluationData: DataPoint[]) => { + var concatenatedEvaluationData = this.ConcatenateTensorData(evaluationData); + const evaluationResult = model.evaluate(concatenatedEvaluationData.concatenatedTensorData, concatenatedEvaluationData.concatenatedLables) as tensorflow.Tensor[]; + + const error = evaluationResult[0].dataSync()[0]; + const accuracy = evaluationResult[1].dataSync()[0]; + return {error: error, accuracy: accuracy} as DomainPointValue; + } + } \ No newline at end of file diff --git a/src/priors.ts b/src/priors.ts index 563ac06..2cd7785 100644 --- a/src/priors.ts +++ b/src/priors.ts @@ -62,7 +62,6 @@ class Priors { } } } - // TODO: test matrix operations and casts to matrix cov /= (this.observedValues[point].length * this.observedValues[point2].length) this.kernel = math.subset(this.kernel, math.index(idx, idx2), cov) as math.Matrix; this.kernel = math.subset(this.kernel, math.index(idx2, idx), cov) as math.Matrix; diff --git a/src/util.ts b/src/util.ts index eea0de6..7af4555 100644 --- a/src/util.ts +++ b/src/util.ts @@ -18,7 +18,6 @@ const argmax = (array: number[]) => { * @param {Array.} std Standard deviation values of the probability distribution over a given domain. * @return {Array.} Values of the expected improvement for all points of the mean and std. */ -// TODO: change type of parameters, use math.matrix, adjust usages const expectedImprovement = (bestObjective: number, mean: math.Matrix, std: math.Matrix) => { var mean: math.Matrix; var std: math.Matrix; diff --git a/tests/runExampleAutotuner.ts b/tests/runExampleAutotuner.ts index 3703fff..d676a2a 100644 --- a/tests/runExampleAutotuner.ts +++ b/tests/runExampleAutotuner.ts @@ -5,20 +5,26 @@ import * as tensorflow from '@tensorflow/tfjs'; import { Classifier } from '@piximi/types'; const runExampleAutotuner = async () => { - var fs = require("fs"); - var stringContent = fs.readFileSync("C:/Users/m_lev/Projects/BA/piximi/autotuner/tests/data/smallMNISTTest.piximi"); - var classifier = JSON.parse(stringContent) as Classifier; + var path = require("path"); + var fs = require("fs"); + var testFilePath = path.resolve('tests', 'data', 'smallMNISTTest.piximi'); + var stringContent = fs.readFileSync(testFilePath); + var classifier = JSON.parse(stringContent) as Classifier; - const dataset = await createDataset(classifier.categories, classifier.images); + const dataset = await createDataset(classifier.categories, classifier.images); - var autotuner = new TensorflowlModelAutotuner(['accuracy'], dataset.dataSet as DataPoint[], dataset.numberOfCategories); - - const testModel = await createModel(); + var autotuner = new TensorflowlModelAutotuner(['accuracy'], dataset.dataSet as DataPoint[], dataset.numberOfCategories); + + const testModel = await createModel(); - const parameters = { lossfunction: [LossFunction.categoricalCrossentropy], optimizerAlgorithm: [tensorflow.train.adadelta()], batchSize: [10], epochs: [5,10] }; - autotuner.addModel('testModel', testModel, parameters); + const parameters = { lossfunction: [LossFunction.categoricalCrossentropy], optimizerAlgorithm: [tensorflow.train.adadelta()], batchSize: [10], epochs: [5,10] }; + autotuner.addModel('testModel', testModel, parameters); - autotuner.bayesianOptimization(); + // tune the hyperparameters + await autotuner.bayesianOptimization(); + + // evaluate the best parameters found on the test set + autotuner.evaluateBestParameter('error', true) }; runExampleAutotuner(); diff --git a/types/types.ts b/types/types.ts index 7ab0dfd..db61fa4 100644 --- a/types/types.ts +++ b/types/types.ts @@ -37,4 +37,6 @@ export type ModelDict = { [identifier: string] : Model}; export type NullableMatrix = math.Matrix | null; -export type DomainPointValue = { error: number, metricScores: number[] } \ No newline at end of file +export type DomainPointValue = { error: number, accuracy: number } + +export type ObservedValues = { [identifier: string]: number[] } \ No newline at end of file