From 00fcddc77dc6dd75830dc944f9de3f64d9989214 Mon Sep 17 00:00:00 2001 From: Michael BAUDIN Date: Sat, 1 Jul 2023 18:35:37 +0200 Subject: [PATCH] Refactor FunctionalChaosValidation based on MetaModelValidation. --- lib/etc/openturns.conf.in | 2 +- .../FunctionalChaos/FunctionalChaosResult.cxx | 8 + .../FunctionalChaosValidation.cxx | 227 +++++++----------- .../openturns/FunctionalChaosResult.hxx | 3 + .../openturns/FunctionalChaosValidation.hxx | 58 +++-- .../LinearModel/LinearModelValidation.cxx | 35 ++- .../openturns/LinearModelValidation.hxx | 11 +- lib/test/t_FunctionalChaosValidation_2d.cxx | 14 +- .../t_FunctionalChaosValidation_ishigami.cxx | 22 +- lib/test/t_LinearModelValidation_std.cxx | 2 +- 10 files changed, 181 insertions(+), 201 deletions(-) diff --git a/lib/etc/openturns.conf.in b/lib/etc/openturns.conf.in index a12de603623..06ee998a42a 100644 --- a/lib/etc/openturns.conf.in +++ b/lib/etc/openturns.conf.in @@ -782,7 +782,7 @@ - + diff --git a/lib/src/Uncertainty/Algorithm/MetaModel/FunctionalChaos/FunctionalChaosResult.cxx b/lib/src/Uncertainty/Algorithm/MetaModel/FunctionalChaos/FunctionalChaosResult.cxx index 1e78ad379fa..b1833e465ab 100644 --- a/lib/src/Uncertainty/Algorithm/MetaModel/FunctionalChaos/FunctionalChaosResult.cxx +++ b/lib/src/Uncertainty/Algorithm/MetaModel/FunctionalChaos/FunctionalChaosResult.cxx @@ -270,6 +270,14 @@ Function FunctionalChaosResult::getComposedMetaModel() const return composedMetaModel_; } +/** Residuals accessor */ +Sample FunctionalChaosResult::getSampleResiduals() const +{ + const Sample predictionsSample(metaModel_(inputSample_)); + const Sample residualsSample(outputSample_ - predictionsSample); + return residualsSample; +} + /* Method save() stores the object through the StorageManager */ void FunctionalChaosResult::save(Advocate & adv) const { diff --git a/lib/src/Uncertainty/Algorithm/MetaModel/FunctionalChaos/FunctionalChaosValidation.cxx b/lib/src/Uncertainty/Algorithm/MetaModel/FunctionalChaos/FunctionalChaosValidation.cxx index 773b975231d..05ead0cd65a 100644 --- a/lib/src/Uncertainty/Algorithm/MetaModel/FunctionalChaos/FunctionalChaosValidation.cxx +++ b/lib/src/Uncertainty/Algorithm/MetaModel/FunctionalChaos/FunctionalChaosValidation.cxx @@ -42,14 +42,26 @@ FunctionalChaosValidation::FunctionalChaosValidation() } /* Default constructor */ -FunctionalChaosValidation::FunctionalChaosValidation(const FunctionalChaosResult & functionalChaosResult) + +FunctionalChaosValidation::FunctionalChaosValidation(const FunctionalChaosResult & functionalChaosResult, + const CrossValidationMethod method, + const UnsignedInteger & kParameter) : PersistentObject() , functionalChaosResult_(functionalChaosResult) + , isInitialized_(false) { - // Nothing to do + if ((method != LEAVEONEOUT) and (method != KFOLD)) + throw InvalidArgumentException(HERE) << "The method " << method << " is not available."; + cvMethod_ = method; + if (kParameter < 1) + throw InvalidArgumentException(HERE) << "Cannot set k parameter of K-Fold method to " << kParameter << " which is lower than 1"; + const UnsignedInteger sampleSize = functionalChaosResult_.getInputSample().getSize(); + if (kParameter > sampleSize) + throw InvalidArgumentException(HERE) << "Cannot set k parameter of K-Fold method to " << kParameter + << " which is larger than the sample size =" << sampleSize; + kParameter_ = kParameter; } - /* Virtual constructor */ FunctionalChaosValidation * FunctionalChaosValidation::clone() const { @@ -65,114 +77,64 @@ String FunctionalChaosValidation::__repr__() const return oss; } +/* Initialize */ +void FunctionalChaosValidation::initialize() +{ + if (!isInitialized_) + { + const Sample metamodelPredictions(computeMetamodelCrossValidationPredictions()); + Sample outputSample(functionalChaosResult_.getOutputSample()); + validation_ = MetaModelValidation(outputSample, metamodelPredictions); // True for LOO. And K-Fold ? + isInitialized_ = true; + } +} + /* Get result*/ FunctionalChaosResult FunctionalChaosValidation::getResult() const { return functionalChaosResult_; } -/* Compute LOO mean squared error */ -Point FunctionalChaosValidation::computeLeaveOneOutMeanSquaredError() const +/* Compute mean squared error */ +Point FunctionalChaosValidation::computeMeanSquaredError() { - const UnsignedInteger sampleSize = functionalChaosResult_.getInputSample().getSize(); - const UnsignedInteger outputDimension = functionalChaosResult_.getOutputSample().getDimension(); - Point leaveOneOutMSE(outputDimension); - Sample leaveOneOutResiduals(computeLeaveOneOutResiduals()); - for (UnsignedInteger j = 0; j < outputDimension; ++j) - { - Sample squaredCorrectedResiduals(sampleSize, 1); - for (UnsignedInteger i = 0; i < sampleSize; ++i) - squaredCorrectedResiduals(i, 0) = std::pow(leaveOneOutResiduals(i, j), 2); - leaveOneOutMSE[j] = squaredCorrectedResiduals.computeMean()[0]; - } // For output marginal indices - return leaveOneOutMSE; + initialize(); + return validation_.computeMeanSquaredError(); } -/* Compute K-Fold mean squared error */ -Point FunctionalChaosValidation::computeKFoldMeanSquaredError(const UnsignedInteger & kParameter) const +/* Compute R2 score */ +Point FunctionalChaosValidation::computeR2Score() { - const UnsignedInteger sampleSize = functionalChaosResult_.getInputSample().getSize(); - const UnsignedInteger outputDimension = functionalChaosResult_.getOutputSample().getDimension(); - // Compute K-Fold error - Sample foldSquaredError(kParameter, outputDimension); - Sample residualsKFold(computeKFoldResiduals(kParameter)); - Point kFoldMSE(outputDimension); - KFoldSplitter splitter(sampleSize, kParameter); - for (UnsignedInteger foldIndex = 0; foldIndex < kParameter; ++foldIndex) - { - Indices indicesTest; - const Indices indicesTrain(splitter.generate(indicesTest)); - const UnsignedInteger foldDimension = indicesTest.getSize(); - for (UnsignedInteger j = 0; j < outputDimension; ++j) - { - for (UnsignedInteger i = 0; i < foldDimension; ++i) - foldSquaredError(foldIndex, j) += std::pow(residualsKFold(indicesTest[i], j), 2); - foldSquaredError(foldIndex, j) /= foldDimension; - } // For output marginal indices - } // For fold indices - kFoldMSE = foldSquaredError.computeMean(); - return kFoldMSE; + initialize(); + return validation_.computeR2Score(); } -/* Compute LOO R2 score */ -Point FunctionalChaosValidation::computeLeaveOneOutR2Score() const +/* Compute residuals */ +Sample FunctionalChaosValidation::getResidualSample() { - Point meanSquaredError(computeLeaveOneOutMeanSquaredError()); - const Sample outputSample(functionalChaosResult_.getOutputSample()); - Point r2Score(MetaModelValidation::ComputeR2Score(meanSquaredError, outputSample)); - return r2Score; + initialize(); + return validation_.getResidualSample(); } -/* Compute K-Fold R2 score */ -Point FunctionalChaosValidation::computeKFoldR2Score(const UnsignedInteger & kParameter) const +/* Get residual distribution */ +Distribution FunctionalChaosValidation::getResidualDistribution(const Bool smooth) { - Point meanSquaredError(computeKFoldMeanSquaredError(kParameter)); - const Sample outputSample(functionalChaosResult_.getOutputSample()); - Point r2Score(MetaModelValidation::ComputeR2Score(meanSquaredError, outputSample)); - return r2Score; + initialize(); + return validation_.getResidualDistribution(smooth); } -/* Compute leave-one-out residuals */ -Sample FunctionalChaosValidation::computeLeaveOneOutResiduals() const +/* Draw */ +GridLayout FunctionalChaosValidation::drawValidation() { - const Sample inputSample(functionalChaosResult_.getInputSample()); - const UnsignedInteger sampleSize = inputSample.getSize(); - const FunctionCollection reducedBasis(functionalChaosResult_.getReducedBasis()); - const UnsignedInteger reducedBasisSize = reducedBasis.getSize(); - if (reducedBasisSize > sampleSize) - throw InvalidArgumentException(HERE) << "Error: the sample size is: " << sampleSize << - " which is lower than the basis size: " << reducedBasisSize; - const Function transformation(functionalChaosResult_.getTransformation()); - const Sample standardSample(transformation(inputSample)); - DesignProxy designProxy(standardSample, reducedBasis); - Indices allIndices(reducedBasisSize); - allIndices.fill(); - const MatrixImplementation designMatrix(designProxy.computeDesign(allIndices)); - // The method name is set to the default one, given by ResourceMap - const String methodName(ResourceMap::GetAsString("LeastSquaresExpansion-DecompositionMethod")); - LeastSquaresMethod leastSquaresMethod(LeastSquaresMethod::Build(methodName, designProxy, allIndices)); - leastSquaresMethod.update(Indices(0), allIndices, Indices(0)); - const Point diagonalH = leastSquaresMethod.getHDiag(); - const Sample outputSample(functionalChaosResult_.getOutputSample()); - const Function metamodel(functionalChaosResult_.getMetaModel()); - const Sample outputMetamodel(metamodel(inputSample)); - const UnsignedInteger outputDimension = outputSample.getDimension(); - Sample residualsLOO(sampleSize, outputDimension); - for (UnsignedInteger j = 0; j < outputDimension; ++j) - { - const Sample marginalOutputSample(outputSample.getMarginal(j)); - const Point marginalOutputPoint(marginalOutputSample.asPoint()); - const Sample outputMetamodelMarginal(outputMetamodel.getMarginal(j)); - const Point residuals(marginalOutputPoint - outputMetamodelMarginal.asPoint()); - for (UnsignedInteger i = 0; i < sampleSize; ++i) - residualsLOO(i, j) = residuals[i] / (1.0 - diagonalH[i]); - } // For output marginal indices - return residualsLOO; + initialize(); + return validation_.drawValidation(); } -/* Compute K-Fold residuals */ -Sample FunctionalChaosValidation::computeKFoldResiduals(const UnsignedInteger & kParameter) const + +/** Compute cross-validation metamodel predictions */ +Sample FunctionalChaosValidation::computeMetamodelCrossValidationPredictions() const { + const Sample residualsSample(functionalChaosResult_.getSampleResiduals()); const Sample inputSample(functionalChaosResult_.getInputSample()); const UnsignedInteger sampleSize = inputSample.getSize(); const FunctionCollection reducedBasis(functionalChaosResult_.getReducedBasis()); @@ -188,59 +150,52 @@ Sample FunctionalChaosValidation::computeKFoldResiduals(const UnsignedInteger & DesignProxy designProxy(standardSample, reducedBasis); Indices allIndices(reducedBasisSize); allIndices.fill(); + const MatrixImplementation designMatrix(designProxy.computeDesign(allIndices)); // The method name is set to the default one, given by ResourceMap const String methodName(ResourceMap::GetAsString("LeastSquaresExpansion-DecompositionMethod")); LeastSquaresMethod leastSquaresMethod(LeastSquaresMethod::Build(methodName, designProxy, allIndices)); leastSquaresMethod.update(Indices(0), allIndices, Indices(0)); - const SymmetricMatrix projectionMatrix(leastSquaresMethod.getH()); - // Compute K-Fold error - Sample residualsKFold(sampleSize, outputDimension); - Point kFoldMSE(outputDimension); - KFoldSplitter splitter(sampleSize, kParameter); - for (UnsignedInteger foldIndex = 0; foldIndex < kParameter; ++foldIndex) + Sample cvPredictions(sampleSize, outputDimension); + if (cvMethod_ == LEAVEONEOUT) { - Indices indicesTest; - const Indices indicesTrain(splitter.generate(indicesTest)); - const Sample inputTestKFold(inputSample.select(indicesTest)); - const Sample outpoutTestKFold(outputSample.select(indicesTest)); - const Sample predictionKFold(metamodel(inputTestKFold)); - const UnsignedInteger foldDimension = indicesTest.getSize(); - SymmetricMatrix projectionKFoldMatrix(foldDimension); - for (UnsignedInteger i1 = 0; i1 < foldDimension; ++i1) - for (UnsignedInteger i2 = 0; i2 < 1 + i1; ++i2) - projectionKFoldMatrix(i1, i2) = projectionMatrix(indicesTest[i1], indicesTest[i2]); - const IdentityMatrix identityMatrix(foldDimension); - const SymmetricMatrix reducedMatrix(identityMatrix - projectionKFoldMatrix); - Matrix multipleRightHandSide(foldDimension, outputDimension); - for (UnsignedInteger j = 0; j < outputDimension; ++j) - for (UnsignedInteger i = 0; i < foldDimension; ++i) - multipleRightHandSide(i, j) = outpoutTestKFold(i, j) - predictionKFold(i, j); - const Matrix correctedKFoldResidualsMatrix(reducedMatrix.solveLinearSystem(multipleRightHandSide)); + const Point diagonalH = leastSquaresMethod.getHDiag(); for (UnsignedInteger j = 0; j < outputDimension; ++j) - for (UnsignedInteger i = 0; i < foldDimension; ++i) - residualsKFold(indicesTest[i], j) = correctedKFoldResidualsMatrix(i, j); - } // For fold indices - return residualsKFold; -} - - -/* Draw model vs metamodel validation graph */ -GridLayout FunctionalChaosValidation::drawLeaveOneOutValidation() const -{ - // Compute leave-one-out predictions - const UnsignedInteger sampleSize = functionalChaosResult_.getInputSample().getSize(); - const Sample outputSample(functionalChaosResult_.getOutputSample()); - const UnsignedInteger outputDimension = outputSample.getDimension(); - Sample leaveOneOutResiduals(computeLeaveOneOutResiduals()); - Sample leaveOneOutPredictions(sampleSize, outputDimension); - for (UnsignedInteger j = 0; j < outputDimension; ++j) - for (UnsignedInteger i = 0; i < sampleSize; ++i) - leaveOneOutPredictions(i, j) = outputSample(i, j) - leaveOneOutResiduals(i, j); - - // Draw graph - MetaModelValidation validation(outputSample, leaveOneOutPredictions); - GridLayout grid(validation.drawValidation()); - return grid; + for (UnsignedInteger i = 0; i < sampleSize; ++i) + cvPredictions(i, j) = outputSample(i, j) - residualsSample(i, j) / (1.0 - diagonalH[i]); + } + else if (cvMethod_ == KFOLD) + { + const SymmetricMatrix projectionMatrix(leastSquaresMethod.getH()); + // Compute K-Fold error + Sample residualsKFold(sampleSize, outputDimension); + KFoldSplitter splitter(sampleSize, kParameter_); + for (UnsignedInteger foldIndex = 0; foldIndex < kParameter_; ++foldIndex) + { + Indices indicesTest; + const Indices indicesTrain(splitter.generate(indicesTest)); + const UnsignedInteger foldSize = indicesTest.getSize(); + // Take into account for different fold sizes + const Scalar foldCorrection = std::sqrt(float(sampleSize) / (kParameter_ * foldSize)); + SymmetricMatrix projectionKFoldMatrix(foldSize); + for (UnsignedInteger i1 = 0; i1 < foldSize; ++i1) + for (UnsignedInteger i2 = 0; i2 < 1 + i1; ++i2) + projectionKFoldMatrix(i1, i2) = projectionMatrix(indicesTest[i1], indicesTest[i2]); + const IdentityMatrix identityMatrix(foldSize); + const SymmetricMatrix reducedMatrix(identityMatrix - projectionKFoldMatrix); + const Sample residualsSampleKFoldTest(residualsSample.select(indicesTest)); + Matrix multipleRightHandSide(foldSize, outputDimension); + for (UnsignedInteger j = 0; j < outputDimension; ++j) + for (UnsignedInteger i = 0; i < foldSize; ++i) + multipleRightHandSide(i, j) = residualsSampleKFoldTest(i, j); + const Matrix correctedKFoldResidualsMatrix(reducedMatrix.solveLinearSystem(multipleRightHandSide)); + for (UnsignedInteger j = 0; j < outputDimension; ++j) + for (UnsignedInteger i = 0; i < foldSize; ++i) + cvPredictions(indicesTest[i], j) = outputSample(indicesTest[i], j) - foldCorrection * correctedKFoldResidualsMatrix(i, j); + } // For fold indices + } + else + throw InvalidArgumentException(HERE) << "The method " << cvMethod_ << " is not available."; + return cvPredictions; } /* Method save() stores the object through the StorageManager */ diff --git a/lib/src/Uncertainty/Algorithm/MetaModel/FunctionalChaos/openturns/FunctionalChaosResult.hxx b/lib/src/Uncertainty/Algorithm/MetaModel/FunctionalChaos/openturns/FunctionalChaosResult.hxx index acee9279d6f..18327b9dd7a 100644 --- a/lib/src/Uncertainty/Algorithm/MetaModel/FunctionalChaos/openturns/FunctionalChaosResult.hxx +++ b/lib/src/Uncertainty/Algorithm/MetaModel/FunctionalChaos/openturns/FunctionalChaosResult.hxx @@ -100,6 +100,9 @@ public: /** Composed meta model accessor */ virtual Function getComposedMetaModel() const; + /** Residuals accessor */ + virtual Sample getSampleResiduals() const; + /** Method save() stores the object through the StorageManager */ void save(Advocate & adv) const override; diff --git a/lib/src/Uncertainty/Algorithm/MetaModel/FunctionalChaos/openturns/FunctionalChaosValidation.hxx b/lib/src/Uncertainty/Algorithm/MetaModel/FunctionalChaos/openturns/FunctionalChaosValidation.hxx index 18a9273cea7..36485d1dec0 100644 --- a/lib/src/Uncertainty/Algorithm/MetaModel/FunctionalChaos/openturns/FunctionalChaosValidation.hxx +++ b/lib/src/Uncertainty/Algorithm/MetaModel/FunctionalChaos/openturns/FunctionalChaosValidation.hxx @@ -23,7 +23,7 @@ #include "openturns/FunctionalChaosResult.hxx" #include "openturns/GridLayout.hxx" - +#include "openturns/MetaModelValidation.hxx" BEGIN_NAMESPACE_OPENTURNS @@ -40,12 +40,15 @@ class OT_API FunctionalChaosValidation public: typedef Collection FunctionCollection; + enum CrossValidationMethod {LEAVEONEOUT = 0, KFOLD = 1}; /** Default constructor */ FunctionalChaosValidation(); /** Parameter constructor */ - FunctionalChaosValidation(const FunctionalChaosResult & chaosResult); + FunctionalChaosValidation(const FunctionalChaosResult & chaosResult, + const CrossValidationMethod method, + const UnsignedInteger & kParameter = ResourceMap::GetAsUnsignedInteger("FunctionalChaosValidation-DefaultKFoldParameter")); /** Virtual constructor */ FunctionalChaosValidation * clone() const override; @@ -56,40 +59,55 @@ public: /** Result accessor */ FunctionalChaosResult getResult() const; - /** Compute leave-one-out error */ - Point computeLeaveOneOutMeanSquaredError() const; - - /** Compute K-Fold error */ - Point computeKFoldMeanSquaredError(const UnsignedInteger & k = ResourceMap::GetAsUnsignedInteger("FunctionalChaosValidation-DefaultKFoldParameter")) const; - - /** Compute leave-one-out R2 score */ - Point computeLeaveOneOutR2Score() const; - - /** Compute K-Fold R2 score */ - Point computeKFoldR2Score(const UnsignedInteger & k = ResourceMap::GetAsUnsignedInteger("FunctionalChaosValidation-DefaultKFoldParameter")) const; + /** Mean squared error accessor */ + Point computeMeanSquaredError(); + + /** Get R2 score */ + Point computeR2Score(); - /** Draw leave-one-out validation */ - GridLayout drawLeaveOneOutValidation() const; + /** Get residuals */ + Sample getResidualSample(); + /** Get residual distribution */ + Distribution getResidualDistribution(const Bool smooth = true); + + /** Draw */ + GridLayout drawValidation(); + + /** Get the K-parameter */ + UnsignedInteger getKParameter() const; + /** Method save() stores the object through the StorageManager */ void save(Advocate & adv) const override; /** Method load() reloads the object from the StorageManager */ void load(Advocate & adv) override; - protected: private: - /** Compute leave-one-out residuals */ - Sample computeLeaveOneOutResiduals() const; + + /** Initialize the object*/ + void initialize(); - /** Compute K-Fold residuals */ - Sample computeKFoldResiduals(const UnsignedInteger & k) const; + /** Compute cross-validation metamodel predictions */ + Sample computeMetamodelCrossValidationPredictions() const; + + /** Initialized ? */ + Bool isInitialized_; /** The functional chaos result */ FunctionalChaosResult functionalChaosResult_; + /** K-parameter */ + UnsignedInteger kParameter_; + + /** Cross-validation method */ + CrossValidationMethod cvMethod_; + + /** MetaModelValidation */ + MetaModelValidation validation_; + } ; /* class FunctionalChaosValidation */ diff --git a/lib/src/Uncertainty/Algorithm/MetaModel/LinearModel/LinearModelValidation.cxx b/lib/src/Uncertainty/Algorithm/MetaModel/LinearModel/LinearModelValidation.cxx index fd79150eb52..bf1cb9520d2 100644 --- a/lib/src/Uncertainty/Algorithm/MetaModel/LinearModel/LinearModelValidation.cxx +++ b/lib/src/Uncertainty/Algorithm/MetaModel/LinearModel/LinearModelValidation.cxx @@ -49,18 +49,6 @@ LinearModelValidation::LinearModelValidation() } /* Parameter constructor */ -LinearModelValidation::LinearModelValidation(const LinearModelResult & linearModelResult, - const CrossValidationMethod method) - : PersistentObject() - , linearModelResult_(linearModelResult) - , isInitialized_(false) - , kParameter_(ResourceMap::GetAsUnsignedInteger("LinearModelValidation-DefaultKFoldParameter")) -{ - if ((method != LEAVEONEOUT) and (method != KFOLD)) - throw InvalidArgumentException(HERE) << "The method " << method << " is not available."; - cvMethod_ = method; -} - LinearModelValidation::LinearModelValidation(const LinearModelResult & linearModelResult, const CrossValidationMethod method, const UnsignedInteger & kParameter) @@ -112,9 +100,8 @@ Sample LinearModelValidation::computeMetamodelCrossValidationPredictions() const if (cvMethod_ == LEAVEONEOUT) { const Point diagonalH(linearModelResult_.getLeverages()); - const Sample residuals(linearModelResult_.getSampleResiduals()); for (UnsignedInteger i = 0; i < sampleSize; ++i) - cvPredictions(i, 0) = outputSample(i, 0) - residuals(i, 0) / (1.0 - diagonalH[i]); + cvPredictions(i, 0) = outputSample(i, 0) - residualsSample(i, 0) / (1.0 - diagonalH[i]); } else if (cvMethod_ == KFOLD) { @@ -134,8 +121,8 @@ Sample LinearModelValidation::computeMetamodelCrossValidationPredictions() const const SymmetricMatrix reducedMatrix(IdentityMatrix(foldSize) - projectionKFoldMatrix); const Sample residualsSampleKFoldTest(residualsSample.select(indicesTest)); const Point residualsKFold(reducedMatrix.solveLinearSystem(residualsSampleKFoldTest.asPoint())); - for (UnsignedInteger i1 = 0; i1 < foldSize; ++i1) - cvPredictions(indicesTest[i1], 0) = outputSample(indicesTest[i1], 0) - foldCorrection * residualsKFold[i1]; + for (UnsignedInteger i = 0; i < foldSize; ++i) + cvPredictions(indicesTest[i], 0) = outputSample(indicesTest[i], 0) - foldCorrection * residualsKFold[i]; } // Loop over the folds } else @@ -155,21 +142,21 @@ void LinearModelValidation::initialize() } } -/* Compute LOO mean squared error */ +/* Compute mean squared error */ Point LinearModelValidation::computeMeanSquaredError() { initialize(); return validation_.computeMeanSquaredError(); } -/* Compute LOO residuals */ +/* Compute residuals */ Sample LinearModelValidation::getResidualSample() { initialize(); return validation_.getResidualSample(); } -/* Compute LOO R2 score */ +/* Compute R2 score */ Point LinearModelValidation::computeR2Score() { initialize(); @@ -183,13 +170,19 @@ GridLayout LinearModelValidation::drawValidation() return validation_.drawValidation(); } -/* Get the K-parameter */ +/* Get residual distribution */ +Distribution LinearModelValidation::getResidualDistribution(const Bool smooth) +{ + initialize(); + return validation_.getResidualDistribution(smooth); +} + +/* Get the K parameter */ UnsignedInteger LinearModelValidation::getKParameter() const { return kParameter_; } - /* Linear model accessor */ LinearModelResult LinearModelValidation::getLinearModelResult() const { diff --git a/lib/src/Uncertainty/Algorithm/MetaModel/LinearModel/openturns/LinearModelValidation.hxx b/lib/src/Uncertainty/Algorithm/MetaModel/LinearModel/openturns/LinearModelValidation.hxx index 9103d3b18a7..b619c1d2465 100644 --- a/lib/src/Uncertainty/Algorithm/MetaModel/LinearModel/openturns/LinearModelValidation.hxx +++ b/lib/src/Uncertainty/Algorithm/MetaModel/LinearModel/openturns/LinearModelValidation.hxx @@ -29,7 +29,6 @@ #include "openturns/LinearModelResult.hxx" #include "openturns/MetaModelValidation.hxx" - BEGIN_NAMESPACE_OPENTURNS /** @@ -50,9 +49,6 @@ public: LinearModelValidation(); /** Parameter constructor */ - explicit LinearModelValidation(const LinearModelResult & linearModelResult, - const CrossValidationMethod method = LEAVEONEOUT); - explicit LinearModelValidation(const LinearModelResult & linearModelResult, const CrossValidationMethod method, const UnsignedInteger & kParameter = ResourceMap::GetAsUnsignedInteger("LinearModelValidation-DefaultKFoldParameter")); @@ -75,10 +71,13 @@ public: /** Get residuals */ Sample getResidualSample(); + /** Get residual distribution */ + Distribution getResidualDistribution(const Bool smooth = true); + /** Draw */ GridLayout drawValidation(); - /** Get the K-parameter */ + /** Get the K parameter */ UnsignedInteger getKParameter() const; /** Method save() stores the object through the StorageManager */ @@ -107,7 +106,7 @@ private: /** Cross-validation method */ CrossValidationMethod cvMethod_; - /** MetaModelValidation */ + /** MetaModelValidation */ MetaModelValidation validation_; }; /* class LinearModelValidation */ diff --git a/lib/test/t_FunctionalChaosValidation_2d.cxx b/lib/test/t_FunctionalChaosValidation_2d.cxx index 64df51e35b5..04e87cf32c3 100644 --- a/lib/test/t_FunctionalChaosValidation_2d.cxx +++ b/lib/test/t_FunctionalChaosValidation_2d.cxx @@ -78,13 +78,13 @@ int main(int, char *[]) algo.run(); // Examine the results - const FunctionalChaosResult result = algo.getResult(); + const FunctionalChaosResult chaosResult = algo.getResult(); ResourceMap::SetAsUnsignedInteger("FunctionalChaosResult-PrintEllipsisThreshold", 20); - fullprint << result.__repr_markdown__() << std::endl; + fullprint << chaosResult.__repr_markdown__() << std::endl; // Analytical leave-one-out - FunctionalChaosValidation validation(result); - const Point mseLOOAnalytical(validation.computeLeaveOneOutMeanSquaredError()); + FunctionalChaosValidation validationLOO(chaosResult, FunctionalChaosValidation::LEAVEONEOUT); + const Point mseLOOAnalytical(validationLOO.computeMeanSquaredError()); fullprint << "Analytical LOO MSE = " << mseLOOAnalytical << std::endl; // Naive leave-one-out @@ -120,8 +120,9 @@ int main(int, char *[]) assert_almost_equal(mseLOOAnalytical, mseLOOnaive, rtolLOO, atolLOO); // Analytical K-Fold + FunctionalChaosValidation validationKFold(chaosResult, FunctionalChaosValidation::KFOLD, kFoldParameter); fullprint << "KFold with K = " << kFoldParameter << std::endl; - const Point mseKFoldAnalytical(validation.computeKFoldMeanSquaredError(kFoldParameter)); + const Point mseKFoldAnalytical(validationKFold.computeMeanSquaredError()); fullprint << "Analytical KFold MSE = " << mseKFoldAnalytical << std::endl; // Naive KFold @@ -131,6 +132,7 @@ int main(int, char *[]) { Indices indicesTest; const Indices indicesTrain(splitter.generate(indicesTest)); + const UnsignedInteger foldSize = indicesTest.getSize(); const Sample inputSampleKFoldTrain(inputSample.select(indicesTrain)); const Sample outputSampleKFoldTrain(outputSample.select(indicesTrain)); const Sample inputSampleKFoldTest(inputSample.select(indicesTest)); @@ -145,7 +147,7 @@ int main(int, char *[]) const Sample marginalPredictionKFoldTest(marginalMetamodel(inputSampleKFoldTest)); const Sample marginalOutputSampleKFoldTest = outputSampleKFoldTest.getMarginal(k); const Point residualsKFoldTest(marginalPredictionKFoldTest.asPoint() - marginalOutputSampleKFoldTest.asPoint()); - residualsKFold(foldIndex, k) = residualsKFoldTest.normSquare() / foldSampleSize; + residualsKFold(foldIndex, k) = residualsKFoldTest.normSquare() / foldSize; } } Point mseKFoldnaive(residualsKFold.computeMean()); diff --git a/lib/test/t_FunctionalChaosValidation_ishigami.cxx b/lib/test/t_FunctionalChaosValidation_ishigami.cxx index 18313ea8109..71d9318213d 100644 --- a/lib/test/t_FunctionalChaosValidation_ishigami.cxx +++ b/lib/test/t_FunctionalChaosValidation_ishigami.cxx @@ -56,11 +56,11 @@ int main(int, char *[]) const UnsignedInteger basisDimension = enumerateFunction.getBasisSizeFromTotalDegree(totalDegree); fullprint << "basisDimension = " << basisDimension << std::endl; const FixedStrategy adaptiveStrategy(productBasis, basisDimension); - // Compute the sample size from number of folds to guarantee a constant integer + // Compute the sample size from number of folds to guarantee a non constant integer // number of points per fold const UnsignedInteger kFoldParameter = 10; UnsignedInteger foldSampleSize = 20; - const UnsignedInteger samplingSize = foldSampleSize * kFoldParameter; + const UnsignedInteger samplingSize = foldSampleSize * kFoldParameter + 1; fullprint << "samplingSize = " << samplingSize << std::endl; // Create the projection strategy const MonteCarloExperiment experiment(distribution, samplingSize); @@ -73,13 +73,13 @@ int main(int, char *[]) algo.run(); // Examine the results - const FunctionalChaosResult result = algo.getResult(); + const FunctionalChaosResult chaosResult = algo.getResult(); ResourceMap::SetAsUnsignedInteger("FunctionalChaosResult-PrintEllipsisThreshold", 20); - fullprint << result.__repr_markdown__() << std::endl; + fullprint << chaosResult.__repr_markdown__() << std::endl; // Analytical leave-one-out - FunctionalChaosValidation validation(result); - const Point mseLOOAnalytical(validation.computeLeaveOneOutMeanSquaredError()); + FunctionalChaosValidation validationLOO(chaosResult, FunctionalChaosValidation::LEAVEONEOUT); + const Point mseLOOAnalytical(validationLOO.computeMeanSquaredError()); fullprint << "Analytical LOO MSE = " << mseLOOAnalytical << std::endl; // Naive leave-one-out @@ -111,7 +111,7 @@ int main(int, char *[]) assert_almost_equal(mseLOOAnalytical, mseLOOnaive, rtolLOO, atolLOO); // Check LOO R2 - const Point r2ScoreLOO(validation.computeLeaveOneOutR2Score()); + const Point r2ScoreLOO(validationLOO.computeR2Score()); fullprint << "Analytical LOO R2 score = " << r2ScoreLOO << std::endl; const Point sampleVariance(outputSample.computeCentralMoment(2)); fullprint << "sampleVariance = " << sampleVariance << std::endl; @@ -123,8 +123,9 @@ int main(int, char *[]) assert_almost_equal(r2ScoreReference, r2ScoreLOO, rtolLOO, atolLOO); // Analytical K-Fold + FunctionalChaosValidation validationKFold(chaosResult, FunctionalChaosValidation::KFOLD, kFoldParameter); fullprint << "KFold with K = " << kFoldParameter << std::endl; - const Point mseKFoldAnalytical(validation.computeKFoldMeanSquaredError(kFoldParameter)); + const Point mseKFoldAnalytical(validationKFold.computeMeanSquaredError()); fullprint << "Analytical KFold MSE = " << mseKFoldAnalytical << std::endl; // Naive KFold @@ -134,6 +135,7 @@ int main(int, char *[]) { Indices indicesTest; const Indices indicesTrain(splitter.generate(indicesTest)); + const UnsignedInteger foldSize = indicesTest.getSize(); const Sample inputSampleKFoldTrain(inputSample.select(indicesTrain)); const Sample outputSampleKFoldTrain(outputSample.select(indicesTrain)); const Sample inputSampleKFoldTest(inputSample.select(indicesTest)); @@ -144,7 +146,7 @@ int main(int, char *[]) const Function metamodelKFold(resultKFold.getMetaModel()); const Sample predictionKFoldTest(metamodelKFold(inputSampleKFoldTest)); const Point residualsKFoldTest(predictionKFoldTest.asPoint() - outputSampleKFoldTest.asPoint()); - residualsKFold(foldIndex, 0) = residualsKFoldTest.normSquare() / foldSampleSize; + residualsKFold(foldIndex, 0) = residualsKFoldTest.normSquare() / foldSize; } Point mseKFoldnaive(residualsKFold.computeMean()); fullprint << "Naive KFold MSE = " << mseKFoldnaive << std::endl; @@ -155,7 +157,7 @@ int main(int, char *[]) assert_almost_equal(mseKFoldAnalytical, mseKFoldnaive, rtolKFold, atolKFold); // Check K-Fold R2 - const Point r2ScoreKFold(validation.computeKFoldR2Score()); + const Point r2ScoreKFold(validationKFold.computeR2Score()); fullprint << "Analytical K-Fold R2 score = " << r2ScoreKFold << std::endl; r2ScoreReference[0] = 1.0 - mseKFoldAnalytical[0] / sampleVariance[0]; fullprint << "Computed R2 score = " << r2ScoreReference << std::endl; diff --git a/lib/test/t_LinearModelValidation_std.cxx b/lib/test/t_LinearModelValidation_std.cxx index a95df0fa8c7..338434d0d37 100644 --- a/lib/test/t_LinearModelValidation_std.cxx +++ b/lib/test/t_LinearModelValidation_std.cxx @@ -61,7 +61,7 @@ int main(int, char *[]) LinearModelResult result(lmAlgo.getResult()); // Create LOO validation - LinearModelValidation validationLOO(result); + LinearModelValidation validationLOO(result, LinearModelValidation::LEAVEONEOUT); fullprint << validationLOO.__str__() << std::endl; // Compute analytical LOO MSE