From 493a150f05a598ef3b4a64e86c417d1fb3f88a94 Mon Sep 17 00:00:00 2001 From: Michael BAUDIN Date: Thu, 21 Mar 2024 17:07:18 +0100 Subject: [PATCH] Moved FCValidation and LMValidation into experimental --- .../openturns/BoxCoxFactory.hxx | 4 ++-- python/doc/pyplots/FunctionalChaosValidation.py | 3 ++- python/doc/pyplots/LinearModelValidation.py | 3 ++- .../doc/theory/meta_modeling/cross_validation.rst | 8 ++++---- .../theory/meta_modeling/pce_cross_validation.rst | 4 ++-- .../functional_chaos_expansion.rst | 2 +- python/doc/user_manual/response_surface/lm.rst | 2 +- python/src/BoxCoxFactory.i | 4 ++-- python/src/FunctionalChaosValidation_doc.i.in | 13 +++++++------ python/src/LinearModelValidation_doc.i.in | 15 ++++++++------- python/src/experimental_module.i | 4 ++++ python/src/metamodel_module.i | 2 -- python/test/t_FunctionalChaosValidation_std.py | 9 ++++----- python/test/t_LinearModelValidation_std.py | 5 +++-- 14 files changed, 42 insertions(+), 36 deletions(-) diff --git a/lib/src/Uncertainty/Algorithm/Transformation/ProcessTransformation/openturns/BoxCoxFactory.hxx b/lib/src/Uncertainty/Algorithm/Transformation/ProcessTransformation/openturns/BoxCoxFactory.hxx index f09fb01587..e15eef76e0 100644 --- a/lib/src/Uncertainty/Algorithm/Transformation/ProcessTransformation/openturns/BoxCoxFactory.hxx +++ b/lib/src/Uncertainty/Algorithm/Transformation/ProcessTransformation/openturns/BoxCoxFactory.hxx @@ -96,12 +96,12 @@ public: const Sample &outputSample, const Basis &basis, const Point &shift, - LinearModelResult &linearModelResult); + LinearModelResult & linearModelResultOut); BoxCoxTransform buildWithLM(const Sample &inputSample, const Sample &outputSample, const Point &shift, - LinearModelResult &linearModelResult); + LinearModelResult & linearModelResultOut); /** Optimization solver accessor */ OptimizationAlgorithm getOptimizationAlgorithm() const; diff --git a/python/doc/pyplots/FunctionalChaosValidation.py b/python/doc/pyplots/FunctionalChaosValidation.py index 4d9a8dde96..64cafd89ef 100644 --- a/python/doc/pyplots/FunctionalChaosValidation.py +++ b/python/doc/pyplots/FunctionalChaosValidation.py @@ -1,6 +1,7 @@ import openturns as ot from openturns.viewer import View from openturns.usecases import ishigami_function +import openturns.experimental as otexp im = ishigami_function.IshigamiModel() sampleSize = 500 @@ -20,6 +21,6 @@ ) chaosalgo.run() result = chaosalgo.getResult() -validation = ot.FunctionalChaosValidation(result) +validation = otexp.FunctionalChaosValidation(result) graph = validation.drawValidation() View(graph) diff --git a/python/doc/pyplots/LinearModelValidation.py b/python/doc/pyplots/LinearModelValidation.py index 301c5bd891..c077b8522c 100644 --- a/python/doc/pyplots/LinearModelValidation.py +++ b/python/doc/pyplots/LinearModelValidation.py @@ -1,5 +1,6 @@ import openturns as ot from openturns.viewer import View +import openturns.experimental as otexp func = ot.SymbolicFunction( ['x1', 'x2', 'x3'], @@ -16,6 +17,6 @@ algo.run() result = algo.getResult() splitter = ot.LeaveOneOutSplitter(sampleSize) -validation = ot.LinearModelValidation(result, splitter) +validation = otexp.LinearModelValidation(result, splitter) graph = validation.drawValidation() View(graph) diff --git a/python/doc/theory/meta_modeling/cross_validation.rst b/python/doc/theory/meta_modeling/cross_validation.rst index 114300853d..d8adcd16e2 100644 --- a/python/doc/theory/meta_modeling/cross_validation.rst +++ b/python/doc/theory/meta_modeling/cross_validation.rst @@ -635,16 +635,16 @@ The generic cross-validation method can be implemented using the following class to split the data set. Since the :class:`~openturns.LinearModelResult` is based on linear least -squares, fast methods are implemented in the :class:`~openturns.LinearModelValidation`. +squares, fast methods are implemented in the :class:`~openturns.experimental.LinearModelValidation`. -See :ref:`pce_cross_validation` and :class:`~openturns.FunctionalChaosValidation` +See :ref:`pce_cross_validation` and :class:`~openturns.experimental.FunctionalChaosValidation` for specific methods for the the cross-validation of a polynomial chaos expansion. .. topic:: API: - See :class:`~openturns.MetaModelValidation` - - See :class:`~openturns.LinearModelValidation` - - See :class:`~openturns.FunctionalChaosValidation` + - See :class:`~openturns.experimental.LinearModelValidation` + - See :class:`~openturns.experimental.FunctionalChaosValidation` - See :class:`~openturns.KFoldSplitter` - See :class:`~openturns.LeaveOneOutSplitter` diff --git a/python/doc/theory/meta_modeling/pce_cross_validation.rst b/python/doc/theory/meta_modeling/pce_cross_validation.rst index 669c8b0d4e..53f45ebf55 100644 --- a/python/doc/theory/meta_modeling/pce_cross_validation.rst +++ b/python/doc/theory/meta_modeling/pce_cross_validation.rst @@ -52,11 +52,11 @@ then the fast methods presented in :ref:`cross_validation` can be applied: - the fast leave-one-out cross-validation, - the fast K-Fold cross-validation. -Fast methods are implemented in :class:`~openturns.FunctionalChaosValidation`. +Fast methods are implemented in :class:`~openturns.experimental.FunctionalChaosValidation`. .. topic:: API: - - See :class:`~openturns.FunctionalChaosValidation` + - See :class:`~openturns.experimental.FunctionalChaosValidation` .. topic:: References: diff --git a/python/doc/user_manual/response_surface/functional_chaos_expansion.rst b/python/doc/user_manual/response_surface/functional_chaos_expansion.rst index 857f4158ee..ad533136f5 100644 --- a/python/doc/user_manual/response_surface/functional_chaos_expansion.rst +++ b/python/doc/user_manual/response_surface/functional_chaos_expansion.rst @@ -107,7 +107,7 @@ Results FunctionalChaosSobolIndices :template: classWithPlot.rst_t - FunctionalChaosValidation + experimental.FunctionalChaosValidation Functional chaos on fields ========================== diff --git a/python/doc/user_manual/response_surface/lm.rst b/python/doc/user_manual/response_surface/lm.rst index 47fe895057..f04817c452 100644 --- a/python/doc/user_manual/response_surface/lm.rst +++ b/python/doc/user_manual/response_surface/lm.rst @@ -26,4 +26,4 @@ Post-processing :template: classWithPlot.rst_t - LinearModelValidation + experimental.LinearModelValidation diff --git a/python/src/BoxCoxFactory.i b/python/src/BoxCoxFactory.i index 6aac6f38c9..544acc70ea 100644 --- a/python/src/BoxCoxFactory.i +++ b/python/src/BoxCoxFactory.i @@ -1,8 +1,8 @@ // SWIG file BoxCoxFactory.i // do not pass argument by reference, return it as tuple item -%typemap(in, numinputs=0) OT::LinearModelResult & linearModelResult ($*ltype temp) %{ temp = OT::LinearModelResult(); $1 = &temp; %} -%typemap(argout) OT::LinearModelResult & linearModelResult %{ $result = SWIG_Python_AppendOutput($result, SWIG_NewPointerObj(new OT::LinearModelResult(*$1), SWIGTYPE_p_OT__LinearModelResult, SWIG_POINTER_OWN | 0 )); %} +%typemap(in, numinputs=0) OT::LinearModelResult & linearModelResultOut ($*ltype temp) %{ temp = OT::LinearModelResult(); $1 = &temp; %} +%typemap(argout) OT::LinearModelResult & linearModelResultOut %{ $result = SWIG_Python_AppendOutput($result, SWIG_NewPointerObj(new OT::LinearModelResult(*$1), SWIGTYPE_p_OT__LinearModelResult, SWIG_POINTER_OWN | 0 )); %} %typemap(in, numinputs=0) OT::GeneralLinearModelResult & generalLinearModelResult ($*ltype temp) %{ temp = OT::GeneralLinearModelResult(); $1 = &temp; %} %typemap(argout) OT::GeneralLinearModelResult & generalLinearModelResult %{ $result = SWIG_Python_AppendOutput($result, SWIG_NewPointerObj(new OT::GeneralLinearModelResult(*$1), SWIGTYPE_p_OT__GeneralLinearModelResult, SWIG_POINTER_OWN | 0 )); %} diff --git a/python/src/FunctionalChaosValidation_doc.i.in b/python/src/FunctionalChaosValidation_doc.i.in index 08fc1c4837..07b574fdb3 100644 --- a/python/src/FunctionalChaosValidation_doc.i.in +++ b/python/src/FunctionalChaosValidation_doc.i.in @@ -51,7 +51,7 @@ conditions are met. If model selection is involved, the naive methods based on the :class:`~openturns.LeaveOneOutSplitter` and :class:`~openturns.KFoldSplitter` classes can be used, but this can be much slower than the - analytical methods implemented in the :class:`~openturns.FunctionalChaosValidation` + analytical methods implemented in the :class:`~openturns.experimental.FunctionalChaosValidation` class. In many cases, however, the order of magnitude of the estimate from the analytical formula applied to a sparse model is correct: the estimate of @@ -72,15 +72,15 @@ the :math:`i`-th prediction is the prediction of the linear model trained using the hold-out sample where the :math:`i`-th observation was removed. This produces a sample of residuals which can be retrieved using -the :class:`~openturns.FunctionalChaosValidation.getResidualSample` method. -The :class:`~openturns.FunctionalChaosValidation.drawValidation` performs +the :class:`~openturns.experimental.FunctionalChaosValidation.getResidualSample` method. +The :class:`~openturns.experimental.FunctionalChaosValidation.drawValidation` performs similarly. If the weights of the observations are not equal, the analytical method may not necessarily provide an accurate estimator of the mean squared error (MSE). This is because LOO and K-Fold cross-validation do not take the weights into account. -Since the :class:`~openturns.FunctionalChaosResult` object does not know +Since the :class:`~openturns.experimental.FunctionalChaosResult` object does not know if the weights are equal, no exception can be generated. If the sample was not produced from Monte Carlo, then the leave-one-out @@ -106,6 +106,7 @@ prevents us from using the fast analytical formulas and get an accurate estimator of the MSE. >>> import openturns as ot +>>> import openturns.experimental as otexp >>> from math import pi >>> from openturns.usecases import ishigami_function >>> im = ishigami_function.IshigamiModel() @@ -128,7 +129,7 @@ accurate estimator of the MSE. Validate the polynomial chaos expansion using leave-one-out cross-validation. >>> splitterLOO = ot.LeaveOneOutSplitter(sampleSize) ->>> validation = ot.FunctionalChaosValidation(result, splitterLOO) +>>> validation = otexp.FunctionalChaosValidation(result, splitterLOO) >>> r2Score = validation.computeR2Score() >>> print('R2 = ', r2Score[0]) R2 = 0.99... @@ -138,7 +139,7 @@ and set the :math:`k` parameter. >>> kParameter = 10 >>> splitterKF = ot.KFoldSplitter(sampleSize, kParameter) ->>> validation = ot.FunctionalChaosValidation( +>>> validation = otexp.FunctionalChaosValidation( ... result, splitterKF ... ) diff --git a/python/src/LinearModelValidation_doc.i.in b/python/src/LinearModelValidation_doc.i.in index f833f24803..9bff154c19 100644 --- a/python/src/LinearModelValidation_doc.i.in +++ b/python/src/LinearModelValidation_doc.i.in @@ -43,7 +43,7 @@ cross-validation methods can be used. If model selection is involved, the naive methods based on the :class:`~openturns.LeaveOneOutSplitter` and :class:`~openturns.KFoldSplitter` classes can be used directly, but this can be much slower than the -analytical methods implemented in the :class:`~openturns.LinearModelValidation` +analytical methods implemented in the :class:`~openturns.experimental.LinearModelValidation` class. In many cases, however, the order of magnitude of the estimate from the analytical formula applied to a sparse model is correct: the estimate of @@ -64,8 +64,8 @@ the :math:`i`-th prediction is the prediction of the linear model trained using the hold-out sample where the :math:`i`-th observation was removed. This produces a sample of residuals which can be retrieved using -the :class:`~openturns.LinearModelValidation.getResidualSample` method. -The :class:`~openturns.LinearModelValidation.drawValidation` performs +the :class:`~openturns.experimental.LinearModelValidation.getResidualSample` method. +The :class:`~openturns.experimental.LinearModelValidation.drawValidation` performs similarly. Examples @@ -73,6 +73,7 @@ Examples Create a linear model. >>> import openturns as ot +>>> import openturns.experimental as otexp >>> func = ot.SymbolicFunction( ... ['x1', 'x2', 'x3'], ... ['x1 + x2 + sin(x2 * 2 * pi_) / 5 + 1e-3 * x3^2'] @@ -88,12 +89,12 @@ Create a linear model. Validate the linear model using leave-one-out cross-validation. ->>> validation = ot.LinearModelValidation(result) +>>> validation = otexp.LinearModelValidation(result) We can use a specific cross-validation splitter if needed. >>> splitterLOO = ot.LeaveOneOutSplitter(sampleSize) ->>> validation = ot.LinearModelValidation(result, splitterLOO) +>>> validation = otexp.LinearModelValidation(result, splitterLOO) >>> r2Score = validation.computeR2Score() >>> print('R2 = ', r2Score[0]) R2 = 0.98... @@ -101,13 +102,13 @@ R2 = 0.98... Validate the linear model using K-Fold cross-validation. >>> splitterKFold = ot.KFoldSplitter(sampleSize) ->>> validation = ot.LinearModelValidation(result, splitterKFold) +>>> validation = otexp.LinearModelValidation(result, splitterKFold) Validate the linear model using K-Fold cross-validation and set K. >>> kFoldParameter = 10 >>> splitterKFold = ot.KFoldSplitter(sampleSize, kFoldParameter) ->>> validation = ot.LinearModelValidation(result, splitterKFold) +>>> validation = otexp.LinearModelValidation(result, splitterKFold) Draw the validation graph. diff --git a/python/src/experimental_module.i b/python/src/experimental_module.i index 835e9bf7fd..4e3cbc6970 100644 --- a/python/src/experimental_module.i +++ b/python/src/experimental_module.i @@ -83,3 +83,7 @@ %include UniformOrderStatistics.i %include GeneralizedExtremeValueValidation.i %include GeneralizedParetoValidation.i + +/* Uncertainty/Algorithm/Metamodel */ +%include FunctionalChaosValidation.i +%include LinearModelValidation.i diff --git a/python/src/metamodel_module.i b/python/src/metamodel_module.i index 7c9081e94b..41a387c357 100644 --- a/python/src/metamodel_module.i +++ b/python/src/metamodel_module.i @@ -60,14 +60,12 @@ %include FunctionalChaosAlgorithm.i %include FunctionalChaosSobolIndices.i %include MetaModelValidation.i -%include FunctionalChaosValidation.i %include GeneralLinearModelResult.i %include GeneralLinearModelAlgorithm.i %include KrigingAlgorithm.i %include LinearModelStepwiseAlgorithm.i %include LinearModelAlgorithm.i %include LinearModelAnalysis.i -%include LinearModelValidation.i /* Uncertainty/Model */ %include RandomVector.i diff --git a/python/test/t_FunctionalChaosValidation_std.py b/python/test/t_FunctionalChaosValidation_std.py index 4e7663f65b..8e70da8836 100644 --- a/python/test/t_FunctionalChaosValidation_std.py +++ b/python/test/t_FunctionalChaosValidation_std.py @@ -3,6 +3,7 @@ import openturns as ot from openturns.usecases import ishigami_function from openturns.testing import assert_almost_equal +import openturns.experimental as otexp def computeMSENaiveLOO( @@ -170,7 +171,7 @@ def computeMSENaiveKFold( # print("1. Analytical leave-one-out") splitterLOO = ot.LeaveOneOutSplitter(sampleSize) -validationLOO = ot.FunctionalChaosValidation( +validationLOO = otexp.FunctionalChaosValidation( chaosResult, splitterLOO ) mseLOOAnalytical = validationLOO.computeMeanSquaredError() @@ -206,13 +207,11 @@ def computeMSENaiveKFold( # print("2. Analytical K-Fold") splitterKF = ot.KFoldSplitter(sampleSize, kFoldParameter) -validationKFold = ot.FunctionalChaosValidation( +validationKFold = otexp.FunctionalChaosValidation( chaosResult, splitterKF ) print("KFold with K = ", kFoldParameter) assert validationKFold.getSplitter().getN() == sampleSize -# TODO: fix this -# assert validationKFold.getSplitter().getSize() == kFoldParameter # Compute mean squared error mseKFoldAnalytical = validationKFold.computeMeanSquaredError() @@ -261,7 +260,7 @@ def computeMSENaiveKFold( # Analytical leave-one-out splitterLOO = ot.LeaveOneOutSplitter(sampleSize) -validationLOO = ot.FunctionalChaosValidation( +validationLOO = otexp.FunctionalChaosValidation( chaosResult, splitterLOO ) mseLOOAnalytical = validationLOO.computeMeanSquaredError() diff --git a/python/test/t_LinearModelValidation_std.py b/python/test/t_LinearModelValidation_std.py index 642baa94e8..4e7f503f40 100644 --- a/python/test/t_LinearModelValidation_std.py +++ b/python/test/t_LinearModelValidation_std.py @@ -2,6 +2,7 @@ import openturns as ot from openturns.testing import assert_almost_equal +import openturns.experimental as otexp ot.TESTPREAMBLE() @@ -29,7 +30,7 @@ # Create LOO validation splitterLOO = ot.LeaveOneOutSplitter(sampleSize) -validationLOO = ot.LinearModelValidation(result, splitterLOO) +validationLOO = otexp.LinearModelValidation(result, splitterLOO) print(validationLOO) # Compute analytical LOO MSE @@ -73,7 +74,7 @@ # Create KFold validation splitterKFold = ot.KFoldSplitter(sampleSize, kFoldParameter) -validationKFold = ot.LinearModelValidation( +validationKFold = otexp.LinearModelValidation( result, splitterKFold ) print(validationKFold)