From 34bb8ba234eb0ec84262fa98a73bbe374514ee6b Mon Sep 17 00:00:00 2001 From: FWuellhorst Date: Fri, 21 Jun 2024 15:27:00 +0200 Subject: [PATCH] Issue253 coverage (#557) * add coverage script from Mans with changes based on review in #315 #243 * add unit-test and fix minor bug * Catch case for no examples --- buildingspy/development/regressiontest.py | 118 +++++++++++++++++- .../tests/test_development_regressiontest.py | 35 ++++++ 2 files changed, 152 insertions(+), 1 deletion(-) diff --git a/buildingspy/development/regressiontest.py b/buildingspy/development/regressiontest.py index 1deb9451..0f6af5a1 100644 --- a/buildingspy/development/regressiontest.py +++ b/buildingspy/development/regressiontest.py @@ -351,6 +351,9 @@ def __init__( self._data = [] self._reporter = rep.Reporter(os.path.join(os.getcwd(), "unitTests-{}.log".format(tool))) + # List to store tested packages, used for coverage report + self._packages = [] + # By default, include export of FMUs. self._include_fmu_test = True @@ -589,6 +592,7 @@ def getModelicaCommand(self): elif self._modelica_tool != 'dymola': return 'jm_ipython.sh' else: + return "C://Program Files//Dymola 2023x//bin64//Dymola" return self._modelica_tool def isExecutable(self, program): @@ -771,11 +775,12 @@ def setSinglePackage(self, packageName): # Set data dictionary as it may have been generated earlier for the whole library. self._data = [] - + self._packages = [] for pac in packages: pacSep = pac.find('.') pacPat = pac[pacSep + 1:] pacPat = pacPat.replace('.', os.sep) + self._packages.append(pacPat) rooPat = os.path.join(self._libHome, 'Resources', 'Scripts', 'Dymola', pacPat) # Verify that the directory indeed exists if not os.path.isdir(rooPat): @@ -4288,3 +4293,114 @@ def _model_from_mo(self, mo_file): model = '.'.join(splt[root:]) # remove the '.mo' at the end return model[:-3] + + def getCoverage(self): + """ + Analyse how many examples are tested. + If ``setSinglePackage`` is called before this function, + only packages set will be included. Else, the whole library + will be checked. + + Returns: + - The coverage rate in percent as float + - The number of examples tested as int + - The total number of examples as int + - The list of models not tested as List[str] + - The list of packages included in the analysis as List[str] + + Example: + >>> from buildingspy.development.regressiontest import Tester + >>> import os + >>> ut = Tester(tool='dymola') + >>> myMoLib = os.path.join("buildingspy", "tests", "MyModelicaLibrary") + >>> ut.setLibraryRoot(myMoLib) + >>> ut.setSinglePackage('Examples') + >>> coverage_result = ut.getCoverage() + """ + # first lines copy and paste from run function + if self.get_number_of_tests() == 0: + self.setDataDictionary(self._rootPackage) + + # Remove all data that do not require a simulation or an FMU export. + # Otherwise, some processes may have no simulation to run and then + # the json output file would have an invalid syntax + + # now we got clean _data to compare + # next step get all examples in the package (whether whole library or + # single package) + if self._packages: + packages = self._packages + else: + packages = list(dict.fromkeys( + [pac['ScriptFile'].split(os.sep)[0] for pac in self._data]) + ) + + all_examples = [] + for package in packages: + package_path = os.path.join(self._libHome, package) + for dirpath, dirnames, filenames in os.walk(package_path): + for filename in filenames: + filepath = os.path.abspath(os.path.join(dirpath, filename)) + if any( + xs in filepath for xs in ['Examples', 'Validation'] + ) and not filepath.endswith(('package.mo', '.order')): + all_examples.append(filepath) + + n_tested_examples = len(temp_data) + n_examples = len(all_examples) + if n_examples > 0: + coverage = round(n_tested_examples / n_examples, 2) * 100 + else: + coverage = 100 + + tested_model_names = [ + nam['ScriptFile'].split(os.sep)[-1][:-1] for nam in self._data + ] + + missing_examples = [ + i for i in all_examples if not any( + xs in i for xs in tested_model_names) + ] + + return coverage, n_tested_examples, n_examples, missing_examples, packages + + def printCoverage( + self, + coverage: float, + n_tested_examples: int, + n_examples: int, + missing_examples: list, + packages: list, + printer: callable = None + ) -> None: + """ + Print the output of getCoverage to inform about + coverage rate and missing models. + The default printer is the ``reporter.writeOutput``. + If another printing method is required, e.g. ``print`` or + ``logging.info``, it may be passed via the ``printer`` argument. + + Example: + >>> from buildingspy.development.regressiontest import Tester + >>> import os + >>> ut = Tester(tool='dymola') + >>> myMoLib = os.path.join("buildingspy", "tests", "MyModelicaLibrary") + >>> ut.setLibraryRoot(myMoLib) + >>> ut.setSinglePackage('Examples') + >>> coverage_result = ut.getCoverage() + >>> ut.printCoverage(*coverage_result, printer=print) + """ + if printer is None: + printer = self._reporter.writeOutput + printer(f'***\nModel Coverage: {int(coverage)} %') + printer( + f'***\nYou are testing: {n_tested_examples} ' + f'out of {n_examples} examples in package{"s" if len(packages) > 1 else ""}:', + ) + for package in packages: + printer(package) + + if missing_examples: + print('***\nThe following examples are not tested\n') + for i in missing_examples: + print(i.split(self._libHome)[1]) diff --git a/buildingspy/tests/test_development_regressiontest.py b/buildingspy/tests/test_development_regressiontest.py index 6d48fbe3..bf34f718 100644 --- a/buildingspy/tests/test_development_regressiontest.py +++ b/buildingspy/tests/test_development_regressiontest.py @@ -326,6 +326,41 @@ def test_expand_packages(self): self.assertRaises(ValueError, r.Tester.expand_packages, "AB}a{") + def test_get_coverage_single_package(self): + coverage_result = self._test_get_and_print_coverage(package="Examples") + self.assertEqual(coverage_result[0], 88) + self.assertEqual(coverage_result[1], 7) + self.assertEqual(coverage_result[2], 8) + self.assertTrue(coverage_result[3][0].endswith("ParameterEvaluation.mo")) + self.assertEqual(coverage_result[4], ["Examples"]) + + def test_get_coverage_all_packages(self): + coverage_result = self._test_get_and_print_coverage(package=None) + self.assertEqual(coverage_result[0], 89) + self.assertEqual(coverage_result[1], 8) + self.assertEqual(coverage_result[2], 9) + self.assertEqual(len(coverage_result[3]), 1) + self.assertEqual(len(coverage_result[4]), 2) + + def _test_get_and_print_coverage(self, package: str = None): + import buildingspy.development.regressiontest as r + ut = r.Tester(tool='dymola') + myMoLib = os.path.join("buildingspy", "tests", "MyModelicaLibrary") + ut.setLibraryRoot(myMoLib) + if package is not None: + ut.setSinglePackage(package) + coverage_result = ut.getCoverage() + self.assertIsInstance(coverage_result, tuple) + self.assertIsInstance(coverage_result[0], float) + self.assertIsInstance(coverage_result[1], int) + self.assertIsInstance(coverage_result[2], int) + self.assertIsInstance(coverage_result[3], list) + self.assertIsInstance(coverage_result[4], list) + # Check print with both custom and standard printer + ut.printCoverage(*coverage_result, printer=print) + ut.printCoverage(*coverage_result) + return coverage_result + if __name__ == '__main__': unittest.main()