From b0169d8ba6259b025912be63229651fe3479d49b Mon Sep 17 00:00:00 2001 From: Manzai Date: Fri, 29 Mar 2019 09:17:57 +0100 Subject: [PATCH 1/9] added a coverage function --- buildingspy/development/regressiontest.py | 68 +++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/buildingspy/development/regressiontest.py b/buildingspy/development/regressiontest.py index 2adf08a7..a9f3c931 100644 --- a/buildingspy/development/regressiontest.py +++ b/buildingspy/development/regressiontest.py @@ -3130,3 +3130,71 @@ def _analyseOMStats(self, lines=None, models=None, simulate=False): print("\nMore detailed information is stored in self._omstats") print(70 * '#') + + def get_test_example_coverage(self): + """ + Analyse how many examples are tested. + + Returns the share of tested examples + """ + # first lines copy paste from run function + if self.get_number_of_tests() == 0: + self.setDataDictionary(self._rootPackage) + + # Remove all data that do not require a simulation or an FMU export. + # Otherwise, some processes may have no simulation to run and then + # the json output file would have an invalid syntax + for ele in self._data[:]: + if not (ele['mustSimulate'] or ele['mustExportFMU']): + self._data.remove(ele) + + # now we got clean _data to compare + # next step get all examples in the package (whether whole library or + # single package) + + packages = list(dict.fromkeys( + [pac['ScriptFile'].split('/')[0] for pac in self._data])) + + tested_model_names = [ + nam['ScriptFile'].split('/')[-1][:-1] for nam in self._data] + + # this needs to be adjusted for also subdirectories + total_examples = [] + for pac in packages: + for (dirpath, dirnames, filenames) in os.walk( + os.path.join(self._libHome, pac)): + for f in filenames: + total_examples.append(os.path.abspath( + os.path.join(dirpath, f))) + + # list filtering only relevant examples, that could be nicer for sure + total_examples = [ + i for i in total_examples if any( + xs in i for xs in ['Examples', 'Validation'])] + total_examples = [i for i in total_examples if not i.endswith( + ('package.mo', '.order'))] + + coverage = round(len(self._data) / len(total_examples), 4) + + print('***\n\nCoverage: ', coverage * 100.0, '%\n') + print( + '***\n\nYou are testing : ', + len(self._data), + ' out of ', + len(total_examples), + 'total examples in ') + for pac in packages: + print(pac) + print('\n') + + tested_model_names = [ + nam['ScriptFile'].split('/')[-1][:-1] for nam in self._data] + + missing_examples = [ + i for i in total_examples if not any( + xs in i for xs in tested_model_names)] + + if missing_examples: + print('***\n\nThe following examples are not tested\n') + for i in missing_examples: + print(i.split(self._libHome)[1]) From cd7e55cc7a5a81a5526f08f49eed7bc95724f7fa Mon Sep 17 00:00:00 2001 From: Manzai Date: Sun, 31 Mar 2019 21:35:27 +0200 Subject: [PATCH 2/9] adjusted the ouput format --- buildingspy/development/regressiontest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/buildingspy/development/regressiontest.py b/buildingspy/development/regressiontest.py index a9f3c931..4b555a71 100644 --- a/buildingspy/development/regressiontest.py +++ b/buildingspy/development/regressiontest.py @@ -3174,9 +3174,9 @@ def get_test_example_coverage(self): total_examples = [i for i in total_examples if not i.endswith( ('package.mo', '.order'))] - coverage = round(len(self._data) / len(total_examples), 4) + coverage = round(len(self._data) / len(total_examples), 4) * 100.0 - print('***\n\nCoverage: ', coverage * 100.0, '%\n') + print('***\n\nCoverage: ', '%.2f' % coverage, '%\n') print( '***\n\nYou are testing : ', len(self._data), From 16b2ee13cea8d926d72ef68544593a311cf21836 Mon Sep 17 00:00:00 2001 From: MichaMans Date: Mon, 1 Apr 2019 10:32:51 +0200 Subject: [PATCH 3/9] refactors the coverage statement and made it ru in on multiple os --- buildingspy/development/regressiontest.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/buildingspy/development/regressiontest.py b/buildingspy/development/regressiontest.py index 4b555a71..91678759 100644 --- a/buildingspy/development/regressiontest.py +++ b/buildingspy/development/regressiontest.py @@ -3153,10 +3153,10 @@ def get_test_example_coverage(self): # single package) packages = list(dict.fromkeys( - [pac['ScriptFile'].split('/')[0] for pac in self._data])) + [pac['ScriptFile'].split(os.sep)[0] for pac in self._data])) tested_model_names = [ - nam['ScriptFile'].split('/')[-1][:-1] for nam in self._data] + nam['ScriptFile'].split(os.sep)[-1][:-1] for nam in self._data] # this needs to be adjusted for also subdirectories total_examples = [] @@ -3176,7 +3176,7 @@ def get_test_example_coverage(self): coverage = round(len(self._data) / len(total_examples), 4) * 100.0 - print('***\n\nCoverage: ', '%.2f' % coverage, '%\n') + print('***\n\nCoverage: ', '%.2f' % coverage + '%') print( '***\n\nYou are testing : ', len(self._data), From 7b7af29a2be78e5c36dae280cad19f3ae18640b4 Mon Sep 17 00:00:00 2001 From: MichaMans Date: Mon, 1 Apr 2019 10:34:22 +0200 Subject: [PATCH 4/9] refactors the print statement for the coverage --- buildingspy/development/regressiontest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildingspy/development/regressiontest.py b/buildingspy/development/regressiontest.py index 91678759..3d093e38 100644 --- a/buildingspy/development/regressiontest.py +++ b/buildingspy/development/regressiontest.py @@ -3188,7 +3188,7 @@ def get_test_example_coverage(self): print('\n') tested_model_names = [ - nam['ScriptFile'].split('/')[-1][:-1] for nam in self._data] + nam['ScriptFile'].split(os.sep)[-1][:-1] for nam in self._data] missing_examples = [ i for i in total_examples if not any( From 1fc7c7c362422fe7ca0f6f0069c2ce1b9ab62f80 Mon Sep 17 00:00:00 2001 From: MichaMans Date: Mon, 1 Apr 2019 16:25:37 +0200 Subject: [PATCH 5/9] refactored the coverage print --- buildingspy/development/regressiontest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/buildingspy/development/regressiontest.py b/buildingspy/development/regressiontest.py index 3d093e38..95cc6a96 100644 --- a/buildingspy/development/regressiontest.py +++ b/buildingspy/development/regressiontest.py @@ -3174,9 +3174,9 @@ def get_test_example_coverage(self): total_examples = [i for i in total_examples if not i.endswith( ('package.mo', '.order'))] - coverage = round(len(self._data) / len(total_examples), 4) * 100.0 + coverage = round(len(self._data) / len(total_examples), 2) * 100 - print('***\n\nCoverage: ', '%.2f' % coverage + '%') + print('***\n\nCoverage: ', str(int(coverage)) + '%') print( '***\n\nYou are testing : ', len(self._data), From 21817e5c6cbcee7f7bc6aa84c3ad65b250347cb0 Mon Sep 17 00:00:00 2001 From: MichaMans Date: Sat, 7 Dec 2019 11:22:47 +0100 Subject: [PATCH 6/9] revised model coverage function --- buildingspy/development/regressiontest.py | 35 +++++++++++++---------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/buildingspy/development/regressiontest.py b/buildingspy/development/regressiontest.py index b1492256..594d8de9 100644 --- a/buildingspy/development/regressiontest.py +++ b/buildingspy/development/regressiontest.py @@ -340,6 +340,9 @@ def __init__( self._data = [] self._reporter = rep.Reporter(os.path.join(os.getcwd(), "unitTests-{}.log".format(tool))) + # packages set by test single packages flag + self._packages = [] + # By default, include export of FMUs. self._include_fmu_test = True @@ -757,11 +760,12 @@ def setSinglePackage(self, packageName): # Set data dictionary as it may have been generated earlier for the whole library. self._data = [] - + self._packages = [] for pac in packages: pacSep = pac.find('.') pacPat = pac[pacSep + 1:] pacPat = pacPat.replace('.', os.sep) + self._packages.append(pacPat) rooPat = os.path.join(self._libHome, 'Resources', 'Scripts', 'Dymola', pacPat) # Verify that the directory indeed exists if not os.path.isdir(rooPat): @@ -3660,7 +3664,7 @@ def _analyseOMStats(self, lines=None, models=None, simulate=False): print("\nMore detailed information is stored in self._omstats") print(70 * '#') - def get_test_example_coverage(self): + def get_models_coverage(self): """ Analyse how many examples are tested. @@ -3673,23 +3677,24 @@ def get_test_example_coverage(self): # Remove all data that do not require a simulation or an FMU export. # Otherwise, some processes may have no simulation to run and then # the json output file would have an invalid syntax + + # to not interact with other code here, we use the temp_data list + + temp_data = [] + for ele in self._data[:]: - if not (ele['mustSimulate'] or ele['mustExportFMU']): - self._data.remove(ele) + if (ele['mustSimulate'] or ele['mustExportFMU']): + temp_data.append(ele) # now we got clean _data to compare # next step get all examples in the package (whether whole library or # single package) - packages = list(dict.fromkeys( - [pac['ScriptFile'].split(os.sep)[0] for pac in self._data])) - tested_model_names = [ - nam['ScriptFile'].split(os.sep)[-1][:-1] for nam in self._data] + nam['ScriptFile'].split(os.sep)[-1][:-1] for nam in temp_data] - # this needs to be adjusted for also subdirectories total_examples = [] - for pac in packages: + for pac in self._packages: for (dirpath, dirnames, filenames) in os.walk( os.path.join(self._libHome, pac)): for f in filenames: @@ -3703,21 +3708,21 @@ def get_test_example_coverage(self): total_examples = [i for i in total_examples if not i.endswith( ('package.mo', '.order'))] - coverage = round(len(self._data) / len(total_examples), 2) * 100 + coverage = round(len(temp_data) / len(total_examples), 2) * 100 - print('***\n\nCoverage: ', str(int(coverage)) + '%') + print('***\n\nModel Coverage: ', str(int(coverage)) + '%') print( '***\n\nYou are testing : ', - len(self._data), + len(temp_data), ' out of ', len(total_examples), 'total examples in ') - for pac in packages: + for pac in self._packages: print(pac) print('\n') tested_model_names = [ - nam['ScriptFile'].split(os.sep)[-1][:-1] for nam in self._data] + nam['ScriptFile'].split(os.sep)[-1][:-1] for nam in temp_data] missing_examples = [ i for i in total_examples if not any( From f8c893341e465419ed2e4e90f7fa9508998a2b05 Mon Sep 17 00:00:00 2001 From: MichaMans Date: Sat, 7 Dec 2019 11:26:19 +0100 Subject: [PATCH 7/9] renamed the function to work again in current ci --- buildingspy/development/regressiontest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildingspy/development/regressiontest.py b/buildingspy/development/regressiontest.py index 594d8de9..91d99a45 100644 --- a/buildingspy/development/regressiontest.py +++ b/buildingspy/development/regressiontest.py @@ -3664,7 +3664,7 @@ def _analyseOMStats(self, lines=None, models=None, simulate=False): print("\nMore detailed information is stored in self._omstats") print(70 * '#') - def get_models_coverage(self): + def get_test_example_coverage(self): """ Analyse how many examples are tested. From ecfa9efbd3fc401dcf8c6f12607677926f603808 Mon Sep 17 00:00:00 2001 From: MichaMans Date: Sat, 7 Dec 2019 11:36:04 +0100 Subject: [PATCH 8/9] debugged function to work with no single package set --- buildingspy/development/regressiontest.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/buildingspy/development/regressiontest.py b/buildingspy/development/regressiontest.py index 91d99a45..8bf42d3c 100644 --- a/buildingspy/development/regressiontest.py +++ b/buildingspy/development/regressiontest.py @@ -3690,11 +3690,17 @@ def get_test_example_coverage(self): # next step get all examples in the package (whether whole library or # single package) + if self._packages: + packs = self._packages + else: + packs = list(dict.fromkeys( + [pac['ScriptFile'].split(os.sep)[0] for pac in self._data])) + tested_model_names = [ nam['ScriptFile'].split(os.sep)[-1][:-1] for nam in temp_data] total_examples = [] - for pac in self._packages: + for pac in packs: for (dirpath, dirnames, filenames) in os.walk( os.path.join(self._libHome, pac)): for f in filenames: @@ -3717,7 +3723,7 @@ def get_test_example_coverage(self): ' out of ', len(total_examples), 'total examples in ') - for pac in self._packages: + for pac in packs: print(pac) print('\n') From 096c751cac3e4a721ecedd43d08e15fbed2863c5 Mon Sep 17 00:00:00 2001 From: MichaMans Date: Sat, 7 Dec 2019 11:46:37 +0100 Subject: [PATCH 9/9] renamed function --- buildingspy/development/regressiontest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildingspy/development/regressiontest.py b/buildingspy/development/regressiontest.py index 60fca044..347c2278 100644 --- a/buildingspy/development/regressiontest.py +++ b/buildingspy/development/regressiontest.py @@ -3679,7 +3679,7 @@ def _analyseOMStats(self, lines=None, models=None, simulate=False): print("\nMore detailed information is stored in self._omstats") print(70 * '#') - def get_test_example_coverage(self): + def get_models_coverage(self): """ Analyse how many examples are tested.