Adds build type preference selection and tablediff case insensitivity

energyplus_regressions/__init__.py

@@ -1,2 +1,2 @@
 NAME = 'energyplus_regressions'
-VERSION = '2.1.2'
+VERSION = '2.1.3'

energyplus_regressions/builds/base.py

@@ -87,10 +87,10 @@ def should_keep(file_path: Path):
         filtered_list = filter(should_keep, all_idfs_relative_path)
         return set(filtered_list)
 
-    def set_build_directory(self, build_directory: Path):
+    def set_build_directory(self, build_directory: Path) -> None:
         raise NotImplementedError('Must implement set_build_directory(str) in derived classes')
 
-    def verify(self):
+    def verify(self) -> list[tuple[str, Path, bool]]:
         results: list[tuple[str, Path, bool]] = []
         if not self.build_directory:
             raise Exception('Build directory has not been set with set_build_directory()')
@@ -140,5 +140,5 @@ def verify(self):
     def get_build_tree(self) -> BuildTree:
         raise NotImplementedError('Must implement get_build_tree() in derived classes')
 
-    def get_idf_directory(self):
+    def get_idf_directory(self) -> Path:
         raise NotImplementedError()

energyplus_regressions/builds/install.py

@@ -21,7 +21,7 @@ def set_build_directory(self, build_directory: Path):
         # For an E+ install, the source directory is kinda just the root repo
         self.source_directory = build_directory
 
-    def get_idf_directory(self):
+    def get_idf_directory(self) -> Path:
         if not self.build_directory:
             raise Exception('Build directory has not been set with set_build_directory()')
         return self.source_directory / 'ExampleFiles'

energyplus_regressions/builds/makefile.py

@@ -30,7 +30,7 @@ def set_build_directory(self, build_directory: Path):
             else:
                 raise Exception('Could not find source directory spec in the CMakeCache file')
 
-    def get_idf_directory(self):
+    def get_idf_directory(self) -> Path:
         if not self.build_directory:
             raise Exception('Build directory has not been set with set_build_directory()')
         return self.source_directory / 'testfiles'

energyplus_regressions/builds/visualstudio.py

@@ -1,6 +1,8 @@
 from pathlib import Path
+from typing import Callable
 
 from energyplus_regressions.builds.base import BaseBuildDirectoryStructure, BuildTree
+from energyplus_regressions.structures import ConfigType
 
 
 class CMakeCacheVisualStudioBuildDirectory(BaseBuildDirectoryStructure):
@@ -11,12 +13,27 @@ class CMakeCacheVisualStudioBuildDirectory(BaseBuildDirectoryStructure):
 
     def __init__(self):
         super(CMakeCacheVisualStudioBuildDirectory, self).__init__()
-        self.build_mode: str = 'Release'
+        self.build_mode: str = ConfigType.RELEASE.value
 
-    def set_build_mode(self, debug):
-        self.build_mode = 'Debug' if debug else 'Release'
+    def set_build_mode(self, config: ConfigType, error_callback: Callable[[str], None] | None = None) -> None:
+        build_mode_folder = config.value
+        desired_build_directory = self.build_directory / 'Products' / build_mode_folder
+        if desired_build_directory.exists():
+            self.build_mode = config.value
+        else:
+            if error_callback:
+                error_callback(
+                    f"Attempted to set build mode as {config.value} but did not detect build dir, use caution! :)"
+                )
+            build_mode_folder = 'Release'
+            release_folder = self.build_directory / 'Products' / build_mode_folder
+            release_folder_exists = release_folder.exists()
+            if release_folder_exists:
+                self.build_mode = ConfigType.RELEASE.value
+            else:  # Finally, if we can't find release either, just set it to debug and let the user deal with it
+                self.build_mode = ConfigType.DEBUG.value
 
-    def set_build_directory(self, build_directory: Path):
+    def set_build_directory(self, build_directory: Path) -> None:
         """
         This method takes a build directory, and updates any dependent member variables, in this case the source dir.
         This method *does* allow an invalid build_directory, as could happen during program initialization
@@ -37,15 +54,8 @@ def set_build_directory(self, build_directory: Path):
                     break
             else:
                 raise Exception('Could not find source directory spec in the CMakeCache file')
-        build_mode_folder = 'Release'
-        release_folder = self.build_directory / 'Products' / build_mode_folder
-        release_folder_exists = release_folder.exists()
-        if release_folder_exists:
-            self.set_build_mode(debug=False)
-        else:
-            self.set_build_mode(debug=True)
 
-    def get_idf_directory(self):
+    def get_idf_directory(self) -> Path:
         if not self.build_directory:
             raise Exception('Build directory has not been set with set_build_directory()')
         return self.source_directory / 'testfiles'

energyplus_regressions/diffs/table_diff.py

@@ -121,8 +121,12 @@ def thresh_abs_rel_diff(abs_thresh, rel_thresh, x, y):
         # else:
         #     diff = 'equal'
         return abs_diff, rel_diff, diff
-    except:
-        return '%s vs %s' % (x, y), '%s vs %s' % (x, y), 'stringdiff'
+    except ValueError:
+        # if we couldn't get a float out of it, we are doing string comparison, check case-insensitively before leaving
+        if x.lower().strip() == y.lower().strip():
+            return 0, 0, 'equal'
+        else:
+            return '%s vs %s' % (x, y), '%s vs %s' % (x, y), 'stringdiff'
 
 
 def prev_sib(entity):

energyplus_regressions/energyplus.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 import glob
 from os import chdir, getcwd, rename, environ
 from pathlib import Path
@@ -8,8 +7,6 @@
 from energyplus_regressions.builds.base import BuildTree
 from energyplus_regressions.structures import ForceRunType
 
-script_dir = Path(__file__).resolve().parent
-
 
 class ExecutionArguments:
     def __init__(self, build_tree: BuildTree, entry_name: str, test_run_directory: Path,
@@ -24,17 +21,17 @@ def __init__(self, build_tree: BuildTree, entry_name: str, test_run_directory: P
 
 
 # noinspection PyBroadException
-def execute_energyplus(e_args: ExecutionArguments):
+def execute_energyplus(e_args: ExecutionArguments) -> tuple[Path, str, bool, bool, str]:
     # set up a few paths
     energyplus = e_args.build_tree.energyplus
     basement = e_args.build_tree.basement
     idd_path = e_args.build_tree.idd_path
     slab = e_args.build_tree.slab
-    basementidd = e_args.build_tree.basementidd
-    slabidd = e_args.build_tree.slabidd
-    expandobjects = e_args.build_tree.expandobjects
-    epmacro = e_args.build_tree.epmacro
-    readvars = e_args.build_tree.readvars
+    basement_idd = e_args.build_tree.basementidd
+    slab_idd = e_args.build_tree.slabidd
+    expand_objects = e_args.build_tree.expandobjects
+    ep_macro = e_args.build_tree.epmacro
+    read_vars = e_args.build_tree.readvars
     parametric = e_args.build_tree.parametric
 
     # Save the current path so we can go back here
@@ -61,7 +58,7 @@ def execute_energyplus(e_args: ExecutionArguments):
         imf_path = e_args.test_run_directory / 'in.imf'
         ght_file = e_args.test_run_directory / 'GHTIn.idf'
         basement_file = e_args.test_run_directory / 'BasementGHTIn.idf'
-        epjson_file = e_args.test_run_directory / 'in.epJSON'
+        ep_json_file = e_args.test_run_directory / 'in.epJSON'
         rvi_file = e_args.test_run_directory / 'in.rvi'
         mvi_file = e_args.test_run_directory / 'in.mvi'
 
@@ -79,7 +76,7 @@ def execute_energyplus(e_args: ExecutionArguments):
                 for line in newlines:
                     f.write(line)
             macro_run = subprocess.Popen(
-                str(epmacro), shell=True, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE
+                str(ep_macro), shell=True, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE
             )
             o, e = macro_run.communicate()
             std_out += o
@@ -102,12 +99,13 @@ def execute_energyplus(e_args: ExecutionArguments):
                     idf_file.unlink()
                 rename(file_to_run_here, idf_file)
             else:
-                return [e_args.build_tree.build_dir, e_args.entry_name, False, False, "Issue with Parametrics"]
+                return e_args.build_tree.build_dir, e_args.entry_name, False, False, "Issue with Parametric"
 
         # Run ExpandObjects and process as necessary, but not for epJSON files!
         if idf_file.exists():
             expand_objects_run = subprocess.Popen(
-                str(expandobjects), shell=True, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE
+                str(expand_objects), shell=True, stdin=subprocess.DEVNULL,
+                stdout=subprocess.PIPE, stderr=subprocess.PIPE
             )
             o, e = expand_objects_run.communicate()
             std_out += o
@@ -118,7 +116,7 @@ def execute_energyplus(e_args: ExecutionArguments):
                 rename(expanded_file, idf_file)
 
                 if basement_file.exists():
-                    shutil.copy(basementidd, e_args.test_run_directory)
+                    shutil.copy(basement_idd, e_args.test_run_directory)
                     basement_environment = environ.copy()
                     basement_environment['CI_BASEMENT_NUMYEARS'] = '2'
                     basement_run = subprocess.Popen(
@@ -140,7 +138,7 @@ def execute_energyplus(e_args: ExecutionArguments):
                     (e_args.test_run_directory / 'BasementGHT.idd').unlink()
 
                 if ght_file.exists():
-                    shutil.copy(slabidd, e_args.test_run_directory)
+                    shutil.copy(slab_idd, e_args.test_run_directory)
                     slab_run = subprocess.Popen(
                         str(slab), shell=True, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE
                     )
@@ -184,7 +182,7 @@ def execute_energyplus(e_args: ExecutionArguments):
         # Execute EnergyPlus
         try:
             command_line = str(energyplus)
-            if epjson_file.exists():
+            if ep_json_file.exists():
                 command_line += ' in.epJSON'
             std_out += subprocess.check_output(
                 command_line, shell=True, stdin=subprocess.DEVNULL, stderr=subprocess.PIPE
@@ -193,31 +191,31 @@ def execute_energyplus(e_args: ExecutionArguments):
             ...
             # so I can verify that I hit this during the test_case_b_crash test, but if I just have the return in
             #  here alone, it shows as missing on the coverage...wonky
-            return [e_args.build_tree.build_dir, e_args.entry_name, False, False, str(e)]
+            return e_args.build_tree.build_dir, e_args.entry_name, False, False, str(e)
 
-        # Execute readvars
+        # Execute read-vars
         if rvi_file.exists():
             csv_run = subprocess.Popen(
-                str(readvars) + ' in.rvi', shell=True, stdin=subprocess.DEVNULL,
+                str(read_vars) + ' in.rvi', shell=True, stdin=subprocess.DEVNULL,
                 stdout=subprocess.PIPE, stderr=subprocess.PIPE
             )
         else:
             csv_run = subprocess.Popen(
-                str(readvars), shell=True, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+                str(read_vars), shell=True, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         o, e = csv_run.communicate()
         std_out += o
         std_err += e
         if mvi_file.exists():
             mtr_run = subprocess.Popen(
-                str(readvars) + ' in.mvi', shell=True, stdin=subprocess.DEVNULL,
+                str(read_vars) + ' in.mvi', shell=True, stdin=subprocess.DEVNULL,
                 stdout=subprocess.PIPE, stderr=subprocess.PIPE
             )
         else:
             with mvi_file.open('w') as f:
                 f.write("eplusout.mtr\n")
                 f.write("eplusmtr.csv\n")
             mtr_run = subprocess.Popen(
-                str(readvars) + ' in.mvi', shell=True, stdin=subprocess.DEVNULL,
+                str(read_vars) + ' in.mvi', shell=True, stdin=subprocess.DEVNULL,
                 stdout=subprocess.PIPE, stderr=subprocess.PIPE
             )
         o, e = mtr_run.communicate()
@@ -232,11 +230,10 @@ def execute_energyplus(e_args: ExecutionArguments):
                 f.write(std_err.decode('utf-8'))
 
         new_idd_path.unlink()
-        return [e_args.build_tree.build_dir, e_args.entry_name, True, False]
+        chdir(start_path)
+        return e_args.build_tree.build_dir, e_args.entry_name, True, False, ""
 
     except Exception as e:
         print("**" + str(e))
-        return [e_args.build_tree.build_dir, e_args.entry_name, False, False, str(e)]
-
-    finally:
         chdir(start_path)
+        return e_args.build_tree.build_dir, e_args.entry_name, False, False, str(e)

energyplus_regressions/runtests.py

@@ -17,7 +17,7 @@
 
 from difflib import unified_diff  # python's own diff library
 
-from energyplus_regressions.builds.base import BuildTree
+from energyplus_regressions.builds.base import BuildTree, BaseBuildDirectoryStructure
 from energyplus_regressions.diffs import math_diff, table_diff, thresh_dict as td
 from energyplus_regressions.energyplus import ExecutionArguments, execute_energyplus
 from energyplus_regressions.structures import (
@@ -41,8 +41,9 @@
 class TestRunConfiguration:
     __test__ = False  # so that PyTest doesn't try to run this as a class fixture
 
-    def __init__(self, force_run_type, num_threads, report_freq, build_a, build_b, single_test_run=False,
-                 force_output_sql: ForceOutputSQL = ForceOutputSQL.NOFORCE,
+    def __init__(self, force_run_type: str, num_threads: int, report_freq: str,
+                 build_a: BaseBuildDirectoryStructure, build_b: BaseBuildDirectoryStructure,
+                 single_test_run: bool = False, force_output_sql: ForceOutputSQL = ForceOutputSQL.NOFORCE,
                  force_output_sql_unitconv: ForceOutputSQLUnitConversion = ForceOutputSQLUnitConversion.NOFORCE):
         self.force_run_type = force_run_type
         self.TestOneFile = single_test_run
@@ -57,7 +58,7 @@ def __init__(self, force_run_type, num_threads, report_freq, build_a, build_b, s
 class TestCaseCompleted:
     __test__ = False  # so that PyTest doesn't try to run this as a class fixture
 
-    def __init__(self, run_directory, case_name, run_status, error_msg_reported_already, extra_message=""):
+    def __init__(self, run_directory: str, case_name: str, run_status, error_msg_reported_already, extra_message=""):
         self.run_directory = run_directory
         self.case_name = case_name
         self.run_success = run_status
@@ -68,7 +69,7 @@ def __init__(self, run_directory, case_name, run_status, error_msg_reported_alre
 # the actual main test suite run class
 class SuiteRunner:
 
-    def __init__(self, run_config, these_entries, mute=False):
+    def __init__(self, run_config: TestRunConfiguration, these_entries, mute=False):
 
         # initialize the master mute button -- this is overridden by registering callbacks
         self.mute = mute
@@ -446,7 +447,7 @@ def run_build(self, build_tree: BuildTree):
                     local_run_type,
                     self.min_reporting_freq,
                     parametric_file,
-                    epw_path
+                    str(epw_path)
                 )
             )
 

energyplus_regressions/structures.py

@@ -52,6 +52,11 @@ class ForceOutputSQLUnitConversion(Enum):
     InchPound = 'InchPound'
 
 
+class ConfigType(Enum):
+    RELEASE = "Release"
+    DEBUG = "Debug"
+
+
 class Results:
     def __init__(self):
         self.descriptions = {}

energyplus_regressions/tests/builds/test_visualstudio.py

@@ -5,6 +5,7 @@
 
 from energyplus_regressions.builds.base import BuildTree
 from energyplus_regressions.builds.visualstudio import CMakeCacheVisualStudioBuildDirectory
+from energyplus_regressions.structures import ConfigType
 
 
 class TestVisualStudioBuildMethods(unittest.TestCase):
@@ -13,6 +14,7 @@ def setUp(self):
         self.build = CMakeCacheVisualStudioBuildDirectory()
         self.run_dir = Path(tempfile.mkdtemp())
         self.dummy_source_dir = Path('/dummy/source/dir')
+        self.output_message: str = ""
 
     def set_cache_file(self):
         with open(os.path.join(self.run_dir, 'CMakeCache.txt'), 'w') as f:
@@ -77,3 +79,31 @@ def test_get_idf_dir(self):
         self.build.set_build_directory(self.run_dir)
         idf_dir = self.build.get_idf_directory()
         self.assertEqual(self.dummy_source_dir / 'testfiles', idf_dir)
+
+    def test_set_build_mode_folder_exists(self):
+        self.set_cache_file()
+        self.build.set_build_directory(self.run_dir)
+        os.makedirs(os.path.join(self.run_dir, 'Products', 'Release'))
+        self.build.set_build_mode(ConfigType.RELEASE)
+        self.assertEqual(ConfigType.RELEASE.value, self.build.build_mode)
+        os.makedirs(os.path.join(self.run_dir, 'Products', 'Debug'))
+        self.build.set_build_mode(ConfigType.DEBUG)
+        self.assertEqual(ConfigType.DEBUG.value, self.build.build_mode)
+
+    def _message_capture(self, s: str):
+        self.output_message = s
+
+    def test_set_build_debug_does_not_exist(self):
+        self.set_cache_file()
+        self.build.set_build_directory(self.run_dir)
+        self.build.set_build_mode(ConfigType.DEBUG, self._message_capture)
+        self.assertIn("caution", self.output_message.lower())
+        self.assertEqual(ConfigType.DEBUG.value, self.build.build_mode)  # should fall to debug type
+
+    def test_set_build_debug_does_not_exist_release_does(self):
+        self.set_cache_file()
+        self.build.set_build_directory(self.run_dir)
+        os.makedirs(os.path.join(self.run_dir, 'Products', 'Release'))
+        self.build.set_build_mode(ConfigType.DEBUG, self._message_capture)
+        self.assertIn("caution", self.output_message.lower())
+        self.assertEqual(ConfigType.RELEASE.value, self.build.build_mode)  # should fall to release type