From e19219ed7a4a28d43323d5f5f2a04223b82a57ec Mon Sep 17 00:00:00 2001 From: Henry LE BERRE Date: Sun, 19 Nov 2023 15:56:29 -0800 Subject: [PATCH] Use CMake to obtain configuration information --- CMakeLists.txt | 8 ++++- toolchain/cmake/configuration.cmake.in | 30 ++++++++++++++++ toolchain/mfc/build.py | 22 ++++++++++-- toolchain/mfc/common.py | 47 -------------------------- toolchain/mfc/packer/pack.py | 38 ++++++++++++++++++--- toolchain/mfc/test/test.py | 7 ++-- 6 files changed, 93 insertions(+), 59 deletions(-) create mode 100644 toolchain/cmake/configuration.cmake.in diff --git a/CMakeLists.txt b/CMakeLists.txt index e8f39f745..7eb2546b1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -26,7 +26,7 @@ option(MFC_SYSCHECK "Build syscheck" OFF) option(MFC_DOCUMENTATION "Build documentation" OFF) option(MFC_ALL "Build everything" OFF) -if (MFC_BUILD_ALL) +if (MFC_ALL) set(MFC_PRE_PROCESS ON FORCE) set(MFC_SIMULATION ON FORCE) set(MFC_POST_PROCESS ON FORCE) @@ -513,3 +513,9 @@ if (MFC_DOCUMENTATION) install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/docs/index.html" DESTINATION "docs/mfc") endif() + +site_name(SITE_NAME) + +configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/toolchain/cmake/configuration.cmake.in" + "${CMAKE_CURRENT_BINARY_DIR}/configuration.txt") diff --git a/toolchain/cmake/configuration.cmake.in b/toolchain/cmake/configuration.cmake.in new file mode 100644 index 000000000..a9b9b5399 --- /dev/null +++ b/toolchain/cmake/configuration.cmake.in @@ -0,0 +1,30 @@ +CMake Configuration: + + CMake v${CMAKE_VERSION} on ${SITE_NAME} + + C : ${CMAKE_C_COMPILER_ID} v${CMAKE_C_COMPILER_VERSION} (${CMAKE_C_COMPILER}) + Fortran : ${CMAKE_Fortran_COMPILER_ID} v${CMAKE_Fortran_COMPILER_VERSION} (${CMAKE_Fortran_COMPILER}) + + PRE_PROCESS : ${MFC_PRE_PROCESS} + SIMULATION : ${MFC_SIMULATION} + POST_PROCESS : ${MFC_POST_PROCESS} + SYSCHECK : ${MFC_SYSCHECK} + DOCUMENTATION : ${MFC_DOCUMENTATION} + ALL : ${MFC_ALL} + + MPI : ${MFC_MPI} + OpenACC : ${MFC_OpenACC} + + Fypp : ${FYPP_EXE} + Doxygen : ${DOXYGEN_EXECUTABLE} + + Build Type : ${CMAKE_BUILD_TYPE} + +Configuration Environment: + + CC : $ENV{CC} + CXX : $ENV{CXX} + FC : $ENV{FC} + OMPI_CC : $ENV{OMPI_CC} + OMPI_CXX : $ENV{OMPI_CXX} + OMPI_FC : $ENV{OMPI_FC} diff --git a/toolchain/mfc/build.py b/toolchain/mfc/build.py index 7505b03a9..a3db1ff8c 100644 --- a/toolchain/mfc/build.py +++ b/toolchain/mfc/build.py @@ -1,4 +1,4 @@ -import os, typing, dataclasses +import re, os, typing, dataclasses from .common import MFCException, system, delete_directory, create_directory from .state import ARG, CFG @@ -70,7 +70,21 @@ def is_configured(self) -> bool: return os.path.isfile( os.sep.join([self.get_build_dirpath(), "CMakeCache.txt"]) ) - + + def get_configuration_txt(self) -> typing.Optional[dict]: + if not self.is_configured(): + return None + + configpath = os.path.join(self.get_build_dirpath(), "configuration.txt") + if not os.path.exists(configpath): + return None + + with open(configpath) as f: + return f.read() + + return None + + def build(self, history: typing.Set[str] = None): if history is None: history = set() @@ -229,6 +243,10 @@ def clean_targets(targets: typing.Iterable[typing.Union[MFCTarget, str]]): get_target(target).clean() +def get_configured_targets() -> typing.List[MFCTarget]: + return [ target for target in TARGETS if target.is_configured() ] + + def build(): build_targets(ARG("targets")) diff --git a/toolchain/mfc/common.py b/toolchain/mfc/common.py index 6d55daa9e..b6a475fe7 100644 --- a/toolchain/mfc/common.py +++ b/toolchain/mfc/common.py @@ -240,50 +240,3 @@ def get_cpuinfo(): else: proc = "No CPU info found" return "CPU Info: \n" + proc - -def get_envinfo(): - myenv = ['FC','CC','CXX','OMPI_FC', 'OMPI_CC', 'OMPI_CXX'] - env = [f"{x} -> {os.getenv(x,' ')}" for x in myenv] - return "Environment Info: \n" + format_list_to_string(env) - -def get_mpiinfo(): - ret = "" - if (does_command_exist("mpif90")): - proc = subprocess.Popen(['which','mpif90'], stdout=subprocess.PIPE,universal_newlines=True) - out, err = proc.communicate() - ret = ret + "mpif90 -> " + out - - if (does_command_exist("mpicc")): - proc = subprocess.Popen(['which','mpicc'], stdout=subprocess.PIPE, - universal_newlines=True) - out, err = proc.communicate() - ret = ret + "mpicc -> " + out - return "MPI Info: \n" + ret - -def get_hostinfo(): - if (does_command_exist("hostname")): - proc = subprocess.Popen(['hostname'], stdout=subprocess.PIPE,universal_newlines=True) - out, err = proc.communicate() - return "Hostname -> " + out - else: - return "No hostname info found" - -def get_lockfile(): - lockfile="build/lock.yaml" - if (os.path.isfile(lockfile)): - # found MFC lockfile - with open(lockfile,"r") as file: - lock = str(yaml.safe_load(file)) - return "Lockfile: \n" + lock + "\n" - else: - return "No lockfile found" - -def get_sysinfo(): - time = "This file was created on: \n" + str(datetime.now()) - host = get_hostinfo() - cpu = get_cpuinfo() - env = get_envinfo() - mod = "Modules: \n" + format_list_to_string(get_loaded_modules()) - mpi = get_mpiinfo() - lock = get_lockfile() - return time + "\n\n" + host + "\n\n" + cpu + "\n\n" + env + "\n\n" + mod + "\n\n" + mpi + "\n\n" + lock diff --git a/toolchain/mfc/packer/pack.py b/toolchain/mfc/packer/pack.py index 695d52eea..6e232854b 100644 --- a/toolchain/mfc/packer/pack.py +++ b/toolchain/mfc/packer/pack.py @@ -1,7 +1,10 @@ -import dataclasses, typing, os, re, math +import dataclasses, typing, sys, os, re, math from .. import common +from ..build import get_configured_targets +from ..state import CFG from ..common import MFCException +from datetime import datetime from pathlib import Path @@ -46,10 +49,37 @@ def save(self, filepath: str): common.file_write(filepath, '\n'.join([ str(e) for e in sorted(self.entries.values(), key=lambda x: x.filepath) ])) - def save_metadata(self, filepath: str, sysinfo): - common.file_write(filepath, sysinfo) + metadata = f"""\ +This file was created on {str(datetime.now())}. - def hash_NaNs(self) -> bool: +mfc.sh: + + Invocation: {' '.join(sys.argv[1:])} + Lock: {CFG()} + +""" + + for target in get_configured_targets(): + cfg = target.get_configuration_txt() + + if cfg is None: + continue + + metadata += f"""\ +{target.name}: + + {' '.join(cfg.splitlines(keepends=True))} +""" + + metadata += f"""\ +CPU: + + {' '.join(common.get_cpuinfo().splitlines(keepends=True))} +""" + + common.file_write(f"{filepath.rstrip('.txt')}-metadata.txt", metadata) + + def has_NaNs(self) -> bool: for entry in self.entries.values(): for double in entry.doubles: if math.isnan(double): diff --git a/toolchain/mfc/test/test.py b/toolchain/mfc/test/test.py index 47775cebc..a68eefa3c 100644 --- a/toolchain/mfc/test/test.py +++ b/toolchain/mfc/test/test.py @@ -7,7 +7,7 @@ from .case import TestCase from .cases import generate_cases from .. import sched -from ..common import MFCException, does_command_exist, format_list_to_string, get_program_output, get_sysinfo +from ..common import MFCException, does_command_exist, format_list_to_string, get_program_output from ..build import build_targets, HDF5, PRE_PROCESS, SIMULATION, POST_PROCESS from ..packer import tol as packtol @@ -160,16 +160,14 @@ def _handle_case(test: TestCase, devices: typing.Set[int]): if err is not None: raise MFCException(f"Test {test}: {err}") - if pack.hash_NaNs(): + if pack.has_NaNs(): raise MFCException(f"Test {test}: NaNs detected in the case.") golden_filepath = os.path.join(test.get_dirpath(), "golden.txt") golden_meta_filepath = os.path.join(test.get_dirpath(), "golden-metadata.txt") - sysinfo = common.get_sysinfo() if ARG("generate"): common.delete_file(golden_filepath) pack.save(golden_filepath) - pack.save_metadata(golden_meta_filepath,sysinfo) else: if not os.path.isfile(golden_filepath): raise MFCException(f"Test {test}: The golden file does not exist! To generate golden files, use the '--generate' flag.") @@ -182,7 +180,6 @@ def _handle_case(test: TestCase, devices: typing.Set[int]): golden.set(pentry) golden.save(golden_filepath) - pack.save_metadata(golden_meta_filepath,sysinfo) else: err, msg = packtol.compare(pack, packer.load(golden_filepath), packtol.Tolerance(tol, tol)) if msg is not None: