From 73e56a7fa14c553f0d64003b88c8e9118f088bea Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Tue, 10 Sep 2024 10:50:33 +0000 Subject: [PATCH] move radiance bias correction staging to jedi class (#2868) --- ush/python/pygfs/jedi/jedi.py | 29 ++++++++++++++++-------- ush/python/pygfs/task/atm_analysis.py | 20 +--------------- ush/python/pygfs/task/atmens_analysis.py | 20 +--------------- 3 files changed, 21 insertions(+), 48 deletions(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index dace86c4fe..4e527460ab 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -1,10 +1,13 @@ #!/usr/bin/env python3 import os +import tarfile from logging import getLogger +from pprint import pformat from typing import List, Dict, Any, Optional from jcb import render from wxflow import (AttrDict, + FileHandler, chdir, rm_p, parse_j2yaml, logit, @@ -188,14 +191,10 @@ def get_obs_dict(self, task_config: AttrDict) -> Dict[str, Any]: return obs_dict @logit(logger) - def get_bias_dict(self, task_config: AttrDict) -> Dict[str, Any]: + def get_bias(self, task_config: AttrDict) -> Dict[str, Any]: """Compile a dictionary of observation files to copy - This method extracts 'observers' from the JEDI yaml and from that list, extracts a list of - observation bias correction files that are to be copied to the run directory - from the component directory. - TODO: COM_ATMOS_ANALYSIS_PREV is hardwired here and this method is not appropriate in - `analysis.py` and should be implemented in the component where this is applicable. + This method stages radiance bias correction files in the obs sub-diretory of the run directory Parameters ---------- @@ -204,8 +203,7 @@ def get_bias_dict(self, task_config: AttrDict) -> Dict[str, Any]: Returns ---------- - bias_dict: Dict - a dictionary containing the list of observation bias files to copy for FileHandler + None """ observations = find_value_in_nested_dict(self.config, 'observations') @@ -218,14 +216,25 @@ def get_bias_dict(self, task_config: AttrDict) -> Dict[str, Any]: basename = os.path.basename(obfile) prefix = '.'.join(basename.split('.')[:-3]) bfile = f"{prefix}.rad_varbc_params.tar" - copylist.append([os.path.join(task_config.COM_ATMOS_ANALYSIS_PREV, bfile), os.path.join(obdir, bfile)]) + radtar = os.path.join(obdir, bfile) + copylist.append([os.path.join(task_config.COM_ATMOS_ANALYSIS_PREV, bfile), radtar]) break bias_dict = { 'mkdir': [os.path.join(task_config.DATA, 'bc')], 'copy': copylist } - return bias_dict + + # stage bias corrections + FileHandler(bias_dict).sync() + logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") + + # extract radiance bias correction files from tarball + radtar = os.path.join(obdir, bfile) + with tarfile.open(radtar, "r") as radbcor: + radbcor.extractall(path=os.path.join(task_config.DATA, 'obs')) + logger.info(f"Extract {radbcor.getnames()}") + radbcor.close() @logit(logger) diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 36f768fa13..c1a6558ade 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 import os -import re import glob import gzip import tarfile @@ -140,24 +139,7 @@ def initialize_analysis(self) -> None: # stage bias corrections logger.info(f"Staging list of bias correction files generated from JEDI config") - bias_dict = self.jedi.get_bias_dict(self.task_config) - FileHandler(bias_dict).sync() - logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") - - # Extract radiance bias correction files from tarball - for action, filelist in bias_dict.items(): - if 'copy' in action: - for sublist in filelist: - if len(sublist) != 2: - raise Exception( - f"List must be of the form ['src', 'dest'], not {sublist}") - src = sublist[0] - if re.search(".tar", src): - radtar = src - with tarfile.open(radtar, "r") as radbcor: - radbcor.extractall(path=os.path.join(self.task_config.DATA, 'obs')) - logger.info(f"Extract {radbcor.getnames()}") - radbcor.close() + self.jedi.get_bias(self.task_config) # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 7004bf059c..1f3a065133 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 import os -import re import glob import gzip import tarfile @@ -139,24 +138,7 @@ def initialize_analysis(self) -> None: # stage bias corrections logger.info(f"Staging list of bias correction files generated from JEDI config") - bias_dict = self.jedi.get_bias_dict(self.task_config) - FileHandler(bias_dict).sync() - logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") - - # Extract radiance bias correction files from tarball - for action, filelist in bias_dict.items(): - if 'copy' in action: - for sublist in filelist: - if len(sublist) != 2: - raise Exception( - f"List must be of the form ['src', 'dest'], not {sublist}") - src = sublist[0] - if re.search(".tar", src): - radtar = src - with tarfile.open(radtar, "r") as radbcor: - radbcor.extractall(path=os.path.join(self.task_config.DATA, 'obs')) - logger.info(f"Extract {radbcor.getnames()}") - radbcor.close() + self.jedi.get_bias(self.task_config) # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}")