diff --git a/docs/uvcal_tutorial.rst b/docs/uvcal_tutorial.rst index 2a95e3eddc..cfa13be261 100644 --- a/docs/uvcal_tutorial.rst +++ b/docs/uvcal_tutorial.rst @@ -148,10 +148,10 @@ b) FHD cal to cal fits >>> import os >>> from pyuvdata import UVCal >>> from pyuvdata.data import DATA_PATH - >>> obs_testfile = os.path.join(DATA_PATH, 'fhd_cal_data/1061316296_obs.sav') - >>> cal_testfile = os.path.join(DATA_PATH, 'fhd_cal_data/1061316296_cal.sav') - >>> settings_testfile = os.path.join(DATA_PATH, 'fhd_cal_data/1061316296_settings.txt') - >>> layout_testfile = os.path.join(DATA_PATH, 'fhd_cal_data/1061316296_layout.sav') + >>> obs_testfile = os.path.join(DATA_PATH, 'fhd_cal_data/metadata/1061316296_obs.sav') + >>> cal_testfile = os.path.join(DATA_PATH, 'fhd_cal_data/calibration/1061316296_cal.sav') + >>> settings_testfile = os.path.join(DATA_PATH, 'fhd_cal_data/metadata/1061316296_settings.txt') + >>> layout_testfile = os.path.join(DATA_PATH, 'fhd_cal_data/metadata/1061316296_layout.sav') >>> # Here we use the ``from_file`` class method, can also use the ``read`` method. >>> # Can optionally specify the ``file_type`` to either method @@ -499,20 +499,20 @@ with the previous file(s). >>> # For FHD cal datasets pass lists for each file type >>> obs_testfiles = [ - ... os.path.join(DATA_PATH, 'fhd_cal_data/1061316296_obs.sav'), + ... os.path.join(DATA_PATH, 'fhd_cal_data/metadata/1061316296_obs.sav'), ... os.path.join(DATA_PATH, 'fhd_cal_data/set2/1061316296_obs.sav'), ... ] >>> cal_testfiles = [ - ... os.path.join(DATA_PATH, 'fhd_cal_data/1061316296_cal.sav'), + ... os.path.join(DATA_PATH, 'fhd_cal_data/calibration/1061316296_cal.sav'), ... os.path.join(DATA_PATH, 'fhd_cal_data/set2/1061316296_cal.sav'), ... ] >>> settings_testfiles = [ - ... os.path.join(DATA_PATH, 'fhd_cal_data/1061316296_settings.txt'), + ... os.path.join(DATA_PATH, 'fhd_cal_data/metadata/1061316296_settings.txt'), ... os.path.join(DATA_PATH, 'fhd_cal_data/set2/1061316296_settings.txt'), ... ] >>> layout_testfiles = [ - ... os.path.join(DATA_PATH, 'fhd_cal_data/1061316296_layout.sav'), - ... os.path.join(DATA_PATH, 'fhd_cal_data/1061316296_layout.sav'), + ... os.path.join(DATA_PATH, 'fhd_cal_data/metadata/1061316296_layout.sav'), + ... os.path.join(DATA_PATH, 'fhd_cal_data/metadata/1061316296_layout.sav'), ... ] >>> fhd_cal = UVCal.from_file( ... cal_testfiles, diff --git a/docs/uvdata_tutorial.rst b/docs/uvdata_tutorial.rst index 86cfefa427..3044275661 100644 --- a/docs/uvdata_tutorial.rst +++ b/docs/uvdata_tutorial.rst @@ -145,16 +145,20 @@ When reading FHD format, we need to point to several files for each observation. >>> uvd = UVData() >>> # Construct the list of files - >>> fhd_prefix = os.path.join(DATA_PATH, 'fhd_vis_data/1061316296_') - >>> fhd_vis_files = [fhd_prefix + f for f in ['vis_XX.sav', 'vis_YY.sav']] + >>> fhd_prefix = '1061316296_' + >>> fhd_vis_files = [os.path.join(DATA_PATH, 'fhd_vis_data', 'vis_data', fhd_prefix + f) for f in ['vis_XX.sav', 'vis_YY.sav']] + >>> flags_file = os.path.join(DATA_PATH, 'fhd_vis_data', 'vis_data', fhd_prefix + 'flags.sav') + >>> layout_file = os.path.join(DATA_PATH, 'fhd_vis_data', 'metadata', fhd_prefix + 'layout.sav') + >>> params_file = os.path.join(DATA_PATH, 'fhd_vis_data', 'metadata', fhd_prefix + 'params.sav') + >>> settings_file = os.path.join(DATA_PATH, 'fhd_vis_data', 'metadata', fhd_prefix + 'settings.txt') # Use the `read` or `from_file` method, optionally specify the file type. >>> uvd = UVData.from_file( ... fhd_vis_files, - ... params_file=fhd_prefix + 'params.sav', - ... flag_file=fhd_prefix + 'flags.sav', - ... layout_file=fhd_prefix + 'layout.sav', - ... settings_file=fhd_prefix + 'settings.txt', + ... flags_file=flags_file, + ... layout_file=layout_file, + ... params_file=params_file, + ... settings_file=settings_file, ... use_future_array_shapes=True ... ) >>> write_file = os.path.join('.', 'tutorial.uvfits') @@ -171,16 +175,21 @@ d) FHD -> miriad >>> import os >>> # Construct the list of files - >>> fhd_prefix = os.path.join(DATA_PATH, 'fhd_vis_data/1061316296_') - >>> fhd_vis_files = [fhd_prefix + f for f in ['vis_XX.sav', 'vis_YY.sav']] + >>> fhd_prefix = '1061316296_' + >>> fhd_vis_files = [os.path.join(DATA_PATH, 'fhd_vis_data', 'vis_data', fhd_prefix + f) for f in ['vis_XX.sav', 'vis_YY.sav']] + >>> flags_file = os.path.join(DATA_PATH, 'fhd_vis_data', 'vis_data', fhd_prefix + 'flags.sav') + >>> layout_file = os.path.join(DATA_PATH, 'fhd_vis_data', 'metadata', fhd_prefix + 'layout.sav') + >>> params_file = os.path.join(DATA_PATH, 'fhd_vis_data', 'metadata', fhd_prefix + 'params.sav') + >>> settings_file = os.path.join(DATA_PATH, 'fhd_vis_data', 'metadata', fhd_prefix + 'settings.txt') + # Use the `read` or `from_file` method, optionally specify the file type. >>> uvd = UVData.from_file( ... fhd_vis_files, - ... params_file=fhd_prefix + 'params.sav', - ... flag_file=fhd_prefix + 'flags.sav', - ... layout_file=fhd_prefix + 'layout.sav', - ... settings_file=fhd_prefix + 'settings.txt', + ... flags_file=flags_file, + ... layout_file=layout_file, + ... params_file=params_file, + ... settings_file=settings_file, ... use_future_array_shapes=True ... ) >>> write_file = os.path.join('.','tutorial.uv') diff --git a/pyuvdata/data/fhd_cal_data/1061316296_cal.sav b/pyuvdata/data/fhd_cal_data/calibration/1061316296_cal.sav similarity index 100% rename from pyuvdata/data/fhd_cal_data/1061316296_cal.sav rename to pyuvdata/data/fhd_cal_data/calibration/1061316296_cal.sav diff --git a/pyuvdata/data/fhd_cal_data/1061316296_layout.sav b/pyuvdata/data/fhd_cal_data/metadata/1061316296_layout.sav similarity index 100% rename from pyuvdata/data/fhd_cal_data/1061316296_layout.sav rename to pyuvdata/data/fhd_cal_data/metadata/1061316296_layout.sav diff --git a/pyuvdata/data/fhd_cal_data/1061316296_obs.sav b/pyuvdata/data/fhd_cal_data/metadata/1061316296_obs.sav similarity index 100% rename from pyuvdata/data/fhd_cal_data/1061316296_obs.sav rename to pyuvdata/data/fhd_cal_data/metadata/1061316296_obs.sav diff --git a/pyuvdata/data/fhd_cal_data/1061316296_settings.txt b/pyuvdata/data/fhd_cal_data/metadata/1061316296_settings.txt similarity index 100% rename from pyuvdata/data/fhd_cal_data/1061316296_settings.txt rename to pyuvdata/data/fhd_cal_data/metadata/1061316296_settings.txt diff --git a/pyuvdata/data/fhd_vis_data/1061316296_layout.sav b/pyuvdata/data/fhd_vis_data/metadata/1061316296_layout.sav similarity index 100% rename from pyuvdata/data/fhd_vis_data/1061316296_layout.sav rename to pyuvdata/data/fhd_vis_data/metadata/1061316296_layout.sav diff --git a/pyuvdata/data/fhd_vis_data/1061316296_obs.sav b/pyuvdata/data/fhd_vis_data/metadata/1061316296_obs.sav similarity index 100% rename from pyuvdata/data/fhd_vis_data/1061316296_obs.sav rename to pyuvdata/data/fhd_vis_data/metadata/1061316296_obs.sav diff --git a/pyuvdata/data/fhd_vis_data/1061316296_params.sav b/pyuvdata/data/fhd_vis_data/metadata/1061316296_params.sav similarity index 100% rename from pyuvdata/data/fhd_vis_data/1061316296_params.sav rename to pyuvdata/data/fhd_vis_data/metadata/1061316296_params.sav diff --git a/pyuvdata/data/fhd_vis_data/1061316296_ps_settings.txt b/pyuvdata/data/fhd_vis_data/metadata/1061316296_ps_settings.txt similarity index 100% rename from pyuvdata/data/fhd_vis_data/1061316296_ps_settings.txt rename to pyuvdata/data/fhd_vis_data/metadata/1061316296_ps_settings.txt diff --git a/pyuvdata/data/fhd_vis_data/1061316296_settings.txt b/pyuvdata/data/fhd_vis_data/metadata/1061316296_settings.txt similarity index 100% rename from pyuvdata/data/fhd_vis_data/1061316296_settings.txt rename to pyuvdata/data/fhd_vis_data/metadata/1061316296_settings.txt diff --git a/pyuvdata/data/fhd_vis_data/1061316296_status.sav b/pyuvdata/data/fhd_vis_data/metadata/1061316296_status.sav similarity index 100% rename from pyuvdata/data/fhd_vis_data/1061316296_status.sav rename to pyuvdata/data/fhd_vis_data/metadata/1061316296_status.sav diff --git a/pyuvdata/data/fhd_vis_data/1061316296_status.txt b/pyuvdata/data/fhd_vis_data/metadata/1061316296_status.txt similarity index 100% rename from pyuvdata/data/fhd_vis_data/1061316296_status.txt rename to pyuvdata/data/fhd_vis_data/metadata/1061316296_status.txt diff --git a/pyuvdata/data/fhd_vis_data/1061316296_flags.sav b/pyuvdata/data/fhd_vis_data/vis_data/1061316296_flags.sav similarity index 100% rename from pyuvdata/data/fhd_vis_data/1061316296_flags.sav rename to pyuvdata/data/fhd_vis_data/vis_data/1061316296_flags.sav diff --git a/pyuvdata/data/fhd_vis_data/1061316296_vis_XX.sav b/pyuvdata/data/fhd_vis_data/vis_data/1061316296_vis_XX.sav similarity index 100% rename from pyuvdata/data/fhd_vis_data/1061316296_vis_XX.sav rename to pyuvdata/data/fhd_vis_data/vis_data/1061316296_vis_XX.sav diff --git a/pyuvdata/data/fhd_vis_data/1061316296_vis_YY.sav b/pyuvdata/data/fhd_vis_data/vis_data/1061316296_vis_YY.sav similarity index 100% rename from pyuvdata/data/fhd_vis_data/1061316296_vis_YY.sav rename to pyuvdata/data/fhd_vis_data/vis_data/1061316296_vis_YY.sav diff --git a/pyuvdata/data/fhd_vis_data/1061316296_vis_model_XX.sav b/pyuvdata/data/fhd_vis_data/vis_data/1061316296_vis_model_XX.sav similarity index 100% rename from pyuvdata/data/fhd_vis_data/1061316296_vis_model_XX.sav rename to pyuvdata/data/fhd_vis_data/vis_data/1061316296_vis_model_XX.sav diff --git a/pyuvdata/data/fhd_vis_data/1061316296_vis_model_YY.sav b/pyuvdata/data/fhd_vis_data/vis_data/1061316296_vis_model_YY.sav similarity index 100% rename from pyuvdata/data/fhd_vis_data/1061316296_vis_model_YY.sav rename to pyuvdata/data/fhd_vis_data/vis_data/1061316296_vis_model_YY.sav diff --git a/pyuvdata/uvcal/fhd_cal.py b/pyuvdata/uvcal/fhd_cal.py index be02b0e6db..96d8299945 100644 --- a/pyuvdata/uvcal/fhd_cal.py +++ b/pyuvdata/uvcal/fhd_cal.py @@ -12,7 +12,7 @@ from .. import utils as uvutils from ..docstrings import copy_replace_short_description -from ..uvdata.fhd import get_fhd_history, get_fhd_layout_info +from ..uvdata.fhd import fhd_filenames, get_fhd_history, get_fhd_layout_info from .uvcal import UVCal, _future_array_shapes_warning __all__ = ["FHDCal"] @@ -49,16 +49,14 @@ def read_fhd_cal( if not read_data and settings_file is None: raise ValueError("A settings_file must be provided if read_data is False.") - filelist = [cal_file, obs_file] - if layout_file is not None: - filelist.append(layout_file) - if settings_file is not None: - filelist.append(settings_file) - for filename in filelist: - # update filelist - basename = os.path.basename(filename) - self.filename = uvutils._combine_filenames(self.filename, [basename]) - self._filename.form = (len(self.filename),) + filenames = fhd_filenames( + obs_file=obs_file, + layout_file=layout_file, + settings_file=settings_file, + cal_file=cal_file, + ) + self.filename = filenames + self._filename.form = (len(self.filename),) this_dict = readsav(obs_file, python_dict=True) obs_data = this_dict["obs"] diff --git a/pyuvdata/uvcal/tests/test_fhd_cal.py b/pyuvdata/uvcal/tests/test_fhd_cal.py index d4e7dd03ab..8fe5610cfd 100644 --- a/pyuvdata/uvcal/tests/test_fhd_cal.py +++ b/pyuvdata/uvcal/tests/test_fhd_cal.py @@ -16,13 +16,13 @@ # set up FHD files testdir = os.path.join(DATA_PATH, "fhd_cal_data/") testfile_prefix = "1061316296_" -obs_testfile = os.path.join(testdir, testfile_prefix + "obs.sav") -cal_testfile = os.path.join(testdir, testfile_prefix + "cal.sav") -settings_testfile = os.path.join(testdir, testfile_prefix + "settings.txt") +obs_testfile = os.path.join(testdir, "metadata", testfile_prefix + "obs.sav") +cal_testfile = os.path.join(testdir, "calibration", testfile_prefix + "cal.sav") +settings_testfile = os.path.join(testdir, "metadata", testfile_prefix + "settings.txt") settings_testfile_nodiffuse = os.path.join( testdir, testfile_prefix + "nodiffuse_settings.txt" ) -layout_testfile = os.path.join(testdir, testfile_prefix + "layout.sav") +layout_testfile = os.path.join(testdir, "metadata", testfile_prefix + "layout.sav") testdir2 = os.path.join(DATA_PATH, "fhd_cal_data/set2") obs_file_multi = [obs_testfile, os.path.join(testdir2, testfile_prefix + "obs.sav")] @@ -48,8 +48,8 @@ def test_read_fhdcal_write_read_calfits(raw, fhd_cal_raw, fhd_cal_fit, tmp_path) fhd_cal = fhd_cal_fit filelist = [cal_testfile, obs_testfile, layout_testfile, settings_testfile] + assert set(fhd_cal.filename) == {os.path.basename(fn) for fn in filelist} - assert fhd_cal.filename == sorted(os.path.basename(file) for file in filelist) assert np.max(fhd_cal.gain_array) < 2.0 outfile = str(tmp_path / "outtest_FHDcal_1061311664.calfits") @@ -109,48 +109,86 @@ def test_read_fhdcal_metadata(raw, fhd_cal_raw, fhd_cal_fit): # test that no diffuse is properly picked up from the settings file when # read_data is False - fhd_cal = UVCal.from_file( - cal_testfile, - obs_file=obs_testfile, - layout_file=layout_testfile, - settings_file=settings_testfile_nodiffuse, - raw=raw, - read_data=False, - use_future_array_shapes=True, - ) + with uvtest.check_warnings( + UserWarning, + match=[ + "Some FHD input files do not have the expected subfolder so FHD folder " + "matching could not be done. The affected file types are: ['settings']", + "The FHD input files do not all have matching prefixes, so they may not be " + "for the same data.", + "Telescope location derived from obs lat/lon/alt values does not match the " + "location in the layout file. Using the value from known_telescopes.", + ], + ): + fhd_cal = UVCal.from_file( + cal_testfile, + obs_file=obs_testfile, + layout_file=layout_testfile, + settings_file=settings_testfile_nodiffuse, + raw=raw, + read_data=False, + use_future_array_shapes=True, + ) assert fhd_cal.diffuse_model is None return -@pytest.mark.filterwarnings("ignore:Telescope location derived from obs lat/lon/alt") def test_read_fhdcal_multimode(): """ Read cal with multiple mode_fit values. """ - fhd_cal = UVCal.from_file( - os.path.join(testdir, testfile_prefix + "multimode_cal.sav"), - obs_file=obs_testfile, - layout_file=layout_testfile, - settings_file=os.path.join(testdir, testfile_prefix + "multimode_settings.txt"), - raw=False, - use_future_array_shapes=True, - ) + with uvtest.check_warnings( + UserWarning, + match=[ + "Some FHD input files do not have the expected subfolder so FHD folder " + "matching could not be done. The affected file types are: " + "['cal', 'settings']", + "The FHD input files do not all have matching prefixes, so they may not be " + "for the same data.", + "Telescope location derived from obs lat/lon/alt values does not match the " + "location in the layout file. Using the value from known_telescopes.", + ], + ): + fhd_cal = UVCal.from_file( + os.path.join(testdir, testfile_prefix + "multimode_cal.sav"), + obs_file=obs_testfile, + layout_file=layout_testfile, + settings_file=os.path.join( + testdir, testfile_prefix + "multimode_settings.txt" + ), + raw=False, + use_future_array_shapes=True, + ) assert fhd_cal.extra_keywords["MODE_FIT"] == "[90, 150, 230, 320, 400, 524]" fhd_cal2 = fhd_cal.copy(metadata_only=True) # check metadata only read - fhd_cal = UVCal.from_file( - os.path.join(testdir, testfile_prefix + "multimode_cal.sav"), - obs_file=obs_testfile, - layout_file=layout_testfile, - settings_file=os.path.join(testdir, testfile_prefix + "multimode_settings.txt"), - raw=False, - read_data=False, - use_future_array_shapes=True, - ) + with uvtest.check_warnings( + UserWarning, + match=[ + "Some FHD input files do not have the expected subfolder so FHD folder " + "matching could not be done. The affected file types are: " + "['cal', 'settings']", + "The FHD input files do not all have matching prefixes, so they may not be " + "for the same data.", + "Telescope location derived from obs lat/lon/alt values does not match the " + "location in the layout file. Using the value from known_telescopes.", + ], + ): + fhd_cal = UVCal.from_file( + os.path.join(testdir, testfile_prefix + "multimode_cal.sav"), + obs_file=obs_testfile, + layout_file=layout_testfile, + settings_file=os.path.join( + testdir, testfile_prefix + "multimode_settings.txt" + ), + raw=False, + read_data=False, + use_future_array_shapes=True, + ) # this file set has a mismatch in Nsources between the cal file & settings # file for some reason. I think it's just an issue with the files chosen assert fhd_cal.Nsources != fhd_cal2.Nsources @@ -213,6 +251,9 @@ def test_flags_galaxy(tmp_path): match=[ "tile_names from obs structure does not match", "Telescope location derived from obs lat/lon/alt", + "Some FHD input files do not have the expected subfolder so FHD folder " + "matching could not be done. The affected file types are: ['cal', " + "'layout', 'obs', 'settings']", ], ): fhd_cal = UVCal.from_file( @@ -235,6 +276,10 @@ def test_unknown_telescope(): match=[ "Telescope foo is not in known_telescopes.", "Telescope location derived from obs lat/lon/alt", + "Some FHD input files do not have the expected subfolder so FHD folder " + "matching could not be done. The affected file types are: ['obs']", + "The FHD input files do not all have matching prefixes, so they may not be " + "for the same data.", ], ): fhd_cal = UVCal.from_file( @@ -275,6 +320,10 @@ def test_break_read_fhdcal(cal_file, obs_file, layout_file, settings_file, nfile if nfiles > 1: message_list *= 2 message_list.append("UVParameter diffuse_model does not match") + message_list.append( + "Some FHD input files do not have the expected subfolder so FHD folder " + "matching could not be done. The affected file types are: ['cal', 'obs']" + ) with uvtest.check_warnings(UserWarning, message_list): fhd_cal = UVCal.from_file( @@ -299,12 +348,17 @@ def test_break_read_fhdcal(cal_file, obs_file, layout_file, settings_file, nfile warning_list = [UserWarning] * (3 * nfiles - 1) if nfiles > 1: - warning_list += [DeprecationWarning] + warning_list += [DeprecationWarning, UserWarning] message_list += [ "Reading multiple files from file specific read methods is deprecated. Use " "the generic `UVCal.read` method instead. This will become an error in " "version 2.5" ] + message_list.append( + "Some FHD input files do not have the expected subfolder so FHD folder " + "matching could not be done. The affected file types are: " + "['cal', 'obs', 'settings']" + ) with uvtest.check_warnings(warning_list, match=message_list): fhd_cal.read_fhd_cal( @@ -338,13 +392,16 @@ def test_break_read_fhdcal(cal_file, obs_file, layout_file, settings_file, nfile ) def test_read_multi(tmp_path, concat_method, read_method): """Test reading in multiple files.""" - warn_type = [UserWarning] * 3 + warn_type = [UserWarning] * 4 msg = [ "UVParameter diffuse_model does not match", "Telescope location derived from obs lat/lon/alt values does not match the " "location in the layout file.", "Telescope location derived from obs lat/lon/alt values does not match the " "location in the layout file.", + "Some FHD input files do not have the expected subfolder so FHD folder " + "matching could not be done. The affected file types are: " + "['cal', 'obs', 'settings']", ] if concat_method == "fast_concat": diff --git a/pyuvdata/uvdata/fhd.py b/pyuvdata/uvdata/fhd.py index 959935861c..bc9cd5f1f4 100644 --- a/pyuvdata/uvdata/fhd.py +++ b/pyuvdata/uvdata/fhd.py @@ -21,6 +21,110 @@ __all__ = ["get_fhd_history", "get_fhd_layout_info", "FHD"] +def fhd_filenames( + *, + vis_files: list[str] | np.ndarray | str | None = None, + params_file: str | None = None, + obs_file: str | None = None, + flags_file: str | None = None, + layout_file: str | None = None, + settings_file: str | None = None, + cal_file: str | None = None, +): + """ + Check the FHD input files for matching prefixes and folders. + + Parameters + ---------- + vis_files : str or array-like of str, optional + FHD visibility save file names, can be data or model visibilities. + params_file : str + FHD params save file name. + obs_file : str + FHD obs save file name. + flags_file : str + FHD flag save file name. + layout_file : str + FHD layout save file name. + layout_file : str + FHD layout save file name. + settings_file : str + FHD settings text file name. + cal_file : str + FHD cal save file name. + + Returns + ------- + A list of file basenames to be used in the object `filename` attribute. + + """ + file_types = { + "vis": {"files": vis_files, "suffix": "_vis", "sub_folder": "vis_data"}, + "cal": {"files": cal_file, "suffix": "_cal", "sub_folder": "calibration"}, + "flags": {"files": flags_file, "suffix": "_flags", "sub_folder": "vis_data"}, + "layout": {"files": layout_file, "suffix": "_layout", "sub_folder": "metadata"}, + "obs": {"files": obs_file, "suffix": "_obs", "sub_folder": "metadata"}, + "params": {"files": params_file, "suffix": "_params", "sub_folder": "metadata"}, + "settings": { + "files": settings_file, + "suffix": "_settings", + "sub_folder": "metadata", + }, + } + + basename_list = [] + prefix_list = [] + folder_list = [] + missing_suffix = [] + missing_subfolder = [] + for ftype, fdict in file_types.items(): + if fdict["files"] is None: + continue + if isinstance(fdict["files"], (list, np.ndarray)): + these_files = fdict["files"] + else: + these_files = [fdict["files"]] + + for fname in these_files: + dirname, basename = os.path.split(fname) + basename_list.append(basename) + if fdict["suffix"] in basename: + suffix_loc = basename.find(fdict["suffix"]) + prefix_list.append(basename[:suffix_loc]) + else: + missing_suffix.append(ftype) + fhd_folder, subfolder = os.path.split(dirname) + if subfolder == fdict["sub_folder"]: + folder_list.append(fhd_folder) + else: + missing_subfolder.append(ftype) + + if len(missing_suffix) > 0: + warnings.warn( + "Some FHD input files do not have the expected suffix so prefix " + f"matching could not be done. The affected file types are: {missing_suffix}" + ) + if len(missing_subfolder) > 0: + warnings.warn( + "Some FHD input files do not have the expected subfolder so FHD " + "folder matching could not be done. The affected file types are: " + f"{missing_subfolder}" + ) + + if np.unique(prefix_list).size > 1: + warnings.warn( + "The FHD input files do not all have matching prefixes, so they " + "may not be for the same data." + ) + if np.unique(folder_list).size > 1: + warnings.warn( + "The FHD input files do not all have the same parent folder, so " + "they may not be for the same FHD run." + ) + + return basename_list + + def get_fhd_history(settings_file, *, return_user=False): """ Small function to get the important history from an FHD settings text file. @@ -323,7 +427,7 @@ def read_fhd( *, params_file: str, obs_file: str | None = None, - flag_file: str | None = None, + flags_file: str | None = None, layout_file: str | None = None, settings_file: str | None = None, background_lsts=True, @@ -364,10 +468,6 @@ def read_fhd( else: raise ValueError("unrecognized file in vis_files") - basename = os.path.basename(filename) - self.filename = uvutils._combine_filenames(self.filename, [basename]) - self._filename.form = (len(self.filename),) - if "_vis_model_" in filename: this_model = True else: @@ -377,16 +477,16 @@ def read_fhd( use_model = this_model elif this_model != use_model: raise ValueError( - "The vis_files parameter has a mix of model and in data files." + "The vis_files parameter has a mix of model and data files." ) if len(datafiles_dict) < 1 and read_data is True: raise ValueError( "The vis_files parameter must be passed if read_data is True" ) - if flag_file is None and read_data is True: + if flags_file is None and read_data is True: raise ValueError( - "The flag_file parameter must be passed if read_data is True" + "The flags_file parameter must be passed if read_data is True" ) if obs_file is None and read_data is False: @@ -406,20 +506,23 @@ def read_fhd( "information will be missing." ) - for fname in [params_file, obs_file, flag_file, layout_file, settings_file]: - if fname is None: - continue - basename = os.path.basename(fname) - self.filename = uvutils._combine_filenames(self.filename, [basename]) - self._filename.form = (len(self.filename),) + filenames = fhd_filenames( + vis_files=vis_files, + params_file=params_file, + obs_file=obs_file, + flags_file=flags_file, + layout_file=layout_file, + settings_file=settings_file, + ) + + self.filename = filenames + self._filename.form = (len(self.filename),) if not read_data: obs_dict = readsav(obs_file, python_dict=True) this_obs = obs_dict["obs"] self.Npols = int(this_obs[0]["N_POL"]) else: - # TODO: add checking to make sure params, flags and datafiles are - # consistent with each other vis_data = {} for pol, file in datafiles_dict.items(): this_dict = readsav(file, python_dict=True) @@ -441,7 +544,7 @@ def read_fhd( params = params_dict["params"] if read_data: - flag_file_dict = readsav(flag_file, python_dict=True) + flag_file_dict = readsav(flags_file, python_dict=True) # The name for this variable changed recently (July 2016). Test for both. vis_weights_data = {} if "flag_arr" in flag_file_dict: @@ -450,7 +553,7 @@ def read_fhd( weights_key = "vis_weights" else: raise ValueError( - "No recognized key for visibility weights in flag_file." + "No recognized key for visibility weights in flags_file." ) for index, w in enumerate(flag_file_dict[weights_key]): vis_weights_data[fhd_pol_list[index]] = w diff --git a/pyuvdata/uvdata/tests/conftest.py b/pyuvdata/uvdata/tests/conftest.py index 99e7b9f735..af9f4b5911 100644 --- a/pyuvdata/uvdata/tests/conftest.py +++ b/pyuvdata/uvdata/tests/conftest.py @@ -3,6 +3,7 @@ # Licensed under the 2-clause BSD License """pytest fixtures for UVData tests.""" +import copy import os import pytest @@ -11,7 +12,6 @@ from pyuvdata import UVData from pyuvdata.data import DATA_PATH from pyuvdata.uvdata.mir_parser import MirParser -from pyuvdata.uvdata.tests.test_fhd import get_fhd_files casa_tutorial_uvfits = os.path.join( DATA_PATH, "day2_TDEM0003_10s_norx_1src_1spw.uvfits" @@ -143,23 +143,45 @@ def mir_data(mir_data_main): def fhd_test_files(): # set up FHD files testdir = os.path.join(DATA_PATH, "fhd_vis_data/") + tf_prefix = "1061316296_" - tf_suffix = [ - "flags.sav", - "vis_XX.sav", - "params.sav", - "vis_YY.sav", - "vis_model_XX.sav", - "vis_model_YY.sav", - "layout.sav", - "settings.txt", - "obs.sav", - ] - testfiles = [] - for s in tf_suffix: - testfiles.append(testdir + tf_prefix + s) - - return get_fhd_files(testfiles) + tf_dict = { + "data_files": [ + os.path.join(testdir, "vis_data", tf_prefix + fname) + for fname in ["vis_XX.sav", "vis_YY.sav"] + ], + "model_files": [ + os.path.join(testdir, "vis_data", tf_prefix + fname) + for fname in ["vis_model_XX.sav", "vis_model_YY.sav"] + ], + "flags_file": os.path.join(testdir, "vis_data", tf_prefix + "flags.sav"), + "params_file": os.path.join(testdir, "metadata", tf_prefix + "params.sav"), + "layout_file": os.path.join(testdir, "metadata", tf_prefix + "layout.sav"), + "settings_file": os.path.join(testdir, "metadata", tf_prefix + "settings.txt"), + "obs_file": os.path.join(testdir, "metadata", tf_prefix + "obs.sav"), + } + + return tf_dict + + +@pytest.fixture(scope="function") +def fhd_data_files(fhd_test_files): + file_dict = copy.deepcopy(fhd_test_files) + file_dict["filename"] = file_dict["data_files"] + del file_dict["data_files"] + del file_dict["model_files"] + + return file_dict + + +@pytest.fixture(scope="function") +def fhd_model_files(fhd_test_files): + file_dict = copy.deepcopy(fhd_test_files) + file_dict["filename"] = file_dict["model_files"] + del file_dict["data_files"] + del file_dict["model_files"] + + return file_dict @pytest.fixture(scope="session") diff --git a/pyuvdata/uvdata/tests/test_fhd.py b/pyuvdata/uvdata/tests/test_fhd.py index 79f181a376..e03b8166b9 100644 --- a/pyuvdata/uvdata/tests/test_fhd.py +++ b/pyuvdata/uvdata/tests/test_fhd.py @@ -5,6 +5,7 @@ """Tests for FHD object. """ +import copy import glob import os from shutil import copyfile @@ -24,7 +25,7 @@ def get_fhd_files(filelist): model_files = [] params_file = None obs_file = None - flag_file = None + flags_file = None layout_file = None settings_file = None for fname in filelist: @@ -38,59 +39,41 @@ def get_fhd_files(filelist): elif "obs" in basename: obs_file = fname elif "flag" in basename: - flag_file = fname + flags_file = fname elif "layout" in basename: layout_file = fname elif "settings" in basename: settings_file = fname - return ( - data_files, - model_files, - params_file, - obs_file, - flag_file, - layout_file, - settings_file, - ) + return { + "data_files": data_files, + "model_files": model_files, + "params_file": params_file, + "obs_file": obs_file, + "flags_file": flags_file, + "layout_file": layout_file, + "settings_file": settings_file, + } @pytest.fixture(scope="function") -def fhd_data(fhd_test_files): +def fhd_data(fhd_data_files): fhd_uv = UVData() - tf_data, tf_model, tf_params, tf_obs, tf_flag, tf_layout, tf_stngs = fhd_test_files - fhd_uv.read( - tf_data, - params_file=tf_params, - obs_file=tf_obs, - flag_file=tf_flag, - layout_file=tf_layout, - settings_file=tf_stngs, - use_future_array_shapes=True, - ) + fhd_uv.read(**fhd_data_files, use_future_array_shapes=True) return fhd_uv @pytest.fixture(scope="function") -def fhd_model(fhd_test_files): +def fhd_model(fhd_model_files): fhd_uv = UVData() - tf_data, tf_model, tf_params, tf_obs, tf_flag, tf_layout, tf_stngs = fhd_test_files - fhd_uv.read( - tf_model, - params_file=tf_params, - obs_file=tf_obs, - flag_file=tf_flag, - layout_file=tf_layout, - settings_file=tf_stngs, - use_future_array_shapes=True, - ) + fhd_uv.read(**fhd_model_files, use_future_array_shapes=True) return fhd_uv @pytest.mark.filterwarnings("ignore:Telescope location derived from obs") -def test_read_fhd_write_read_uvfits(fhd_data, tmp_path, fhd_test_files): +def test_read_fhd_write_read_uvfits(fhd_data, tmp_path, fhd_data_files): """ FHD to uvfits loopback test. @@ -104,20 +87,16 @@ def test_read_fhd_write_read_uvfits(fhd_data, tmp_path, fhd_test_files): fhd_uv.write_uvfits(outfile) uvfits_uv.read_uvfits(outfile, use_future_array_shapes=True) - all_testfiles = list(fhd_test_files) - for fname in fhd_test_files: + all_testfiles = [] + for fname in fhd_data_files.values(): if isinstance(fname, list): - all_testfiles.remove(fname) - all_testfiles.extend(fname) - - all_testfiles_data = [] - for fname in all_testfiles: - temp = os.path.basename(fname) - if "vis_model" not in temp: - all_testfiles_data.append(temp) + for temp in fname: + all_testfiles.append(temp) + else: + all_testfiles.append(fname) # make sure filename attributes are correct - assert set(fhd_uv.filename) == {os.path.basename(fn) for fn in all_testfiles_data} + assert set(fhd_uv.filename) == {os.path.basename(fn) for fn in all_testfiles} assert uvfits_uv.filename == [os.path.basename(outfile)] fhd_uv.filename = uvfits_uv.filename fhd_uv._filename.form = (1,) @@ -130,19 +109,9 @@ def test_read_fhd_write_read_uvfits(fhd_data, tmp_path, fhd_test_files): @pytest.mark.filterwarnings("ignore:Telescope location derived from obs") -def test_read_fhd_metadata_only(fhd_data, fhd_test_files): +def test_read_fhd_metadata_only(fhd_data, fhd_data_files): fhd_uv = UVData() - tf_data, tf_model, tf_params, tf_obs, tf_flag, tf_layout, tf_stngs = fhd_test_files - fhd_uv.read( - tf_data, - params_file=tf_params, - obs_file=tf_obs, - flag_file=tf_flag, - layout_file=tf_layout, - settings_file=tf_stngs, - read_data=False, - use_future_array_shapes=True, - ) + fhd_uv.read(**fhd_data_files, read_data=False, use_future_array_shapes=True) assert fhd_uv.metadata_only @@ -153,32 +122,25 @@ def test_read_fhd_metadata_only(fhd_data, fhd_test_files): @pytest.mark.parametrize("multi", [True, False]) -def test_read_fhd_metadata_only_error(fhd_test_files, multi): +def test_read_fhd_metadata_only_error(fhd_data_files, multi): fhd_uv = UVData() - tf_data, tf_model, tf_params, tf_obs, tf_flag, tf_layout, tf_stngs = fhd_test_files + + del fhd_data_files["obs_file"] if multi: - tf_data = [[tf_data[0]], [tf_data[1]]] - tf_params = [tf_params] * 2 - tf_flag = [tf_flag] * 2 - tf_layout = [tf_layout] * 2 - tf_stngs = [tf_stngs] * 2 + for ftype, fnames in fhd_data_files.items(): + if isinstance(fnames, list): + fhd_data_files[ftype] = [[fnames[0]], [fnames[1]]] + else: + fhd_data_files[ftype] = [fnames] * 2 with pytest.raises( ValueError, match="The obs_file parameter must be passed if read_data is False." ): - fhd_uv.read( - tf_data, - params_file=tf_params, - flag_file=tf_flag, - layout_file=tf_layout, - settings_file=tf_stngs, - read_data=False, - use_future_array_shapes=True, - ) + fhd_uv.read(**fhd_data_files, read_data=False, use_future_array_shapes=True) -def test_read_fhd_select(fhd_test_files): +def test_read_fhd_select(fhd_data_files): """ test select on read with FHD files. @@ -187,7 +149,7 @@ def test_read_fhd_select(fhd_test_files): """ fhd_uv = UVData() fhd_uv2 = UVData() - tf_data, tf_model, tf_params, tf_obs, tf_flag, tf_layout, tf_stngs = fhd_test_files + with uvtest.check_warnings( [UserWarning, UserWarning, DeprecationWarning], [ @@ -199,15 +161,7 @@ def test_read_fhd_select(fhd_test_files): _future_array_shapes_warning, ], ): - fhd_uv2.read( - tf_data, - params_file=tf_params, - obs_file=tf_obs, - flag_file=tf_flag, - layout_file=tf_layout, - settings_file=tf_stngs, - freq_chans=np.arange(2), - ) + fhd_uv2.read(**fhd_data_files, freq_chans=np.arange(2)) with uvtest.check_warnings( [UserWarning, DeprecationWarning], @@ -217,33 +171,27 @@ def test_read_fhd_select(fhd_test_files): _future_array_shapes_warning, ], ): - fhd_uv.read( - tf_data, - params_file=tf_params, - obs_file=tf_obs, - flag_file=tf_flag, - layout_file=tf_layout, - settings_file=tf_stngs, - ) + fhd_uv.read(**fhd_data_files) fhd_uv.select(freq_chans=np.arange(2)) assert fhd_uv == fhd_uv2 @pytest.mark.parametrize("multi", [True, False]) -def test_read_fhd_write_read_uvfits_no_layout(fhd_test_files, multi): +def test_read_fhd_write_read_uvfits_no_layout(fhd_data_files, multi): """ Test errors/warnings with with no layout file. """ fhd_uv = UVData() - tf_data, tf_model, tf_params, tf_obs, tf_flag, tf_layout, tf_stngs = fhd_test_files + + del fhd_data_files["layout_file"] if multi: - tf_data = [[tf_data[0]], [tf_data[1]]] - tf_params = [tf_params] * 2 - tf_obs = [tf_obs] * 2 - tf_flag = [tf_flag] * 2 - tf_stngs = [tf_stngs] * 2 + for ftype, fnames in fhd_data_files.items(): + if isinstance(fnames, list): + fhd_data_files[ftype] = [[fnames[0]], [fnames[1]]] + else: + fhd_data_files[ftype] = [fnames] * 2 if not multi: # check warning raised @@ -252,28 +200,13 @@ def test_read_fhd_write_read_uvfits_no_layout(fhd_test_files, multi): match="The layout_file parameter was not passed, so antenna_postions will " "not be defined and antenna names and numbers might be incorrect.", ): - fhd_uv.read( - tf_data, - params_file=tf_params, - obs_file=tf_obs, - flag_file=tf_flag, - settings_file=tf_stngs, - run_check=False, - use_future_array_shapes=True, - ) + fhd_uv.read(**fhd_data_files, run_check=False, use_future_array_shapes=True) with pytest.raises( ValueError, match="Required UVParameter _antenna_positions has not been set" ): with uvtest.check_warnings(UserWarning, "No layout file"): - fhd_uv.read( - tf_data, - params_file=tf_params, - obs_file=tf_obs, - flag_file=tf_flag, - settings_file=tf_stngs, - use_future_array_shapes=True, - ) + fhd_uv.read(**fhd_data_files, use_future_array_shapes=True) @pytest.mark.filterwarnings("ignore:Telescope location derived from obs") @@ -316,8 +249,7 @@ def test_fhd_antenna_pos(fhd_data): assert mwa_corr_obj._antenna_positions == cotter_obj._antenna_positions -@pytest.mark.filterwarnings("ignore:Telescope location derived from obs") -def test_read_fhd_write_read_uvfits_variant_flag(tmp_path, fhd_test_files): +def test_read_fhd_write_read_uvfits_variant_flag(tmp_path, fhd_data_files): """ FHD to uvfits loopback test with variant flag file. @@ -326,19 +258,44 @@ def test_read_fhd_write_read_uvfits_variant_flag(tmp_path, fhd_test_files): """ fhd_uv = UVData() uvfits_uv = UVData() - variant_flag_file = os.path.join( + + variant_flags_file = os.path.join( DATA_PATH, "fhd_vis_data/", "1061316296_variant_flags.sav" ) - tf_data, tf_model, tf_params, tf_obs, tf_flag, tf_layout, tf_stngs = fhd_test_files - fhd_uv.read( - tf_data, - params_file=tf_params, - obs_file=tf_obs, - flag_file=variant_flag_file, - layout_file=tf_layout, - settings_file=tf_stngs, - use_future_array_shapes=True, + fhd_data_files["flags_file"] = variant_flags_file + + with uvtest.check_warnings( + UserWarning, + match=[ + "Some FHD input files do not have the expected subfolder so FHD folder " + "matching could not be done. The affected file types are: ['flags']", + "The FHD input files do not all have matching prefixes, so they may not be " + "for the same data.", + "Telescope location derived from obs lat/lon/alt values does not match the " + "location in the layout file. Using the value from known_telescopes.", + ], + ): + fhd_uv.read(**fhd_data_files, use_future_array_shapes=True) + + os.makedirs(os.path.join(tmp_path, "vis_data")) + temp_flag_file = copyfile( + variant_flags_file, os.path.join(tmp_path, "vis_data", "foo.sav") ) + fhd_data_files["flags_file"] = temp_flag_file + with uvtest.check_warnings( + UserWarning, + match=[ + "Some FHD input files do not have the expected suffix so prefix matching " + "could not be done. The affected file types are: ['flags']", + "The FHD input files do not all have the same parent folder, so they may " + "not be for the same FHD run.", + "Telescope location derived from obs lat/lon/alt values does not match the " + "location in the layout file. Using the value from known_telescopes.", + ], + ): + fhd_uv2 = UVData.from_file(**fhd_data_files, use_future_array_shapes=True) + + assert fhd_uv == fhd_uv2 outfile = str(tmp_path / "outtest_FHD_1061316296.uvfits") fhd_uv.write_uvfits(outfile) @@ -351,8 +308,7 @@ def test_read_fhd_write_read_uvfits_variant_flag(tmp_path, fhd_test_files): assert fhd_uv == uvfits_uv -@pytest.mark.filterwarnings("ignore:Telescope location derived from obs lat/lon/alt") -def test_read_fhd_write_read_uvfits_fix_layout(tmp_path, fhd_test_files): +def test_read_fhd_write_read_uvfits_fix_layout(tmp_path, fhd_data_files): """ FHD to uvfits loopback test with fixed array center layout file. @@ -361,20 +317,48 @@ def test_read_fhd_write_read_uvfits_fix_layout(tmp_path, fhd_test_files): """ fhd_uv = UVData() uvfits_uv = UVData() + layout_fixed_file = os.path.join( DATA_PATH, "fhd_vis_data/", "1061316296_fixed_arr_center_layout.sav" ) - tf_data, tf_model, tf_params, tf_obs, tf_flag, tf_layout, tf_stngs = fhd_test_files - fhd_uv.read( - tf_data, - params_file=tf_params, - obs_file=tf_obs, - flag_file=tf_flag, - layout_file=layout_fixed_file, - settings_file=tf_stngs, - use_future_array_shapes=True, + fhd_data_files["layout_file"] = layout_fixed_file + with uvtest.check_warnings( + UserWarning, + match=[ + "Some FHD input files do not have the expected subfolder so FHD folder " + "matching could not be done. The affected file types are: ['layout']", + "The FHD input files do not all have matching prefixes, so they may not be " + "for the same data.", + "Telescope location derived from obs lat/lon/alt values does not match the " + "location in the layout file. Using the value from known_telescopes.", + ], + ): + fhd_uv.read(**fhd_data_files, use_future_array_shapes=True) + + os.makedirs(os.path.join(tmp_path, "fhd_vis_data2", "metadata")) + temp_layout_file = copyfile( + layout_fixed_file, + os.path.join( + tmp_path, + "fhd_vis_data2", + "metadata", + "1061316296_fixed_arr_center_layout.sav", + ), ) + fhd_data_files["layout_file"] = temp_layout_file + with uvtest.check_warnings( + UserWarning, + match=[ + "The FHD input files do not all have the same parent folder, so they may " + "not be for the same FHD run.", + "The FHD input files do not all have matching prefixes, so they may not be " + "for the same data.", + "Telescope location derived from obs lat/lon/alt values does not match the " + "location in the layout file. Using the value from known_telescopes.", + ], + ): + fhd_uv.read(**fhd_data_files, use_future_array_shapes=True) outfile = str(tmp_path / "outtest_FHD_1061316296.uvfits") @@ -388,7 +372,7 @@ def test_read_fhd_write_read_uvfits_fix_layout(tmp_path, fhd_test_files): assert fhd_uv == uvfits_uv -def test_read_fhd_write_read_uvfits_fix_layout_bad_obs_loc(tmp_path, fhd_test_files): +def test_read_fhd_write_read_uvfits_fix_layout_bad_obs_loc(tmp_path, fhd_data_files): """ FHD to uvfits loopback test with fixed array center layout file, bad obs location. @@ -406,19 +390,17 @@ def test_read_fhd_write_read_uvfits_fix_layout_bad_obs_loc(tmp_path, fhd_test_fi messages = [ "Telescope location derived from obs", "tile_names from obs structure does not match", + "Some FHD input files do not have the expected subfolder so FHD folder " + "matching could not be done. The affected file types are: ['vis', 'layout']", + "The FHD input files do not all have matching prefixes, so they may not be " + "for the same data.", ] - tf_data, tf_model, tf_params, tf_obs, tf_flag, tf_layout, tf_stngs = fhd_test_files - with uvtest.check_warnings(UserWarning, messages): - fhd_uv.read( - [bad_obs_loc_file], - params_file=tf_params, - obs_file=tf_obs, - flag_file=tf_flag, - layout_file=layout_fixed_file, - settings_file=tf_stngs, - use_future_array_shapes=True, - ) + fhd_data_files["filename"] = [bad_obs_loc_file] + fhd_data_files["layout_file"] = layout_fixed_file + + with uvtest.check_warnings(UserWarning, match=messages): + fhd_uv.read(**fhd_data_files, use_future_array_shapes=True) outfile = str(tmp_path / "outtest_FHD_1061316296.uvfits") fhd_uv.write_uvfits(outfile) @@ -430,7 +412,7 @@ def test_read_fhd_write_read_uvfits_fix_layout_bad_obs_loc(tmp_path, fhd_test_fi assert fhd_uv == uvfits_uv -def test_read_fhd_write_read_uvfits_bad_obs_loc(tmp_path, fhd_test_files): +def test_read_fhd_write_read_uvfits_bad_obs_loc(tmp_path, fhd_data_files): """ FHD to uvfits loopback test with bad obs location (and bad layout location). @@ -445,18 +427,16 @@ def test_read_fhd_write_read_uvfits_bad_obs_loc(tmp_path, fhd_test_files): messages = [ "Telescope location derived from obs", "tile_names from obs structure does not match", + "Some FHD input files do not have the expected subfolder so FHD folder " + "matching could not be done. The affected file types are: ['vis']", + "The FHD input files do not all have matching prefixes, so they may not be " + "for the same data.", ] - tf_data, tf_model, tf_params, tf_obs, tf_flag, tf_layout, tf_stngs = fhd_test_files - with uvtest.check_warnings(UserWarning, messages): - fhd_uv.read( - [bad_obs_loc_file], - params_file=tf_params, - obs_file=tf_obs, - flag_file=tf_flag, - layout_file=tf_layout, - settings_file=tf_stngs, - use_future_array_shapes=True, - ) + + fhd_data_files["filename"] = [bad_obs_loc_file] + + with uvtest.check_warnings(UserWarning, match=messages): + fhd_uv.read(**fhd_data_files, use_future_array_shapes=True) outfile = str(tmp_path / "outtest_FHD_1061316296.uvfits") fhd_uv.write_uvfits(outfile) @@ -468,7 +448,7 @@ def test_read_fhd_write_read_uvfits_bad_obs_loc(tmp_path, fhd_test_files): assert fhd_uv == uvfits_uv -def test_read_fhd_write_read_uvfits_altered_layout(tmp_path, fhd_test_files): +def test_read_fhd_write_read_uvfits_altered_layout(tmp_path, fhd_data_files): """ FHD to uvfits loopback test with altered layout file. @@ -482,16 +462,19 @@ def test_read_fhd_write_read_uvfits_altered_layout(tmp_path, fhd_test_files): altered_layout_file = os.path.join( DATA_PATH, "fhd_vis_data/", "1061316296_broken_layout.sav" ) - tf_data, tf_model, tf_params, tf_obs, tf_flag, tf_layout, tf_stngs = fhd_test_files - fhd_uv.read( - tf_data, - params_file=tf_params, - obs_file=tf_obs, - flag_file=tf_flag, - layout_file=altered_layout_file, - settings_file=tf_stngs, - use_future_array_shapes=True, - ) + + fhd_data_files["layout_file"] = altered_layout_file + + with uvtest.check_warnings( + UserWarning, + match=[ + "Some FHD input files do not have the expected subfolder so FHD folder " + "matching could not be done. The affected file types are: ['layout']", + "The FHD input files do not all have matching prefixes, so they may not be " + "for the same data.", + ], + ): + fhd_uv.read(**fhd_data_files, use_future_array_shapes=True) outfile = str(tmp_path / "outtest_FHD_1061316296.uvfits") fhd_uv.write_uvfits(outfile) @@ -504,7 +487,7 @@ def test_read_fhd_write_read_uvfits_altered_layout(tmp_path, fhd_test_files): @pytest.mark.parametrize("multi", [True, False]) -def test_read_fhd_write_read_uvfits_no_settings(tmp_path, fhd_test_files, multi): +def test_read_fhd_write_read_uvfits_no_settings(tmp_path, fhd_data_files, multi): """ FHD to uvfits loopback test with no settings file. @@ -519,25 +502,17 @@ def test_read_fhd_write_read_uvfits_no_settings(tmp_path, fhd_test_files, multi) "Telescope location derived from obs lat/lon/alt values does not match the " "location in the layout file. Using the value from known_telescopes.", ] - tf_data, tf_model, tf_params, tf_obs, tf_flag, tf_layout, tf_stngs = fhd_test_files - + del fhd_data_files["settings_file"] if multi: messages *= 2 - tf_data = [[tf_data[0]], [tf_data[1]]] - tf_params = [tf_params] * 2 - tf_obs = [tf_obs] * 2 - tf_flag = [tf_flag] * 2 - tf_layout = [tf_layout] * 2 - - with uvtest.check_warnings(UserWarning, messages): - fhd_uv.read( - tf_data, - params_file=tf_params, - obs_file=tf_obs, - flag_file=tf_flag, - layout_file=tf_layout, - use_future_array_shapes=True, - ) + for ftype, fnames in fhd_data_files.items(): + if isinstance(fnames, list): + fhd_data_files[ftype] = [[fnames[0]], [fnames[1]]] + else: + fhd_data_files[ftype] = [fnames] * 2 + + with uvtest.check_warnings(UserWarning, match=messages): + fhd_uv.read(**fhd_data_files, use_future_array_shapes=True) if not multi: # Check only pyuvdata history with no settings file @@ -553,93 +528,63 @@ def test_read_fhd_write_read_uvfits_no_settings(tmp_path, fhd_test_files, multi) assert fhd_uv == uvfits_uv -def test_break_read_fhd(fhd_test_files): +def test_break_read_fhd(fhd_data_files, fhd_model_files): """Try various cases of incomplete file lists.""" fhd_uv = UVData() # missing flags - tf_data, tf_model, tf_params, tf_obs, tf_flag, tf_layout, tf_stngs = fhd_test_files + + file_dict = copy.deepcopy(fhd_data_files) + del file_dict["flags_file"] with pytest.raises( - ValueError, match="The flag_file parameter must be passed if read_data is True" + ValueError, match="The flags_file parameter must be passed if read_data is True" ): - fhd_uv.read( - tf_data, - params_file=tf_params, - obs_file=tf_obs, - layout_file=tf_layout, - settings_file=tf_stngs, - use_future_array_shapes=True, - ) + fhd_uv.read(**file_dict, use_future_array_shapes=True) + + for ftype, fnames in file_dict.items(): + if isinstance(fnames, list): + file_dict[ftype] = [[fnames[0]], [fnames[1]]] + else: + file_dict[ftype] = [fnames] * 2 with pytest.raises( - ValueError, match="The flag_file parameter must be passed if read_data is True" + ValueError, match="The flags_file parameter must be passed if read_data is True" ): - fhd_uv.read( - [[tf_data[0]], [tf_data[1]]], - params_file=[tf_params] * 2, - obs_file=[tf_obs] * 2, - layout_file=[tf_layout] * 2, - settings_file=[tf_stngs] * 2, - use_future_array_shapes=True, - ) + fhd_uv.read(**file_dict, use_future_array_shapes=True) + file_dict = copy.deepcopy(fhd_data_files) + del file_dict["params_file"] # Missing params with pytest.raises( ValueError, match="The params_file must be passed for FHD files." ): - fhd_uv.read( - tf_data, - flag_file=tf_flag, - obs_file=tf_obs, - layout_file=tf_layout, - settings_file=tf_stngs, - use_future_array_shapes=True, - ) + fhd_uv.read(**file_dict, use_future_array_shapes=True) + file_dict = copy.deepcopy(fhd_data_files) + file_dict["filename"] = ["foo.sav"] # No data files with pytest.raises(ValueError, match="unrecognized file in vis_files"): - fhd_uv.read( - ["foo.sav"], - params_file=tf_params, - flag_file=tf_flag, - obs_file=tf_obs, - layout_file=tf_layout, - settings_file=tf_stngs, - use_future_array_shapes=True, - ) + fhd_uv.read(**file_dict, use_future_array_shapes=True) + file_dict["filename"] = [None] # No data files with pytest.raises( ValueError, match="The vis_files parameter must be passed if read_data is True" ): - fhd_uv.read( - [None], - params_file=tf_params, - flag_file=tf_flag, - obs_file=tf_obs, - layout_file=tf_layout, - settings_file=tf_stngs, - file_type="fhd", - use_future_array_shapes=True, - ) + fhd_uv.read(**file_dict, file_type="fhd", use_future_array_shapes=True) # mix of model & data files + + file_dict["filename"] = [ + fhd_data_files["filename"][0], + fhd_model_files["filename"][1], + ] with pytest.raises( - ValueError, - match="The vis_files parameter has a mix of model and in data files.", + ValueError, match="The vis_files parameter has a mix of model and data files." ): - fhd_uv.read( - [tf_data[0], tf_model[1]], - params_file=tf_params, - flag_file=tf_flag, - obs_file=tf_obs, - layout_file=tf_layout, - settings_file=tf_stngs, - file_type="fhd", - use_future_array_shapes=True, - ) + fhd_uv.read(**file_dict, file_type="fhd", use_future_array_shapes=True) -def test_read_fhd_warnings(fhd_test_files): +def test_read_fhd_warnings(fhd_data_files): """Test warnings with various broken inputs.""" # bad obs structure values broken_data_file = os.path.join( @@ -651,38 +596,38 @@ def test_read_fhd_warnings(fhd_test_files): "Telescope location derived from obs", "These visibilities may have been phased improperly", "Nbls does not match", + "Some FHD input files do not have the expected subfolder so FHD folder " + "matching could not be done. The affected file types are: ['vis']", + "The FHD input files do not all have matching prefixes, so they may not be for " + "the same data.", ] fhd_uv = UVData() - tf_data, tf_model, tf_params, tf_obs, tf_flag, tf_layout, tf_stngs = fhd_test_files - with uvtest.check_warnings(UserWarning, warn_messages): - fhd_uv.read( - broken_data_file, - params_file=tf_params, - flag_file=tf_flag, - obs_file=tf_obs, - layout_file=tf_layout, - settings_file=tf_stngs, - run_check=False, - use_future_array_shapes=True, - ) + + fhd_data_files["filename"] = broken_data_file + + with uvtest.check_warnings(UserWarning, match=warn_messages): + fhd_uv.read(**fhd_data_files, run_check=False, use_future_array_shapes=True) # bad flag file - broken_flag_file = os.path.join( + broken_flags_file = os.path.join( DATA_PATH, "fhd_vis_data/", "1061316296_broken_flags.sav" ) + + fhd_data_files["flags_file"] = broken_flags_file fhd_uv = UVData() - with pytest.raises( - ValueError, match="No recognized key for visibility weights in flag_file." + with uvtest.check_warnings( + UserWarning, + match=[ + "Some FHD input files do not have the expected subfolder so FHD folder " + "matching could not be done. The affected file types are: ['vis', 'flags']", + "The FHD input files do not all have matching prefixes, so they may not be " + "for the same data.", + ], ): - fhd_uv.read( - tf_data, - params_file=tf_params, - flag_file=broken_flag_file, - obs_file=tf_obs, - layout_file=tf_layout, - settings_file=tf_stngs, - use_future_array_shapes=True, - ) + with pytest.raises( + ValueError, match="No recognized key for visibility weights in flags_file." + ): + fhd_uv.read(**fhd_data_files, use_future_array_shapes=True) @pytest.mark.parametrize( @@ -694,27 +639,20 @@ def test_read_fhd_warnings(fhd_test_files): (["vis_YX.sav", "extra_vis_YX.sav"], 0, "multiple yx datafiles in vis_files"), ], ) -def test_read_fhd_extra_files(new_file_end, file_copy_ind, message, fhd_test_files): - tf_data, tf_model, tf_params, tf_obs, tf_flag, tf_layout, tf_stngs = fhd_test_files +def test_read_fhd_extra_files( + tmp_path, new_file_end, file_copy_ind, message, fhd_data_files +): # try cases with extra files of each type new_files = [] for file_end in new_file_end: - extra_file = os.path.join(DATA_PATH, "fhd_vis_data/", "1061316296_" + file_end) + extra_file = os.path.join(tmp_path, "1061316296_" + file_end) new_files.append(extra_file) - copyfile(tf_data[file_copy_ind], extra_file) + copyfile(fhd_data_files["filename"][file_copy_ind], extra_file) + + fhd_data_files["filename"].extend(new_files) fhd_uv = UVData() with pytest.raises(ValueError, match=message): - fhd_uv.read( - tf_data + new_files, - params_file=tf_params, - flag_file=tf_flag, - obs_file=tf_obs, - layout_file=tf_layout, - settings_file=tf_stngs, - use_future_array_shapes=True, - ) - for extra_file in new_files: - os.remove(extra_file) + fhd_uv.read(**fhd_data_files, use_future_array_shapes=True) @pytest.mark.filterwarnings("ignore:Telescope location derived from obs") @@ -735,23 +673,20 @@ def test_read_fhd_model(tmp_path, fhd_model): @pytest.mark.filterwarnings("ignore:Telescope location derived from obs") @pytest.mark.parametrize("axis", [None, "polarization"]) -def test_multi_files(fhd_model, axis, fhd_test_files): +def test_multi_files(fhd_model, axis, fhd_model_files): """Read multiple files at once.""" fhd_uv1 = UVData() fhd_uv2 = UVData() - tf_data, tf_model, tf_params, tf_obs, tf_flag, tf_layout, tf_stngs = fhd_test_files - test1 = [tf_model[0]] - test2 = [tf_model[1]] + for ftype, fnames in fhd_model_files.items(): + if isinstance(fnames, list): + fhd_model_files[ftype] = [[fnames[0]], [fnames[1]]] + else: + fhd_model_files[ftype] = [fnames] * 2 + fhd_model_files["filename"] = np.array( + [fhd_model_files["filename"][0], fhd_model_files["filename"][1]] + ) fhd_uv1.read( - np.array([test1, test2]), - params_file=[tf_params, tf_params], - flag_file=[tf_flag, tf_flag], - obs_file=[tf_obs, tf_obs], - layout_file=[tf_layout, tf_layout], - settings_file=[tf_stngs, tf_stngs], - file_type="fhd", - axis=axis, - use_future_array_shapes=True, + **fhd_model_files, file_type="fhd", axis=axis, use_future_array_shapes=True ) fhd_uv2 = fhd_model @@ -767,34 +702,35 @@ def test_multi_files(fhd_model, axis, fhd_test_files): @pytest.mark.filterwarnings("ignore:Telescope location derived from obs") -@pytest.mark.parametrize("ftype_err", ["params", "obs", "flag", "layout", "settings"]) -def test_multi_files_errors(fhd_model, fhd_test_files, ftype_err): +@pytest.mark.parametrize( + "ftype_err", + ["params_file", "obs_file", "flags_file", "layout_file", "settings_file"], +) +def test_multi_files_errors(fhd_model, fhd_model_files, ftype_err): fhd_uv1 = UVData() - tf_data, tf_model, tf_params, tf_obs, tf_flag, tf_layout, tf_stngs = fhd_test_files - test1 = [tf_model[0]] - test2 = [tf_model[1]] + for ftype, fnames in fhd_model_files.items(): + if isinstance(fnames, list): + fhd_model_files[ftype] = [[fnames[0]], [fnames[1]]] + else: + fhd_model_files[ftype] = [fnames] * 2 + fhd_model_files["filename"] = np.array( + [fhd_model_files["filename"][0], fhd_model_files["filename"][1]] + ) - n_files_per_type = np.zeros(5, dtype="int") + 2 - ftype_ind_dict = {"params": 0, "flag": 1, "obs": 2, "layout": 3, "settings": 4} - n_files_per_type[ftype_ind_dict[ftype_err]] = 3 + fhd_model_files[ftype_err] = [fhd_model_files[ftype_err]] * 3 msg = "For multiple FHD files, " - if ftype_err == "params": + if ftype_err == "params_file": msg += "the number of params_file" else: - ftype_name = ftype_err + "_file" + ftype_name = ftype_err msg += "if " + ftype_name + " is passed, the number of " + ftype_name msg += " values must match the number of data file sets." with pytest.raises(ValueError, match=msg): fhd_uv1.read( - np.array([test1, test2]), - params_file=[tf_params] * n_files_per_type[0], - flag_file=[tf_flag] * n_files_per_type[1], - obs_file=[tf_obs] * n_files_per_type[2], - layout_file=[tf_layout] * n_files_per_type[3], - settings_file=[tf_stngs] * n_files_per_type[4], + **fhd_model_files, file_type="fhd", axis="polarization", use_future_array_shapes=True, @@ -806,15 +742,10 @@ def test_single_time(): test reading in a file with a single time. """ single_time_filelist = glob.glob(os.path.join(DATA_PATH, "refsim1.1_fhd/*")) - ( - data_files, - model_files, - params_file, - obs_file, - flag_file, - layout_file, - settings_file, - ) = get_fhd_files(single_time_filelist) + file_dict = get_fhd_files(single_time_filelist) + file_dict["filename"] = file_dict["data_files"] + del file_dict["data_files"] + del file_dict["model_files"] fhd_uv = UVData() with uvtest.check_warnings( @@ -822,17 +753,12 @@ def test_single_time(): [ "tile_names from obs structure does not match", "Telescope location derived from obs lat/lon/alt", + "Some FHD input files do not have the expected subfolder so FHD folder " + "matching could not be done. The affected file types are: ['vis', 'vis', " + "'flags', 'layout', 'params', 'settings']", ], ): - fhd_uv.read( - data_files, - params_file=params_file, - flag_file=flag_file, - obs_file=obs_file, - layout_file=layout_file, - settings_file=settings_file, - use_future_array_shapes=True, - ) + fhd_uv.read(**file_dict, use_future_array_shapes=True) assert np.unique(fhd_uv.time_array).size == 1 @@ -841,15 +767,11 @@ def test_conjugation(): """test uvfits vs fhd conjugation""" uvfits_file = os.path.join(DATA_PATH, "ref_1.1_uniform.uvfits") single_time_filelist = glob.glob(os.path.join(DATA_PATH, "refsim1.1_fhd/*")) - ( - data_files, - model_files, - params_file, - obs_file, - flag_file, - layout_file, - settings_file, - ) = get_fhd_files(single_time_filelist) + file_dict = get_fhd_files(single_time_filelist) + file_dict["filename"] = file_dict["data_files"] + del file_dict["data_files"] + del file_dict["model_files"] + uvfits_uv = UVData() uvfits_uv.read(uvfits_file, use_future_array_shapes=True) @@ -859,17 +781,12 @@ def test_conjugation(): [ "tile_names from obs structure does not match", "Telescope location derived from obs lat/lon/alt", + "Some FHD input files do not have the expected subfolder so FHD folder " + "matching could not be done. The affected file types are: ['vis', 'vis', " + "'flags', 'layout', 'params', 'settings']", ], ): - fhd_uv.read( - data_files, - params_file=params_file, - flag_file=flag_file, - obs_file=obs_file, - layout_file=layout_file, - settings_file=settings_file, - use_future_array_shapes=True, - ) + fhd_uv.read(**file_dict, use_future_array_shapes=True) uvfits_uv.select(polarizations=fhd_uv.polarization_array) diff --git a/pyuvdata/uvdata/tests/test_uvdata.py b/pyuvdata/uvdata/tests/test_uvdata.py index 038319d617..e04aebe705 100644 --- a/pyuvdata/uvdata/tests/test_uvdata.py +++ b/pyuvdata/uvdata/tests/test_uvdata.py @@ -9736,27 +9736,13 @@ def test_remove_eq_coeffs_errors(casa_uvfits): ("read_fhd", []), ], ) -def test_multifile_read_errors(read_func, filelist, fhd_test_files): +def test_multifile_read_errors(read_func, filelist, fhd_data_files): uv = UVData() kwargs = {} if "fhd" in read_func: - ( - tf_data, - tf_model, - tf_params, - tf_obs, - tf_flag, - tf_layout, - tf_stngs, - ) = fhd_test_files - filelist = [[tf_data[0]], [tf_data[1]]] - kwargs = { - "params_file": tf_params, - "obs_file": tf_obs, - "flag_file": tf_flag, - "layout_file": tf_layout, - "settings_file": tf_stngs, - } + filelist = [[fhd_data_files["filename"][0]], [fhd_data_files["filename"][1]]] + kwargs = fhd_data_files + del kwargs["filename"] with pytest.raises( ValueError, match="Reading multiple files from class specific read functions is no " @@ -11968,28 +11954,14 @@ def test_set_nsamples_wrong_shape_error(hera_uvh5): ], ], ) -def test_from_file(filename, msg, fhd_test_files): +def test_from_file(filename, msg, fhd_data_files): kwargs = {} if "miriad" in filename: pytest.importorskip("pyuvdata._miriad") elif "fhd" in filename: - ( - tf_data, - tf_model, - tf_params, - tf_obs, - tf_flag, - tf_layout, - tf_stngs, - ) = fhd_test_files - filename = tf_data - kwargs = { - "params_file": tf_params, - "obs_file": tf_obs, - "flag_file": tf_flag, - "layout_file": tf_layout, - "settings_file": tf_stngs, - } + filename = fhd_data_files["filename"] + kwargs = fhd_data_files + del kwargs["filename"] if isinstance(filename, str): testfile = os.path.join(DATA_PATH, filename) diff --git a/pyuvdata/uvdata/uvdata.py b/pyuvdata/uvdata/uvdata.py index ce0dbb8a80..05bd01ef10 100644 --- a/pyuvdata/uvdata/uvdata.py +++ b/pyuvdata/uvdata/uvdata.py @@ -10499,7 +10499,7 @@ def read_fhd(self, vis_files, *, params_file, **kwargs): The FHD params save file. Required. obs_file : str The FHD obs save file. Required if `read_data` is False. - flag_file : str + flags_file : str The FHD data (or model) flag save file. Required if `read_data` is True. layout_file : str The FHD layout save file. Required for correct antenna metadata. @@ -11436,7 +11436,7 @@ def read( # FHD params_file=None, obs_file=None, - flag_file=None, + flags_file=None, layout_file=None, settings_file=None, # MS @@ -11693,7 +11693,7 @@ def read( The FHD params save file. Required. obs_file : str The FHD obs save file. Required if `read_data` is False. - flag_file : str + flags_file : str The FHD data (or model) flag save file. Required if `read_data` is True. layout_file : str The FHD layout save file. Required for correct antenna metadata. @@ -12002,7 +12002,7 @@ def read( f = filename[file_num] params_file_use = None obs_file_use = None - flag_file_use = None + flags_file_use = None layout_file_use = None settings_file_use = None if file_type == "fhd": @@ -12028,18 +12028,18 @@ def read( else: obs_file = [None] * n_files - if flag_file is not None: + if flags_file is not None: if ( - not isinstance(flag_file, (list, tuple, np.ndarray)) - or len(flag_file) != n_files + not isinstance(flags_file, (list, tuple, np.ndarray)) + or len(flags_file) != n_files ): raise ValueError( - "For multiple FHD files, if flag_file is passed, the " - "number of flag_file values must match the number of data " + "For multiple FHD files, if flags_file is passed, the " + "number of flags_file values must match the number of data " "file sets." ) else: - flag_file = [None] * n_files + flags_file = [None] * n_files if layout_file is not None: if ( @@ -12069,7 +12069,7 @@ def read( params_file_use = params_file[file_num] obs_file_use = obs_file[file_num] - flag_file_use = flag_file[file_num] + flags_file_use = flags_file[file_num] layout_file_use = layout_file[file_num] settings_file_use = settings_file[file_num] @@ -12117,7 +12117,7 @@ def read( # FHD params_file=params_file_use, obs_file=obs_file_use, - flag_file=flag_file_use, + flags_file=flags_file_use, layout_file=layout_file_use, settings_file=settings_file_use, # MS @@ -12200,7 +12200,7 @@ def read( if file_type == "fhd": params_file_use = params_file[file_num] obs_file_use = obs_file[file_num] - flag_file_use = flag_file[file_num] + flags_file_use = flags_file[file_num] layout_file_use = layout_file[file_num] settings_file_use = settings_file[file_num] @@ -12250,7 +12250,7 @@ def read( # FHD params_file=params_file_use, obs_file=obs_file_use, - flag_file=flag_file_use, + flags_file=flags_file_use, layout_file=layout_file_use, settings_file=settings_file_use, # MS @@ -12612,7 +12612,7 @@ def read( vis_files=filename, params_file=params_file, obs_file=obs_file, - flag_file=flag_file, + flags_file=flags_file, layout_file=layout_file, settings_file=settings_file, background_lsts=background_lsts, diff --git a/pyuvdata/uvflag/uvflag.py b/pyuvdata/uvflag/uvflag.py index 946737a03d..0334217675 100644 --- a/pyuvdata/uvflag/uvflag.py +++ b/pyuvdata/uvflag/uvflag.py @@ -482,7 +482,7 @@ def __init__( ) self._telescope_location = uvp.LocationParameter( - "telescope_location", description=desc, tols=1e-3, frame="itrs", + "telescope_location", description=desc, tols=1e-3, frame="itrs" ) self._history = uvp.UVParameter(