diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5765314..cd912d0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -77,6 +77,7 @@ jobs: finish: name: Finish Coverage Analysis needs: build + if: ${{ always() }} runs-on: ubuntu-latest steps: - name: Coveralls Finished diff --git a/CHANGELOG.md b/CHANGELOG.md index 799fb67..8bdc0c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,13 @@ Change Log All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](https://semver.org/). +[0.X.X] - 2024-XX-XX +-------------------- +* Enhancements + * Added more partial load options for the GNSS line of site data. +* Maintenance + * Updated datetime calls to `utcnow` to be `now` with timezone UTC. + [0.2.0] - 2024-03-15 -------------------- * Enhancements diff --git a/docs/examples/ex_gnss_tec.rst b/docs/examples/ex_gnss_tec.rst index aecc920..524c89d 100644 --- a/docs/examples/ex_gnss_tec.rst +++ b/docs/examples/ex_gnss_tec.rst @@ -1,18 +1,21 @@ .. _ex-gnss-tec: -Plot GNSS TEC -============= +GNSS TEC +======== The Global Navigation Satellte System (GNSS) Total Electron Content (TEC) is one of the most valuable ionospheric data sets, given its long and continuous operational duration and expansive coverage. :py:mod:`pysatMadrigal` currently -only supports Vertical TEC (VTEC) data handling through -:py:mod:`pysatMadrigal.instruments.gnss_tec`. +supports Vertical TEC (VTEC), Line-of-Site (LoS) Slant TEC, and ground receiver +data handling through :py:mod:`pysatMadrigal.instruments.gnss_tec`. -The VTEC measurements are median filtered to fill 1 degree latitude by 1 -degree longitude bins. This can be represented with spatially representative -coverage as shown in the example below. Start by obtaining the desired data -and loading it. +Plot VTEC Maps +-------------- + +The Madrigal VTEC maps are made up of median filtered data placed in 1 degree +latitude by 1 degree longitude bins. This can be represented with spatially +representative coverage as shown in the example below. Start by obtaining the +desired data and loading it. .. code:: @@ -81,3 +84,45 @@ a regular grid with VTEC value indicated by color. :width: 800px :align: center :alt: Mapped median Vertical Total Electron Content over the globe. + + +Load LoS TEC +------------ + +The data used to create the Madrigal VTEC maps is available in the LoS TEC +files. These files contain both the Vertical and Slant TEC from the available +GNSS satellite networks. These files are large (several GB) and may not +successfully download through the MadrigalWeb interface. In such instances, it +may be simpler to download the desired data directly from the Open Madrigal +website. + +Once downloaded, the data is best loaded in subsets. Current load options +include loading data by a unique receiver name ('site'), a unique satellite ID +('prn', 'sat', 'sat_id'), a time ('time', 'unix', 'ut1_unix', or 'ut2_unix'), +an orientation ('azm', 'elm'), or a location ('gdlatr', 'gdlat', 'gdlonr', +'glon'). If specifying a time, orientation, or location a range may also be +specified, as illustrated in the following example. + +.. code:: + + tec = pysat.Instrument(inst_module=pysat_mad.instruments.gnss_tec, + tag='los') + ftime = dt.datetime(2013, 1, 1) + + # If this fails, access data at: + # http://cedar.openmadrigal.org/single?isGlobal=on&categories=17&instruments=8000&years=2013&months=1&days=1 + # Then save the data with a '.hdf5' extension in the directory found by: + # print(tec.files.data_path) + if not ftime in tec.files.files.index: + tec.download(start=ftime, user='firstname+lastname', password='myname@email.address') + + # Load only the GPS data from -40 to -20 degrees longitude + tec.load(date=ftime, los_method='glon', los_value=-30, los_range=10, + gnss_network='gps') + + +When loading data by a specific time or receiver, it may be desirable to +determine what times and receivers are available. The functions +:py:func:`pysatMadrigal.instruments.methods.gnss.get_los_receiver_sites` and +:py:func:`pysatMadrigal.instruments.methods.gnss.get_los_times` can be used on +a list of filenames to determine what loading options are available. diff --git a/pysatMadrigal/instruments/gnss_tec.py b/pysatMadrigal/instruments/gnss_tec.py index b922096..9f42ad5 100644 --- a/pysatMadrigal/instruments/gnss_tec.py +++ b/pysatMadrigal/instruments/gnss_tec.py @@ -264,7 +264,7 @@ def download(date_array, tag='', inst_id='', data_path=None, user=None, def load(fnames, tag='', inst_id='', los_method='site', los_value=None, - gnss_network='all'): + los_range=0, gnss_network='all'): """Load the GNSS TEC data. Parameters @@ -278,10 +278,18 @@ def load(fnames, tag='', inst_id='', los_method='site', los_value=None, Instrument ID used to identify particular data set to be loaded. This input is nominally provided by pysat itself. (default='') los_method : str - For 'los' tag only, load data for a unique GNSS receiver site ('site') - or at a unique time ('time') (default='site') - los_value : str, dt.datetime, or NoneType - For 'los' tag only, load data at this unique site or time (default=None) + Load data for a unique GNSS receiver site ('site'), a unique time + ('time'), a unique unix time ('unix', 'ut1_unix', 'ut2_unix'), a unique + PRN ('prn', 'sat', 'sat_id'), a unique orientation ('azm', 'elm'), or + a unique location ('gdlatr', 'gdlonr', 'gdlat', 'glon'). + los_value : int, float, str, or dt.datetime + For 'los' tag only, load data at this unique site, PRN, time, + orientation, or location. + los_range : int or float + For time, orientation, or location methods specifiy a range that will + be included in the output. Expects a single value and will return + values from `los_value - los_range <= value <= los_value + los_range` + (default=0) gnss_nework : bool For 'los' tag only, limit data by GNSS network if not 'all'. Currently supports 'all', 'gps', and 'glonass' (default='all') @@ -317,8 +325,9 @@ def load(fnames, tag='', inst_id='', los_method='site', los_value=None, if los_value is None: raise ValueError('must specify a valid {:}'.format(los_method)) - data, meta, lat_keys, lon_keys = gnss.load_los(fnames, los_method, - los_value, gnss_network) + data, meta, lat_keys, lon_keys = gnss.load_los( + fnames, los_method, los_value, los_range=los_range, + gnss_network=gnss_network) if len(data.dims.keys()) > 0: # Squeeze the kindat and kinst 'coordinates', but keep them as floats @@ -345,7 +354,8 @@ def load(fnames, tag='', inst_id='', los_method='site', los_value=None, def list_remote_files(tag, inst_id, start=dt.datetime(1998, 10, 15), - stop=dt.datetime.utcnow(), user=None, password=None): + stop=dt.datetime.now(tz=dt.timezone.utc), user=None, + password=None): """Create a Pandas Series of every file for chosen remote data. Parameters @@ -358,10 +368,10 @@ def list_remote_files(tag, inst_id, start=dt.datetime(1998, 10, 15), provided by pysat itself. start : dt.datetime or NoneType Starting time for file list. If None, replaced with default. - (default=10-15-1998) + (default=dt.datetime(1998, 10, 15)) stop : dt.datetime or NoneType Ending time for the file list. If None, replaced with default. - (default=time of run) + (default=dt.datetime.now(tz=dt.timezone.utc)) user : str or NoneType Username to be passed along to resource with relevant data. (default=None) diff --git a/pysatMadrigal/instruments/madrigal_dst.py b/pysatMadrigal/instruments/madrigal_dst.py index 1df06e1..083da4d 100644 --- a/pysatMadrigal/instruments/madrigal_dst.py +++ b/pysatMadrigal/instruments/madrigal_dst.py @@ -67,7 +67,8 @@ madrigal_inst_code = 212 madrigal_tag = {'': {'': "30006"}} madrigal_start = dt.datetime(1957, 1, 1) -madrigal_end = dt.datetime.utcnow() +madrigal_end = pysat.utils.time.filter_datetime_input( + dt.datetime.now(tz=dt.timezone.utc)) # Filters out timezone # Local attributes # diff --git a/pysatMadrigal/instruments/madrigal_geoind.py b/pysatMadrigal/instruments/madrigal_geoind.py index 97e888e..7207b36 100644 --- a/pysatMadrigal/instruments/madrigal_geoind.py +++ b/pysatMadrigal/instruments/madrigal_geoind.py @@ -67,7 +67,8 @@ madrigal_inst_code = 210 madrigal_tag = {'': {'': "30007"}} madrigal_start = dt.datetime(1950, 1, 1) -madrigal_end = dt.datetime.utcnow() +madrigal_end = pysat.utils.time.filter_datetime_input( + dt.datetime.now(tz=dt.timezone.utc)) # Filters out timezone # Local attributes # diff --git a/pysatMadrigal/instruments/madrigal_pandas.py b/pysatMadrigal/instruments/madrigal_pandas.py index 574f09f..1519d31 100644 --- a/pysatMadrigal/instruments/madrigal_pandas.py +++ b/pysatMadrigal/instruments/madrigal_pandas.py @@ -64,7 +64,7 @@ import datetime as dt -from pysat import logger +import pysat from pysatMadrigal.instruments.methods import general @@ -135,10 +135,11 @@ def init(self, kindat=''): # If the kindat (madrigal tag) is not known, advise user self.kindat = kindat if self.kindat == '': - logger.warning('`inst_id` did not supply KINDAT, all will be returned.') + pysat.logger.warning( + '`inst_id` did not supply KINDAT, all will be returned.') # Remind the user of the Rules of the Road - logger.info(self.acknowledgements) + pysat.logger.info(self.acknowledgements) return @@ -155,8 +156,9 @@ def clean(self): """ if self.clean_level in ['clean', 'dusty', 'dirty']: - logger.warning(''.join(["The generalized Madrigal data Instrument ", - "can't support instrument-specific cleaning."])) + pysat.logger.warning(''.join(["The generalized Madrigal data ", + "Instrument can't support instrument-", + "specific cleaning."])) return @@ -292,7 +294,8 @@ def download(date_array, tag, inst_id, data_path, user=None, password=None, def list_remote_files(tag, inst_id, kindat='', user=None, password=None, url="http://cedar.openmadrigal.org", - start=dt.datetime(1900, 1, 1), stop=dt.datetime.utcnow()): + start=dt.datetime(1900, 1, 1), + stop=dt.datetime.now(tz=dt.timezone.utc)): """List files available from Madrigal. Parameters @@ -320,7 +323,8 @@ def list_remote_files(tag, inst_id, kindat='', user=None, password=None, start : dt.datetime Starting time for file list (default=dt.datetime(1900, 1, 1)) stop : dt.datetime - Ending time for the file list (default=dt.datetime.utcnow()) + Ending time for the file list + (default=dt.datetime.now(tz=dt.timezone.utc)) Returns ------- diff --git a/pysatMadrigal/instruments/methods/general.py b/pysatMadrigal/instruments/methods/general.py index 4ca23bd..ea72d58 100644 --- a/pysatMadrigal/instruments/methods/general.py +++ b/pysatMadrigal/instruments/methods/general.py @@ -1022,7 +1022,8 @@ def download(date_array, inst_code=None, kindat=None, data_path=None, def get_remote_filenames(inst_code=None, kindat='', user=None, password=None, web_data=None, url="http://cedar.openmadrigal.org", - start=dt.datetime(1900, 1, 1), stop=dt.datetime.now(), + start=dt.datetime(1900, 1, 1), + stop=dt.datetime.now(tz=dt.timezone.utc), date_array=None): """Retrieve the remote filenames for a specified Madrigal experiment. @@ -1053,7 +1054,7 @@ def get_remote_filenames(inst_code=None, kindat='', user=None, password=None, (default=dt.datetime(1900, 1, 1)) stop : dt.datetime or NoneType Ending time for the file list, None reverts to default - (default=dt.datetime.utcnow()) + (default=dt.datetime.now(tz=dt.timezone.utc)) date_array : dt.datetime or NoneType Array of datetimes to download data for. The sequence of dates need not be contiguous and will be used instead of start and stop if supplied. @@ -1102,7 +1103,10 @@ def get_remote_filenames(inst_code=None, kindat='', user=None, password=None, start = dt.datetime(1900, 1, 1) if stop is None: - stop = dt.datetime.utcnow() + stop = dt.datetime.now(tz=dt.timezone.utc) + + # Ensure the end time does not have timezone information + stop = pysat.utils.time.filter_datetime_input(stop) # If start and stop are identical, increment if start == stop: @@ -1178,7 +1182,7 @@ def list_remote_files(tag, inst_id, inst_code=None, kindats=None, user=None, password=None, supported_tags=None, url="http://cedar.openmadrigal.org", two_digit_year_break=None, start=dt.datetime(1900, 1, 1), - stop=dt.datetime.utcnow()): + stop=dt.datetime.now(tz=dt.timezone.utc)): """List files available from Madrigal. Parameters diff --git a/pysatMadrigal/instruments/methods/gnss.py b/pysatMadrigal/instruments/methods/gnss.py index 0283a18..9809332 100644 --- a/pysatMadrigal/instruments/methods/gnss.py +++ b/pysatMadrigal/instruments/methods/gnss.py @@ -156,7 +156,7 @@ def load_site(fnames): return data, meta, lat_keys, lon_keys -def load_los(fnames, los_method, los_value, gnss_network='all'): +def load_los(fnames, los_method, los_value, los_range=0, gnss_network='all'): """Load the GNSS slant TEC data. Parameters @@ -164,10 +164,18 @@ def load_los(fnames, los_method, los_value, gnss_network='all'): fnames : list List of filenames los_method : str - For 'los' tag only, load data for a unique GNSS receiver site ('site') - or at a unique time ('time') - los_value : str or dt.datetime - For 'los' tag only, load data at this unique site or time + Load data for a unique GNSS receiver site ('site'), a unique time + ('time'), a unique unix time ('unix', 'ut1_unix', 'ut2_unix'), a unique + PRN ('prn', 'sat', 'sat_id'), a unique orientation ('azm', 'elm'), or + a unique location ('gdlatr', 'gdlonr', 'gdlat', 'glon'). + los_value : int, float, str, or dt.datetime + For 'los' tag only, load data at this unique site, PRN, time, + orientation, or location. + los_range : int or float + For time, orientation, or location methods specifiy a range that will + be included in the output. Expects a single value and will return + values from `los_value - los_range <= value <= los_value + los_range` + (default=0) gnss_network : bool Limit data by GNSS network, if not 'all'. Currently supports 'all', 'gps', and 'glonass' (default='all') @@ -185,12 +193,11 @@ def load_los(fnames, los_method, los_value, gnss_network='all'): """ # Define the xarray coordinate dimensions and lat/lon keys - xcoords = {('time', 'gps_site', 'sat_id', 'kindat', 'kinst'): + xcoords = {('time', 'gps_site', 'sat_id', 'gnss_type', 'kindat', 'kinst'): ['pierce_alt', 'los_tec', 'dlos_tec', 'tec', 'azm', 'elm', 'gdlat', 'glon', 'rec_bias', 'drec_bias'], ('time', ): ['year', 'month', 'day', 'hour', 'min', 'sec', 'ut1_unix', 'ut2_unix', 'recno'], - ('time', 'sat_id', ): ['gnss_type'], ('time', 'gps_site', ): ['gdlatr', 'gdlonr']} lat_keys = ['gdlatr', 'gdlat'] lon_keys = ['gdlonr', 'glon'] @@ -208,16 +215,37 @@ def load_los(fnames, los_method, los_value, gnss_network='all'): meta = pysat.Meta() # Load the data using the desired method - if los_method.lower() == 'site': + if los_method.lower() in ['site', 'gps_site']: sel_key = 'gps_site' + sel_range = None # Convert the site to bytes los_value = np.bytes_(los_value) - elif los_method.lower() == 'time': - sel_key = 'ut1_unix' - - # Convert the input datetime to UNIX seconds - los_value = (los_value - dt.datetime(1970, 1, 1)).total_seconds() + elif los_method.lower() in ['time', 'unix', 'ut1_unix', 'ut2_unix']: + sel_key = los_method.lower() if los_method.lower().find( + 'ut') == 0 else 'ut1_unix' + + # Convert the input datetime to UNIX seconds if needed + if isinstance(los_value, dt.datetime): + los_value = (los_value - dt.datetime(1970, 1, 1)).total_seconds() + + # Determine selection range + if los_range == 0: + sel_range = None + else: + sel_range = [los_value - los_range, los_value + los_range] + elif los_method.lower() in ['sat', 'sat_id', 'prn']: + sel_key = 'sat_id' + sel_range = None + elif los_method.lower() in ['azm', 'elm', 'gdlatr', 'gdlat', 'gdlonr', + 'glon']: + sel_key = los_method.lower() + + # Determine selection range + if los_range == 0: + sel_range = None + else: + sel_range = [los_value - los_range, los_value + los_range] else: raise ValueError('unsupported selection type: {:}'.format(los_method)) @@ -227,7 +255,11 @@ def load_los(fnames, los_method, los_value, gnss_network='all'): for fname in load_file_types['hdf5']: with h5py.File(fname, 'r') as fin: sel_arr = fin['Data']['Table Layout'][sel_key] - sel_mask = sel_arr == los_value + + if sel_range is None: + sel_mask = sel_arr == los_value + else: + sel_mask = (sel_arr >= sel_range[0]) & (sel_arr <= sel_range[1]) if gnss_network.lower() != 'all': # Redefine the selection mask to include network as well @@ -258,13 +290,17 @@ def load_los(fnames, los_method, los_value, gnss_network='all'): # Enforce lowercase variable names data.columns = [item.lower() for item in data.columns] - # Convert the data to an xarray Dataset - time_ind = general.build_madrigal_datetime_index(data) + # Get the time index saved in the correct location + if los_method.lower() in ['time', 'unix', 'ut1_unix', 'ut2_unix']: + data['time'] = general.build_madrigal_datetime_index(data) + time_ind = None + else: + time_ind = general.build_madrigal_datetime_index(data) else: time_ind = None - # Convert the output to xarray - data = general.convert_pandas_to_xarray(xcoords, data, time_ind) + # Convert the output to xarray + data = general.convert_pandas_to_xarray(xcoords, data, time_ind) return data, meta, lat_keys, lon_keys diff --git a/pysatMadrigal/tests/test_methods_general.py b/pysatMadrigal/tests/test_methods_general.py index dffee57..cbdb39a 100644 --- a/pysatMadrigal/tests/test_methods_general.py +++ b/pysatMadrigal/tests/test_methods_general.py @@ -817,7 +817,7 @@ def setup_method(self): """Create a clean testing environment.""" self.exp = None self.start = dt.datetime(1950, 1, 1) - self.stop = dt.datetime.utcnow() + self.stop = dt.datetime.now(tz=dt.timezone.utc) return def teardown_method(self):