diff --git a/icesat2_toolkit/convert.py b/icesat2_toolkit/convert.py index fad5baf..64a0ee2 100644 --- a/icesat2_toolkit/convert.py +++ b/icesat2_toolkit/convert.py @@ -1,6 +1,6 @@ """ convert.py -Written by Tyler Sutterley (12/2022) +Written by Tyler Sutterley (03/2024) Utilities for converting ICESat-2 HDF5 files into different formats PYTHON DEPENDENCIES: @@ -17,6 +17,7 @@ https://pandas.pydata.org/ UPDATE HISTORY: + Updated 03/2024: use pathlib to define and operate on paths Updated 12/2022: place some imports behind try/except statements Updated 06/2022: place zarr and pandas imports behind try/except statements Updated 04/2022: updated docstrings to numpy documentation format diff --git a/icesat2_toolkit/io/ATL03.py b/icesat2_toolkit/io/ATL03.py index 65dab2f..59d6750 100644 --- a/icesat2_toolkit/io/ATL03.py +++ b/icesat2_toolkit/io/ATL03.py @@ -1,6 +1,6 @@ #!/usr/bin/env python u""" -ATL03.py (11/2023) +ATL03.py (03/2024) Read ICESat-2 ATL03 and ATL09 data files to calculate average segment surfaces ATL03 datasets: Global Geolocated Photons ATL09 datasets: Atmospheric Characteristics @@ -15,6 +15,7 @@ https://www.h5py.org/ UPDATE HISTORY: + Updated 03/2024: use pathlib to define and operate on paths Updated 11/2023: drop DIMENSION_LIST, CLASS and NAME attributes Updated 12/2022: place some imports behind try/except statements refactor ICESat-2 data product read programs under io diff --git a/icesat2_toolkit/io/ATL06.py b/icesat2_toolkit/io/ATL06.py index 58a0963..4bd1bcc 100644 --- a/icesat2_toolkit/io/ATL06.py +++ b/icesat2_toolkit/io/ATL06.py @@ -1,6 +1,6 @@ #!/usr/bin/env python u""" -ATL06.py (11/2023) +ATL06.py (03/2024) Read ICESat-2 ATL06 (Land Ice Along-Track Height Product) data files OPTIONS: @@ -16,6 +16,7 @@ https://www.h5py.org/ UPDATE HISTORY: + Updated 03/2024: use pathlib to define and operate on paths Updated 11/2023: drop DIMENSION_LIST, CLASS and NAME attributes Updated 05/2023: extract more ancillary data from ATL06 files Updated 12/2022: place some imports behind try/except statements diff --git a/icesat2_toolkit/io/ATL07.py b/icesat2_toolkit/io/ATL07.py index a38204b..cbe9fdb 100644 --- a/icesat2_toolkit/io/ATL07.py +++ b/icesat2_toolkit/io/ATL07.py @@ -1,6 +1,6 @@ #!/usr/bin/env python u""" -ATL07.py (11/2023) +ATL07.py (03/2024) Read ICESat-2 ATL07 (Sea Ice Height) data files PYTHON DEPENDENCIES: @@ -11,6 +11,7 @@ https://www.h5py.org/ UPDATE HISTORY: + Updated 03/2024: use pathlib to define and operate on paths Updated 11/2023: drop DIMENSION_LIST, CLASS and NAME attributes Updated 05/2023: extract more ancillary data from ATL07 files Updated 12/2022: place some imports behind try/except statements diff --git a/icesat2_toolkit/io/ATL10.py b/icesat2_toolkit/io/ATL10.py index 85bc7f3..2ad9ed1 100644 --- a/icesat2_toolkit/io/ATL10.py +++ b/icesat2_toolkit/io/ATL10.py @@ -1,6 +1,6 @@ #!/usr/bin/env python u""" -ATL10.py (11/2023) +ATL10.py (03/2024) Read ICESat-2 ATL10 (Sea Ice Freeboard) data files PYTHON DEPENDENCIES: @@ -11,6 +11,7 @@ https://www.h5py.org/ UPDATE HISTORY: + Updated 03/2024: use pathlib to define and operate on paths Updated 11/2023: drop DIMENSION_LIST, CLASS and NAME attributes Updated 05/2023: extract more ancillary data from ATL10 files Updated 12/2022: place some imports behind try/except statements diff --git a/icesat2_toolkit/io/ATL11.py b/icesat2_toolkit/io/ATL11.py index 080b332..41dc33e 100644 --- a/icesat2_toolkit/io/ATL11.py +++ b/icesat2_toolkit/io/ATL11.py @@ -1,6 +1,6 @@ #!/usr/bin/env python u""" -ATL11.py (11/2023) +ATL11.py (03/2024) Read ICESat-2 ATL11 (Annual Land Ice Height) data files OPTIONS: @@ -18,6 +18,7 @@ https://www.h5py.org/ UPDATE HISTORY: + Updated 03/2024: use pathlib to define and operate on paths Updated 11/2023: drop DIMENSION_LIST, CLASS and NAME attributes Updated 05/2023: extract more ancillary data from ATL11 files Updated 12/2022: place some imports behind try/except statements diff --git a/icesat2_toolkit/io/ATL12.py b/icesat2_toolkit/io/ATL12.py index b7ff28b..a200b10 100644 --- a/icesat2_toolkit/io/ATL12.py +++ b/icesat2_toolkit/io/ATL12.py @@ -1,6 +1,6 @@ #!/usr/bin/env python u""" -ATL12.py (11/2023) +ATL12.py (03/2024) Read ICESat-2 ATL12 (Ocean Surface Height) data files PYTHON DEPENDENCIES: @@ -11,6 +11,7 @@ https://www.h5py.org/ UPDATE HISTORY: + Updated 03/2024: use pathlib to define and operate on paths Updated 11/2023: drop DIMENSION_LIST, CLASS and NAME attributes Updated 05/2023: extract more ancillary data from ATL12 files Updated 12/2022: place some imports behind try/except statements diff --git a/icesat2_toolkit/tools.py b/icesat2_toolkit/tools.py index eb9bc64..75f85bf 100644 --- a/icesat2_toolkit/tools.py +++ b/icesat2_toolkit/tools.py @@ -1,7 +1,7 @@ #!/usr/bin/env python u""" tools.py -Written by Tyler Sutterley (04/2022) +Written by Tyler Sutterley (03/2024) Plotting tools and utilities PYTHON DEPENDENCIES: @@ -13,6 +13,7 @@ https://github.com/matplotlib/matplotlib UPDATE HISTORY: + Updated 03/2024: use pathlib to define and operate on paths Updated 04/2022: updated docstrings to numpy documentation format Updated 12/2021: added custom colormap function for some common scales Written 09/2021 diff --git a/icesat2_toolkit/utilities.py b/icesat2_toolkit/utilities.py index b7c72da..8562150 100644 --- a/icesat2_toolkit/utilities.py +++ b/icesat2_toolkit/utilities.py @@ -1,7 +1,7 @@ #!/usr/bin/env python u""" utilities.py -Written by Tyler Sutterley (11/2023) +Written by Tyler Sutterley (03/2024) Download and management utilities for syncing time and auxiliary files PYTHON DEPENDENCIES: @@ -9,6 +9,7 @@ https://pypi.python.org/pypi/lxml UPDATE HISTORY: + Updated 03/2024: can use regions to filter sea ice products in cmr queries Updated 11/2023: updated ssl context to fix deprecation error Updated 07/2023: add function for S3 filesystem add s3 and opendap endpoint options to cmr query functions @@ -1608,36 +1609,44 @@ def cmr_granules(granule: str | int | list | None): logging.warning("Listed granule region is not presently available") return granule_list -# PURPOSE: check if the submitted ATL14/ATL15 regions are valid -def cmr_regions(region: str | list | None): +# PURPOSE: check if the submitted regions are valid +def cmr_regions(region: str | int | list | None): """ - Check if the submitted ATL14/ATL15 regions are valid + Check if the submitted ATL07 or ATL14/ATL15 regions are valid Parameters ---------- - region: str, list or NoneType - ICESat-2 ATL14/ATL15 region + region: str, int, list or NoneType + ICESat-2 ATL07 or ATL14/ATL15 region Returns ------- region_list: list - formatted available ATL14/ATL15 regions + formatted available regions """ + # all available ICESat-2 sea ice regions + ATL07_regions = ['01','02'] # all available ICESat-2 ATL14/15 regions - all_regions = ['AA','AK','CN','CS','GL','IS','SV','RA'] + ATL1415_regions = ['AA','A1','A2','A3','A4','AK', + 'CN','CS','GL','IS','SV','RA'] + # combined regions + all_regions = ATL07_regions + ATL1415_regions if region is None: return ["??"] else: if isinstance(region, str): assert region in all_regions region_list = [str(region)] + elif isinstance(region, int): + assert int(region) > 0, "Sea ice hemisphere region must be positive" + region_list = [str(region).zfill(2)] elif isinstance(region, list): region_list = [] for r in region: assert r in all_regions region_list.append(str(r)) else: - raise TypeError("Please enter the region as a list or string") + raise TypeError("Please enter the region as a list, int or string") # check if user-entered region is currently not available if not set(all_regions) & set(region_list): logging.warning("Listed region is not presently available") @@ -1693,7 +1702,7 @@ def cmr_readable_granules(product: str, **kwargs): granules: str, list or NoneType, default None ICESat-2 granule region strings to query regions: str, list or NoneType, default None - ICESat-2 ATL14/ATL15 region + ICESat-2 ATL07 or ATL14/ATL15 region resolutions: str, list or NoneType, default None ICESat-2 ATL14/ATL15 spatial resolution @@ -1719,6 +1728,19 @@ def cmr_readable_granules(product: str, **kwargs): # append the granule pattern pattern = f"{product}_{r}_????_{s}_*" readable_granule_list.append(pattern) + elif product in ("ATL07", "ATL10", "ATL20", "ATL21"): + # along-track sea ice products + # for each sea ice hemisphere flag + for r in cmr_regions(kwargs["regions"]): + # for each available cycle of interest + for c in cmr_cycles(kwargs["cycles"]): + # for each available track of interest + for t in cmr_tracks(kwargs["tracks"]): + # use single character wildcards "?" for date strings, + # unused granule numbers, and any unset parameters + pattern = f"{product}-{r}_{14 * '?'}_{t}{c}??_*" + # append the granule pattern + readable_granule_list.append(pattern) else: # along-track products # for each available cycle of interest @@ -1728,10 +1750,7 @@ def cmr_readable_granules(product: str, **kwargs): # for each available granule region of interest for g in cmr_granules(kwargs["granules"]): # use single character wildcards "?" for date strings, - # sea ice product hemispheres, and any unset parameters - if product in ("ATL07", "ATL10", "ATL20", "ATL21"): - pattern = f"{product}-??_{14 * '?'}_{t}{c}{g}_*" - elif product in ("ATL11",): + if product in ("ATL11",): pattern = f"{product}_{t}{g}_*" else: pattern = f"{product}_{14 * '?'}_{t}{c}{g}_*" @@ -1831,7 +1850,7 @@ def cmr( granules: str, list or NoneType, default None ICESat-2 granule region strings to query regions: str, list or NoneType, default None - ICESat-2 ATL14/15 region strings to query + ICESat-2 ATL07 or ATL14/15 region strings to query resolutions: str, list or NoneType, default None ICESat-2 ATL14/15 resolution strings to query bbox: list or NoneType, default None diff --git a/scripts/MPI_DEM_ICESat2_ATL03.py b/scripts/MPI_DEM_ICESat2_ATL03.py index 812fa1f..aac2e08 100644 --- a/scripts/MPI_DEM_ICESat2_ATL03.py +++ b/scripts/MPI_DEM_ICESat2_ATL03.py @@ -1,7 +1,7 @@ #!/usr/bin/env python u""" MPI_DEM_ICESat2_ATL03.py -Written by Tyler Sutterley (12/2022) +Written by Tyler Sutterley (03/2024) Determines which digital elevation model tiles to read for a given ATL03 file Reads 3x3 array of tiles for points within bounding box of central mosaic tile Interpolates digital elevation model to ICESat-2 ATL03 photon event locations @@ -60,6 +60,7 @@ https://nsidc.org/data/nsidc-0645/versions/1 UPDATE HISTORY: + Updated 03/2024: use pathlib to define and operate on paths Updated 12/2022: single implicit import of altimetry tools Updated 11/2022: new ArcticDEM and REMA mosaic index shapefiles verify coordinate reference system attribute from shapefile diff --git a/scripts/MPI_DEM_ICESat2_ATL06.py b/scripts/MPI_DEM_ICESat2_ATL06.py index 7e61b07..12e50dc 100644 --- a/scripts/MPI_DEM_ICESat2_ATL06.py +++ b/scripts/MPI_DEM_ICESat2_ATL06.py @@ -1,7 +1,7 @@ #!/usr/bin/env python u""" MPI_DEM_ICESat2_ATL06.py -Written by Tyler Sutterley (12/2022) +Written by Tyler Sutterley (03/2024) Determines which digital elevation model tiles to read for a given ATL06 file Reads 3x3 array of tiles for points within bounding box of central mosaic tile Interpolates digital elevation model to locations of ICESat-2 ATL06 segments @@ -60,6 +60,7 @@ https://nsidc.org/data/nsidc-0645/versions/1 UPDATE HISTORY: + Updated 03/2024: use pathlib to define and operate on paths Updated 12/2022: single implicit import of altimetry tools Updated 11/2022: new ArcticDEM and REMA mosaic index shapefiles verify coordinate reference system attribute from shapefile diff --git a/scripts/MPI_DEM_ICESat2_ATL11.py b/scripts/MPI_DEM_ICESat2_ATL11.py index 581d65b..8e300ad 100644 --- a/scripts/MPI_DEM_ICESat2_ATL11.py +++ b/scripts/MPI_DEM_ICESat2_ATL11.py @@ -1,7 +1,7 @@ #!/usr/bin/env python u""" MPI_DEM_ICESat2_ATL11.py -Written by Tyler Sutterley (12/2022) +Written by Tyler Sutterley (03/2024) Determines which digital elevation model tiles to read for a given ATL11 file Reads 3x3 array of tiles for points within bounding box of central mosaic tile Interpolates digital elevation model to locations of ICESat-2 ATL11 segments @@ -60,6 +60,7 @@ https://nsidc.org/data/nsidc-0645/versions/1 UPDATE HISTORY: + Updated 03/2024: use pathlib to define and operate on paths Updated 12/2022: single implicit import of altimetry tools Updated 11/2022: new ArcticDEM and REMA mosaic index shapefiles verify coordinate reference system attribute from shapefile diff --git a/scripts/MPI_ICESat2_ATL03.py b/scripts/MPI_ICESat2_ATL03.py index c93749d..efd800b 100644 --- a/scripts/MPI_ICESat2_ATL03.py +++ b/scripts/MPI_ICESat2_ATL03.py @@ -1,6 +1,6 @@ #!/usr/bin/env python u""" -MPI_ICESat2_ATL03.py (12/2022) +MPI_ICESat2_ATL03.py (03/2024) Read ICESat-2 ATL03 and ATL09 data files to calculate average segment surfaces ATL03 datasets: Global Geolocated Photons ATL09 datasets: Atmospheric Characteristics @@ -42,6 +42,7 @@ classify_photons.py: Yet Another Photon Classifier for Geolocated Photon Data UPDATE HISTORY: + Updated 03/2024: use pathlib to define and operate on paths Updated 12/2022: single implicit import of altimetry tools Updated 06/2022: update classify photons to match current GSFC version Updated 05/2022: use argparse descriptions within sphinx documentation diff --git a/scripts/MPI_ICESat2_ATL03_histogram.py b/scripts/MPI_ICESat2_ATL03_histogram.py index 6b57ad7..47dc95e 100644 --- a/scripts/MPI_ICESat2_ATL03_histogram.py +++ b/scripts/MPI_ICESat2_ATL03_histogram.py @@ -1,6 +1,6 @@ #!/usr/bin/env python u""" -MPI_ICESat2_ATL03_histogram.py (12/2022) +MPI_ICESat2_ATL03_histogram.py (03/2024) Read ICESat-2 ATL03 and ATL09 data files to calculate average segment surfaces ATL03 datasets: Global Geolocated Photons ATL09 datasets: Atmospheric Characteristics @@ -75,6 +75,7 @@ Geophysical Journal International (1997) 131, 267-280 UPDATE HISTORY: + Updated 03/2024: use pathlib to define and operate on paths Updated 12/2022: single implicit import of altimetry tools Updated 06/2022: update classify photons to match current GSFC version Updated 05/2022: use argparse descriptions within sphinx documentation diff --git a/scripts/MPI_reduce_ICESat2_ATL03_RGI.py b/scripts/MPI_reduce_ICESat2_ATL03_RGI.py index bd346db..e3cfb9c 100644 --- a/scripts/MPI_reduce_ICESat2_ATL03_RGI.py +++ b/scripts/MPI_reduce_ICESat2_ATL03_RGI.py @@ -1,7 +1,7 @@ #!/usr/bin/env python u""" MPI_reduce_ICESat2_ATL03_RGI.py -Written by Tyler Sutterley (12/2022) +Written by Tyler Sutterley (03/2024) Create masks for reducing ICESat-2 data to the Randolph Glacier Inventory https://www.glims.org/RGI/rgi60_dl.html @@ -57,6 +57,7 @@ utilities.py: download and management utilities for syncing files UPDATE HISTORY: + Updated 03/2024: use pathlib to define and operate on paths Updated 12/2022: single implicit import of altimetry tools Updated 07/2022: place some imports behind try/except statements Updated 05/2022: use argparse descriptions within sphinx documentation diff --git a/scripts/MPI_reduce_ICESat2_ATL06_RGI.py b/scripts/MPI_reduce_ICESat2_ATL06_RGI.py index c3e4214..b2bd3ad 100644 --- a/scripts/MPI_reduce_ICESat2_ATL06_RGI.py +++ b/scripts/MPI_reduce_ICESat2_ATL06_RGI.py @@ -1,7 +1,7 @@ #!/usr/bin/env python u""" MPI_reduce_ICESat2_ATL06_RGI.py -Written by Tyler Sutterley (12/2022) +Written by Tyler Sutterley (03/2024) Create masks for reducing ICESat-2 data to the Randolph Glacier Inventory https://www.glims.org/RGI/rgi60_dl.html @@ -57,6 +57,7 @@ utilities.py: download and management utilities for syncing files UPDATE HISTORY: + Updated 03/2024: use pathlib to define and operate on paths Updated 12/2022: single implicit import of altimetry tools Updated 07/2022: place some imports behind try/except statements Updated 05/2022: use argparse descriptions within sphinx documentation diff --git a/scripts/MPI_reduce_ICESat2_ATL06_drainages.py b/scripts/MPI_reduce_ICESat2_ATL06_drainages.py index 941314f..9b80831 100644 --- a/scripts/MPI_reduce_ICESat2_ATL06_drainages.py +++ b/scripts/MPI_reduce_ICESat2_ATL06_drainages.py @@ -1,7 +1,7 @@ #!/usr/bin/env python u""" MPI_reduce_ICESat2_ATL06_drainages.py -Written by Tyler Sutterley (12/2022) +Written by Tyler Sutterley (03/2024) Create masks for reducing ICESat-2 data into IMBIE-2 drainage regions @@ -38,6 +38,7 @@ utilities.py: download and management utilities for syncing files UPDATE HISTORY: + Updated 03/2024: use pathlib to define and operate on paths Updated 12/2022: single implicit import of altimetry tools Updated 07/2022: place some imports behind try/except statements Updated 05/2022: use argparse descriptions within sphinx documentation diff --git a/scripts/MPI_reduce_ICESat2_ATL06_grounded.py b/scripts/MPI_reduce_ICESat2_ATL06_grounded.py index 9116624..3a3ffac 100644 --- a/scripts/MPI_reduce_ICESat2_ATL06_grounded.py +++ b/scripts/MPI_reduce_ICESat2_ATL06_grounded.py @@ -1,7 +1,7 @@ #!/usr/bin/env python u""" MPI_reduce_ICESat2_ATL06_grounded.py -Written by Tyler Sutterley (12/2022) +Written by Tyler Sutterley (03/2024) Create masks for reducing ICESat-2 data into grounded ice regions @@ -40,6 +40,7 @@ utilities.py: download and management utilities for syncing files UPDATE HISTORY: + Updated 03/2024: use pathlib to define and operate on paths Updated 12/2022: single implicit import of altimetry tools Updated 07/2022: place some imports behind try/except statements Updated 05/2022: use argparse descriptions within sphinx documentation diff --git a/scripts/MPI_reduce_ICESat2_ATL06_ice_shelves.py b/scripts/MPI_reduce_ICESat2_ATL06_ice_shelves.py index e524d04..ee49d79 100644 --- a/scripts/MPI_reduce_ICESat2_ATL06_ice_shelves.py +++ b/scripts/MPI_reduce_ICESat2_ATL06_ice_shelves.py @@ -1,7 +1,7 @@ #!/usr/bin/env python u""" MPI_reduce_ICESat2_ATL06_ice_shelves.py -Written by Tyler Sutterley (12/2022) +Written by Tyler Sutterley (03/2024) Create masks for reducing ICESat-2 data into regions of floating ice shelves @@ -39,6 +39,7 @@ utilities.py: download and management utilities for syncing files UPDATE HISTORY: + Updated 03/2024: use pathlib to define and operate on paths Updated 12/2022: single implicit import of altimetry tools Updated 07/2022: place some imports behind try/except statements Updated 05/2022: use argparse descriptions within sphinx documentation diff --git a/scripts/MPI_reduce_ICESat2_ATL11_RGI.py b/scripts/MPI_reduce_ICESat2_ATL11_RGI.py index 864d6c8..ae7cb88 100644 --- a/scripts/MPI_reduce_ICESat2_ATL11_RGI.py +++ b/scripts/MPI_reduce_ICESat2_ATL11_RGI.py @@ -1,7 +1,7 @@ #!/usr/bin/env python u""" MPI_reduce_ICESat2_ATL11_RGI.py -Written by Tyler Sutterley (12/2022) +Written by Tyler Sutterley (03/2024) Create masks for reducing ICESat-2 data to the Randolph Glacier Inventory https://www.glims.org/RGI/rgi60_dl.html @@ -59,6 +59,7 @@ utilities.py: download and management utilities for syncing files UPDATE HISTORY: + Updated 03/2024: use pathlib to define and operate on paths Updated 12/2022: single implicit import of altimetry tools Updated 07/2022: place some imports behind try/except statements Updated 05/2022: use argparse descriptions within sphinx documentation diff --git a/scripts/MPI_reduce_ICESat2_ATL11_drainages.py b/scripts/MPI_reduce_ICESat2_ATL11_drainages.py index d236047..9e93e29 100644 --- a/scripts/MPI_reduce_ICESat2_ATL11_drainages.py +++ b/scripts/MPI_reduce_ICESat2_ATL11_drainages.py @@ -1,7 +1,7 @@ #!/usr/bin/env python u""" MPI_reduce_ICESat2_ATL11_drainages.py -Written by Tyler Sutterley (12/2022) +Written by Tyler Sutterley (03/2024) Create masks for reducing ICESat-2 data into IMBIE-2 drainage regions @@ -38,6 +38,7 @@ utilities.py: download and management utilities for syncing files UPDATE HISTORY: + Updated 03/2024: use pathlib to define and operate on paths Updated 12/2022: single implicit import of altimetry tools Updated 07/2022: place some imports behind try/except statements Updated 05/2022: use argparse descriptions within sphinx documentation diff --git a/scripts/MPI_reduce_ICESat2_ATL11_grounded.py b/scripts/MPI_reduce_ICESat2_ATL11_grounded.py index e9fce47..018fd4d 100644 --- a/scripts/MPI_reduce_ICESat2_ATL11_grounded.py +++ b/scripts/MPI_reduce_ICESat2_ATL11_grounded.py @@ -1,7 +1,7 @@ #!/usr/bin/env python u""" MPI_reduce_ICESat2_ATL11_grounded.py -Written by Tyler Sutterley (12/2022) +Written by Tyler Sutterley (03/2024) Create masks for reducing ICESat-2 data into grounded ice regions @@ -40,6 +40,7 @@ utilities.py: download and management utilities for syncing files UPDATE HISTORY: + Updated 03/2024: use pathlib to define and operate on paths Updated 12/2022: single implicit import of altimetry tools Updated 07/2022: place some imports behind try/except statements Updated 05/2022: use argparse descriptions within sphinx documentation diff --git a/scripts/MPI_reduce_ICESat2_ATL11_ice_shelves.py b/scripts/MPI_reduce_ICESat2_ATL11_ice_shelves.py index bced256..14525a3 100644 --- a/scripts/MPI_reduce_ICESat2_ATL11_ice_shelves.py +++ b/scripts/MPI_reduce_ICESat2_ATL11_ice_shelves.py @@ -1,7 +1,7 @@ #!/usr/bin/env python u""" MPI_reduce_ICESat2_ATL11_ice_shelves.py -Written by Tyler Sutterley (12/2022) +Written by Tyler Sutterley (03/2024) Create masks for reducing ICESat-2 data into regions of floating ice shelves @@ -39,6 +39,7 @@ utilities.py: download and management utilities for syncing files UPDATE HISTORY: + Updated 03/2024: use pathlib to define and operate on paths Updated 12/2022: single implicit import of altimetry tools Updated 07/2022: place some imports behind try/except statements Updated 05/2022: use argparse descriptions within sphinx documentation diff --git a/scripts/nsidc_icesat2_associated.py b/scripts/nsidc_icesat2_associated.py index c126c01..4599751 100644 --- a/scripts/nsidc_icesat2_associated.py +++ b/scripts/nsidc_icesat2_associated.py @@ -1,7 +1,7 @@ #!/usr/bin/env python u""" nsidc_icesat2_associated.py -Written by Tyler Sutterley (12/2022) +Written by Tyler Sutterley (03/2024) Acquires ICESat-2 datafiles from the National Snow and Ice Data Center (NSIDC) server that is associated with an input file @@ -57,6 +57,7 @@ utilities.py: download and management utilities for syncing files UPDATE HISTORY: + Updated 03/2024: use pathlib to define and operate on paths Updated 12/2022: single implicit import of altimetry tools Updated 05/2022: use argparse descriptions within sphinx documentation Updated 03/2022: use attempt login function to check credentials diff --git a/scripts/nsidc_icesat2_convert.py b/scripts/nsidc_icesat2_convert.py index f7fd48b..5606388 100644 --- a/scripts/nsidc_icesat2_convert.py +++ b/scripts/nsidc_icesat2_convert.py @@ -1,7 +1,7 @@ #!/usr/bin/env python u""" nsidc_icesat2_convert.py -Written by Tyler Sutterley (09/2023) +Written by Tyler Sutterley (03/2024) Acquires ICESat-2 datafiles from NSIDC and directly converts to zarr datafiles or rechunked HDF5 files @@ -85,6 +85,7 @@ utilities.py: download and management utilities for syncing files UPDATE HISTORY: + Updated 03/2024: use pathlib to define and operate on paths Updated 09/2023: generalized regular expressions for non-entered cases Updated 12/2022: single implicit import of altimetry tools Updated 05/2022: use argparse descriptions within sphinx documentation diff --git a/scripts/nsidc_icesat2_dragann.py b/scripts/nsidc_icesat2_dragann.py index 8ad7506..82e8a28 100644 --- a/scripts/nsidc_icesat2_dragann.py +++ b/scripts/nsidc_icesat2_dragann.py @@ -1,7 +1,7 @@ #!/usr/bin/env python u""" nsidc_icesat2_dragann.py -Written by Tyler Sutterley (09/2023) +Written by Tyler Sutterley (03/2024) Acquires the ATL03 geolocated photon height product and appends the ATL08 DRAGANN classifications from NSIDC @@ -57,6 +57,7 @@ utilities.py: download and management utilities for syncing files UPDATE HISTORY: + Updated 03/2024: use pathlib to define and operate on paths Updated 09/2023: generalized regular expressions for non-entered cases Updated 12/2022: single implicit import of altimetry tools Updated 05/2022: use argparse descriptions within sphinx documentation diff --git a/scripts/nsidc_icesat2_sync.py b/scripts/nsidc_icesat2_sync.py index b774f19..9201d2c 100644 --- a/scripts/nsidc_icesat2_sync.py +++ b/scripts/nsidc_icesat2_sync.py @@ -1,7 +1,7 @@ #!/usr/bin/env python u""" nsidc_icesat2_sync.py -Written by Tyler Sutterley (09/2023) +Written by Tyler Sutterley (03/2024) Acquires ICESat-2 datafiles from the National Snow and Ice Data Center (NSIDC) @@ -69,6 +69,7 @@ utilities.py: download and management utilities for syncing files UPDATE HISTORY: + Updated 03/2024: use pathlib to define and operate on paths Updated 09/2023: generalized regular expressions for non-entered cases Updated 12/2022: single implicit import of altimetry tools Updated 05/2022: use argparse descriptions within sphinx documentation diff --git a/scripts/nsidc_icesat2_sync_s3.py b/scripts/nsidc_icesat2_sync_s3.py index 0f5b7c7..b27580a 100644 --- a/scripts/nsidc_icesat2_sync_s3.py +++ b/scripts/nsidc_icesat2_sync_s3.py @@ -1,7 +1,7 @@ #!/usr/bin/env python u""" nsidc_icesat2_sync_s3.py -Written by Tyler Sutterley (09/2023) +Written by Tyler Sutterley (03/2024) Acquires ICESat-2 datafiles from the National Snow and Ice Data Center (NSIDC) and transfers to an AWS S3 bucket using a local machine as pass through @@ -72,6 +72,7 @@ utilities.py: download and management utilities for syncing files UPDATE HISTORY: + Updated 03/2024: use pathlib to define and operate on paths Updated 09/2023: generalized regular expressions for non-entered cases Updated 12/2022: single implicit import of altimetry tools Updated 05/2022: use argparse descriptions within sphinx documentation diff --git a/scripts/reduce_ICESat2_ATL06_raster.py b/scripts/reduce_ICESat2_ATL06_raster.py index f297795..471a245 100644 --- a/scripts/reduce_ICESat2_ATL06_raster.py +++ b/scripts/reduce_ICESat2_ATL06_raster.py @@ -1,7 +1,7 @@ #!/usr/bin/env python u""" reduce_ICESat2_ATL06_raster.py -Written by Tyler Sutterley (12/2022) +Written by Tyler Sutterley (03/2024) Create masks for reducing ICESat-2 ATL06 data using raster imagery @@ -43,6 +43,7 @@ utilities.py: download and management utilities for syncing files UPDATE HISTORY: + Updated 03/2024: use pathlib to define and operate on paths Updated 12/2022: single implicit import of altimetry tools refactored ICESat-2 data product read programs under io Updated 06/2022: added option sigma to Gaussian filter raster images diff --git a/scripts/reduce_ICESat2_ATL07_raster.py b/scripts/reduce_ICESat2_ATL07_raster.py index ef95e5e..6abc533 100644 --- a/scripts/reduce_ICESat2_ATL07_raster.py +++ b/scripts/reduce_ICESat2_ATL07_raster.py @@ -1,7 +1,7 @@ #!/usr/bin/env python u""" reduce_ICESat2_ATL07_raster.py -Written by Tyler Sutterley (12/2022) +Written by Tyler Sutterley (03/2024) Create masks for reducing ICESat-2 ATL07 data using raster imagery @@ -43,6 +43,7 @@ utilities.py: download and management utilities for syncing files UPDATE HISTORY: + Updated 03/2024: use pathlib to define and operate on paths Updated 12/2022: single implicit import of altimetry tools refactored ICESat-2 data product read programs under io Updated 06/2022: added option sigma to Gaussian filter raster images diff --git a/scripts/reduce_ICESat2_ATL10_raster.py b/scripts/reduce_ICESat2_ATL10_raster.py index 2f57406..97800b4 100644 --- a/scripts/reduce_ICESat2_ATL10_raster.py +++ b/scripts/reduce_ICESat2_ATL10_raster.py @@ -1,7 +1,7 @@ #!/usr/bin/env python u""" reduce_ICESat2_ATL10_raster.py -Written by Tyler Sutterley (12/2022) +Written by Tyler Sutterley (03/2024) Create masks for reducing ICESat-2 ATL10 data using raster imagery @@ -43,6 +43,7 @@ utilities.py: download and management utilities for syncing files UPDATE HISTORY: + Updated 03/2024: use pathlib to define and operate on paths Updated 12/2022: single implicit import of altimetry tools refactored ICESat-2 data product read programs under io Updated 06/2022: added option sigma to Gaussian filter raster images diff --git a/scripts/reduce_ICESat2_ATL11_raster.py b/scripts/reduce_ICESat2_ATL11_raster.py index 38f888c..203cf94 100644 --- a/scripts/reduce_ICESat2_ATL11_raster.py +++ b/scripts/reduce_ICESat2_ATL11_raster.py @@ -1,7 +1,7 @@ #!/usr/bin/env python u""" reduce_ICESat2_ATL11_raster.py -Written by Tyler Sutterley (12/2022) +Written by Tyler Sutterley (03/2024) Create masks for reducing ICESat-2 ATL11 data using raster imagery @@ -43,6 +43,7 @@ utilities.py: download and management utilities for syncing files UPDATE HISTORY: + Updated 03/2024: use pathlib to define and operate on paths Updated 12/2022: single implicit import of altimetry tools refactored ICESat-2 data product read programs under io Updated 06/2022: added option sigma to Gaussian filter raster images diff --git a/scripts/scp_ICESat2_files.py b/scripts/scp_ICESat2_files.py index e252afb..8565b12 100644 --- a/scripts/scp_ICESat2_files.py +++ b/scripts/scp_ICESat2_files.py @@ -1,7 +1,7 @@ #!/usr/bin/env python u""" scp_ICESat2_files.py -Written by Tyler Sutterley (09/2023) +Written by Tyler Sutterley (03/2024) Copies ICESat-2 HDF5 data from between a local host and a remote host can switch between pushing and pulling to/from remote PUSH to remote: s.put(local_file, remote_file) @@ -38,6 +38,7 @@ https://github.com/jbardin/scp.py UPDATE HISTORY: + Updated 03/2024: use pathlib to define and operate on paths Updated 09/2023: generalized regular expressions for non-entered cases Updated 12/2022: use f-strings for ascii and verbose outputs Updated 05/2022: use argparse descriptions within sphinx documentation diff --git a/scripts/scp_scf_ICESat2_files.py b/scripts/scp_scf_ICESat2_files.py index 1b52046..4f39346 100644 --- a/scripts/scp_scf_ICESat2_files.py +++ b/scripts/scp_scf_ICESat2_files.py @@ -1,7 +1,7 @@ #!/usr/bin/env python u""" scp_scf_ICESat2_files.py -Written by Tyler Sutterley (09/2023) +Written by Tyler Sutterley (03/2024) Copies ICESat-2 HDF5 files from the SCF server to a remote host using the SCF-authorized local computer as a proxy server @@ -40,6 +40,7 @@ https://github.com/jbardin/scp.py UPDATE HISTORY: + Updated 03/2024: use pathlib to define and operate on paths Updated 09/2023: generalized regular expressions for non-entered cases Updated 12/2022: use f-strings for ascii and verbose outputs Updated 05/2022: use argparse descriptions within sphinx documentation diff --git a/scripts/symbolic_ICESat2_files.py b/scripts/symbolic_ICESat2_files.py index e8cac65..d6a2f19 100644 --- a/scripts/symbolic_ICESat2_files.py +++ b/scripts/symbolic_ICESat2_files.py @@ -1,7 +1,7 @@ #!/usr/bin/env python u""" symbolic_ICESat2_files.py -Written by Tyler Sutterley (09/2023) +Written by Tyler Sutterley (03/2024) Creates symbolic links for ICESat-2 HDF5 files organized by date CALLING SEQUENCE: @@ -25,6 +25,7 @@ -M X, --mode X: permission mode of directories UPDATE HISTORY: + Updated 03/2024: use pathlib to define and operate on paths Updated 09/2023: generalized regular expressions for non-entered cases Updated 12/2022: use f-strings for ascii and verbose outputs Updated 11/2022: added option if SCF outgoing is a flattened structure diff --git a/test/test_download_and_read.py b/test/test_download_and_read.py index a2dce47..2789b15 100644 --- a/test/test_download_and_read.py +++ b/test/test_download_and_read.py @@ -1,8 +1,9 @@ #!/usr/bin/env python u""" -test_download_and_read.py (03/2023) +test_download_and_read.py (03/2024) UPDATE HISTORY: + Updated 03/2024: use CMR queries to find granules in each download test Updated 03/2023: replace deprecated functions with new io functions Updated 08/2022: added backup AWS access to files Updated 02/2022: add CMR query tests for cycles, tracks and granules @@ -16,154 +17,172 @@ import icesat2_toolkit as is2tk # PURPOSE: Download an ATL03 file from NSIDC and check that read program runs -def test_ATL03_download_and_read(username,password): - HOST = ['https://n5eil01u.ecs.nsidc.org','ATLAS','ATL03.005','2018.10.14', - 'ATL03_20181014000347_02350101_005_01.h5'] - buffer, error = is2tk.utilities.from_nsidc(HOST, +def test_ATL03_download_and_read(username, password): + # query CMR for ATL03 data + ids, urls = is2tk.utilities.cmr(product='ATL03', + release='006', cycles=1, tracks=235, granules=1, + verbose=False) + # attempt download from on-prem NSIDC + buffer, error = is2tk.utilities.from_nsidc(urls[0], username=username, password=password, - local=HOST[-1], verbose=True) + local=ids[0], verbose=True) # attempt download from AWS if not buffer: url = posixpath.dirname(is2tk.utilities._s3_endpoints['nsidc']) bucket = is2tk.utilities._s3_buckets['nsidc'] - key = is2tk.utilities.s3_key(posixpath.join(*HOST)) + key = is2tk.utilities.s3_key(urls[0]) buffer, error = is2tk.utilities.from_nsidc([url,bucket,key], - username=username, password=password, local=HOST[-1], + username=username, password=password, local=ids[0], verbose=True) # raise exception if download error if not buffer: raise Exception(error) # read ATL03 data from downloaded HDF5 file - mds,attrs,beams = is2tk.io.ATL03.read_granule(HOST[-1], + mds,attrs,beams = is2tk.io.ATL03.read_granule(ids[0], ATTRIBUTES=False, VERBOSE=True ) assert all(gtx in mds.keys() for gtx in beams) # PURPOSE: Download an ATL06 file from NSIDC and check that read program runs -def test_ATL06_download_and_read(username,password): - HOST = ['https://n5eil01u.ecs.nsidc.org','ATLAS','ATL06.005','2018.10.14', - 'ATL06_20181014001049_02350102_005_01.h5'] - buffer, error = is2tk.utilities.from_nsidc(HOST, +def test_ATL06_download_and_read(username, password): + # query CMR for ATL06 data + ids, urls = is2tk.utilities.cmr(product='ATL06', + release='006', cycles=1, tracks=235, granules=2, + verbose=False) + # attempt download from on-prem NSIDC + buffer, error = is2tk.utilities.from_nsidc(urls[0], username=username, password=password, - local=HOST[-1], verbose=True) + local=ids[0], verbose=True) # attempt download from AWS if not buffer: url = posixpath.dirname(is2tk.utilities._s3_endpoints['nsidc']) bucket = is2tk.utilities._s3_buckets['nsidc'] - key = is2tk.utilities.s3_key(posixpath.join(*HOST)) + key = is2tk.utilities.s3_key(urls[0]) buffer, error = is2tk.utilities.from_nsidc([url,bucket,key], - username=username, password=password, local=HOST[-1], + username=username, password=password, local=ids[0], verbose=True) # raise exception if download error if not buffer: raise Exception(error) # read ATL06 data from downloaded HDF5 file - mds,attrs,beams = is2tk.io.ATL06.read_granule(HOST[-1], + mds,attrs,beams = is2tk.io.ATL06.read_granule(ids[0], ATTRIBUTES=False, - HISTOGRAM=False, - QUALITY=False, VERBOSE=True ) assert all(gtx in mds.keys() for gtx in beams) # PURPOSE: Download an ATL07 file from NSIDC and check that read program runs def test_ATL07_download_and_read(username,password): - HOST = ['https://n5eil01u.ecs.nsidc.org','ATLAS','ATL07.005','2018.10.14', - 'ATL07-01_20181014000347_02350101_005_03.h5'] - buffer, error = is2tk.utilities.from_nsidc(HOST, + # query CMR for ATL07 data + ids, urls = is2tk.utilities.cmr(product='ATL07', + release='006', cycles=1, tracks=235, regions=1, + verbose=False) + # attempt download from on-prem NSIDC + buffer, error = is2tk.utilities.from_nsidc(urls[0], username=username, password=password, - local=HOST[-1], verbose=True) + local=ids[0], verbose=True) # attempt download from AWS if not buffer: url = posixpath.dirname(is2tk.utilities._s3_endpoints['nsidc']) bucket = is2tk.utilities._s3_buckets['nsidc'] - key = is2tk.utilities.s3_key(posixpath.join(*HOST)) + key = is2tk.utilities.s3_key(urls[0]) buffer, error = is2tk.utilities.from_nsidc([url,bucket,key], - username=username, password=password, local=HOST[-1], + username=username, password=password, local=ids[0], verbose=True) # raise exception if download error if not buffer: raise Exception(error) # read ATL07 data from downloaded HDF5 file - mds,attrs,beams = is2tk.io.ATL07.read_granule(HOST[-1], - ATTRIBUTES=False, VERBOSE=True) + mds,attrs,beams = is2tk.io.ATL07.read_granule(ids[0], + ATTRIBUTES=False, + VERBOSE=True + ) assert all(gtx in mds.keys() for gtx in beams) # PURPOSE: Download an ATL11 file from NSIDC and check that read program runs def test_ATL11_download_and_read(username,password): + # query CMR for ATL11 data + ids, urls = is2tk.utilities.cmr(product='ATL11', + release='006', tracks=1, granules=3, verbose=False) # attempt download from on-prem NSIDC - HOST = ['https://n5eil01u.ecs.nsidc.org','ATLAS','ATL11.005','2019.03.29', - 'ATL11_000103_0315_005_03.h5'] - buffer, error = is2tk.utilities.from_nsidc(HOST, + buffer, error = is2tk.utilities.from_nsidc(urls[0], username=username, password=password, - local=HOST[-1], verbose=True) + local=ids[0], verbose=True) # attempt download from AWS if not buffer: url = posixpath.dirname(is2tk.utilities._s3_endpoints['nsidc']) bucket = is2tk.utilities._s3_buckets['nsidc'] - key = is2tk.utilities.s3_key(posixpath.join(*HOST)) + key = is2tk.utilities.s3_key(urls[0]) buffer, error = is2tk.utilities.from_nsidc([url,bucket,key], - username=username, password=password, local=HOST[-1], + username=username, password=password, local=ids[0], verbose=True) # raise exception if download error if not buffer: raise Exception(error) - # read ATL12 data from downloaded HDF5 file + # read ATL11 data from downloaded HDF5 file GROUPS = ['cycle_stats','ref_surf','crossing_track_data'] - mds,attrs,pairs = is2tk.io.ATL11.read_granule(HOST[-1], - ATTRIBUTES=False, GROUPS=GROUPS, VERBOSE=True) + mds,attrs,pairs = is2tk.io.ATL11.read_granule(ids[0], + ATTRIBUTES=False, + GROUPS=GROUPS, + VERBOSE=True + ) assert all(ptx in mds.keys() for ptx in pairs) ptx = pairs[0] assert all(group in mds[ptx].keys() for group in GROUPS) # PURPOSE: Download an ATL12 file from NSIDC and check that read program runs def test_ATL12_download_and_read(username,password): - HOST = ['https://n5eil01u.ecs.nsidc.org','ATLAS','ATL12.005','2018.10.14', - 'ATL12_20181014031222_02370101_005_02.h5'] - buffer, error = is2tk.utilities.from_nsidc(HOST, + # query CMR for ATL12 data + ids, urls = is2tk.utilities.cmr(product='ATL12', + release='006', cycles=1, tracks=237, granules=1, + verbose=False) + # attempt download from on-prem NSIDC + buffer, error = is2tk.utilities.from_nsidc(urls[0], username=username, password=password, - local=HOST[-1], verbose=True) + local=ids[0], verbose=True) # attempt download from AWS if not buffer: url = posixpath.dirname(is2tk.utilities._s3_endpoints['nsidc']) bucket = is2tk.utilities._s3_buckets['nsidc'] - key = is2tk.utilities.s3_key(posixpath.join(*HOST)) + key = is2tk.utilities.s3_key(urls[0]) buffer, error = is2tk.utilities.from_nsidc([url,bucket,key], - username=username, password=password, local=HOST[-1], + username=username, password=password, local=ids[0], verbose=True) # raise exception if download error if not buffer: raise Exception(error) # read ATL12 data from downloaded HDF5 file - mds,attrs,beams = is2tk.io.ATL12.read_granule(HOST[-1], - ATTRIBUTES=False, VERBOSE=True) + mds,attrs,beams = is2tk.io.ATL12.read_granule(ids[0], + ATTRIBUTES=False, + VERBOSE=True + ) assert all(gtx in mds.keys() for gtx in beams) # PURPOSE: test CMR queries for specific cycles def test_cmr_query_cycles(): ids,urls = is2tk.utilities.cmr(product='ATL06', - release='005',cycles=[2,3],tracks=752,granules=10, + release='006',cycles=[2,3],tracks=752,granules=10, verbose=False) - valid = ['ATL06_20190215171140_07520210_005_01.h5', - 'ATL06_20190517125119_07520310_005_01.h5'] + valid = ['ATL06_20190215171140_07520210_006_02.h5', + 'ATL06_20190517125119_07520310_006_02.h5'] assert all(id in valid for id in ids) # PURPOSE: test CMR queries for specific tracks def test_cmr_query_tracks(): ids,urls = is2tk.utilities.cmr(product='ATL06', - release='005',cycles=2,tracks=[752,753],granules=10, + release='006',cycles=2,tracks=[752,753],granules=10, verbose=False) - valid = ['ATL06_20190215171140_07520210_005_01.h5', - 'ATL06_20190215184558_07530210_005_01.h5'] + valid = ['ATL06_20190215171140_07520210_006_02.h5', + 'ATL06_20190215184558_07530210_006_02.h5'] assert all(id in valid for id in ids) # PURPOSE: test CMR queries for specific granules def test_cmr_query_granules(): ids,urls = is2tk.utilities.cmr(product='ATL06', - release='005',cycles=2,tracks=752,granules=[10,11,12], + release='006',cycles=2,tracks=752,granules=[10,11,12], verbose=False) - valid = ['ATL06_20190215171140_07520210_005_01.h5', - 'ATL06_20190215171921_07520211_005_01.h5', - 'ATL06_20190215172504_07520212_005_01.h5'] + valid = ['ATL06_20190215171140_07520210_006_02.h5', + 'ATL06_20190215171921_07520211_006_02.h5', + 'ATL06_20190215172504_07520212_006_02.h5'] assert all(id in valid for id in ids)