diff --git a/src/imagery/i.earthaccess.download/i.earthaccess.download.html b/src/imagery/i.earthaccess.download/i.earthaccess.download.html index 428c318..ec092bc 100644 --- a/src/imagery/i.earthaccess.download/i.earthaccess.download.html +++ b/src/imagery/i.earthaccess.download/i.earthaccess.download.html @@ -8,7 +8,7 @@

DESCRIPTION

archives (granules).

-The temporal search parameters (temporal, created_at, +The optional temporal search parameters (temporal, created_at, production_date, revision_date) are given as a comma separated pair of ISO-formated datetime strings. @@ -16,7 +16,7 @@

DESCRIPTION

In order to be able to download data the user has to create an EarthData account and provide credentials to the module add earthaccess library respectively. Credentials can be provided in form of a ~/.netrc -file, environment variables named EARTHDATA_USERNAME and +file or using environment variables named EARTHDATA_USERNAME and EARTHDATA_PASSWORD. @@ -68,7 +68,7 @@

REQUIREMENTS

REFERENCES

-asf_search Basics +earthaccess documentation

AUTHOR

diff --git a/src/imagery/i.earthaccess.download/i.earthaccess.download.py b/src/imagery/i.earthaccess.download/i.earthaccess.download.py index 2d5066f..a058443 100644 --- a/src/imagery/i.earthaccess.download/i.earthaccess.download.py +++ b/src/imagery/i.earthaccess.download/i.earthaccess.download.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 -""" -MODULE: i.earthaccess.download +"""MODULE: i.earthaccess.download AUTHOR(S): Stefan Blumentrath -PURPOSE: Searches and Downloads SAR data from the Alaska Satellite Facility +PURPOSE: Searches and Downloads earth observation data using the + EarthAccess library for NASA Earthdata APIs. COPYRIGHT: (C) 2024 by NVE, Stefan Blumentrath This program is free software; you can redistribute it and/or modify @@ -16,22 +16,6 @@ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - def day_night_flag(self, day_night_flag: str) -> Self: ... - def ( - min_cover: Optional[FloatLike] = ..., - max_cover: Optional[FloatLike] = ..., - def (self, instrument: str) -> Self: ... - def (self, platform: str) -> Self: ... - def sort_key(self, sort_key: str) -> Self: ... - def (self, granule_ur: str) -> Self: ... - -class CollectionQuery(GranuleCollectionBaseQuery): - def archive_center(self, center: str) -> Self: ... - def keyword(self, text: str) -> Self: ... - def native_id(self, native_ids: Sequence[str]) -> Self: ... - def tool_concept_id(self, ids: Sequence[str]) -> Self: ... - def service_concept_id(self, ids: Sequence[str]) -> Self: ... - """ # %Module @@ -145,6 +129,7 @@ def service_concept_id(self, ids: Sequence[str]) -> Self: ... # %option G_OPT_F_OUTPUT # % key: file +# % required: no # % description: Write search result to file # %end @@ -168,7 +153,7 @@ def service_concept_id(self, ids: Sequence[str]) -> Self: ... # % required: no # % multiple: no # % description: Comma separated list of scenes or file with scenes (one per row) -# % label: Selected scenes to download from ASF +# % label: Selected scenes to download using earthaccess # %end # %option @@ -203,7 +188,7 @@ def service_concept_id(self, ids: Sequence[str]) -> Self: ... from grass.pygrass.vector import VectorTopo -def get_spatial_query_parameter(aoi): +def get_spatial_query_parameter(aoi: str) -> dict: """Generate the spatial query parameter from user input The input aoi (=area of interest) can be: @@ -232,6 +217,7 @@ def get_spatial_query_parameter(aoi): reg = gs.parse_command("g.region", flags="gl", quiet=True) if not aoi: + gs.debug(_("Using the bounding box from computational region as AOI")) # Use bounding box from computational region if reg: return { @@ -240,7 +226,7 @@ def get_spatial_query_parameter(aoi): (min(float(reg["sw_lat"]), float(reg["se_lat"]))), max(float(reg["se_long"]), float(reg["ne_long"])), (max(float(reg["nw_lat"]), float(reg["ne_lat"]))), - ) + ), } reg = gs.parse_command("g.region", flags="g", quiet=True) return {"bounding_box": (reg["w"], reg["s"], reg["e"], reg["n"])} @@ -250,22 +236,49 @@ def get_spatial_query_parameter(aoi): wgs_84.ImportFromEPSG(4326) wgs_84.SetAxisMappingStrategy(osr.OAMS_TRADITIONAL_GIS_ORDER) + # Try GeoJSON / AOI file with OGR + if Path(aoi).exists(): + gs.debug(_("Reading AOI from file {}").format(aoi)) + try: + ogr_dataset = ogr.Open(aoi) + except OSError: + gs.fatal(_("Failed to open AOI file {}").format(aoi)) + if not ogr_dataset: + gs.fatal(_("Could not read AOI file {}").format(aoi)) + if ogr_dataset.GetLayerCount() > 1: + gs.warning(_("Input file contains more than one layer")) + ogr_layer = ogr_dataset.GetLayerByIndex(0) + if ogr_layer.GetGeomType() != 3: + gs.warning(_("GeoJSON does not contain polygons")) + if ogr_layer.GetFeatureCount() > 1: + gs.warning( + _("GeoJSON contains more than one geometry. Using only the first one."), + ) + layer_crs = ogr_layer.GetSpatialRef() + ogr_feature = ogr_layer.GetFeature(0) + geom = ogr_feature.GetGeometryRef() + if not layer_crs.IsSame(wgs_84): + geom.TransformTo(wgs_84) + ring = geom.GetGeometryRef(0) + + return {"polygon": ring.GetPoints()} + # Try GRASS GIS vector map - try: + if gs.legal_name(aoi): aoi_map = VectorTopo(aoi) if aoi_map.exist(): aoi_map.open("r") if aoi_map.number_of("areas") > 1: gs.warning( _( - "GeoJSON contains more than one geometry. Using only the first one." - ) + "GeoJSON contains more than one geometry. Using only the first one.", + ), ) if aoi_map.number_of("areas") > 1: gs.warning( _( - "GeoJSON contains more than one geometry. Using only the first one." - ) + "GeoJSON contains more than one geometry. Using only the first one.", + ), ) area = aoi_map.viter("areas") coordinate_pairs = area.points().to_list() @@ -281,64 +294,40 @@ def get_spatial_query_parameter(aoi): else: coordinate_pairs = tuple(coordinate_pairs) return {"polygon": coordinate_pairs} - except Exception: - gs.fatal(_("Cannot get AOI from vector map <{}>")) - # Try GeoJSON / AOI file with OGR - try: - ogr_dataset = ogr.Open(aoi) - except OSError: - gs.fatal(_("Failed to open AOI file {}").format(aoi)) - if not ogr_dataset: - gs.fatal(_("Could not read AOI file {}").format(aoi)) - if ogr_dataset.GetLayerCount() > 1: - gs.warning(_("Input file contains more than one layer")) - ogr_layer = ogr_dataset.GetLayerByIndex(0) - if ogr_layer.GetGeomType() != 3: - gs.warning(_("GeoJSON does not contain polygons")) - if ogr_layer.GetFeatureCount() > 1: - gs.warning( - _("GeoJSON contains more than one geometry. Using only the first one.") - ) - layer_crs = ogr_layer.GetSpatialRef() - ogr_feature = ogr_layer.GetFeature(0) - geom = ogr_feature.GetGeometryRef() - if not layer_crs.IsSame(wgs_84): - geom.TransformTo(wgs_84) - ring = geom.GetGeometryRef(0) + gs.debug(_("AOI vector map <{}> not found.").format(aoi)) + gs.fatal(_("Invalid input for AOI option")) - return {"polygon": ring.GetPoints()} - -def get_temporal_query_parameters(user_options): - """""" +def get_temporal_query_parameters(user_options: dict) -> dict: + """Extract temporal query parameters from user given module options.""" temporal_filters = {} # Set keyword arguments with temporal range for search for search_option in ("temporal", "production_date", "created_at", "revision_date"): if not user_options[search_option]: continue filter_values = user_options[search_option].split(",") + if not filter_values: + continue if len(filter_values) > 2: gs.fatal(_("Too many input values for <{}>. It cannot be more than two.")) try: - filter_values = ( - datetime.fromisoformat(time_stamp) - for time_stamp in filter_values - if filter_values + filter_values = tuple( + datetime.fromisoformat(time_stamp) for time_stamp in filter_values ) except: gs.fatal( _( - "Invalid input for <{}>. It must be a sing or pair of ISO-formated datetime(s)" - ) + "Invalid input for <{}>. It must be a sing or pair of ISO-formated datetime(s)", + ), ) - if len(filter_values) < 2: + if len(list(filter_values)) < 2: filter_values = (filter_values[0], None) temporal_filters[search_option] = filter_values return temporal_filters -def extract_core_umm_metadata(dataset_dict): +def extract_core_umm_metadata(dataset_dict: dict) -> dict: umm_keys = { "Abstract", "AccessConstraints", @@ -385,39 +374,54 @@ def extract_core_umm_metadata(dataset_dict): "VersionDescription", } - def _get_spatial_extent(dataset_dict): - """gsr = set() - ...: for d in datasets: - ...: # keys.update(list(d["umm"].keys())) - ...: # gsr.update(d["umm"].get('SpatialExtent').keys()) - ...: spatial_representation = d["umm"].get('SpatialExtent').get('GranuleSpatialRepresentation') - ...: if spatial_representation == 'CARTESIAN': - ...: spatial_domain = d["umm"].get('SpatialExtent').get('HorizontalSpatialDomain') - ...: if spatial_domain: - ...: {'Geometry', 'ResolutionAndCoordinateSystem', 'ZoneIdentifier'} - ...: if d["umm"].get('SpatialExtent').get('HorizontalSpatialDomain').get('Geometry'): - ...: gsr.update(d["umm"].get('SpatialExtent').get('HorizontalSpatialDomain').get('Geometry').keys()) - ...: print(d["umm"].get('SpatialExtent').get('HorizontalSpatialDomain').get('ResolutionAndCoordinateSystem')) - ...: print(d["umm"].get('SpatialExtent').get('HorizontalSpatialDomain').get('ZoneIdentifier')) - ...:""" - spatial_dict = {} - spatial_representation = ( - dataset_dict["umm"].get("SpatialExtent").get("GranuleSpatialRepresentation") - ) - if spatial_representation == "CARTESIAN": - print( - dataset_dict["umm"].get("SpatialExtent").get("HorizontalSpatialDomain") + def _get_spatial_extent(dataset_dict: dict) -> str: + """Extract spatial extent from UMM metadata.""" + spatial_representation = dataset_dict["umm"].get("SpatialExtent") + if not spatial_representation: + return None + # if spatial_representation == "CARTESIAN": + # return dataset_dict["umm"].get("SpatialExtent").get("HorizontalSpatialDomain", None) + # elif spatial_representation in {"GEODETIC", "NO_SPATIAL", "ORBIT"}: + # return None + spatial_representation = spatial_representation.get("HorizontalSpatialDomain") + if not spatial_representation: + return None + spatial_representation = spatial_representation.get("Geometry") + if not spatial_representation: + return None + spatial_representation = spatial_representation.get("BoundingRectangles") + if not spatial_representation: + return None + return tuple(spatial_representation[0].values()) + + def _get_cycle(dataset_dict: dict) -> str: + """Extract spatial extent from UMM metadata.""" + try: + return ( + dataset_dict["umm"] + .get("SpatialExtent") + .get("HorizontalSpatialDomain") + .get("Track") + .get("Cycle") ) - elif spatial_representation in {"GEODETIC", "NO_SPATIAL", "ORBIT"}: - pass - return ( - dataset_dict["umm"] - .get("SpatialExtent") - .get("HorizontalSpatialDomain") - .get("Geometry") - ) + except KeyError: + return None - def _get_temporal_extent(dataset_dict): + def _get_pass(dataset_dict: dict) -> str: + """Extract spatial extent from UMM metadata.""" + try: + return ( + dataset_dict["umm"] + .get("SpatialExtent") + .get("HorizontalSpatialDomain") + .get("Track") + .get("Passes")[0]["Pass"] + ) + except KeyError: + return None + + def _get_temporal_extent(dataset_dict: dict) -> tuple: + """Return the temporal extent of a granule / dataset.""" datetime_range = ( dataset_dict["umm"].get("TemporalExtents")[0].get("RangeDateTimes", "") ) @@ -425,48 +429,93 @@ def _get_temporal_extent(dataset_dict): return datetime_range[0].get("BeginningDateTime", ""), datetime_range[ 0 ].get("BeginningDateTime", "") - return "", "" + return None - def _get_temporal_extent(dataset_dict): - datetime_range = ( - dataset_dict["umm"].get("TemporalExtents")[0].get("RangeDateTimes", "") - ) + def _get_temporal_extent(dataset_dict: dict) -> tuple: + """Return the temporal extent of a granule / dataset.""" + datetime_range = dataset_dict["umm"].get("TemporalExtent") if datetime_range: - return datetime_range[0].get("BeginningDateTime", ""), datetime_range[ - 0 - ].get("BeginningDateTime", "") - return "", "" + datetime_range = datetime_range.get("RangeDateTime", "") + if datetime_range: + return datetime_range.get("BeginningDateTime", ""), datetime_range.get( + "EndingDateTime", "" + ) + return None - def _get_doi(dataset_dict): + def _get_doi(dataset_dict: dict) -> str: + """Return DOI of the dataset if available, otherwise return empty string.""" + doi = dataset_dict["umm"].get("DOI") + if not doi: + return None return ( dataset_dict["umm"].get("DOI").get("Authority", "") + "/" + dataset_dict["umm"].get("DOI").get("DOI", "") ) - def _get_iso_categories(dataset_dict): + def _get_iso_categories(dataset_dict: dict) -> str: iso_cats = dataset_dict["umm"].get("ISOTopicCategories") - return "|".join(iso_cats) if iso_cats else "" - - return { - "ShortName": dataset_dict["umm"].get("ShortName", ""), - "Version": dataset_dict["umm"].get("Version", ""), - "EntryTitle": dataset_dict["umm"].get("EntryTitle", ""), - "Abstract": dataset_dict["umm"].get("Abstract", ""), - "Purpose": dataset_dict["umm"].get("Purpose", ""), - "ProcessingLevel": dataset_dict["umm"].get("ProcessingLevel").get("Id", ""), - "TemporalExtent": _get_temporal_extent(dataset_dict), - "TemporalExtent_1": _get_doi(dataset_dict), - "TemporalExtent_2": _get_iso_categories(dataset_dict), - "TemporalExtent_3": _get_spatial_extent(dataset_dict), + return "|".join(iso_cats) if iso_cats else None + + def _get_processing_level(dataset_dict: dict) -> str: + """Return processing level of the dataset.""" + processing_level = dataset_dict["umm"].get("ProcessingLevel") + if processing_level: + return processing_level.get("Id", "") + return None + + def _get_data_link(dataset_dict: dict) -> list: + """Return processing level of the dataset.""" + return dataset_dict.data_links() + + def _get_size(dataset_dict: dict) -> list: + """Return processing level of the dataset.""" + return dataset_dict.size() + + def _get_short_name(dataset_dict: dict) -> str: + """Return short name of the dataset.""" + if "CollectionReference" in dataset_dict["umm"]: + return dataset_dict["umm"]["CollectionReference"].get("ShortName", None) + return dataset_dict["umm"].get("ShortName", None) + + def _get_version(dataset_dict: dict) -> str: + """Return version of the dataset.""" + if "CollectionReference" in dataset_dict["umm"]: + return dataset_dict["umm"]["CollectionReference"].get("Version", None) + return dataset_dict["umm"].get("Version", None) + + def _get_platform(dataset_dict: dict) -> list: + """Return platforms of the dataset.""" + platform_info = dataset_dict["umm"].get("Platforms")[0] + return platform_info["ShortName"] + + def _get_instruments(dataset_dict: dict) -> list: + """Return platforms of the dataset.""" + platform_info = dataset_dict["umm"].get("Platforms")[0] + return [instr.get("ShortName") for instr in platform_info["Instruments"]] + + metadata_conversion = { + "ShortName": _get_short_name, + "Platform": _get_platform, + "Instruments": _get_instruments, + "Version": _get_version, + "TemporalExtent": _get_temporal_extent, + "SpatialExtent": _get_spatial_extent, + "Cycle": _get_cycle, + "Pass": _get_pass, + "DOI": _get_doi, + "ISOTopicCategories": _get_iso_categories, + "data_link": _get_data_link, + "size": _get_size, } + return {k: metadata_conversion[k](dataset_dict) for k in metadata_conversion} -def main(): - """Search and download data products from ASF""" +def main(): + """Search and download data products using earthaccess API.""" check_scenes = options["check_scenes"] - skip = flags["s"] or check_scenes in ["all", "existing"] + skip = flags["s"] or check_scenes in {"all", "existing"} # Extract AOI for geo_search search_options = get_spatial_query_parameter(options["aoi"]) @@ -480,7 +529,7 @@ def main(): search_option: options[search_option] for search_option in ("provider", "keyword", "short_name", "granule_name") if options[search_option] - } + }, ) if options["limit"]: @@ -502,8 +551,8 @@ def main(): except Exception: gs.warning( _( - "Login to EarthData failed. Download may fail or search may return incomplete results." - ) + "Login to EarthData failed. Download may fail or search may return incomplete results.", + ), ) # https://github.com/nsidc/earthaccess/blob/0385d126695807f5c865076350b7def04109e088/earthaccess/api.py#L35 @@ -516,8 +565,8 @@ def main(): except: gs.fatal( _( - "Collection search failed. Please check the search parameters and login information." - ) + "Collection search failed. Please check the search parameters and login information.", + ), ) if options["print"] == "collections": if options["format"] == "json": @@ -540,11 +589,11 @@ def main(): if d["umm"].get("DOI").get("Authority") else "" ), - ] + ], ) for d in datasets - ] - ) + ], + ), ) sys.exit(0) else: # if options["print"] == "collection_names": @@ -558,7 +607,7 @@ def main(): if "keyword" in search_options: search_options.pop("keyword") gs.warning( - _("'keyword' is not a supported parameter for granule search. Ignoring...") + _("'keyword' is not a supported parameter for granule search. Ignoring..."), ) # https://github.com/podaac/tutorials/blob/master/notebooks/SearchDownload_SWOTviaCMR.ipynb @@ -566,9 +615,28 @@ def main(): if options["print"] == "granule_metadata": if format == "json": - print(json.dumps(data_granules, indent=2)) + print( + json.dumps( + [extract_core_umm_metadata(dg) for dg in data_granules], indent=2 + ) + ) else: - print(data_granules) + metadata = [extract_core_umm_metadata(dg) for dg in data_granules] + print(",".join(list(metadata[0].keys()))) + for dg in metadata: + print( + ",".join( + [ + ( + "|".join(map(str, k)) + if isinstance(k, (list, tuple)) # noqa: UP038 + else str(k) + ) + for k in dg.values() + ] + ) + ) + sys.exit(0) if flags["s"]: @@ -582,8 +650,9 @@ def main(): nprocs = int(options["nprocs"]) gs.verbose( _("Start downloading {n} granules using {p} threads.").format( - n=len(data_granules), p=nprocs - ) + n=len(data_granules), + p=nprocs, + ), ) try: @@ -591,8 +660,8 @@ def main(): except: gs.fatal( _( - "Downloading data failed. Please check search parameters and login information." - ) + "Downloading data failed. Please check search parameters and login information.", + ), ) @@ -605,18 +674,21 @@ def main(): gs.fatal( _( "Can not import the earthaccess library. " - "Please install it with 'pip install earthaccess'" - ) + "Please install it with 'pip install earthaccess'", + ), ) try: from osgeo import ogr, osr + + ogr.UseExceptions() + osr.UseExceptions() except ImportError: gs.fatal( _( "Can not import the GDAL python library. " - "Please install it with 'pip install GDAL==$(gdal-config --version)'" - ) + "Please install it with 'pip install GDAL==$(gdal-config --version)'", + ), ) main()