From 0842f4d75424f746583ba4ef94bee5030743f1e2 Mon Sep 17 00:00:00 2001 From: Reid Hewitt Date: Fri, 15 Dec 2023 15:14:26 -0700 Subject: [PATCH 01/13] add beautiful soup --- poetry.lock | 29 +++++++++++++++++++++++++++++ pyproject.toml | 3 ++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index 40277d18..9a25d9b7 100644 --- a/poetry.lock +++ b/poetry.lock @@ -18,6 +18,24 @@ docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib- tests = ["attrs[tests-no-zope]", "zope-interface"] tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +[[package]] +name = "beautifulsoup4" +version = "4.12.2" +description = "Screen-scraping library" +optional = false +python-versions = ">=3.6.0" +files = [ + {file = "beautifulsoup4-4.12.2-py3-none-any.whl", hash = "sha256:bd2520ca0d9d7d12694a53d44ac482d181b4ec1888909b035a3dbf40d0f57d4a"}, + {file = "beautifulsoup4-4.12.2.tar.gz", hash = "sha256:492bbc69dca35d12daac71c4db1bfff0c876c00ef4a2ffacce226d4638eb72da"}, +] + +[package.dependencies] +soupsieve = ">1.2" + +[package.extras] +html5lib = ["html5lib"] +lxml = ["lxml"] + [[package]] name = "certifi" version = "2023.11.17" @@ -738,6 +756,17 @@ files = [ {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] +[[package]] +name = "soupsieve" +version = "2.5" +description = "A modern CSS selector implementation for Beautiful Soup." +optional = false +python-versions = ">=3.8" +files = [ + {file = "soupsieve-2.5-py3-none-any.whl", hash = "sha256:eaa337ff55a1579b6549dc679565eac1e3d000563bcb1c8ab0d0fefbc0c2cdc7"}, + {file = "soupsieve-2.5.tar.gz", hash = "sha256:5663d5a7b3bfaeee0bc4372e7fc48f9cff4940b3eec54a6451cc5299f1097690"}, +] + [[package]] name = "text-unidecode" version = "1.3" diff --git a/pyproject.toml b/pyproject.toml index 22afccc0..c095ab15 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,11 +20,12 @@ repository = "https://github.com/GSA/datagov-harvesting-logic" [tool.poetry.dependencies] python = ">=3.10" jsonschema = ">=4" -requests = ">=2" python-dotenv = ">=1" deepdiff = ">=6" pytest = ">=7.3.2" ckanapi = ">=4.7" +ckanapi = "^4.7" +beautifulsoup4 = "^4.12.2" [tool.poetry.group.dev.dependencies] pytest = "^7.3.0" From 8076a669b25b48c0735f8841317f5acd64fcc478 Mon Sep 17 00:00:00 2001 From: Reid Hewitt Date: Fri, 15 Dec 2023 15:14:44 -0700 Subject: [PATCH 02/13] create waf module and traversal function. --- harvester/waf.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 harvester/waf.py diff --git a/harvester/waf.py b/harvester/waf.py new file mode 100644 index 00000000..3f9fd8b1 --- /dev/null +++ b/harvester/waf.py @@ -0,0 +1,32 @@ +import requests +from bs4 import BeautifulSoup +import os + + +def traverse_waf(url, files=[], file_ext=".xml", folder="/", filters=[]): + parent = os.path.dirname(url.rstrip("/")) + + res = requests.get(url) + if res.status_code == 200: + soup = BeautifulSoup(res.content, "html.parser") + anchors = soup.find_all("a", href=True) + + files += [ + os.path.join(url, anchor["href"]) + for anchor in anchors + if anchor["href"].endswith(file_ext) + ] + + folders = [] + for anchor in anchors: + if ( + anchor["href"].endswith(folder) + and not parent.endswith(anchor["href"].rstrip("/")) + and anchor["href"] not in filters + ): + folders.append(os.path.join(url, anchor["href"])) + + for folder in folders: + traverse_waf(folder, files=files, filters=filters) + + return files From 2b7fb844474c18995a837f8989f0ab4bbee48c70 Mon Sep 17 00:00:00 2001 From: Reid Hewitt Date: Fri, 15 Dec 2023 15:15:28 -0700 Subject: [PATCH 03/13] add waf traversal test and fixture. update dcatus paths. --- tests/extract/conftest.py | 12 +++++++++--- tests/extract/test_waf.py | 6 ++++++ 2 files changed, 15 insertions(+), 3 deletions(-) create mode 100644 tests/extract/test_waf.py diff --git a/tests/extract/conftest.py b/tests/extract/conftest.py index a06d06e3..27d46e42 100644 --- a/tests/extract/conftest.py +++ b/tests/extract/conftest.py @@ -4,7 +4,7 @@ @pytest.fixture def get_dcatus_job(): """example dcatus job payload""" - return "http://localhost/dcatus.json" + return "http://localhost/dcatus/dcatus.json" @pytest.fixture @@ -16,10 +16,16 @@ def get_bad_url(): @pytest.fixture def get_bad_json(): """example bad json with missing enclosing bracket""" - return "http://localhost/unclosed.json" + return "http://localhost/dcatus/unclosed.json" @pytest.fixture def get_no_dataset_key_dcatus_json(): """example dcatus json with no 'dataset' key""" - return "http://localhost/no_dataset_key.json" + return "http://localhost/dcatus/no_dataset_key.json" + + +@pytest.fixture +def get_waf_url(): + """example waf""" + return "http://localhost" diff --git a/tests/extract/test_waf.py b/tests/extract/test_waf.py new file mode 100644 index 00000000..a6bacdc0 --- /dev/null +++ b/tests/extract/test_waf.py @@ -0,0 +1,6 @@ +from harvester.extract.waf import traverse_waf + + +def test_traverse_waf(get_waf_url): + files = traverse_waf(get_waf_url, filters=["../", "dcatus/"]) + assert len(files) == 7 From 455a03bf192c0bbc97f761c5206e806dbc7ebe2e Mon Sep 17 00:00:00 2001 From: Reid Hewitt Date: Fri, 15 Dec 2023 15:15:39 -0700 Subject: [PATCH 04/13] reorganize. --- tests/harvest-sources/{ => dcatus}/dcatus.json | 0 tests/harvest-sources/{ => dcatus}/dcatus_to_ckan.json | 0 tests/harvest-sources/{ => dcatus}/no_dataset_key.json | 0 tests/harvest-sources/{ => dcatus}/unclosed.json | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename tests/harvest-sources/{ => dcatus}/dcatus.json (100%) rename tests/harvest-sources/{ => dcatus}/dcatus_to_ckan.json (100%) rename tests/harvest-sources/{ => dcatus}/no_dataset_key.json (100%) rename tests/harvest-sources/{ => dcatus}/unclosed.json (100%) diff --git a/tests/harvest-sources/dcatus.json b/tests/harvest-sources/dcatus/dcatus.json similarity index 100% rename from tests/harvest-sources/dcatus.json rename to tests/harvest-sources/dcatus/dcatus.json diff --git a/tests/harvest-sources/dcatus_to_ckan.json b/tests/harvest-sources/dcatus/dcatus_to_ckan.json similarity index 100% rename from tests/harvest-sources/dcatus_to_ckan.json rename to tests/harvest-sources/dcatus/dcatus_to_ckan.json diff --git a/tests/harvest-sources/no_dataset_key.json b/tests/harvest-sources/dcatus/no_dataset_key.json similarity index 100% rename from tests/harvest-sources/no_dataset_key.json rename to tests/harvest-sources/dcatus/no_dataset_key.json diff --git a/tests/harvest-sources/unclosed.json b/tests/harvest-sources/dcatus/unclosed.json similarity index 100% rename from tests/harvest-sources/unclosed.json rename to tests/harvest-sources/dcatus/unclosed.json From 83defecce40c83d1ef38f7d8da42dbb799fc761f Mon Sep 17 00:00:00 2001 From: Reid Hewitt Date: Fri, 15 Dec 2023 15:15:50 -0700 Subject: [PATCH 05/13] add example waf files. --- tests/harvest-sources/waf/USGSHydroCached.xml | 237 ++++++++++++++++++ tests/harvest-sources/waf/USGSImageryOnly.xml | 220 ++++++++++++++++ tests/harvest-sources/waf/USGSImageryTopo.xml | 223 ++++++++++++++++ .../waf/USGSShadedReliefOnly.xml | 218 ++++++++++++++++ .../waf/browse/USGSHydroCached.xml | 237 ++++++++++++++++++ .../waf/browse/more/USGSShadedReliefOnly.xml | 218 ++++++++++++++++ .../waf/other/USGSImageryOnly.xml | 220 ++++++++++++++++ 7 files changed, 1573 insertions(+) create mode 100644 tests/harvest-sources/waf/USGSHydroCached.xml create mode 100644 tests/harvest-sources/waf/USGSImageryOnly.xml create mode 100644 tests/harvest-sources/waf/USGSImageryTopo.xml create mode 100644 tests/harvest-sources/waf/USGSShadedReliefOnly.xml create mode 100644 tests/harvest-sources/waf/browse/USGSHydroCached.xml create mode 100644 tests/harvest-sources/waf/browse/more/USGSShadedReliefOnly.xml create mode 100644 tests/harvest-sources/waf/other/USGSImageryOnly.xml diff --git a/tests/harvest-sources/waf/USGSHydroCached.xml b/tests/harvest-sources/waf/USGSHydroCached.xml new file mode 100644 index 00000000..fc47b01a --- /dev/null +++ b/tests/harvest-sources/waf/USGSHydroCached.xml @@ -0,0 +1,237 @@ + + + + + + + U.S. Geological Survey + 2018 + USGS Hydro Cached Base Map Service from The National Map + raster digital data, map service + + Rolla, MO and Denver, CO + USGS - National Geospatial Technical Operations Center (NGTOC) + + https://viewer.nationalmap.gov/viewer + + + + ​This service is a cached overlay of a cartographic representation of the National Hydrography Dataset (NHD). ​The NHD is a comprehensive set of digital ​geo​spatial data that encodes information about naturally occurring and constructed bodies of surface water​, paths through which water flows​, ​related features​ such as stream gages and dams​​, and additional hydrologic information​. ​It is available nationwide ​in a 1:24,000-scale seamless dataset, referred to as high resolution NHD. The NHD supports many applications, such as making maps, geocoding observations, flow modeling, data maintenance and stewardship. For additional information, go to http://nhd.usgs.gov. Additional datasets are used for small-scale hydrography representation as well​, including medium resolution NHDPlus published by EPA​; USGS Small-Scale hydrography; and bathymetry from ETOPO1 Global Relief, provided by NOAA National Centers for Environmental Information, U.S. Coastal Relief Model. + This tile cache base map provides a visualization of the free data that is + available for download from The National Map at https://viewer.nationalmap.gov/viewer. + + + + + + 2018 + + + publication date + + + Complete + As needed + + + + -179.999 + -65 + 71.5 + 17.625 + + + + + NGDA Portfolio Themes + National Geospatial Data Asset + NGDA + Water Inland Theme + + + None + National Hydrography Dataset + NHD + Water + Rivers + Streams + Lakes + The National Map + USGS + + + ISO 19115 Topic Category + imageryBaseMapsEarthCover + boundaries + elevation + inlandWaters + + + The National Map Theme Thesaurus + Hydrography + + + The National Map Type Thesaurus + Base Map Service + + + + Geographic Names Information System + US + United States + + + + None + None. Acknowledgement of the originating agencies would be appreciated in products + derived from these data. + + https://thor-f5.er.usgs.gov/ngtoc/metadata/waf/services/base_maps/browse/basemap-hydro-cached.png + Browse graphic showing US representation at small scale. + JPG + + USGS, NHD Stewards - https://nhd.usgs.gov/stewardship.html#.W2tN29Uzqpo + ArcGIS 10.2 + + + Raster + + + + + + WGS84 Web Mercator (Auxiliary Sphere) + + 0 + 0 + 0 + 0 + + + + coordinate pair + + 1.000000 + 1.000000 + + meters + + + + + + + + + U.S. Geological Survey + Not Provided + + + mailing and physical +
PO Box 25046 Denver Federal Center
+ Lakewood + CO + 80225 + + 1-888-ASK-USGS (1-888-275-8747) + tnm_help@usgs.gov + + + USGS Hydro-NHD Base Map Service + Although these data have been processed successfully on a computer system at the + U.S. Geological Survey, no warranty expressed or implied is made regarding the accuracy or + utility of the data on any other system or for general or scientific purposes, nor shall + the act of distribution constitute any such warranty. This disclaimer applies both to + individual use of the data and aggregate use with other data. It is strongly recommended + that these data are directly acquired from a U.S. Geological Survey server, and not + indirectly through other sources which may have changed the data in some way. It is also + strongly recommended that careful attention be paid to the contents of the metadata file + associated with these data. The U.S. Geological Survey shall not be held liable for + improper or incorrect use of the data described and/or contained herein. + + + + Hydro Cached Base Map Service (ArcGIS) + 10.21 + https://developers.arcgis.com/rest/ + + + + + + https://basemap.nationalmap.gov/arcgis/rest/services/USGSHydroCached/MapServer + + + + + + + + Hydro Cached Base Map Service (WMS) + 1.3.0 + https://www.opengeospatial.org/standards/wms + + + + + + https://basemap.nationalmap.gov/arcgis/services/USGSHydroCached/MapServer/WMSServer?request=GetCapabilities&service=WMS + + + + + + + + Hydro Cached Base Map Service (WMTS) + 1.0.0 + https://www.opengeospatial.org/standards/wmts + + + + + + https://basemap.nationalmap.gov/arcgis/rest/services/USGSHydroCached/MapServer/WMTS/1.0.0/WMTSCapabilities.xml + + + + + + None + + + + 20181128 + + + + U.S. Geological Survey, National Geospatial Technical Operations + Center + Not Provided + + + mailing and physical +
1400 Independence Road
+ Rolla + MO + 65401 + + + mailing and physical +
PO Box 25046 Denver Federal Center
+ Lakewood + CO + 80225 + + 1-888-ASK-USGS (1-888-275-8747) + tnm_help@usgs.gov + Monday through Friday, 8:00 AM to 4:30 PM CT + Metadata information can also be obtained through online services using The + National Map Viewer at https://nationalmap.gov, or EarthExplorer at + https://earthexplorer.usgs.gov, or Ask USGS at https://www.usgs.gov/ask. + + + FGDC Content Standard for Digital Geospatial Metadata + FGDC-STD-001-1998 + + diff --git a/tests/harvest-sources/waf/USGSImageryOnly.xml b/tests/harvest-sources/waf/USGSImageryOnly.xml new file mode 100644 index 00000000..acf08caf --- /dev/null +++ b/tests/harvest-sources/waf/USGSImageryOnly.xml @@ -0,0 +1,220 @@ + + + + + + + U.S. Geological Survey + 2018 + USGS Imagery Only Base Map Service from The National Map + raster digital data, map service + + Rolla, MO and Denver, CO + USGS - National Geospatial Technical Operations Center + (NGTOC) + + https://viewer.nationalmap.gov + + + + USGS Imagery Only is a tile cache base map service of orthoimagery in The National Map visible to the 1:9,028 zoom scale. Orthoimagery data is typically high resolution aerial images that combine the visual attributes of an aerial photograph with the spatial accuracy and reliability of a planimetric map. USGS digital orthoimage resolution may vary from 6 inches to 1 meter. In the former resolution, every pixel in an orthoimage covers a six inch square of the earth's surface, while in the latter resolution, one meter square is represented by each pixel. Blue Marble: Next Generation and Landsat imagery data sources are displayed at small to medium scales, however, the majority of the imagery service source is from the National Agriculture Imagery Program (NAIP) for the conterminous United States. The data is 1 meter pixel resolution collected with "leaf-on" conditions. Collection of NAIP imagery is administered by the U.S. Department of Agriculture's Farm Service Agency (FSA). In areas where NAIP data is not available, other imagery may be acquired through partnerships by the USGS. For Alaska, 10-meter resolution SPOT imagery is provided for viewing. The National Map download client allows free downloads of public domain, 1-meter resolution orthoimagery in JPEG 2000 (jp2) format for the conterminous United States. However, the 10-meter Alaska orthoimagery data will not be available for direct download from the National Map due to license restrictions. For additional information on orthoimagery, go to https://nationalmap.gov/ortho.html. + This tile cache base map provides a visualization of the free data that is + available for download from The National Map at + https://viewer.nationalmap.gov. + + + + + + 2008 + 2016 + + + publication date + + + Complete + As needed + + + + -179.1666667 + 180 + 71.5 + 17.625 + + + + + ISO 19115 Topic Category + imageryBaseMapsEarthCover + + + The National Map Theme Thesaurus + Orthoimagery + + + The National Map Type Thesaurus + Base Map Service + + + Geographic Names Information System + US + United States + + + + None + None. Acknowledgement of the originating agencies would be appreciated in products + derived from these data. + + https://thor-f5.er.usgs.gov/ngtoc/metadata/waf/services/base_maps/browse/basemap-imagery-only.png + Browse graphic showing US representation at small scale. + PNG + + USGS, USDA-FSA + ArcGIS 10.2 + + + Raster + + + + + + WGS84 Web Mercator (Auxiliary Sphere) + + 0 + 0 + 0 + 0 + + + + coordinate pair + + 1.000000 + 1.000000 + + meters + + + + + + + + + U.S. Geological Survey + Not Provided + + + mailing and physical +
PO Box 25046 Denver Federal Center
+ Lakewood + CO + 80225 + + 1-888-ASK-USGS (1-888-275-8747) + tnm_help@usgs.gov + + + USGS Imagery Only Base Map Service + Although these data have been processed successfully on a computer system at the + U.S. Geological Survey, no warranty expressed or implied is made regarding the accuracy + or utility of the data on any other system or for general or scientific purposes, nor + shall the act of distribution constitute any such warranty. This disclaimer applies both + to individual use of the data and aggregate use with other data. It is strongly + recommended that these data are directly acquired from a U.S. Geological Survey server, + and not indirectly through other sources which may have changed the data in some way. It + is also strongly recommended that careful attention be paid to the contents of the + metadata file associated with these data. The U.S. Geological Survey shall not be held + liable for improper or incorrect use of the data described and/or contained + herein. + + + + ImageryOnly Base Map Service (ArcGIS) + 10.21 + https://www.geoplatform.gov/spec/ + + + + + + https://basemap.nationalmap.gov/arcgis/rest/services/USGSImageryOnly/MapServer + + + + + + + + ImageryOnly Base Map Service (WMS) + 1.3.0 + https://www.opengeospatial.org/standards/wms + + + + + + https://basemap.nationalmap.gov/arcgis/services/USGSImageryOnly/MapServer/WMSServer?request=GetCapabilities&service=WMS + + + + + + + + ImageryOnly Base Map Service (WMTS) + 1.0.0 + https://www.opengeospatial.org/standards/wmts + + + + + + https://basemap.nationalmap.gov/arcgis/rest/services/USGSImageryOnly/MapServer/WMTS/1.0.0/WMTSCapabilities.xml + + + + + + None + + + + 20181128 + + + + U.S. Geological Survey, National Geospatial Technical Operations + Center + Not Provided + + + mailing and physical +
1400 Independence Road
+ Rolla + MO + 65401 + + + mailing and physical +
PO Box 25046 Denver Federal Center
+ Lakewood + CO + 80225 + + 1-888-ASK-USGS (1-888-275-8747) + tnm_help@usgs.gov + Monday through Friday, 8:00 AM to 4:30 PM CT + Metadata information can also be obtained through online services using The + National Map Viewer at https://nationalmap.usgs.gov, or EarthExplorer at + https://earthexplorer.usgs.gov, or Ask USGS at https://www.usgs.gov/ask. + + + FGDC Content Standard for Digital Geospatial Metadata + FGDC-STD-001-1998 + + diff --git a/tests/harvest-sources/waf/USGSImageryTopo.xml b/tests/harvest-sources/waf/USGSImageryTopo.xml new file mode 100644 index 00000000..58bf2ea9 --- /dev/null +++ b/tests/harvest-sources/waf/USGSImageryTopo.xml @@ -0,0 +1,223 @@ + + + + + + + U.S. Geological Survey + 2018 + USGS Imagery Topo Base Map Service from The National Map + raster digital data, map service + + Rolla, MO and Denver, CO + USGS - National Geospatial Technical Operations Center (NGTOC) + + https://viewer.nationalmap.gov + + + + USGS Imagery Topo is a tile cache base map of orthoimagery in The National Map and US Topo vector data visible to the 1:9,028 scale. Orthoimagery data are typically high resolution images that combine the visual attributes of an aerial photograph with the spatial accuracy and reliability of a planimetric map. USGS digital orthoimage resolution may vary from 6 inches to 1 meter. In the former resolution, every pixel in an orthoimage covers a six inch square of the earth's surface, while in the latter resolution, one meter square is represented by each pixel. Blue Marble: Next Generation source is displayed at small to medium scales. However, the majority of the imagery service source is from the National Agriculture Imagery Program (NAIP) for the conterminous United States. The data is 1-meter pixel resolution with "leaf-on". Collection of NAIP imagery is administered by the U.S. Department of Agriculture's Farm Service Agency (FSA). In areas where NAIP data is not available, other imagery may be acquired through partnerships by the USGS. The National Map program is working on acquisition of high resolution orthoimagery (HRO) for Alaska. Most of the new Alaska imagery data will not be available in this service due to license restrictions. The National Map viewer allows free downloads of public domain, 1-meter resolution orthoimagery in JPEG 2000 (jp2) format for the conterminous United States. + This tile cache base map provides a visualization of the free data that is available for download from The National Map at + https://viewer.nationalmap.gov. + + + + + + 2018 + + + publication date + + + Complete + As needed + + + + -179.1666667 + 180 + 71.5 + 17.625 + + + + + ISO 19115 Topic Category + imageryBaseMapsEarthCover + boundaries + elevation + inlandWaters + structure + transportation + + + The National Map Theme Thesaurus + Elevation + Orthoimagery + Hydrography + Geographic Names + Boundaries + Transportation + Structures + Land Cover + + + The National Map Type Thesaurus + Base Map Service + + + Geographic Names Information System + US + United States + + + + None + None. Acknowledgement of the originating agencies would be appreciated in products derived from these data. + + https://thor-f5.er.usgs.gov/ngtoc/metadata/waf/services/base_maps/browse/basemap-imagery-topo.png + Browse graphic showing US representation at small scale. + PNG + + USGS, USDA-FSA, NPS, Census, TomTom (TeleAtlas) + ArcGIS 10.2 + + + Raster + + + + + + WGS84 Web Mercator (Auxiliary Sphere) + + 0 + 0 + 0 + 0 + + + + coordinate pair + + 1.000000 + 1.000000 + + meters + + + + + + + + + U.S. Geological Survey + Not Provided + + + mailing and physical +
PO Box 25046 Denver Federal Center
+ Lakewood + CO + 80225 + + 1-888-ASK-USGS (1-888-275-8747) + tnm_help@usgs.gov + + + USGS Imagery Topo Base Map Service + Although these data have been processed successfully on a computer system at the U.S. Geological Survey, no warranty + expressed or implied is made regarding the accuracy or utility of the data on any other system or for general or scientific + purposes, nor shall the act of distribution constitute any such warranty. This disclaimer applies both to individual use of the + data and aggregate use with other data. It is strongly recommended that these data are directly acquired from a U.S. Geological + Survey server, and not indirectly through other sources which may have changed the data in some way. It is also strongly + recommended that careful attention be paid to the contents of the metadata file associated with these data. The U.S. Geological + Survey shall not be held liable for improper or incorrect use of the data described and/or contained herein. + + + + ImageryTopo Base Map Service (ArcGIS) + 10.21 + https://www.geoplatform.gov/spec/ + + + + + + https://basemap.nationalmap.gov/arcgis/rest/services/USGSImageryTopo/MapServer + + + + + + + + ImageryTopo Base Map Service (WMS) + 1.3.0 + https://www.opengeospatial.org/standards/wms + + + + + + https://basemap.nationalmap.gov/arcgis/services/USGSImageryTopo/MapServer/WMSServer?request=GetCapabilities&service=WMS + + + + + + + + ImageryTopo Base Map Service (WMTS) + 1.0.0 + https://www.opengeospatial.org/standards/wmts + + + + + + https://basemap.nationalmap.gov/arcgis/rest/services/USGSImageryTopo/MapServer/WMTS/1.0.0/WMTSCapabilities.xml + + + + + + None + + + + 20181128 + + + + U.S. Geological Survey, National Geospatial Technical Operations Center + Not Provided + + + mailing and physical +
1400 Independence Road
+ Rolla + MO + 65401 + + + mailing and physical +
PO Box 25046 Denver Federal Center
+ Lakewood + CO + 80225 + + 1-888-ASK-USGS (1-888-275-8747) + tnm_help@usgs.gov + Monday through Friday, 8:00 AM to 4:00 PM + Metadata information can also be obtained through online services using The National Map Viewer at + https://nationalmap.usgs.gov, or EarthExplorer at https://earthexplorer.usgs.gov, or Ask USGS at + https://www.usgs.gov/ask. + + + FGDC Content Standard for Digital Geospatial Metadata + FGDC-STD-001-1998 + + diff --git a/tests/harvest-sources/waf/USGSShadedReliefOnly.xml b/tests/harvest-sources/waf/USGSShadedReliefOnly.xml new file mode 100644 index 00000000..1b72d727 --- /dev/null +++ b/tests/harvest-sources/waf/USGSShadedReliefOnly.xml @@ -0,0 +1,218 @@ + + + + + + + U.S. Geological Survey + 2018 + USGS Hill Shade Base Map Service from The National Map + raster digital data, map service + + Rolla, MO and Denver, CO + USGS - National Geospatial Technical Operations Center (NGTOC) + + https://viewer.nationalmap.gov + + + + The USGS Shaded Relief base map service from The National Map was created with data from the 3D Elevation Program (3DEP) for large and medium scales (1:9,028 through 1:1,155,581), and Global Multi-resolution Terrain Elevation Data 2010 (GMTED2010) for small scales (1:2,311,162 through 1:295,828,764). 3DEP maintains a seamless dataset of best available raster elevation data, as LIDAR and as digital elevation models (DEMs), for the conterminous United States, Alaska, Hawaii, and Territorial Islands. Resolutions available include 1-meter, 3-meter (1/9-arc-second), 10-meter (1/3-arc-second), 30-meter (1-arc-second), and 60-meter (2-arc-second; only in Alaska). 3DEP also contains coverage of Canada, Mexico, Central America, and the Caribbean at 30-meter (1-arc-second) resolution. 3DEP shaded relief was derived with a multi-directional hillshade technique. Small-scale shaded relief was created with a single-direction hillshade from GMTED2010 global elevation data at 7.5-, 15-, and 30-arc-second resolutions, and resampled further for scales 1:18,489,298 and smaller. Ocean areas are left unfilled for maximum flexibility by end users. For additional information, go to https://topotools.cr.usgs.gov/gmted_viewer/ , https://nationalmap.gov/3DEP/ , and https://apps.nationalmap.gov/3depdem/ + This cached tiled base map provides a visualization of the free data that is available for download from The National Map + at https://viewer.nationalmap.gov. + + + + + + 2018 + + + publication date + + + Complete + As needed + + + + -179.1666667 + 180 + 71.5 + 17.625 + + + + + ISO 19115 Topic Category + imageryBaseMapsEarthCover + elevation + + + NGDA Portfolio Themes + National Geospatial Data Asset + NGDA + Elevation Theme + + + The National Map Theme Thesaurus + Elevation + + + The National Map Type Thesaurus + Base Map Service + + + Geographic Names Information System + US + United States + + + + None + None. Acknowledgement of the originating agencies would be appreciated in products derived from these data. + + https://thor-f5.er.usgs.gov/ngtoc/metadata/waf/services/base_maps/browse/basemap-hillshade.png + Browse graphic showing US representation at small scale. + JPG + + USGS + ArcGIS 10.5 + + + Raster + + + + + + WGS84 Web Mercator (Auxiliary Sphere) + + 0 + 0 + 0 + 0 + + + + coordinate pair + + 1.000000 + 1.000000 + + meters + + + + + + + + + U.S. Geological Survey + Not Provided + + + mailing and physical +
PO Box 25046 Denver Federal Center
+ Lakewood + CO + 80225 + + 1-888-ASK-USGS (1-888-275-8747) + tnm_help@usgs.gov + + + USGS Hill Shade Base Map Service + Although these data have been processed successfully on a computer system at the U.S. Geological Survey, no warranty + expressed or implied is made regarding the accuracy or utility of the data on any other system or for general or scientific + purposes, nor shall the act of distribution constitute any such warranty. This disclaimer applies both to individual use of the + data and aggregate use with other data. It is strongly recommended that these data are directly acquired from a U.S. Geological + Survey server, and not indirectly through other sources which may have changed the data in some way. It is also strongly + recommended that careful attention be paid to the contents of the metadata file associated with these data. The U.S. Geological + Survey shall not be held liable for improper or incorrect use of the data described and/or contained herein. + + + + ShadedReliefOnly Base Map Service (ArcGIS) + 10.21 + https://www.geoplatform.gov/spec/ + + + + + + https://basemap.nationalmap.gov/arcgis/rest/services/USGSShadedReliefOnly/MapServer + + + + + + + + ShadedReliefOnly Base Map Service (WMS) + 1.3.0 + https://www.opengeospatial.org/standards/wms + + + + + + https://basemap.nationalmap.gov/arcgis/services/USGSShadedReliefOnly/MapServer/WMSServer?request=GetCapabilities&service=WMS + + + + + + + + ShadedReliefOnly Base Map Service (WMTS) + 1.0.0 + https://www.opengeospatial.org/standards/wmts + + + + + + https://basemap.nationalmap.gov/arcgis/rest/services/USGSShadedReliefOnly/MapServer/WMTS/1.0.0/WMTSCapabilities.xml + + + + + + None + + + + 20181128 + + + + U.S. Geological Survey, National Geospatial Technical Operations Center + Not Provided + + + mailing and physical +
1400 Independence Road
+ Rolla + MO + 65401 + + + mailing and physical +
PO Box 25046 Denver Federal Center
+ Lakewood + CO + 80225 + + 1-888-ASK-USGS (1-888-275-8747) + tnm_help@usgs.gov + Monday through Friday, 8:00 AM to 4:30 PM CT + Metadata information can also be obtained through online services using The National Map Viewer at + https://nationalmap.usgs.gov, or EarthExplorer at https://earthexplorer.usgs.gov, or Ask USGS at + https://www.usgs.gov/ask. + + + FGDC Content Standard for Digital Geospatial Metadata + FGDC-STD-001-1998 + + diff --git a/tests/harvest-sources/waf/browse/USGSHydroCached.xml b/tests/harvest-sources/waf/browse/USGSHydroCached.xml new file mode 100644 index 00000000..fc47b01a --- /dev/null +++ b/tests/harvest-sources/waf/browse/USGSHydroCached.xml @@ -0,0 +1,237 @@ + + + + + + + U.S. Geological Survey + 2018 + USGS Hydro Cached Base Map Service from The National Map + raster digital data, map service + + Rolla, MO and Denver, CO + USGS - National Geospatial Technical Operations Center (NGTOC) + + https://viewer.nationalmap.gov/viewer + + + + ​This service is a cached overlay of a cartographic representation of the National Hydrography Dataset (NHD). ​The NHD is a comprehensive set of digital ​geo​spatial data that encodes information about naturally occurring and constructed bodies of surface water​, paths through which water flows​, ​related features​ such as stream gages and dams​​, and additional hydrologic information​. ​It is available nationwide ​in a 1:24,000-scale seamless dataset, referred to as high resolution NHD. The NHD supports many applications, such as making maps, geocoding observations, flow modeling, data maintenance and stewardship. For additional information, go to http://nhd.usgs.gov. Additional datasets are used for small-scale hydrography representation as well​, including medium resolution NHDPlus published by EPA​; USGS Small-Scale hydrography; and bathymetry from ETOPO1 Global Relief, provided by NOAA National Centers for Environmental Information, U.S. Coastal Relief Model. + This tile cache base map provides a visualization of the free data that is + available for download from The National Map at https://viewer.nationalmap.gov/viewer. + + + + + + 2018 + + + publication date + + + Complete + As needed + + + + -179.999 + -65 + 71.5 + 17.625 + + + + + NGDA Portfolio Themes + National Geospatial Data Asset + NGDA + Water Inland Theme + + + None + National Hydrography Dataset + NHD + Water + Rivers + Streams + Lakes + The National Map + USGS + + + ISO 19115 Topic Category + imageryBaseMapsEarthCover + boundaries + elevation + inlandWaters + + + The National Map Theme Thesaurus + Hydrography + + + The National Map Type Thesaurus + Base Map Service + + + + Geographic Names Information System + US + United States + + + + None + None. Acknowledgement of the originating agencies would be appreciated in products + derived from these data. + + https://thor-f5.er.usgs.gov/ngtoc/metadata/waf/services/base_maps/browse/basemap-hydro-cached.png + Browse graphic showing US representation at small scale. + JPG + + USGS, NHD Stewards - https://nhd.usgs.gov/stewardship.html#.W2tN29Uzqpo + ArcGIS 10.2 + + + Raster + + + + + + WGS84 Web Mercator (Auxiliary Sphere) + + 0 + 0 + 0 + 0 + + + + coordinate pair + + 1.000000 + 1.000000 + + meters + + + + + + + + + U.S. Geological Survey + Not Provided + + + mailing and physical +
PO Box 25046 Denver Federal Center
+ Lakewood + CO + 80225 + + 1-888-ASK-USGS (1-888-275-8747) + tnm_help@usgs.gov + + + USGS Hydro-NHD Base Map Service + Although these data have been processed successfully on a computer system at the + U.S. Geological Survey, no warranty expressed or implied is made regarding the accuracy or + utility of the data on any other system or for general or scientific purposes, nor shall + the act of distribution constitute any such warranty. This disclaimer applies both to + individual use of the data and aggregate use with other data. It is strongly recommended + that these data are directly acquired from a U.S. Geological Survey server, and not + indirectly through other sources which may have changed the data in some way. It is also + strongly recommended that careful attention be paid to the contents of the metadata file + associated with these data. The U.S. Geological Survey shall not be held liable for + improper or incorrect use of the data described and/or contained herein. + + + + Hydro Cached Base Map Service (ArcGIS) + 10.21 + https://developers.arcgis.com/rest/ + + + + + + https://basemap.nationalmap.gov/arcgis/rest/services/USGSHydroCached/MapServer + + + + + + + + Hydro Cached Base Map Service (WMS) + 1.3.0 + https://www.opengeospatial.org/standards/wms + + + + + + https://basemap.nationalmap.gov/arcgis/services/USGSHydroCached/MapServer/WMSServer?request=GetCapabilities&service=WMS + + + + + + + + Hydro Cached Base Map Service (WMTS) + 1.0.0 + https://www.opengeospatial.org/standards/wmts + + + + + + https://basemap.nationalmap.gov/arcgis/rest/services/USGSHydroCached/MapServer/WMTS/1.0.0/WMTSCapabilities.xml + + + + + + None + + + + 20181128 + + + + U.S. Geological Survey, National Geospatial Technical Operations + Center + Not Provided + + + mailing and physical +
1400 Independence Road
+ Rolla + MO + 65401 + + + mailing and physical +
PO Box 25046 Denver Federal Center
+ Lakewood + CO + 80225 + + 1-888-ASK-USGS (1-888-275-8747) + tnm_help@usgs.gov + Monday through Friday, 8:00 AM to 4:30 PM CT + Metadata information can also be obtained through online services using The + National Map Viewer at https://nationalmap.gov, or EarthExplorer at + https://earthexplorer.usgs.gov, or Ask USGS at https://www.usgs.gov/ask. + + + FGDC Content Standard for Digital Geospatial Metadata + FGDC-STD-001-1998 + + diff --git a/tests/harvest-sources/waf/browse/more/USGSShadedReliefOnly.xml b/tests/harvest-sources/waf/browse/more/USGSShadedReliefOnly.xml new file mode 100644 index 00000000..1b72d727 --- /dev/null +++ b/tests/harvest-sources/waf/browse/more/USGSShadedReliefOnly.xml @@ -0,0 +1,218 @@ + + + + + + + U.S. Geological Survey + 2018 + USGS Hill Shade Base Map Service from The National Map + raster digital data, map service + + Rolla, MO and Denver, CO + USGS - National Geospatial Technical Operations Center (NGTOC) + + https://viewer.nationalmap.gov + + + + The USGS Shaded Relief base map service from The National Map was created with data from the 3D Elevation Program (3DEP) for large and medium scales (1:9,028 through 1:1,155,581), and Global Multi-resolution Terrain Elevation Data 2010 (GMTED2010) for small scales (1:2,311,162 through 1:295,828,764). 3DEP maintains a seamless dataset of best available raster elevation data, as LIDAR and as digital elevation models (DEMs), for the conterminous United States, Alaska, Hawaii, and Territorial Islands. Resolutions available include 1-meter, 3-meter (1/9-arc-second), 10-meter (1/3-arc-second), 30-meter (1-arc-second), and 60-meter (2-arc-second; only in Alaska). 3DEP also contains coverage of Canada, Mexico, Central America, and the Caribbean at 30-meter (1-arc-second) resolution. 3DEP shaded relief was derived with a multi-directional hillshade technique. Small-scale shaded relief was created with a single-direction hillshade from GMTED2010 global elevation data at 7.5-, 15-, and 30-arc-second resolutions, and resampled further for scales 1:18,489,298 and smaller. Ocean areas are left unfilled for maximum flexibility by end users. For additional information, go to https://topotools.cr.usgs.gov/gmted_viewer/ , https://nationalmap.gov/3DEP/ , and https://apps.nationalmap.gov/3depdem/ + This cached tiled base map provides a visualization of the free data that is available for download from The National Map + at https://viewer.nationalmap.gov. + + + + + + 2018 + + + publication date + + + Complete + As needed + + + + -179.1666667 + 180 + 71.5 + 17.625 + + + + + ISO 19115 Topic Category + imageryBaseMapsEarthCover + elevation + + + NGDA Portfolio Themes + National Geospatial Data Asset + NGDA + Elevation Theme + + + The National Map Theme Thesaurus + Elevation + + + The National Map Type Thesaurus + Base Map Service + + + Geographic Names Information System + US + United States + + + + None + None. Acknowledgement of the originating agencies would be appreciated in products derived from these data. + + https://thor-f5.er.usgs.gov/ngtoc/metadata/waf/services/base_maps/browse/basemap-hillshade.png + Browse graphic showing US representation at small scale. + JPG + + USGS + ArcGIS 10.5 + + + Raster + + + + + + WGS84 Web Mercator (Auxiliary Sphere) + + 0 + 0 + 0 + 0 + + + + coordinate pair + + 1.000000 + 1.000000 + + meters + + + + + + + + + U.S. Geological Survey + Not Provided + + + mailing and physical +
PO Box 25046 Denver Federal Center
+ Lakewood + CO + 80225 + + 1-888-ASK-USGS (1-888-275-8747) + tnm_help@usgs.gov + + + USGS Hill Shade Base Map Service + Although these data have been processed successfully on a computer system at the U.S. Geological Survey, no warranty + expressed or implied is made regarding the accuracy or utility of the data on any other system or for general or scientific + purposes, nor shall the act of distribution constitute any such warranty. This disclaimer applies both to individual use of the + data and aggregate use with other data. It is strongly recommended that these data are directly acquired from a U.S. Geological + Survey server, and not indirectly through other sources which may have changed the data in some way. It is also strongly + recommended that careful attention be paid to the contents of the metadata file associated with these data. The U.S. Geological + Survey shall not be held liable for improper or incorrect use of the data described and/or contained herein. + + + + ShadedReliefOnly Base Map Service (ArcGIS) + 10.21 + https://www.geoplatform.gov/spec/ + + + + + + https://basemap.nationalmap.gov/arcgis/rest/services/USGSShadedReliefOnly/MapServer + + + + + + + + ShadedReliefOnly Base Map Service (WMS) + 1.3.0 + https://www.opengeospatial.org/standards/wms + + + + + + https://basemap.nationalmap.gov/arcgis/services/USGSShadedReliefOnly/MapServer/WMSServer?request=GetCapabilities&service=WMS + + + + + + + + ShadedReliefOnly Base Map Service (WMTS) + 1.0.0 + https://www.opengeospatial.org/standards/wmts + + + + + + https://basemap.nationalmap.gov/arcgis/rest/services/USGSShadedReliefOnly/MapServer/WMTS/1.0.0/WMTSCapabilities.xml + + + + + + None + + + + 20181128 + + + + U.S. Geological Survey, National Geospatial Technical Operations Center + Not Provided + + + mailing and physical +
1400 Independence Road
+ Rolla + MO + 65401 + + + mailing and physical +
PO Box 25046 Denver Federal Center
+ Lakewood + CO + 80225 + + 1-888-ASK-USGS (1-888-275-8747) + tnm_help@usgs.gov + Monday through Friday, 8:00 AM to 4:30 PM CT + Metadata information can also be obtained through online services using The National Map Viewer at + https://nationalmap.usgs.gov, or EarthExplorer at https://earthexplorer.usgs.gov, or Ask USGS at + https://www.usgs.gov/ask. + + + FGDC Content Standard for Digital Geospatial Metadata + FGDC-STD-001-1998 + + diff --git a/tests/harvest-sources/waf/other/USGSImageryOnly.xml b/tests/harvest-sources/waf/other/USGSImageryOnly.xml new file mode 100644 index 00000000..acf08caf --- /dev/null +++ b/tests/harvest-sources/waf/other/USGSImageryOnly.xml @@ -0,0 +1,220 @@ + + + + + + + U.S. Geological Survey + 2018 + USGS Imagery Only Base Map Service from The National Map + raster digital data, map service + + Rolla, MO and Denver, CO + USGS - National Geospatial Technical Operations Center + (NGTOC) + + https://viewer.nationalmap.gov + + + + USGS Imagery Only is a tile cache base map service of orthoimagery in The National Map visible to the 1:9,028 zoom scale. Orthoimagery data is typically high resolution aerial images that combine the visual attributes of an aerial photograph with the spatial accuracy and reliability of a planimetric map. USGS digital orthoimage resolution may vary from 6 inches to 1 meter. In the former resolution, every pixel in an orthoimage covers a six inch square of the earth's surface, while in the latter resolution, one meter square is represented by each pixel. Blue Marble: Next Generation and Landsat imagery data sources are displayed at small to medium scales, however, the majority of the imagery service source is from the National Agriculture Imagery Program (NAIP) for the conterminous United States. The data is 1 meter pixel resolution collected with "leaf-on" conditions. Collection of NAIP imagery is administered by the U.S. Department of Agriculture's Farm Service Agency (FSA). In areas where NAIP data is not available, other imagery may be acquired through partnerships by the USGS. For Alaska, 10-meter resolution SPOT imagery is provided for viewing. The National Map download client allows free downloads of public domain, 1-meter resolution orthoimagery in JPEG 2000 (jp2) format for the conterminous United States. However, the 10-meter Alaska orthoimagery data will not be available for direct download from the National Map due to license restrictions. For additional information on orthoimagery, go to https://nationalmap.gov/ortho.html. + This tile cache base map provides a visualization of the free data that is + available for download from The National Map at + https://viewer.nationalmap.gov. + + + + + + 2008 + 2016 + + + publication date + + + Complete + As needed + + + + -179.1666667 + 180 + 71.5 + 17.625 + + + + + ISO 19115 Topic Category + imageryBaseMapsEarthCover + + + The National Map Theme Thesaurus + Orthoimagery + + + The National Map Type Thesaurus + Base Map Service + + + Geographic Names Information System + US + United States + + + + None + None. Acknowledgement of the originating agencies would be appreciated in products + derived from these data. + + https://thor-f5.er.usgs.gov/ngtoc/metadata/waf/services/base_maps/browse/basemap-imagery-only.png + Browse graphic showing US representation at small scale. + PNG + + USGS, USDA-FSA + ArcGIS 10.2 + + + Raster + + + + + + WGS84 Web Mercator (Auxiliary Sphere) + + 0 + 0 + 0 + 0 + + + + coordinate pair + + 1.000000 + 1.000000 + + meters + + + + + + + + + U.S. Geological Survey + Not Provided + + + mailing and physical +
PO Box 25046 Denver Federal Center
+ Lakewood + CO + 80225 + + 1-888-ASK-USGS (1-888-275-8747) + tnm_help@usgs.gov + + + USGS Imagery Only Base Map Service + Although these data have been processed successfully on a computer system at the + U.S. Geological Survey, no warranty expressed or implied is made regarding the accuracy + or utility of the data on any other system or for general or scientific purposes, nor + shall the act of distribution constitute any such warranty. This disclaimer applies both + to individual use of the data and aggregate use with other data. It is strongly + recommended that these data are directly acquired from a U.S. Geological Survey server, + and not indirectly through other sources which may have changed the data in some way. It + is also strongly recommended that careful attention be paid to the contents of the + metadata file associated with these data. The U.S. Geological Survey shall not be held + liable for improper or incorrect use of the data described and/or contained + herein. + + + + ImageryOnly Base Map Service (ArcGIS) + 10.21 + https://www.geoplatform.gov/spec/ + + + + + + https://basemap.nationalmap.gov/arcgis/rest/services/USGSImageryOnly/MapServer + + + + + + + + ImageryOnly Base Map Service (WMS) + 1.3.0 + https://www.opengeospatial.org/standards/wms + + + + + + https://basemap.nationalmap.gov/arcgis/services/USGSImageryOnly/MapServer/WMSServer?request=GetCapabilities&service=WMS + + + + + + + + ImageryOnly Base Map Service (WMTS) + 1.0.0 + https://www.opengeospatial.org/standards/wmts + + + + + + https://basemap.nationalmap.gov/arcgis/rest/services/USGSImageryOnly/MapServer/WMTS/1.0.0/WMTSCapabilities.xml + + + + + + None + + + + 20181128 + + + + U.S. Geological Survey, National Geospatial Technical Operations + Center + Not Provided + + + mailing and physical +
1400 Independence Road
+ Rolla + MO + 65401 + + + mailing and physical +
PO Box 25046 Denver Federal Center
+ Lakewood + CO + 80225 + + 1-888-ASK-USGS (1-888-275-8747) + tnm_help@usgs.gov + Monday through Friday, 8:00 AM to 4:30 PM CT + Metadata information can also be obtained through online services using The + National Map Viewer at https://nationalmap.usgs.gov, or EarthExplorer at + https://earthexplorer.usgs.gov, or Ask USGS at https://www.usgs.gov/ask. + + + FGDC Content Standard for Digital Geospatial Metadata + FGDC-STD-001-1998 + + From e060bbc0606831dd809a211eafcbc63d76183771 Mon Sep 17 00:00:00 2001 From: Reid Hewitt Date: Fri, 15 Dec 2023 15:21:10 -0700 Subject: [PATCH 06/13] update harvest source location. --- tests/load/ckan/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/load/ckan/conftest.py b/tests/load/ckan/conftest.py index 204a3015..b915f4db 100644 --- a/tests/load/ckan/conftest.py +++ b/tests/load/ckan/conftest.py @@ -28,7 +28,7 @@ def test_ckan_package_id(): @pytest.fixture def test_dcatus_catalog(): - return open_json(HARVEST_SOURCES / "dcatus_to_ckan.json") + return open_json(HARVEST_SOURCES / "dcatus" / "dcatus_to_ckan.json") @pytest.fixture From 2ddaf0165dfa1788b9b1fc2bf2cf006fdf7a77fc Mon Sep 17 00:00:00 2001 From: Reid Hewitt Date: Fri, 15 Dec 2023 17:38:06 -0700 Subject: [PATCH 07/13] avoid looping through the current directory twice. --- harvester/waf.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/harvester/waf.py b/harvester/waf.py index 3f9fd8b1..694a8fd4 100644 --- a/harvester/waf.py +++ b/harvester/waf.py @@ -11,12 +11,6 @@ def traverse_waf(url, files=[], file_ext=".xml", folder="/", filters=[]): soup = BeautifulSoup(res.content, "html.parser") anchors = soup.find_all("a", href=True) - files += [ - os.path.join(url, anchor["href"]) - for anchor in anchors - if anchor["href"].endswith(file_ext) - ] - folders = [] for anchor in anchors: if ( @@ -26,7 +20,10 @@ def traverse_waf(url, files=[], file_ext=".xml", folder="/", filters=[]): ): folders.append(os.path.join(url, anchor["href"])) - for folder in folders: - traverse_waf(folder, files=files, filters=filters) + if anchor["href"].endswith(file_ext): + files.append(os.path.join(url, anchor["href"])) + + for folder in folders: + traverse_waf(folder, files=files, filters=filters) return files From 3a97c4cb4f125f11ee5bf121d87357ce64795e8a Mon Sep 17 00:00:00 2001 From: Reid Hewitt Date: Mon, 18 Dec 2023 10:42:44 -0700 Subject: [PATCH 08/13] add TODO for exception handling. --- harvester/waf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/harvester/waf.py b/harvester/waf.py index 694a8fd4..89b46c54 100644 --- a/harvester/waf.py +++ b/harvester/waf.py @@ -4,6 +4,7 @@ def traverse_waf(url, files=[], file_ext=".xml", folder="/", filters=[]): + # TODO: add exception handling parent = os.path.dirname(url.rstrip("/")) res = requests.get(url) From 027e7940b750fc5abf5647e0d9d9686c079b81b1 Mon Sep 17 00:00:00 2001 From: Reid Hewitt Date: Mon, 18 Dec 2023 11:42:00 -0700 Subject: [PATCH 09/13] add waf download function. --- harvester/waf.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/harvester/waf.py b/harvester/waf.py index 89b46c54..746f3b74 100644 --- a/harvester/waf.py +++ b/harvester/waf.py @@ -28,3 +28,13 @@ def traverse_waf(url, files=[], file_ext=".xml", folder="/", filters=[]): traverse_waf(folder, files=files, filters=filters) return files + + +def download_waf(files): + output = [] + for file in files: + res = requests.get(file) + if res.status_code == 200: + output.append(res.content) + + return output From 16185b6f79d04008892d11735962d5e5763f041b Mon Sep 17 00:00:00 2001 From: Reid Hewitt Date: Mon, 18 Dec 2023 11:42:10 -0700 Subject: [PATCH 10/13] include waf download in test. --- tests/extract/test_waf.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/extract/test_waf.py b/tests/extract/test_waf.py index a6bacdc0..55966642 100644 --- a/tests/extract/test_waf.py +++ b/tests/extract/test_waf.py @@ -1,6 +1,9 @@ -from harvester.extract.waf import traverse_waf +from harvester.extract.waf import traverse_waf, download_waf def test_traverse_waf(get_waf_url): files = traverse_waf(get_waf_url, filters=["../", "dcatus/"]) assert len(files) == 7 + + downloaded_files = download_waf(files) + assert len(downloaded_files) == 7 From 0baafa0d8a83f1be0103acffb129d551d97ffb60 Mon Sep 17 00:00:00 2001 From: Reid Hewitt Date: Mon, 18 Dec 2023 11:48:43 -0700 Subject: [PATCH 11/13] store download content as object. --- harvester/waf.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/harvester/waf.py b/harvester/waf.py index 746f3b74..ed4b456d 100644 --- a/harvester/waf.py +++ b/harvester/waf.py @@ -33,8 +33,11 @@ def traverse_waf(url, files=[], file_ext=".xml", folder="/", filters=[]): def download_waf(files): output = [] for file in files: + data = {} + data["url"] = file res = requests.get(file) if res.status_code == 200: - output.append(res.content) + data["content"] = res.content + output.append(data) return output From eb4b3a1229e172354e2756a2f79ee7d479ca8129 Mon Sep 17 00:00:00 2001 From: robert-bryson Date: Tue, 19 Dec 2023 18:12:25 +0000 Subject: [PATCH 12/13] fixup + poetry update --- poetry.lock | 308 ++++++++++++++++++++++++------------------------- pyproject.toml | 1 - 2 files changed, 154 insertions(+), 155 deletions(-) diff --git a/poetry.lock b/poetry.lock index 9a25d9b7..70e127a8 100644 --- a/poetry.lock +++ b/poetry.lock @@ -177,63 +177,63 @@ files = [ [[package]] name = "coverage" -version = "7.3.2" +version = "7.3.3" description = "Code coverage measurement for Python" optional = false python-versions = ">=3.8" files = [ - {file = "coverage-7.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d872145f3a3231a5f20fd48500274d7df222e291d90baa2026cc5152b7ce86bf"}, - {file = "coverage-7.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:310b3bb9c91ea66d59c53fa4989f57d2436e08f18fb2f421a1b0b6b8cc7fffda"}, - {file = "coverage-7.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f47d39359e2c3779c5331fc740cf4bce6d9d680a7b4b4ead97056a0ae07cb49a"}, - {file = "coverage-7.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aa72dbaf2c2068404b9870d93436e6d23addd8bbe9295f49cbca83f6e278179c"}, - {file = "coverage-7.3.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:beaa5c1b4777f03fc63dfd2a6bd820f73f036bfb10e925fce067b00a340d0f3f"}, - {file = "coverage-7.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:dbc1b46b92186cc8074fee9d9fbb97a9dd06c6cbbef391c2f59d80eabdf0faa6"}, - {file = "coverage-7.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:315a989e861031334d7bee1f9113c8770472db2ac484e5b8c3173428360a9148"}, - {file = "coverage-7.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d1bc430677773397f64a5c88cb522ea43175ff16f8bfcc89d467d974cb2274f9"}, - {file = "coverage-7.3.2-cp310-cp310-win32.whl", hash = "sha256:a889ae02f43aa45032afe364c8ae84ad3c54828c2faa44f3bfcafecb5c96b02f"}, - {file = "coverage-7.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:c0ba320de3fb8c6ec16e0be17ee1d3d69adcda99406c43c0409cb5c41788a611"}, - {file = "coverage-7.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ac8c802fa29843a72d32ec56d0ca792ad15a302b28ca6203389afe21f8fa062c"}, - {file = "coverage-7.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:89a937174104339e3a3ffcf9f446c00e3a806c28b1841c63edb2b369310fd074"}, - {file = "coverage-7.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e267e9e2b574a176ddb983399dec325a80dbe161f1a32715c780b5d14b5f583a"}, - {file = "coverage-7.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2443cbda35df0d35dcfb9bf8f3c02c57c1d6111169e3c85fc1fcc05e0c9f39a3"}, - {file = "coverage-7.3.2-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4175e10cc8dda0265653e8714b3174430b07c1dca8957f4966cbd6c2b1b8065a"}, - {file = "coverage-7.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0cbf38419fb1a347aaf63481c00f0bdc86889d9fbf3f25109cf96c26b403fda1"}, - {file = "coverage-7.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:5c913b556a116b8d5f6ef834038ba983834d887d82187c8f73dec21049abd65c"}, - {file = "coverage-7.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1981f785239e4e39e6444c63a98da3a1db8e971cb9ceb50a945ba6296b43f312"}, - {file = "coverage-7.3.2-cp311-cp311-win32.whl", hash = "sha256:43668cabd5ca8258f5954f27a3aaf78757e6acf13c17604d89648ecc0cc66640"}, - {file = "coverage-7.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:e10c39c0452bf6e694511c901426d6b5ac005acc0f78ff265dbe36bf81f808a2"}, - {file = "coverage-7.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:4cbae1051ab791debecc4a5dcc4a1ff45fc27b91b9aee165c8a27514dd160836"}, - {file = "coverage-7.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:12d15ab5833a997716d76f2ac1e4b4d536814fc213c85ca72756c19e5a6b3d63"}, - {file = "coverage-7.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3c7bba973ebee5e56fe9251300c00f1579652587a9f4a5ed8404b15a0471f216"}, - {file = "coverage-7.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fe494faa90ce6381770746077243231e0b83ff3f17069d748f645617cefe19d4"}, - {file = "coverage-7.3.2-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f6e9589bd04d0461a417562649522575d8752904d35c12907d8c9dfeba588faf"}, - {file = "coverage-7.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d51ac2a26f71da1b57f2dc81d0e108b6ab177e7d30e774db90675467c847bbdf"}, - {file = "coverage-7.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:99b89d9f76070237975b315b3d5f4d6956ae354a4c92ac2388a5695516e47c84"}, - {file = "coverage-7.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:fa28e909776dc69efb6ed975a63691bc8172b64ff357e663a1bb06ff3c9b589a"}, - {file = "coverage-7.3.2-cp312-cp312-win32.whl", hash = "sha256:289fe43bf45a575e3ab10b26d7b6f2ddb9ee2dba447499f5401cfb5ecb8196bb"}, - {file = "coverage-7.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:7dbc3ed60e8659bc59b6b304b43ff9c3ed858da2839c78b804973f613d3e92ed"}, - {file = "coverage-7.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f94b734214ea6a36fe16e96a70d941af80ff3bfd716c141300d95ebc85339738"}, - {file = "coverage-7.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:af3d828d2c1cbae52d34bdbb22fcd94d1ce715d95f1a012354a75e5913f1bda2"}, - {file = "coverage-7.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:630b13e3036e13c7adc480ca42fa7afc2a5d938081d28e20903cf7fd687872e2"}, - {file = "coverage-7.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c9eacf273e885b02a0273bb3a2170f30e2d53a6d53b72dbe02d6701b5296101c"}, - {file = "coverage-7.3.2-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d8f17966e861ff97305e0801134e69db33b143bbfb36436efb9cfff6ec7b2fd9"}, - {file = "coverage-7.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b4275802d16882cf9c8b3d057a0839acb07ee9379fa2749eca54efbce1535b82"}, - {file = "coverage-7.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:72c0cfa5250f483181e677ebc97133ea1ab3eb68645e494775deb6a7f6f83901"}, - {file = "coverage-7.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:cb536f0dcd14149425996821a168f6e269d7dcd2c273a8bff8201e79f5104e76"}, - {file = "coverage-7.3.2-cp38-cp38-win32.whl", hash = "sha256:307adb8bd3abe389a471e649038a71b4eb13bfd6b7dd9a129fa856f5c695cf92"}, - {file = "coverage-7.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:88ed2c30a49ea81ea3b7f172e0269c182a44c236eb394718f976239892c0a27a"}, - {file = "coverage-7.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b631c92dfe601adf8f5ebc7fc13ced6bb6e9609b19d9a8cd59fa47c4186ad1ce"}, - {file = "coverage-7.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d3d9df4051c4a7d13036524b66ecf7a7537d14c18a384043f30a303b146164e9"}, - {file = "coverage-7.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5f7363d3b6a1119ef05015959ca24a9afc0ea8a02c687fe7e2d557705375c01f"}, - {file = "coverage-7.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2f11cc3c967a09d3695d2a6f03fb3e6236622b93be7a4b5dc09166a861be6d25"}, - {file = "coverage-7.3.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:149de1d2401ae4655c436a3dced6dd153f4c3309f599c3d4bd97ab172eaf02d9"}, - {file = "coverage-7.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:3a4006916aa6fee7cd38db3bfc95aa9c54ebb4ffbfc47c677c8bba949ceba0a6"}, - {file = "coverage-7.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9028a3871280110d6e1aa2df1afd5ef003bab5fb1ef421d6dc748ae1c8ef2ebc"}, - {file = "coverage-7.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:9f805d62aec8eb92bab5b61c0f07329275b6f41c97d80e847b03eb894f38d083"}, - {file = "coverage-7.3.2-cp39-cp39-win32.whl", hash = "sha256:d1c88ec1a7ff4ebca0219f5b1ef863451d828cccf889c173e1253aa84b1e07ce"}, - {file = "coverage-7.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b4767da59464bb593c07afceaddea61b154136300881844768037fd5e859353f"}, - {file = "coverage-7.3.2-pp38.pp39.pp310-none-any.whl", hash = "sha256:ae97af89f0fbf373400970c0a21eef5aa941ffeed90aee43650b81f7d7f47637"}, - {file = "coverage-7.3.2.tar.gz", hash = "sha256:be32ad29341b0170e795ca590e1c07e81fc061cb5b10c74ce7203491484404ef"}, + {file = "coverage-7.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d874434e0cb7b90f7af2b6e3309b0733cde8ec1476eb47db148ed7deeb2a9494"}, + {file = "coverage-7.3.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ee6621dccce8af666b8c4651f9f43467bfbf409607c604b840b78f4ff3619aeb"}, + {file = "coverage-7.3.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1367aa411afb4431ab58fd7ee102adb2665894d047c490649e86219327183134"}, + {file = "coverage-7.3.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f0f8f0c497eb9c9f18f21de0750c8d8b4b9c7000b43996a094290b59d0e7523"}, + {file = "coverage-7.3.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db0338c4b0951d93d547e0ff8d8ea340fecf5885f5b00b23be5aa99549e14cfd"}, + {file = "coverage-7.3.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d31650d313bd90d027f4be7663dfa2241079edd780b56ac416b56eebe0a21aab"}, + {file = "coverage-7.3.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:9437a4074b43c177c92c96d051957592afd85ba00d3e92002c8ef45ee75df438"}, + {file = "coverage-7.3.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:9e17d9cb06c13b4f2ef570355fa45797d10f19ca71395910b249e3f77942a837"}, + {file = "coverage-7.3.3-cp310-cp310-win32.whl", hash = "sha256:eee5e741b43ea1b49d98ab6e40f7e299e97715af2488d1c77a90de4a663a86e2"}, + {file = "coverage-7.3.3-cp310-cp310-win_amd64.whl", hash = "sha256:593efa42160c15c59ee9b66c5f27a453ed3968718e6e58431cdfb2d50d5ad284"}, + {file = "coverage-7.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8c944cf1775235c0857829c275c777a2c3e33032e544bcef614036f337ac37bb"}, + {file = "coverage-7.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:eda7f6e92358ac9e1717ce1f0377ed2b9320cea070906ece4e5c11d172a45a39"}, + {file = "coverage-7.3.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3c854c1d2c7d3e47f7120b560d1a30c1ca221e207439608d27bc4d08fd4aeae8"}, + {file = "coverage-7.3.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:222b038f08a7ebed1e4e78ccf3c09a1ca4ac3da16de983e66520973443b546bc"}, + {file = "coverage-7.3.3-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ff4800783d85bff132f2cc7d007426ec698cdce08c3062c8d501ad3f4ea3d16c"}, + {file = "coverage-7.3.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:fc200cec654311ca2c3f5ab3ce2220521b3d4732f68e1b1e79bef8fcfc1f2b97"}, + {file = "coverage-7.3.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:307aecb65bb77cbfebf2eb6e12009e9034d050c6c69d8a5f3f737b329f4f15fb"}, + {file = "coverage-7.3.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:ffb0eacbadb705c0a6969b0adf468f126b064f3362411df95f6d4f31c40d31c1"}, + {file = "coverage-7.3.3-cp311-cp311-win32.whl", hash = "sha256:79c32f875fd7c0ed8d642b221cf81feba98183d2ff14d1f37a1bbce6b0347d9f"}, + {file = "coverage-7.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:243576944f7c1a1205e5cd658533a50eba662c74f9be4c050d51c69bd4532936"}, + {file = "coverage-7.3.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:a2ac4245f18057dfec3b0074c4eb366953bca6787f1ec397c004c78176a23d56"}, + {file = "coverage-7.3.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f9191be7af41f0b54324ded600e8ddbcabea23e1e8ba419d9a53b241dece821d"}, + {file = "coverage-7.3.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:31c0b1b8b5a4aebf8fcd227237fc4263aa7fa0ddcd4d288d42f50eff18b0bac4"}, + {file = "coverage-7.3.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ee453085279df1bac0996bc97004771a4a052b1f1e23f6101213e3796ff3cb85"}, + {file = "coverage-7.3.3-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1191270b06ecd68b1d00897b2daddb98e1719f63750969614ceb3438228c088e"}, + {file = "coverage-7.3.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:007a7e49831cfe387473e92e9ff07377f6121120669ddc39674e7244350a6a29"}, + {file = "coverage-7.3.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:af75cf83c2d57717a8493ed2246d34b1f3398cb8a92b10fd7a1858cad8e78f59"}, + {file = "coverage-7.3.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:811ca7373da32f1ccee2927dc27dc523462fd30674a80102f86c6753d6681bc6"}, + {file = "coverage-7.3.3-cp312-cp312-win32.whl", hash = "sha256:733537a182b5d62184f2a72796eb6901299898231a8e4f84c858c68684b25a70"}, + {file = "coverage-7.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:e995efb191f04b01ced307dbd7407ebf6e6dc209b528d75583277b10fd1800ee"}, + {file = "coverage-7.3.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:fbd8a5fe6c893de21a3c6835071ec116d79334fbdf641743332e442a3466f7ea"}, + {file = "coverage-7.3.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:50c472c1916540f8b2deef10cdc736cd2b3d1464d3945e4da0333862270dcb15"}, + {file = "coverage-7.3.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e9223a18f51d00d3ce239c39fc41410489ec7a248a84fab443fbb39c943616c"}, + {file = "coverage-7.3.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f501e36ac428c1b334c41e196ff6bd550c0353c7314716e80055b1f0a32ba394"}, + {file = "coverage-7.3.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:475de8213ed95a6b6283056d180b2442eee38d5948d735cd3d3b52b86dd65b92"}, + {file = "coverage-7.3.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:afdcc10c01d0db217fc0a64f58c7edd635b8f27787fea0a3054b856a6dff8717"}, + {file = "coverage-7.3.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:fff0b2f249ac642fd735f009b8363c2b46cf406d3caec00e4deeb79b5ff39b40"}, + {file = "coverage-7.3.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:a1f76cfc122c9e0f62dbe0460ec9cc7696fc9a0293931a33b8870f78cf83a327"}, + {file = "coverage-7.3.3-cp38-cp38-win32.whl", hash = "sha256:757453848c18d7ab5d5b5f1827293d580f156f1c2c8cef45bfc21f37d8681069"}, + {file = "coverage-7.3.3-cp38-cp38-win_amd64.whl", hash = "sha256:ad2453b852a1316c8a103c9c970db8fbc262f4f6b930aa6c606df9b2766eee06"}, + {file = "coverage-7.3.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3b15e03b8ee6a908db48eccf4e4e42397f146ab1e91c6324da44197a45cb9132"}, + {file = "coverage-7.3.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:89400aa1752e09f666cc48708eaa171eef0ebe3d5f74044b614729231763ae69"}, + {file = "coverage-7.3.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c59a3e59fb95e6d72e71dc915e6d7fa568863fad0a80b33bc7b82d6e9f844973"}, + {file = "coverage-7.3.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9ede881c7618f9cf93e2df0421ee127afdfd267d1b5d0c59bcea771cf160ea4a"}, + {file = "coverage-7.3.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f3bfd2c2f0e5384276e12b14882bf2c7621f97c35320c3e7132c156ce18436a1"}, + {file = "coverage-7.3.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:7f3bad1a9313401ff2964e411ab7d57fb700a2d5478b727e13f156c8f89774a0"}, + {file = "coverage-7.3.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:65d716b736f16e250435473c5ca01285d73c29f20097decdbb12571d5dfb2c94"}, + {file = "coverage-7.3.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a702e66483b1fe602717020a0e90506e759c84a71dbc1616dd55d29d86a9b91f"}, + {file = "coverage-7.3.3-cp39-cp39-win32.whl", hash = "sha256:7fbf3f5756e7955174a31fb579307d69ffca91ad163467ed123858ce0f3fd4aa"}, + {file = "coverage-7.3.3-cp39-cp39-win_amd64.whl", hash = "sha256:cad9afc1644b979211989ec3ff7d82110b2ed52995c2f7263e7841c846a75348"}, + {file = "coverage-7.3.3-pp38.pp39.pp310-none-any.whl", hash = "sha256:d299d379b676812e142fb57662a8d0d810b859421412b4d7af996154c00c31bb"}, + {file = "coverage-7.3.3.tar.gz", hash = "sha256:df04c64e58df96b4427db8d0559e95e2df3138c9916c96f9f6a4dd220db2fdb7"}, ] [package.dependencies] @@ -490,110 +490,110 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] [[package]] name = "rpds-py" -version = "0.13.2" +version = "0.15.2" description = "Python bindings to Rust's persistent data structures (rpds)" optional = false python-versions = ">=3.8" files = [ - {file = "rpds_py-0.13.2-cp310-cp310-macosx_10_7_x86_64.whl", hash = "sha256:1ceebd0ae4f3e9b2b6b553b51971921853ae4eebf3f54086be0565d59291e53d"}, - {file = "rpds_py-0.13.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:46e1ed994a0920f350a4547a38471217eb86f57377e9314fbaaa329b71b7dfe3"}, - {file = "rpds_py-0.13.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ee353bb51f648924926ed05e0122b6a0b1ae709396a80eb583449d5d477fcdf7"}, - {file = "rpds_py-0.13.2-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:530190eb0cd778363bbb7596612ded0bb9fef662daa98e9d92a0419ab27ae914"}, - {file = "rpds_py-0.13.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:29d311e44dd16d2434d5506d57ef4d7036544fc3c25c14b6992ef41f541b10fb"}, - {file = "rpds_py-0.13.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2e72f750048b32d39e87fc85c225c50b2a6715034848dbb196bf3348aa761fa1"}, - {file = "rpds_py-0.13.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db09b98c7540df69d4b47218da3fbd7cb466db0fb932e971c321f1c76f155266"}, - {file = "rpds_py-0.13.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2ac26f50736324beb0282c819668328d53fc38543fa61eeea2c32ea8ea6eab8d"}, - {file = "rpds_py-0.13.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:12ecf89bd54734c3c2c79898ae2021dca42750c7bcfb67f8fb3315453738ac8f"}, - {file = "rpds_py-0.13.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:3a44c8440183b43167fd1a0819e8356692bf5db1ad14ce140dbd40a1485f2dea"}, - {file = "rpds_py-0.13.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:bcef4f2d3dc603150421de85c916da19471f24d838c3c62a4f04c1eb511642c1"}, - {file = "rpds_py-0.13.2-cp310-none-win32.whl", hash = "sha256:ee6faebb265e28920a6f23a7d4c362414b3f4bb30607141d718b991669e49ddc"}, - {file = "rpds_py-0.13.2-cp310-none-win_amd64.whl", hash = "sha256:ac96d67b37f28e4b6ecf507c3405f52a40658c0a806dffde624a8fcb0314d5fd"}, - {file = "rpds_py-0.13.2-cp311-cp311-macosx_10_7_x86_64.whl", hash = "sha256:b5f6328e8e2ae8238fc767703ab7b95785521c42bb2b8790984e3477d7fa71ad"}, - {file = "rpds_py-0.13.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:729408136ef8d45a28ee9a7411917c9e3459cf266c7e23c2f7d4bb8ef9e0da42"}, - {file = "rpds_py-0.13.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:65cfed9c807c27dee76407e8bb29e6f4e391e436774bcc769a037ff25ad8646e"}, - {file = "rpds_py-0.13.2-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:aefbdc934115d2f9278f153952003ac52cd2650e7313750390b334518c589568"}, - {file = "rpds_py-0.13.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d48db29bd47814671afdd76c7652aefacc25cf96aad6daefa82d738ee87461e2"}, - {file = "rpds_py-0.13.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3c55d7f2d817183d43220738270efd3ce4e7a7b7cbdaefa6d551ed3d6ed89190"}, - {file = "rpds_py-0.13.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6aadae3042f8e6db3376d9e91f194c606c9a45273c170621d46128f35aef7cd0"}, - {file = "rpds_py-0.13.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5feae2f9aa7270e2c071f488fab256d768e88e01b958f123a690f1cc3061a09c"}, - {file = "rpds_py-0.13.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:51967a67ea0d7b9b5cd86036878e2d82c0b6183616961c26d825b8c994d4f2c8"}, - {file = "rpds_py-0.13.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:4d0c10d803549427f427085ed7aebc39832f6e818a011dcd8785e9c6a1ba9b3e"}, - {file = "rpds_py-0.13.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:603d5868f7419081d616dab7ac3cfa285296735e7350f7b1e4f548f6f953ee7d"}, - {file = "rpds_py-0.13.2-cp311-none-win32.whl", hash = "sha256:b8996ffb60c69f677245f5abdbcc623e9442bcc91ed81b6cd6187129ad1fa3e7"}, - {file = "rpds_py-0.13.2-cp311-none-win_amd64.whl", hash = "sha256:5379e49d7e80dca9811b36894493d1c1ecb4c57de05c36f5d0dd09982af20211"}, - {file = "rpds_py-0.13.2-cp312-cp312-macosx_10_7_x86_64.whl", hash = "sha256:8a776a29b77fe0cc28fedfd87277b0d0f7aa930174b7e504d764e0b43a05f381"}, - {file = "rpds_py-0.13.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2a1472956c5bcc49fb0252b965239bffe801acc9394f8b7c1014ae9258e4572b"}, - {file = "rpds_py-0.13.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f252dfb4852a527987a9156cbcae3022a30f86c9d26f4f17b8c967d7580d65d2"}, - {file = "rpds_py-0.13.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f0d320e70b6b2300ff6029e234e79fe44e9dbbfc7b98597ba28e054bd6606a57"}, - {file = "rpds_py-0.13.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ade2ccb937060c299ab0dfb2dea3d2ddf7e098ed63ee3d651ebfc2c8d1e8632a"}, - {file = "rpds_py-0.13.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b9d121be0217787a7d59a5c6195b0842d3f701007333426e5154bf72346aa658"}, - {file = "rpds_py-0.13.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8fa6bd071ec6d90f6e7baa66ae25820d57a8ab1b0a3c6d3edf1834d4b26fafa2"}, - {file = "rpds_py-0.13.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c918621ee0a3d1fe61c313f2489464f2ae3d13633e60f520a8002a5e910982ee"}, - {file = "rpds_py-0.13.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:25b28b3d33ec0a78e944aaaed7e5e2a94ac811bcd68b557ca48a0c30f87497d2"}, - {file = "rpds_py-0.13.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:31e220a040b89a01505128c2f8a59ee74732f666439a03e65ccbf3824cdddae7"}, - {file = "rpds_py-0.13.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:15253fff410873ebf3cfba1cc686a37711efcd9b8cb30ea21bb14a973e393f60"}, - {file = "rpds_py-0.13.2-cp312-none-win32.whl", hash = "sha256:b981a370f8f41c4024c170b42fbe9e691ae2dbc19d1d99151a69e2c84a0d194d"}, - {file = "rpds_py-0.13.2-cp312-none-win_amd64.whl", hash = "sha256:4c4e314d36d4f31236a545696a480aa04ea170a0b021e9a59ab1ed94d4c3ef27"}, - {file = "rpds_py-0.13.2-cp38-cp38-macosx_10_7_x86_64.whl", hash = "sha256:80e5acb81cb49fd9f2d5c08f8b74ffff14ee73b10ca88297ab4619e946bcb1e1"}, - {file = "rpds_py-0.13.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:efe093acc43e869348f6f2224df7f452eab63a2c60a6c6cd6b50fd35c4e075ba"}, - {file = "rpds_py-0.13.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c2a61c0e4811012b0ba9f6cdcb4437865df5d29eab5d6018ba13cee1c3064a0"}, - {file = "rpds_py-0.13.2-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:751758d9dd04d548ec679224cc00e3591f5ebf1ff159ed0d4aba6a0746352452"}, - {file = "rpds_py-0.13.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6ba8858933f0c1a979781272a5f65646fca8c18c93c99c6ddb5513ad96fa54b1"}, - {file = "rpds_py-0.13.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bfdfbe6a36bc3059fff845d64c42f2644cf875c65f5005db54f90cdfdf1df815"}, - {file = "rpds_py-0.13.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa0379c1935c44053c98826bc99ac95f3a5355675a297ac9ce0dfad0ce2d50ca"}, - {file = "rpds_py-0.13.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d5593855b5b2b73dd8413c3fdfa5d95b99d657658f947ba2c4318591e745d083"}, - {file = "rpds_py-0.13.2-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:2a7bef6977043673750a88da064fd513f89505111014b4e00fbdd13329cd4e9a"}, - {file = "rpds_py-0.13.2-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:3ab96754d23372009638a402a1ed12a27711598dd49d8316a22597141962fe66"}, - {file = "rpds_py-0.13.2-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:e06cfea0ece444571d24c18ed465bc93afb8c8d8d74422eb7026662f3d3f779b"}, - {file = "rpds_py-0.13.2-cp38-none-win32.whl", hash = "sha256:5493569f861fb7b05af6d048d00d773c6162415ae521b7010197c98810a14cab"}, - {file = "rpds_py-0.13.2-cp38-none-win_amd64.whl", hash = "sha256:b07501b720cf060c5856f7b5626e75b8e353b5f98b9b354a21eb4bfa47e421b1"}, - {file = "rpds_py-0.13.2-cp39-cp39-macosx_10_7_x86_64.whl", hash = "sha256:881df98f0a8404d32b6de0fd33e91c1b90ed1516a80d4d6dc69d414b8850474c"}, - {file = "rpds_py-0.13.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d79c159adea0f1f4617f54aa156568ac69968f9ef4d1e5fefffc0a180830308e"}, - {file = "rpds_py-0.13.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:38d4f822ee2f338febcc85aaa2547eb5ba31ba6ff68d10b8ec988929d23bb6b4"}, - {file = "rpds_py-0.13.2-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5d75d6d220d55cdced2f32cc22f599475dbe881229aeddba6c79c2e9df35a2b3"}, - {file = "rpds_py-0.13.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5d97e9ae94fb96df1ee3cb09ca376c34e8a122f36927230f4c8a97f469994bff"}, - {file = "rpds_py-0.13.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:67a429520e97621a763cf9b3ba27574779c4e96e49a27ff8a1aa99ee70beb28a"}, - {file = "rpds_py-0.13.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:188435794405c7f0573311747c85a96b63c954a5f2111b1df8018979eca0f2f0"}, - {file = "rpds_py-0.13.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:38f9bf2ad754b4a45b8210a6c732fe876b8a14e14d5992a8c4b7c1ef78740f53"}, - {file = "rpds_py-0.13.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:a6ba2cb7d676e9415b9e9ac7e2aae401dc1b1e666943d1f7bc66223d3d73467b"}, - {file = "rpds_py-0.13.2-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:eaffbd8814bb1b5dc3ea156a4c5928081ba50419f9175f4fc95269e040eff8f0"}, - {file = "rpds_py-0.13.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:5a4c1058cdae6237d97af272b326e5f78ee7ee3bbffa6b24b09db4d828810468"}, - {file = "rpds_py-0.13.2-cp39-none-win32.whl", hash = "sha256:b5267feb19070bef34b8dea27e2b504ebd9d31748e3ecacb3a4101da6fcb255c"}, - {file = "rpds_py-0.13.2-cp39-none-win_amd64.whl", hash = "sha256:ddf23960cb42b69bce13045d5bc66f18c7d53774c66c13f24cf1b9c144ba3141"}, - {file = "rpds_py-0.13.2-pp310-pypy310_pp73-macosx_10_7_x86_64.whl", hash = "sha256:97163a1ab265a1073a6372eca9f4eeb9f8c6327457a0b22ddfc4a17dcd613e74"}, - {file = "rpds_py-0.13.2-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:25ea41635d22b2eb6326f58e608550e55d01df51b8a580ea7e75396bafbb28e9"}, - {file = "rpds_py-0.13.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76d59d4d451ba77f08cb4cd9268dec07be5bc65f73666302dbb5061989b17198"}, - {file = "rpds_py-0.13.2-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e7c564c58cf8f248fe859a4f0fe501b050663f3d7fbc342172f259124fb59933"}, - {file = "rpds_py-0.13.2-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:61dbc1e01dc0c5875da2f7ae36d6e918dc1b8d2ce04e871793976594aad8a57a"}, - {file = "rpds_py-0.13.2-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fdb82eb60d31b0c033a8e8ee9f3fc7dfbaa042211131c29da29aea8531b4f18f"}, - {file = "rpds_py-0.13.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d204957169f0b3511fb95395a9da7d4490fb361763a9f8b32b345a7fe119cb45"}, - {file = "rpds_py-0.13.2-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c45008ca79bad237cbc03c72bc5205e8c6f66403773929b1b50f7d84ef9e4d07"}, - {file = "rpds_py-0.13.2-pp310-pypy310_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:79bf58c08f0756adba691d480b5a20e4ad23f33e1ae121584cf3a21717c36dfa"}, - {file = "rpds_py-0.13.2-pp310-pypy310_pp73-musllinux_1_2_i686.whl", hash = "sha256:e86593bf8637659e6a6ed58854b6c87ec4e9e45ee8a4adfd936831cef55c2d21"}, - {file = "rpds_py-0.13.2-pp310-pypy310_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:d329896c40d9e1e5c7715c98529e4a188a1f2df51212fd65102b32465612b5dc"}, - {file = "rpds_py-0.13.2-pp38-pypy38_pp73-macosx_10_7_x86_64.whl", hash = "sha256:4a5375c5fff13f209527cd886dc75394f040c7d1ecad0a2cb0627f13ebe78a12"}, - {file = "rpds_py-0.13.2-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:06d218e4464d31301e943b65b2c6919318ea6f69703a351961e1baaf60347276"}, - {file = "rpds_py-0.13.2-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1f41d32a2ddc5a94df4b829b395916a4b7f103350fa76ba6de625fcb9e773ac"}, - {file = "rpds_py-0.13.2-pp38-pypy38_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6bc568b05e02cd612be53900c88aaa55012e744930ba2eeb56279db4c6676eb3"}, - {file = "rpds_py-0.13.2-pp38-pypy38_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9d94d78418203904730585efa71002286ac4c8ac0689d0eb61e3c465f9e608ff"}, - {file = "rpds_py-0.13.2-pp38-pypy38_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bed0252c85e21cf73d2d033643c945b460d6a02fc4a7d644e3b2d6f5f2956c64"}, - {file = "rpds_py-0.13.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:244e173bb6d8f3b2f0c4d7370a1aa341f35da3e57ffd1798e5b2917b91731fd3"}, - {file = "rpds_py-0.13.2-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7f55cd9cf1564b7b03f238e4c017ca4794c05b01a783e9291065cb2858d86ce4"}, - {file = "rpds_py-0.13.2-pp38-pypy38_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:f03a1b3a4c03e3e0161642ac5367f08479ab29972ea0ffcd4fa18f729cd2be0a"}, - {file = "rpds_py-0.13.2-pp38-pypy38_pp73-musllinux_1_2_i686.whl", hash = "sha256:f5f4424cb87a20b016bfdc157ff48757b89d2cc426256961643d443c6c277007"}, - {file = "rpds_py-0.13.2-pp38-pypy38_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:c82bbf7e03748417c3a88c1b0b291288ce3e4887a795a3addaa7a1cfd9e7153e"}, - {file = "rpds_py-0.13.2-pp39-pypy39_pp73-macosx_10_7_x86_64.whl", hash = "sha256:c0095b8aa3e432e32d372e9a7737e65b58d5ed23b9620fea7cb81f17672f1fa1"}, - {file = "rpds_py-0.13.2-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:4c2d26aa03d877c9730bf005621c92da263523a1e99247590abbbe252ccb7824"}, - {file = "rpds_py-0.13.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:96f2975fb14f39c5fe75203f33dd3010fe37d1c4e33177feef1107b5ced750e3"}, - {file = "rpds_py-0.13.2-pp39-pypy39_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4dcc5ee1d0275cb78d443fdebd0241e58772a354a6d518b1d7af1580bbd2c4e8"}, - {file = "rpds_py-0.13.2-pp39-pypy39_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:61d42d2b08430854485135504f672c14d4fc644dd243a9c17e7c4e0faf5ed07e"}, - {file = "rpds_py-0.13.2-pp39-pypy39_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d3a61e928feddc458a55110f42f626a2a20bea942ccedb6fb4cee70b4830ed41"}, - {file = "rpds_py-0.13.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7de12b69d95072394998c622cfd7e8cea8f560db5fca6a62a148f902a1029f8b"}, - {file = "rpds_py-0.13.2-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:87a90f5545fd61f6964e65eebde4dc3fa8660bb7d87adb01d4cf17e0a2b484ad"}, - {file = "rpds_py-0.13.2-pp39-pypy39_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:9c95a1a290f9acf7a8f2ebbdd183e99215d491beea52d61aa2a7a7d2c618ddc6"}, - {file = "rpds_py-0.13.2-pp39-pypy39_pp73-musllinux_1_2_i686.whl", hash = "sha256:35f53c76a712e323c779ca39b9a81b13f219a8e3bc15f106ed1e1462d56fcfe9"}, - {file = "rpds_py-0.13.2-pp39-pypy39_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:96fb0899bb2ab353f42e5374c8f0789f54e0a94ef2f02b9ac7149c56622eaf31"}, - {file = "rpds_py-0.13.2.tar.gz", hash = "sha256:f8eae66a1304de7368932b42d801c67969fd090ddb1a7a24f27b435ed4bed68f"}, + {file = "rpds_py-0.15.2-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:337a8653fb11d2fbe7157c961cc78cb3c161d98cf44410ace9a3dc2db4fad882"}, + {file = "rpds_py-0.15.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:813a65f95bfcb7c8f2a70dd6add9b51e9accc3bdb3e03d0ff7a9e6a2d3e174bf"}, + {file = "rpds_py-0.15.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:082e0e55d73690ffb4da4352d1b5bbe1b5c6034eb9dc8c91aa2a3ee15f70d3e2"}, + {file = "rpds_py-0.15.2-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5595c80dd03d7e6c6afb73f3594bf3379a7d79fa57164b591d012d4b71d6ac4c"}, + {file = "rpds_py-0.15.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fb10bb720348fe1647a94eb605accb9ef6a9b1875d8845f9e763d9d71a706387"}, + {file = "rpds_py-0.15.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:53304cc14b1d94487d70086e1cb0cb4c29ec6da994d58ae84a4d7e78c6a6d04d"}, + {file = "rpds_py-0.15.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d64a657de7aae8db2da60dc0c9e4638a0c3893b4d60101fd564a3362b2bfeb34"}, + {file = "rpds_py-0.15.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ee40206d1d6e95eaa2b7b919195e3689a5cf6ded730632de7f187f35a1b6052c"}, + {file = "rpds_py-0.15.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:1607cda6129f815493a3c184492acb5ae4aa6ed61d3a1b3663aa9824ed26f7ac"}, + {file = "rpds_py-0.15.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:f3e6e2e502c4043c52a99316d89dc49f416acda5b0c6886e0dd8ea7bb35859e8"}, + {file = "rpds_py-0.15.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:044f6f46d62444800402851afa3c3ae50141f12013060c1a3a0677e013310d6d"}, + {file = "rpds_py-0.15.2-cp310-none-win32.whl", hash = "sha256:c827a931c6b57f50f1bb5de400dcfb00bad8117e3753e80b96adb72d9d811514"}, + {file = "rpds_py-0.15.2-cp310-none-win_amd64.whl", hash = "sha256:3bbc89ce2a219662ea142f0abcf8d43f04a41d5b1880be17a794c39f0d609cb0"}, + {file = "rpds_py-0.15.2-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:1fd0f0b1ccd7d537b858a56355a250108df692102e08aa2036e1a094fd78b2dc"}, + {file = "rpds_py-0.15.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b414ef79f1f06fb90b5165db8aef77512c1a5e3ed1b4807da8476b7e2c853283"}, + {file = "rpds_py-0.15.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c31272c674f725dfe0f343d73b0abe8c878c646967ec1c6106122faae1efc15b"}, + {file = "rpds_py-0.15.2-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a6945c2d61c42bb7e818677f43638675b8c1c43e858b67a96df3eb2426a86c9d"}, + {file = "rpds_py-0.15.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:02744236ac1895d7be837878e707a5c35fb8edc5137602f253b63623d7ad5c8c"}, + {file = "rpds_py-0.15.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2181e86d4e1cdf49a7320cb72a36c45efcb7670d0a88f09fd2d3a7967c0540fd"}, + {file = "rpds_py-0.15.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6a8ff8e809da81363bffca2b965cb6e4bf6056b495fc3f078467d1f8266fe27f"}, + {file = "rpds_py-0.15.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:97532802f14d383f37d603a56e226909f825a83ff298dc1b6697de00d2243999"}, + {file = "rpds_py-0.15.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:13716e53627ad97babf72ac9e01cf9a7d4af2f75dd5ed7b323a7a9520e948282"}, + {file = "rpds_py-0.15.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:2f1f295a5c28cfa74a7d48c95acc1c8a7acd49d7d9072040d4b694fe11cd7166"}, + {file = "rpds_py-0.15.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:8ec464f20fe803ae00419bd1610934e3bda963aeba1e6181dfc9033dc7e8940c"}, + {file = "rpds_py-0.15.2-cp311-none-win32.whl", hash = "sha256:b61d5096e75fd71018b25da50b82dd70ec39b5e15bb2134daf7eb7bbbc103644"}, + {file = "rpds_py-0.15.2-cp311-none-win_amd64.whl", hash = "sha256:9d41ebb471a6f064c0d1c873c4f7dded733d16ca5db7d551fb04ff3805d87802"}, + {file = "rpds_py-0.15.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:13ff62d3561a23c17341b4afc78e8fcfd799ab67c0b1ca32091d71383a98ba4b"}, + {file = "rpds_py-0.15.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b70b45a40ad0798b69748b34d508259ef2bdc84fb2aad4048bc7c9cafb68ddb3"}, + {file = "rpds_py-0.15.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b4ecbba7efd82bd2a4bb88aab7f984eb5470991c1347bdd1f35fb34ea28dba6e"}, + {file = "rpds_py-0.15.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9d38494a8d21c246c535b41ecdb2d562c4b933cf3d68de03e8bc43a0d41be652"}, + {file = "rpds_py-0.15.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:13152dfe7d7c27c40df8b99ac6aab12b978b546716e99f67e8a67a1d441acbc3"}, + {file = "rpds_py-0.15.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:164fcee32f15d04d61568c9cb0d919e37ff3195919cd604039ff3053ada0461b"}, + {file = "rpds_py-0.15.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6a5122b17a4faf5d7a6d91fa67b479736c0cacc7afe791ddebb7163a8550b799"}, + {file = "rpds_py-0.15.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:46b4f3d47d1033db569173be62365fbf7808c2bd3fb742314d251f130d90d44c"}, + {file = "rpds_py-0.15.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c61e42b4ceb9759727045765e87d51c1bb9f89987aca1fcc8a040232138cad1c"}, + {file = "rpds_py-0.15.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:d2aa3ca9552f83b0b4fa6ca8c6ce08da6580f37e3e0ab7afac73a1cfdc230c0e"}, + {file = "rpds_py-0.15.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ec19e823b4ccd87bd69e990879acbce9e961fc7aebe150156b8f4418d4b27b7f"}, + {file = "rpds_py-0.15.2-cp312-none-win32.whl", hash = "sha256:afeabb382c1256a7477b739820bce7fe782bb807d82927102cee73e79b41b38b"}, + {file = "rpds_py-0.15.2-cp312-none-win_amd64.whl", hash = "sha256:422b0901878a31ef167435c5ad46560362891816a76cc0d150683f3868a6f0d1"}, + {file = "rpds_py-0.15.2-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:baf744e5f9d5ee6531deea443be78b36ed1cd36c65a0b95ea4e8d69fa0102268"}, + {file = "rpds_py-0.15.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7e072f5da38d6428ba1fc1115d3cc0dae895df671cb04c70c019985e8c7606be"}, + {file = "rpds_py-0.15.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f138f550b83554f5b344d6be35d3ed59348510edc3cb96f75309db6e9bfe8210"}, + {file = "rpds_py-0.15.2-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b2a4cd924d0e2f4b1a68034abe4cadc73d69ad5f4cf02db6481c0d4d749f548f"}, + {file = "rpds_py-0.15.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5eb05b654a41e0f81ab27a7c3e88b6590425eb3e934e1d533ecec5dc88a6ffff"}, + {file = "rpds_py-0.15.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2ee066a64f0d2ba45391cac15b3a70dcb549e968a117bd0500634754cfe0e5fc"}, + {file = "rpds_py-0.15.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c51a899792ee2c696072791e56b2020caff58b275abecbc9ae0cb71af0645c95"}, + {file = "rpds_py-0.15.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ac2ac84a4950d627d84b61f082eba61314373cfab4b3c264b62efab02ababe83"}, + {file = "rpds_py-0.15.2-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:62b292fff4739c6be89e6a0240c02bda5a9066a339d90ab191cf66e9fdbdc193"}, + {file = "rpds_py-0.15.2-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:98ee201a52a7f65608e5494518932e1473fd43535f12cade0a1b4ab32737fe28"}, + {file = "rpds_py-0.15.2-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:3d40fb3ca22e3d40f494d577441b263026a3bd8c97ae6ce89b2d3c4b39ac9581"}, + {file = "rpds_py-0.15.2-cp38-none-win32.whl", hash = "sha256:30479a9f1fce47df56b07460b520f49fa2115ec2926d3b1303c85c81f8401ed1"}, + {file = "rpds_py-0.15.2-cp38-none-win_amd64.whl", hash = "sha256:2df3d07a16a3bef0917b28cd564778fbb31f3ffa5b5e33584470e2d1b0f248f0"}, + {file = "rpds_py-0.15.2-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:56b51ba29a18e5f5810224bcf00747ad931c0716e3c09a76b4a1edd3d4aba71f"}, + {file = "rpds_py-0.15.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3c11bc5814554b018f6c5d6ae0969e43766f81e995000b53a5d8c8057055e886"}, + {file = "rpds_py-0.15.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2faa97212b0dc465afeedf49045cdd077f97be1188285e646a9f689cb5dfff9e"}, + {file = "rpds_py-0.15.2-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:86c01299942b0f4b5b5f28c8701689181ad2eab852e65417172dbdd6c5b3ccc8"}, + {file = "rpds_py-0.15.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dd7d3608589072f63078b4063a6c536af832e76b0b3885f1bfe9e892abe6c207"}, + {file = "rpds_py-0.15.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:938518a11780b39998179d07f31a4a468888123f9b00463842cd40f98191f4d3"}, + {file = "rpds_py-0.15.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2dccc623725d0b298f557d869a68496a2fd2a9e9c41107f234fa5f7a37d278ac"}, + {file = "rpds_py-0.15.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d46ee458452727a147d7897bb33886981ae1235775e05decae5d5d07f537695a"}, + {file = "rpds_py-0.15.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:d9d7ebcd11ea76ba0feaae98485cd8e31467c3d7985210fab46983278214736b"}, + {file = "rpds_py-0.15.2-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:8a5f574b92b3ee7d254e56d56e37ec0e1416acb1ae357c4956d76a1788dc58fb"}, + {file = "rpds_py-0.15.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:3db0c998c92b909d7c90b66c965590d4f3cd86157176a6cf14aa1f867b77b889"}, + {file = "rpds_py-0.15.2-cp39-none-win32.whl", hash = "sha256:bbc7421cbd28b4316d1d017db338039a7943f945c6f2bb15e1439b14b5682d28"}, + {file = "rpds_py-0.15.2-cp39-none-win_amd64.whl", hash = "sha256:1c24e30d720c0009b6fb2e1905b025da56103c70a8b31b99138e4ed1c2a6c5b0"}, + {file = "rpds_py-0.15.2-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:1e6fcd0a0f62f2997107f758bb372397b8d5fd5f39cc6dcb86f7cb98a2172d6c"}, + {file = "rpds_py-0.15.2-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:d800a8e2ac62db1b9ea5d6d1724f1a93c53907ca061de4d05ed94e8dfa79050c"}, + {file = "rpds_py-0.15.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9e09d017e3f4d9bd7d17a30d3f59e4d6d9ba2d2ced280eec2425e84112cf623f"}, + {file = "rpds_py-0.15.2-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b88c3ab98556bc351b36d6208a6089de8c8db14a7f6e1f57f82a334bd2c18f0b"}, + {file = "rpds_py-0.15.2-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8f333bfe782a2d05a67cfaa0cc9cd68b36b39ee6acfe099f980541ed973a7093"}, + {file = "rpds_py-0.15.2-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b629db53fe17e6ce478a969d30bd1d0e8b53238c46e3a9c9db39e8b65a9ef973"}, + {file = "rpds_py-0.15.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:485fbdd23becb822804ed05622907ee5c8e8a5f43f6f43894a45f463b2217045"}, + {file = "rpds_py-0.15.2-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:893e38d0f4319dfa70c0f36381a37cc418985c87b11d9784365b1fff4fa6973b"}, + {file = "rpds_py-0.15.2-pp310-pypy310_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:8ffdeb7dbd0160d4e391e1f857477e4762d00aa2199c294eb95dfb9451aa1d9f"}, + {file = "rpds_py-0.15.2-pp310-pypy310_pp73-musllinux_1_2_i686.whl", hash = "sha256:fc33267d58dfbb2361baed52668c5d8c15d24bc0372cecbb79fed77339b55e0d"}, + {file = "rpds_py-0.15.2-pp310-pypy310_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:2e7e5633577b3bd56bf3af2ef6ae3778bbafb83743989d57f0e7edbf6c0980e4"}, + {file = "rpds_py-0.15.2-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:8b9650f92251fdef843e74fc252cdfd6e3c700157ad686eeb0c6d7fdb2d11652"}, + {file = "rpds_py-0.15.2-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:07a2e1d78d382f7181789713cdf0c16edbad4fe14fe1d115526cb6f0eef0daa3"}, + {file = "rpds_py-0.15.2-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:03f9c5875515820633bd7709a25c3e60c1ea9ad1c5d4030ce8a8c203309c36fd"}, + {file = "rpds_py-0.15.2-pp38-pypy38_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:580182fa5b269c2981e9ce9764367cb4edc81982ce289208d4607c203f44ffde"}, + {file = "rpds_py-0.15.2-pp38-pypy38_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aa1e626c524d2c7972c0f3a8a575d654a3a9c008370dc2a97e46abd0eaa749b9"}, + {file = "rpds_py-0.15.2-pp38-pypy38_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ae9d83a81b09ce3a817e2cbb23aabc07f86a3abc664c613cd283ce7a03541e95"}, + {file = "rpds_py-0.15.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9235be95662559141934fced8197de6fee8c58870f36756b0584424b6d708393"}, + {file = "rpds_py-0.15.2-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a72e00826a2b032dda3eb25aa3e3579c6d6773d22d8446089a57a123481cc46c"}, + {file = "rpds_py-0.15.2-pp38-pypy38_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:ab095edf1d840a6a6a4307e1a5b907a299a94e7b90e75436ee770b8c35d22a25"}, + {file = "rpds_py-0.15.2-pp38-pypy38_pp73-musllinux_1_2_i686.whl", hash = "sha256:3b79c63d29101cbaa53a517683557bb550462394fb91044cc5998dd2acff7340"}, + {file = "rpds_py-0.15.2-pp38-pypy38_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:911e600e798374c0d86235e7ef19109cf865d1336942d398ff313375a25a93ba"}, + {file = "rpds_py-0.15.2-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:3cd61e759c4075510052d1eca5cddbd297fe1164efec14ef1fce3f09b974dfe4"}, + {file = "rpds_py-0.15.2-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:9d2ae79f31da5143e020a8d4fc74e1f0cbcb8011bdf97453c140aa616db51406"}, + {file = "rpds_py-0.15.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5e99d6510c8557510c220b865d966b105464740dcbebf9b79ecd4fbab30a13d9"}, + {file = "rpds_py-0.15.2-pp39-pypy39_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6c43e1b89099279cc03eb1c725c5de12af6edcd2f78e2f8a022569efa639ada3"}, + {file = "rpds_py-0.15.2-pp39-pypy39_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ac7187bee72384b9cfedf09a29a3b2b6e8815cc64c095cdc8b5e6aec81e9fd5f"}, + {file = "rpds_py-0.15.2-pp39-pypy39_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3423007fc0661827e06f8a185a3792c73dda41f30f3421562f210cf0c9e49569"}, + {file = "rpds_py-0.15.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2974e6dff38afafd5ccf8f41cb8fc94600b3f4fd9b0a98f6ece6e2219e3158d5"}, + {file = "rpds_py-0.15.2-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:93c18a1696a8e0388ed84b024fe1a188a26ba999b61d1d9a371318cb89885a8c"}, + {file = "rpds_py-0.15.2-pp39-pypy39_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:c7cd0841a586b7105513a7c8c3d5c276f3adc762a072d81ef7fae80632afad1e"}, + {file = "rpds_py-0.15.2-pp39-pypy39_pp73-musllinux_1_2_i686.whl", hash = "sha256:709dc11af2f74ba89c68b1592368c6edcbccdb0a06ba77eb28c8fe08bb6997da"}, + {file = "rpds_py-0.15.2-pp39-pypy39_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:fc066395e6332da1e7525d605b4c96055669f8336600bef8ac569d5226a7c76f"}, + {file = "rpds_py-0.15.2.tar.gz", hash = "sha256:373b76eeb79e8c14f6d82cb1d4d5293f9e4059baec6c1b16dca7ad13b6131b39"}, ] [[package]] @@ -808,4 +808,4 @@ zstd = ["zstandard (>=0.18.0)"] [metadata] lock-version = "2.0" python-versions = ">=3.10" -content-hash = "c9f91c72b39f10497630c0bc1d1c142d94106e0878737423fcbb864ab24bef0c" +content-hash = "05bad374bbff6faf2eb4b3899d561c2e84516799c192ceb398d3ccde90edc41c" diff --git a/pyproject.toml b/pyproject.toml index c095ab15..618cf977 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,6 @@ python-dotenv = ">=1" deepdiff = ">=6" pytest = ">=7.3.2" ckanapi = ">=4.7" -ckanapi = "^4.7" beautifulsoup4 = "^4.12.2" [tool.poetry.group.dev.dependencies] From 7986401b20aebaafd1ce63587de5f92363afb240 Mon Sep 17 00:00:00 2001 From: robert-bryson Date: Tue, 19 Dec 2023 18:25:31 +0000 Subject: [PATCH 13/13] moved waf into extract.py, fixed tests, added make down/clean --- Makefile | 6 ++++++ harvester/extract.py | 42 ++++++++++++++++++++++++++++++++++++++ harvester/waf.py | 43 --------------------------------------- tests/extract/test_waf.py | 2 +- 4 files changed, 49 insertions(+), 44 deletions(-) delete mode 100644 harvester/waf.py diff --git a/Makefile b/Makefile index 5f0a47ad..2dbc8937 100644 --- a/Makefile +++ b/Makefile @@ -16,6 +16,12 @@ test: up ## Runs poetry tests, ignores ckan load up: ## Sets up local docker environment docker compose up -d +down: ## Shuts down local docker instance + docker-compose down + +clean: ## Cleans docker images + docker compose down -v --remove-orphans + lint: ## Lints wtih ruff ruff . diff --git a/harvester/extract.py b/harvester/extract.py index 85cfcfaa..81a2a685 100644 --- a/harvester/extract.py +++ b/harvester/extract.py @@ -1,6 +1,8 @@ import logging +import os import requests +from bs4 import BeautifulSoup from requests.exceptions import JSONDecodeError, RequestException logger = logging.getLogger("harvester") @@ -26,6 +28,46 @@ def download_dcatus_catalog(url): return Exception(e) +def traverse_waf(url, files=[], file_ext=".xml", folder="/", filters=[]): + # TODO: add exception handling + parent = os.path.dirname(url.rstrip("/")) + + res = requests.get(url) + if res.status_code == 200: + soup = BeautifulSoup(res.content, "html.parser") + anchors = soup.find_all("a", href=True) + + folders = [] + for anchor in anchors: + if ( + anchor["href"].endswith(folder) + and not parent.endswith(anchor["href"].rstrip("/")) + and anchor["href"] not in filters + ): + folders.append(os.path.join(url, anchor["href"])) + + if anchor["href"].endswith(file_ext): + files.append(os.path.join(url, anchor["href"])) + + for folder in folders: + traverse_waf(folder, files=files, filters=filters) + + return files + + +def download_waf(files): + output = [] + for file in files: + data = {} + data["url"] = file + res = requests.get(file) + if res.status_code == 200: + data["content"] = res.content + output.append(data) + + return output + + def extract(harvest_source) -> list: """Extracts all records from a harvest_source""" logger.info("Hello from harvester.extract()") diff --git a/harvester/waf.py b/harvester/waf.py deleted file mode 100644 index ed4b456d..00000000 --- a/harvester/waf.py +++ /dev/null @@ -1,43 +0,0 @@ -import requests -from bs4 import BeautifulSoup -import os - - -def traverse_waf(url, files=[], file_ext=".xml", folder="/", filters=[]): - # TODO: add exception handling - parent = os.path.dirname(url.rstrip("/")) - - res = requests.get(url) - if res.status_code == 200: - soup = BeautifulSoup(res.content, "html.parser") - anchors = soup.find_all("a", href=True) - - folders = [] - for anchor in anchors: - if ( - anchor["href"].endswith(folder) - and not parent.endswith(anchor["href"].rstrip("/")) - and anchor["href"] not in filters - ): - folders.append(os.path.join(url, anchor["href"])) - - if anchor["href"].endswith(file_ext): - files.append(os.path.join(url, anchor["href"])) - - for folder in folders: - traverse_waf(folder, files=files, filters=filters) - - return files - - -def download_waf(files): - output = [] - for file in files: - data = {} - data["url"] = file - res = requests.get(file) - if res.status_code == 200: - data["content"] = res.content - output.append(data) - - return output diff --git a/tests/extract/test_waf.py b/tests/extract/test_waf.py index 55966642..c49048ab 100644 --- a/tests/extract/test_waf.py +++ b/tests/extract/test_waf.py @@ -1,4 +1,4 @@ -from harvester.extract.waf import traverse_waf, download_waf +from harvester.extract import download_waf, traverse_waf def test_traverse_waf(get_waf_url):