From 2c56d828e6d22c5c3ffaf5f8e76326c55463386a Mon Sep 17 00:00:00 2001 From: Colton Loftus <70598503+C-Loftus@users.noreply.github.com> Date: Mon, 26 Aug 2024 16:14:36 -0400 Subject: [PATCH] Reduce hard coding and make dev experience better (#5) * Reduce hard coding and make dev experience better * change test dir name * few edits to comments * update comment in volume --- .github/workflows/check_output.yml | 25 ++++ docker-compose.yml | 6 + templates/usgs-location-oriented.j2 | 58 +++++--- tests/runner.py | 40 ++++++ ...ons-usgs-things-items-'USCE-07335390'.json | 132 ++++++++++++++++++ 5 files changed, 243 insertions(+), 18 deletions(-) create mode 100644 .github/workflows/check_output.yml create mode 100644 tests/runner.py create mode 100644 tests/usgs-location-oriented/collections-usgs-things-items-'USCE-07335390'.json diff --git a/.github/workflows/check_output.yml b/.github/workflows/check_output.yml new file mode 100644 index 0000000..8097b69 --- /dev/null +++ b/.github/workflows/check_output.yml @@ -0,0 +1,25 @@ +name: Run Jinja2 Tests + +on: [push] + +jobs: + run-tests: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.12' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install jinja2 + + - name: Run tests + run: | + python ./tests/runner.py \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 1022db3..ea5dfa0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,3 +6,9 @@ services: - PYGEOAPI_URL=http://localhost:5000 ports: - 5000:80 + # volumes: + # # Mounting allows for hot reloading during development. + # # We have to mount these all independently since if we mount the entire directory, it overrides other templates + # - ./templates/ornl-hydrolocation.j2:/pygeoapi/pygeoapi/templates/ornl-hydrolocation.j2 + # - ./templates/usgs-hydrolocation.j2:/pygeoapi/pygeoapi/templates/usgs-hydrolocation.j2 + # - ./templates/usgs-location-oriented.j2:/pygeoapi/pygeoapi/templates/usgs-location-oriented.j2 \ No newline at end of file diff --git a/templates/usgs-location-oriented.j2 b/templates/usgs-location-oriented.j2 index 8bb749b..0a37f18 100644 --- a/templates/usgs-location-oriented.j2 +++ b/templates/usgs-location-oriented.j2 @@ -22,12 +22,23 @@ "@type": [ "hyf:HY_HydrometricFeature", "hyf:HY_HydroLocation", - "locType:stream" + {% if data['monitoringLocationType'] %} + "locType:{{ data['monitoringLocationType'] }}" + {% endif %} ], - "hyf:HydroLocationType": "hydrometric station", + + + {% if data['monitoringLocationType'] %} + "hyf:HydroLocationType": "{{ data['monitoringLocationType'] }}", + {% endif %} + + {#Was told to not worry about changing this for time being, no way to get this without a geospatial query #} + {# "sameAs": { - "@id": "https://geoconnex.us/ref/gages/{# Was told to not worry about changing this for time being, no way to get this without a geospatial query #}" + "@id": "https://geoconnex.us/ref/gages/{{}} }, + #} + "identifier": { "@type": "PropertyValue", "propertyID": "USGS site number", @@ -65,7 +76,7 @@ "hyf:referencedPosition": { "hyf:HY_IndirectPosition": { "hyf:linearElement": { - {# I am assuming we can hard code 0 since this template is location-oriented and thus all date is related to one location#} + {# Assumes we can hard code 0 since this template is location-oriented and thus all date is related to one location#} "@id": "https://geoconnex.us/ref/mainstems/{{ data['Locations'][0]['properties']['mainstemURL'] }}" } } @@ -75,14 +86,16 @@ "subjectOf": [ {% for stream in data['Datastreams'] %} { - {# In production we would have a function to map this to all possible types #} - {% with typeOfData= "Flow rate" if "Flow rate" in stream['description'] else "Precipitation"%} + {# Description ends up looking like the example below. However, we don't need the ID + or units when describing just the type of data so we split on , and use the first part #} + {# "Lake or reservoir water surface elevation above NGVD 1929, feet / USCE-07335390-727586146fff4717b8d772eb32388d85" #} + {% with typeOfData= stream['description'].split(',')[0] %} "@type": "Dataset", {% if data['name'] %} {# We use the name since all datastreams are coming from the same location #} - "name": "Flow rate data from USGS Monitoring Location {{data['name']}}", + "name": "{{data['name']}}", {% endif %} {% if stream['description'] %} @@ -90,15 +103,17 @@ "description": "{{ stream["description"]}}", {% endif %} - {# This is hardcoded, since it is not provided in the raw jsonld#} + {# This is hardcoded, since it is not provided in the raw jsonld, but we can assume everything in the USGS template is from USGS#} "provider": { "url": "https://waterdata.usgs.gov", "@type": "GovernmentOrganization", "name": "U.S. Geological Survey Water Data for the Nation" }, + + {% if data['monitoringLocationNumber'] %} + "url": "https://waterdata.usgs.gov/monitoring-location/{{data["monitoringLocationNumber"]}}/#parameterCode={{ stream["properties"]["ParameterCode"] }}", + {% endif %} - "url": "https://waterdata.usgs.gov/monitoring-location/08282300/#parameterCode={{ stream["properties"]["ParameterCode"] }}", - {# Assume here it is discharge, but ideally we would get this variable #} "variableMeasured": { "@type": "PropertyValue", "name": "{{ typeOfData }}", @@ -107,12 +122,12 @@ "description": "{{ typeOfData }} in {{ stream['unitOfMeasurement']['symbol'] }}", {% endif %} - "propertyID": "https://www.wikidata.org/w/index.php?search={{ typeOfData }}", {# If there are many types of data, we have to just search since we can't be confident that it matches the wikipedia name #} + "propertyID": "https://www.wikidata.org/w/index.php?search={{ typeOfData }}", "url": "https://en.wikipedia.org/w/index.php?search={{ typeOfData }}", {# For USGS, ['unitOfMeasurement']['name'] is not defined, so we use 'symbol' #} @@ -120,17 +135,22 @@ "unitText": "{{ stream['unitOfMeasurement']['symbol'] }}", {% endif %} - {# These need to be hardcoded, qudt does not seem to be output by USGS with pygeoapi #} - {# In production we would map {{ typeOfData }} to a dict that outputs the well defined qudt value #} + {# These need to be hardcoded, qudt does not seem to be output by USGS with pygeoapi. + No clear way to map every permutation to a unique id all in jinja #} + {# "qudt:hasQuantityKind": "qudt-quantkinds:VolumeFlowRate", "unitCode": "qudt-units:FT3-PER-SEC", "measurementTechnique": "observation", + #} + "measurementMethod": { - {# Assume gaging stations but this is not provided in the raw jsonld#} - "name": "{{ typeOfData }} Measurements at Gaging Stations", - "publisher": "U.S. Geological Survey", - "url": "https://doi.org/10.3133/tm3A8" + {# Can't say if it is a gaging station or another type since this is not provided in the raw jsonld. + Also do not have enough info to link to a remote URL for the doi #} + {# "url": "https://doi.org/10.3133/tm3A8", #} + + "name": "{{ typeOfData }} Measurements", + "publisher": "U.S. Geological Survey" } }, {% if stream['observedArea'] is defined and stream['observedArea']['phenomenonTime'] %} @@ -138,8 +158,10 @@ {% endif %} {# dc and dcat are not provided in the raw jsonld, must be hardcoded #} - "dc:accrualPeriodicity": "freq:daily", + {#"dc:accrualPeriodicity": "freq:daily", "dcat:temporalResolution": {"@value":"PT15M","@type":"xsd:duration"}, + #} + "distribution": [ { "@type": "DataDownload", diff --git a/tests/runner.py b/tests/runner.py new file mode 100644 index 0000000..bcae810 --- /dev/null +++ b/tests/runner.py @@ -0,0 +1,40 @@ +import os +import json +from jinja2 import Environment, FileSystemLoader + +template_dir = 'templates' +env = Environment(loader=FileSystemLoader(template_dir)) + +def apply_template_to_json(template_path, json_path): + template = env.get_template(template_path) + + with open(json_path) as json_file: + data = json.load(json_file) + # pygeoapi templates use data as the key where all the properties are stored + # so we just rename the key + data["data"] = data["properties"] + + result = template.render(data) + + return result + +for template_filename in os.listdir(template_dir): + if template_filename.endswith('.j2'): + template_name = template_filename.replace('.j2', '') + + test_dir = os.path.abspath(os.path.join('tests', template_name)) + + if not os.path.isdir(test_dir): + print(f"No test directory found for template {template_name}") + continue + + for file_name in os.listdir(test_dir): + if file_name.endswith('.json'): + json_path = os.path.join(test_dir, file_name) + template_path = template_filename + + result = apply_template_to_json(template_path, json_path) + + print(f"Result for {file_name} using {template_filename}:") + print(result) + print("-" * 40) diff --git a/tests/usgs-location-oriented/collections-usgs-things-items-'USCE-07335390'.json b/tests/usgs-location-oriented/collections-usgs-things-items-'USCE-07335390'.json new file mode 100644 index 0000000..fbeb191 --- /dev/null +++ b/tests/usgs-location-oriented/collections-usgs-things-items-'USCE-07335390'.json @@ -0,0 +1,132 @@ +{ + "type":"Feature", + "id":"'USCE-07335390'", + "properties":{ + "@iot.selfLink":"https://labs.waterdata.usgs.gov/sta/v1.1/Things('USCE-07335390')", + "name":"USCE-07335390", + "description":"Lake, Reservoir, Impoundment", + "Locations":[ + { + "@iot.selfLink":"https://labs.waterdata.usgs.gov/sta/v1.1/Locations('62f14fda-2d2a-11ec-ad43-63929dbd9073')", + "@iot.id":"62f14fda-2d2a-11ec-ad43-63929dbd9073", + "name":"USCE-07335390", + "description":"Lake, Reservoir, Impoundment", + "encodingType":"application/vnd.geo+json", + "location":{ + "type":"Point", + "coordinates":[ + -95.5446813444291, + 33.8528821051941 + ] + } + } + ], + "Datastreams":[ + { + "@iot.selfLink":"https://labs.waterdata.usgs.gov/sta/v1.1/Datastreams('727586146fff4717b8d772eb32388d85')", + "@iot.id":"727586146fff4717b8d772eb32388d85", + "name":"727586146fff4717b8d772eb32388d85", + "description":"Lake or reservoir water surface elevation above NGVD 1929, feet / USCE-07335390-727586146fff4717b8d772eb32388d85", + "observationType":"Instantaneous", + "unitOfMeasurement":{ + "name":"Feet", + "symbol":"ft", + "definition":"" + }, + "observedArea":{ + "type":"Point", + "coordinates":[ + -95.5446813, + 33.8528821 + ] + }, + "phenomenonTime":"2024-08-16T09:30:00Z/2024-08-26T15:00:00Z", + "properties":{ + "Thresholds":[ + { + "Name":"Conservation Pool Elevation", + "Type":"None", + "Periods":[ + { + "EndTime":"9999-12-31T23:59:59.9999999+00:00", + "StartTime":"0001-01-01T00:00:00.0000000+00:00", + "SuppressData":false, + "ReferenceValue":451, + "ReferenceValueToTriggerDisplay":null + } + ], + "ReferenceCode":"NWISWeb ref level - Public" + } + ], + "ParameterCode":"62614", + "StatisticCode":"00011" + } + } + ], + "Locations@iot.navigationLink":"https://labs.waterdata.usgs.gov/sta/v1.1/Things('USCE-07335390')/Locations", + "HistoricalLocations@iot.navigationLink":"https://labs.waterdata.usgs.gov/sta/v1.1/Things('USCE-07335390')/HistoricalLocations", + "Datastreams@iot.navigationLink":"https://labs.waterdata.usgs.gov/sta/v1.1/Things('USCE-07335390')/Datastreams", + "state":"Texas", + "county":"Lamar County", + "country":"United States of America", + "mapScale":24000, + "stateCode":"TX", + "agencyCode":"USCE", + "countryFIPS":"US", + "hydrologicUnit":"11140101", + "decimalLatitude":33.8528821051941, + "decimalLongitude":-95.5446813444291, + "monitoringLocationUrl":"https://waterdata.usgs.gov/monitoring-location/07335390", + "monitoringLocationName":"USCOE Pat Mayse Lk nr Chicota, TX", + "monitoringLocationType":"Lake, Reservoir, Impoundment", + "monitoringLocationNumber":"07335390", + "locationHUCTwelveDigitCode":"111401010606", + "decimalLatitudeStandardized":33.8528821051941, + "decimalLongitudeStandardized":-95.5446813444291 + }, + "geometry":{ + "type":"Point", + "coordinates":[ + -95.5446813444291, + 33.8528821051941 + ] + }, + "links":[ + { + "type":"application/json", + "rel":"root", + "title":"The landing page of this server as JSON", + "href":"http://localhost:5000?f=json" + }, + { + "type":"text/html", + "rel":"root", + "title":"The landing page of this server as HTML", + "href":"http://localhost:5000?f=html" + }, + { + "rel":"self", + "type":"application/geo+json", + "title":"This document as JSON", + "href":"http://localhost:5000/collections/USGS/Things/items/'USCE-07335390'?f=json" + }, + { + "rel":"alternate", + "type":"application/ld+json", + "title":"This document as RDF (JSON-LD)", + "href":"http://localhost:5000/collections/USGS/Things/items/'USCE-07335390'?f=jsonld" + }, + { + "rel":"alternate", + "type":"text/html", + "title":"This document as HTML", + "href":"http://localhost:5000/collections/USGS/Things/items/'USCE-07335390'?f=html" + }, + { + "rel":"collection", + "type":"application/json", + "title":"USGS Things", + "href":"http://localhost:5000/collections/USGS/Things" + } + ] +} \ No newline at end of file