Skip to content

Commit

Permalink
Reduce hard coding and make dev experience better (#5)
Browse files Browse the repository at this point in the history
* Reduce hard coding and make dev experience better

* change test dir name

* few edits to comments

* update comment in volume
  • Loading branch information
C-Loftus authored Aug 26, 2024
1 parent d93b020 commit 2c56d82
Show file tree
Hide file tree
Showing 5 changed files with 243 additions and 18 deletions.
25 changes: 25 additions & 0 deletions .github/workflows/check_output.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
name: Run Jinja2 Tests

on: [push]

jobs:
run-tests:
runs-on: ubuntu-latest

steps:
- name: Checkout code
uses: actions/checkout@v3

- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.12'

- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install jinja2
- name: Run tests
run: |
python ./tests/runner.py
6 changes: 6 additions & 0 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,3 +6,9 @@ services:
- PYGEOAPI_URL=http://localhost:5000
ports:
- 5000:80
# volumes:
# # Mounting allows for hot reloading during development.
# # We have to mount these all independently since if we mount the entire directory, it overrides other templates
# - ./templates/ornl-hydrolocation.j2:/pygeoapi/pygeoapi/templates/ornl-hydrolocation.j2
# - ./templates/usgs-hydrolocation.j2:/pygeoapi/pygeoapi/templates/usgs-hydrolocation.j2
# - ./templates/usgs-location-oriented.j2:/pygeoapi/pygeoapi/templates/usgs-location-oriented.j2
58 changes: 40 additions & 18 deletions templates/usgs-location-oriented.j2
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,23 @@
"@type": [
"hyf:HY_HydrometricFeature",
"hyf:HY_HydroLocation",
"locType:stream"
{% if data['monitoringLocationType'] %}
"locType:{{ data['monitoringLocationType'] }}"
{% endif %}
],
"hyf:HydroLocationType": "hydrometric station",


{% if data['monitoringLocationType'] %}
"hyf:HydroLocationType": "{{ data['monitoringLocationType'] }}",
{% endif %}

{#Was told to not worry about changing this for time being, no way to get this without a geospatial query #}
{#
"sameAs": {
"@id": "https://geoconnex.us/ref/gages/{# Was told to not worry about changing this for time being, no way to get this without a geospatial query #}"
"@id": "https://geoconnex.us/ref/gages/{{}}
},
#}

"identifier": {
"@type": "PropertyValue",
"propertyID": "USGS site number",
Expand Down Expand Up @@ -65,7 +76,7 @@
"hyf:referencedPosition": {
"hyf:HY_IndirectPosition": {
"hyf:linearElement": {
{# I am assuming we can hard code 0 since this template is location-oriented and thus all date is related to one location#}
{# Assumes we can hard code 0 since this template is location-oriented and thus all date is related to one location#}
"@id": "https://geoconnex.us/ref/mainstems/{{ data['Locations'][0]['properties']['mainstemURL'] }}"
}
}
Expand All @@ -75,30 +86,34 @@
"subjectOf": [
{% for stream in data['Datastreams'] %}
{
{# In production we would have a function to map this to all possible types #}
{% with typeOfData= "Flow rate" if "Flow rate" in stream['description'] else "Precipitation"%}
{# Description ends up looking like the example below. However, we don't need the ID
or units when describing just the type of data so we split on , and use the first part #}
{# "Lake or reservoir water surface elevation above NGVD 1929, feet / USCE-07335390-727586146fff4717b8d772eb32388d85" #}
{% with typeOfData= stream['description'].split(',')[0] %}

"@type": "Dataset",

{% if data['name'] %}
{# We use the name since all datastreams are coming from the same location #}
"name": "Flow rate data from USGS Monitoring Location {{data['name']}}",
"name": "{{data['name']}}",
{% endif %}

{% if stream['description'] %}
{# Description describes what type of data that is being collected #}
"description": "{{ stream["description"]}}",
{% endif %}

{# This is hardcoded, since it is not provided in the raw jsonld#}
{# This is hardcoded, since it is not provided in the raw jsonld, but we can assume everything in the USGS template is from USGS#}
"provider": {
"url": "https://waterdata.usgs.gov",
"@type": "GovernmentOrganization",
"name": "U.S. Geological Survey Water Data for the Nation"
},

{% if data['monitoringLocationNumber'] %}
"url": "https://waterdata.usgs.gov/monitoring-location/{{data["monitoringLocationNumber"]}}/#parameterCode={{ stream["properties"]["ParameterCode"] }}",
{% endif %}

"url": "https://waterdata.usgs.gov/monitoring-location/08282300/#parameterCode={{ stream["properties"]["ParameterCode"] }}",
{# Assume here it is discharge, but ideally we would get this variable #}
"variableMeasured": {
"@type": "PropertyValue",
"name": "{{ typeOfData }}",
Expand All @@ -107,39 +122,46 @@
"description": "{{ typeOfData }} in {{ stream['unitOfMeasurement']['symbol'] }}",
{% endif %}

"propertyID": "https://www.wikidata.org/w/index.php?search={{ typeOfData }}",

{#
If there are many types of data, we have to just search since we can't be confident
that it matches the wikipedia name
#}
"propertyID": "https://www.wikidata.org/w/index.php?search={{ typeOfData }}",
"url": "https://en.wikipedia.org/w/index.php?search={{ typeOfData }}",

{# For USGS, ['unitOfMeasurement']['name'] is not defined, so we use 'symbol' #}
{% if stream['unitOfMeasurement']['symbol'] %}
"unitText": "{{ stream['unitOfMeasurement']['symbol'] }}",
{% endif %}

{# These need to be hardcoded, qudt does not seem to be output by USGS with pygeoapi #}
{# In production we would map {{ typeOfData }} to a dict that outputs the well defined qudt value #}
{# These need to be hardcoded, qudt does not seem to be output by USGS with pygeoapi.
No clear way to map every permutation to a unique id all in jinja #}
{#
"qudt:hasQuantityKind": "qudt-quantkinds:VolumeFlowRate",
"unitCode": "qudt-units:FT3-PER-SEC",
"measurementTechnique": "observation",
#}

"measurementMethod": {

{# Assume gaging stations but this is not provided in the raw jsonld#}
"name": "{{ typeOfData }} Measurements at Gaging Stations",
"publisher": "U.S. Geological Survey",
"url": "https://doi.org/10.3133/tm3A8"
{# Can't say if it is a gaging station or another type since this is not provided in the raw jsonld.
Also do not have enough info to link to a remote URL for the doi #}
{# "url": "https://doi.org/10.3133/tm3A8", #}

"name": "{{ typeOfData }} Measurements",
"publisher": "U.S. Geological Survey"
}
},
{% if stream['observedArea'] is defined and stream['observedArea']['phenomenonTime'] %}
"temporalCoverage": "{{ stream['observedArea']['phenomenonTime'] }}",
{% endif %}

{# dc and dcat are not provided in the raw jsonld, must be hardcoded #}
"dc:accrualPeriodicity": "freq:daily",
{#"dc:accrualPeriodicity": "freq:daily",
"dcat:temporalResolution": {"@value":"PT15M","@type":"xsd:duration"},
#}

"distribution": [
{
"@type": "DataDownload",
Expand Down
40 changes: 40 additions & 0 deletions tests/runner.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
import os
import json
from jinja2 import Environment, FileSystemLoader

template_dir = 'templates'
env = Environment(loader=FileSystemLoader(template_dir))

def apply_template_to_json(template_path, json_path):
template = env.get_template(template_path)

with open(json_path) as json_file:
data = json.load(json_file)
# pygeoapi templates use data as the key where all the properties are stored
# so we just rename the key
data["data"] = data["properties"]

result = template.render(data)

return result

for template_filename in os.listdir(template_dir):
if template_filename.endswith('.j2'):
template_name = template_filename.replace('.j2', '')

test_dir = os.path.abspath(os.path.join('tests', template_name))

if not os.path.isdir(test_dir):
print(f"No test directory found for template {template_name}")
continue

for file_name in os.listdir(test_dir):
if file_name.endswith('.json'):
json_path = os.path.join(test_dir, file_name)
template_path = template_filename

result = apply_template_to_json(template_path, json_path)

print(f"Result for {file_name} using {template_filename}:")
print(result)
print("-" * 40)
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
{
"type":"Feature",
"id":"'USCE-07335390'",
"properties":{
"@iot.selfLink":"https://labs.waterdata.usgs.gov/sta/v1.1/Things('USCE-07335390')",
"name":"USCE-07335390",
"description":"Lake, Reservoir, Impoundment",
"Locations":[
{
"@iot.selfLink":"https://labs.waterdata.usgs.gov/sta/v1.1/Locations('62f14fda-2d2a-11ec-ad43-63929dbd9073')",
"@iot.id":"62f14fda-2d2a-11ec-ad43-63929dbd9073",
"name":"USCE-07335390",
"description":"Lake, Reservoir, Impoundment",
"encodingType":"application/vnd.geo+json",
"location":{
"type":"Point",
"coordinates":[
-95.5446813444291,
33.8528821051941
]
}
}
],
"Datastreams":[
{
"@iot.selfLink":"https://labs.waterdata.usgs.gov/sta/v1.1/Datastreams('727586146fff4717b8d772eb32388d85')",
"@iot.id":"727586146fff4717b8d772eb32388d85",
"name":"727586146fff4717b8d772eb32388d85",
"description":"Lake or reservoir water surface elevation above NGVD 1929, feet / USCE-07335390-727586146fff4717b8d772eb32388d85",
"observationType":"Instantaneous",
"unitOfMeasurement":{
"name":"Feet",
"symbol":"ft",
"definition":""
},
"observedArea":{
"type":"Point",
"coordinates":[
-95.5446813,
33.8528821
]
},
"phenomenonTime":"2024-08-16T09:30:00Z/2024-08-26T15:00:00Z",
"properties":{
"Thresholds":[
{
"Name":"Conservation Pool Elevation",
"Type":"None",
"Periods":[
{
"EndTime":"9999-12-31T23:59:59.9999999+00:00",
"StartTime":"0001-01-01T00:00:00.0000000+00:00",
"SuppressData":false,
"ReferenceValue":451,
"ReferenceValueToTriggerDisplay":null
}
],
"ReferenceCode":"NWISWeb ref level - Public"
}
],
"ParameterCode":"62614",
"StatisticCode":"00011"
}
}
],
"Locations@iot.navigationLink":"https://labs.waterdata.usgs.gov/sta/v1.1/Things('USCE-07335390')/Locations",
"HistoricalLocations@iot.navigationLink":"https://labs.waterdata.usgs.gov/sta/v1.1/Things('USCE-07335390')/HistoricalLocations",
"Datastreams@iot.navigationLink":"https://labs.waterdata.usgs.gov/sta/v1.1/Things('USCE-07335390')/Datastreams",
"state":"Texas",
"county":"Lamar County",
"country":"United States of America",
"mapScale":24000,
"stateCode":"TX",
"agencyCode":"USCE",
"countryFIPS":"US",
"hydrologicUnit":"11140101",
"decimalLatitude":33.8528821051941,
"decimalLongitude":-95.5446813444291,
"monitoringLocationUrl":"https://waterdata.usgs.gov/monitoring-location/07335390",
"monitoringLocationName":"USCOE Pat Mayse Lk nr Chicota, TX",
"monitoringLocationType":"Lake, Reservoir, Impoundment",
"monitoringLocationNumber":"07335390",
"locationHUCTwelveDigitCode":"111401010606",
"decimalLatitudeStandardized":33.8528821051941,
"decimalLongitudeStandardized":-95.5446813444291
},
"geometry":{
"type":"Point",
"coordinates":[
-95.5446813444291,
33.8528821051941
]
},
"links":[
{
"type":"application/json",
"rel":"root",
"title":"The landing page of this server as JSON",
"href":"http://localhost:5000?f=json"
},
{
"type":"text/html",
"rel":"root",
"title":"The landing page of this server as HTML",
"href":"http://localhost:5000?f=html"
},
{
"rel":"self",
"type":"application/geo+json",
"title":"This document as JSON",
"href":"http://localhost:5000/collections/USGS/Things/items/'USCE-07335390'?f=json"
},
{
"rel":"alternate",
"type":"application/ld+json",
"title":"This document as RDF (JSON-LD)",
"href":"http://localhost:5000/collections/USGS/Things/items/'USCE-07335390'?f=jsonld"
},
{
"rel":"alternate",
"type":"text/html",
"title":"This document as HTML",
"href":"http://localhost:5000/collections/USGS/Things/items/'USCE-07335390'?f=html"
},
{
"rel":"collection",
"type":"application/json",
"title":"USGS Things",
"href":"http://localhost:5000/collections/USGS/Things"
}
]
}

0 comments on commit 2c56d82

Please sign in to comment.