Skip to content

Commit 0ab7107

Browse files
davenquinndavenquinn
authored
Merge pull request #113 from UW-Macrostrat/integrate-api-v3
Integrate API v3
2 parents 154e1af + 4083ba2 commit 0ab7107

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

86 files changed

+7612
-501
lines changed

‎.github/workflows/build-api-dev.yaml

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
name: Build Staging
2+
3+
on:
4+
push:
5+
tags:
6+
- v[0-9]+.[0-9]+.[0-9]+-** # Semver Pre-Release
7+
pull_request:
8+
branches: [ main ]
9+
paths:
10+
- services/api-v3/**
11+
jobs:
12+
docker:
13+
runs-on: ubuntu-latest
14+
steps:
15+
- name: Checkout
16+
uses: actions/checkout@v3
17+
with:
18+
submodules: 'recursive'
19+
- name: Docker meta
20+
id: meta
21+
uses: docker/metadata-action@v4
22+
with:
23+
images: hub.opensciencegrid.org/macrostrat/api-v3
24+
tags: |
25+
type=semver,pattern={{version}}
26+
type=raw,value=latest-itb
27+
type=ref,event=branch,suffix=-{{date 'YYYYMMDDHHmmss'}}
28+
type=ref,event=tag,suffix=-{{date 'YYYYMMDDHHmmss'}}
29+
type=raw,value=latest-itb-{{date 'YYYYMMDDHHmmss'}}
30+
type=raw,value=sha-{{sha}}
31+
- name: Set up Docker Buildx
32+
uses: docker/setup-buildx-action@v2
33+
- name: Login to OSG DockerHub
34+
uses: docker/login-action@v2
35+
with:
36+
registry: hub.opensciencegrid.org
37+
username: ${{ vars.HARBOR_CLI_NAME }}
38+
password: ${{ secrets.HARBOR_CLI_SECRET }}
39+
- name: Build and push
40+
uses: docker/build-push-action@v5
41+
with:
42+
context: services/api-v3
43+
push: true
44+
tags: ${{ steps.meta.outputs.tags }}
45+
labels: ${{ steps.meta.outputs.labels }}
46+
cache-from: type=gha
47+
cache-to: type=gha,mode=max

‎.github/workflows/build-api-prod.yaml

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
name: Build Production
2+
3+
on:
4+
push:
5+
branches:
6+
- main
7+
tags:
8+
- v[0-9]+.[0-9]+.[0-9]+ # Semver Release
9+
paths:
10+
- services/api-v3/**
11+
12+
jobs:
13+
docker:
14+
runs-on: ubuntu-latest
15+
steps:
16+
- name: Checkout
17+
uses: actions/checkout@v3
18+
with:
19+
submodules: 'recursive'
20+
- name: Docker meta
21+
id: meta
22+
uses: docker/metadata-action@v4
23+
with:
24+
images: hub.opensciencegrid.org/macrostrat/api-v3
25+
tags: |
26+
type=ref,event=pr,suffix=-{{date 'YYYYMMDDHHmmss'}}
27+
type=ref,event=branch,suffix=-{{date 'YYYYMMDDHHmmss'}}
28+
type=semver,pattern={{version}}
29+
type=raw,value=latest,enable={{is_default_branch}}
30+
type=raw,value=sha-{{sha}}
31+
- name: Set up Docker Buildx
32+
uses: docker/setup-buildx-action@v2
33+
- name: Login to OSG DockerHub
34+
uses: docker/login-action@v2
35+
with:
36+
registry: hub.opensciencegrid.org
37+
username: ${{ vars.HARBOR_CLI_NAME }}
38+
password: ${{ secrets.HARBOR_CLI_SECRET }}
39+
- name: Build and push
40+
uses: docker/build-push-action@v5
41+
with:
42+
context: services/api-v3
43+
push: true
44+
tags: ${{ steps.meta.outputs.tags }}
45+
labels: ${{ steps.meta.outputs.labels }}
46+
cache-from: type=gha
47+
cache-to: type=gha,mode=max

‎.github/workflows/continuous-integration.yml

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,16 +13,28 @@ jobs:
1313
with:
1414
submodules: 'recursive'
1515

16+
# Install poetry and fix setuptools error
1617
- name: Install poetry
1718
run: pipx install poetry
1819

20+
# External GDAL is needed for map integration system installation now.
21+
# We might try to remove this in the future, or figure out how to use
22+
# a bundled version of GDAL from the virtual environment.
23+
- name: Install GDAL
24+
run: |
25+
sudo apt-add-repository ppa:ubuntugis/ubuntugis-unstable
26+
sudo apt-get update
27+
sudo apt-get install gdal-bin libgdal-dev
28+
1929
- uses: actions/setup-python@v4
2030
with:
2131
python-version: '3.11'
2232
cache: 'poetry'
2333

2434
- name: Install dependencies
25-
run: poetry install
35+
run: |
36+
poetry install
37+
poetry run pip install --upgrade setuptools
2638
2739
- name: Run Macrostrat tests
28-
run: make test
40+
run: make test-ci

‎.idea/prettier.xml

Lines changed: 7 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎.idea/sqldialects.xml

Lines changed: 1 addition & 9 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Makefile

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,9 @@ all:
33
poetry install --only=dev
44
macrostrat poetry run mono install
55
poetry install
6+
# Install the version of the GDAL bindings that matches the native lib.
7+
# This is breakable and should be replaced with a more robust solution.
8+
#poetry run pip install GDAL==$(shell gdal-config --version | sed 's/\([0-9]*\)\.\([0-9]*\).*/\1.\2/')
69

710
install:
811
ln -sf $(shell pwd)/bin/macrostrat /usr/local/bin/macrostrat
@@ -12,7 +15,19 @@ format:
1215
poetry run isort .
1316

1417
test:
15-
poetry run pytest -s -x cli/tests
18+
# These tests may fail due to an older GDAL version in use.
19+
# We need to figure out how to bundle GDAL or run in a Docker context
20+
poetry run pytest -s -x --ignore=runtime-tests --ignore=services --ignore=v2-transition .
21+
22+
test-ci:
23+
# We need a fairly recent version of GDAL (3.10) for map integration tests to pass.
24+
# For now, we avoid running these tests in CI.
25+
poetry run pytest -s -x \
26+
--ignore=runtime-tests \
27+
--ignore=services \
28+
--ignore=v2-transition \
29+
-m "not requires_gdal" \
30+
.
1631

1732
test-warnings:
1833
poetry run pytest cli/tests -W error

‎README.md

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,22 @@ The core component packaged here is a Python application designed to control
1212
various aspects of Macrostrat's system and supporting processes (e.g., map
1313
caching and harmonization).
1414

15+
## Key components in this repository
16+
17+
### Management utilities
18+
19+
- [Command-line interface](./cli): A Python-based CLI for managing Macrostrat's
20+
system components
21+
- [Map ingestion system](./map-integration): Scripts for importing geologic maps
22+
into Macrostrat's database
23+
24+
### Services
25+
26+
- [API v3](./services/api-v3): An API for managing Macrostrat elements, and authentication
27+
- Macrostrat task runner: *coming soon*
28+
29+
Other Macrostrat services (e.g., read-only APIs) are housed in other repositories.
30+
1531
## Installation
1632

1733
- Running `make` will install the Python dependencies and build the application.

‎cli/macrostrat/cli/database/__init__.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ def apply(self, db, match: str = None):
5757
for f in self.fixtures:
5858
if callable(f):
5959
f(db)
60-
return
60+
continue
6161
# This does the same as the upstream "Apply fixtures" function
6262
# but it has support for a 'match' parameter
6363
files = _match_paths(get_sql_files(f), match)

‎cli/macrostrat/cli/database/migrations/map_ingestion_api/__init__.py

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
from pathlib import Path
2+
3+
from macrostrat.cli.database import setup_postgrest_access
4+
from macrostrat.core.migrations import Migration, view_exists
5+
6+
__dir__ = Path(__file__).parent
7+
8+
9+
class MapsPostgrestAPI(Migration):
10+
name = "map-ingestion-api"
11+
# This partition is required
12+
subsystem = "map-ingestion"
13+
description = """
14+
Add a basic postgrest API for map ingestion schema
15+
"""
16+
depends_on = ["maps-source-operations"]
17+
18+
postconditions = [
19+
view_exists("map_ingestion_api", "line_types", "point_types", "maps"),
20+
]
21+
22+
fixtures = [
23+
__dir__,
24+
setup_postgrest_access(
25+
"map_ingestion_api",
26+
read_user="web_user",
27+
write_user="web_user",
28+
),
29+
]

‎cli/macrostrat/cli/database/migrations/map_ingestion_api/map-ingestion-api.sql

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
CREATE SCHEMA IF NOT EXISTS map_ingestion_api;
2+
3+
CREATE OR REPLACE VIEW map_ingestion_api.line_types AS
4+
SELECT type, count(*) FROM maps.lines
5+
WHERE type IS NOT null
6+
GROUP BY type
7+
ORDER BY count(*) DESC;
8+
9+
CREATE OR REPLACE VIEW map_ingestion_api.point_types AS
10+
SELECT point_type, count(*) FROM maps.points
11+
WHERE point_type IS NOT null
12+
GROUP BY point_type
13+
ORDER BY count(*) DESC;
14+
15+
-- Correct a small data error
16+
UPDATE maps.lines SET type = 'strike-slip fault'
17+
WHERE type = 'strike-slilp fault';
18+
19+
-- Create another view for api
20+
DROP VIEW IF EXISTS map_ingestion_api.maps;
21+
CREATE OR REPLACE VIEW map_ingestion_api.maps AS
22+
SELECT
23+
s.source_id,
24+
s.slug,
25+
name,
26+
url,
27+
ref_year,
28+
scale,
29+
i.state,
30+
(SELECT array_agg(tag) AS tags FROM maps_metadata.ingest_process_tag WHERE ingest_process_id = i.id) AS tags
31+
FROM maps.sources s
32+
LEFT JOIN maps_metadata.ingest_process i
33+
ON s.source_id = i.source_id
34+
ORDER BY s.source_id DESC;
35+

‎cli/macrostrat/cli/database/migrations/map_sources/01-maps-sources.sql

Lines changed: 15 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
11
ALTER TABLE maps.sources ADD COLUMN IF NOT EXISTS raster_url text;
2+
ALTER TABLE maps.sources ADD COLUMN IF NOT EXISTS scale_denominator integer;
3+
ALTER TABLE maps.sources ADD COLUMN IF NOT EXISTS is_finalized boolean DEFAULT false;
24

35
DROP VIEW IF EXISTS macrostrat_api.sources_metadata CASCADE;
46
DROP VIEW IF EXISTS maps.sources_metadata CASCADE;
7+
DROP VIEW IF EXISTS macrostrat_api.sources CASCADE;
58

69
CREATE OR REPLACE VIEW maps.sources_metadata AS
710
SELECT
@@ -22,22 +25,18 @@ SELECT
2225
new_priority priority,
2326
status_code,
2427
raster_url,
25-
CASE
26-
WHEN psi.source_id IS NULL THEN false
27-
ELSE true
28-
END AS is_mapped
28+
scale_denominator,
29+
is_finalized
2930
FROM maps.sources AS s
30-
LEFT JOIN (
31-
SELECT
32-
DISTINCT(polygons.source_id)
33-
FROM maps.polygons
34-
) psi ON s.source_id = psi.source_id
3531
ORDER BY source_id DESC;
3632

3733
COMMENT ON VIEW maps.sources_metadata IS 'Convenience view for maps.sources with only metadata fields';
3834

3935
CREATE OR REPLACE VIEW macrostrat_api.sources_metadata AS
40-
SELECT * FROM maps.sources_metadata;
36+
SELECT *,
37+
-- Legacy column for backwards compatibility.
38+
is_finalized is_mapped
39+
FROM maps.sources_metadata;
4140

4241
CREATE OR REPLACE VIEW macrostrat_api.sources_ingestion AS
4342
SELECT
@@ -62,7 +61,9 @@ SELECT
6261
i.comments,
6362
i.created_on,
6463
i.completed_on,
65-
i.map_id
64+
i.map_id,
65+
s.is_finalized,
66+
s.scale_denominator
6667
FROM maps.sources_metadata s
6768
JOIN maps_metadata.ingest_process i
6869
ON i.source_id = s.source_id;
@@ -86,5 +87,7 @@ SELECT
8687
s.priority,
8788
s.status_code,
8889
s.raster_url,
89-
s.web_geom envelope
90+
s.web_geom envelope,
91+
s.is_finalized,
92+
s.scale_denominator
9093
FROM maps.sources s;

‎cli/macrostrat/cli/database/migrations/map_sources/__init__.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
from macrostrat.core.migrations import Migration, view_exists
1+
from macrostrat.core.migrations import Migration, has_columns, view_exists
22

33

44
class MapsSourcesMetadataMigration(Migration):
@@ -15,4 +15,5 @@ class MapsSourcesMetadataMigration(Migration):
1515
view_exists(
1616
"macrostrat_api", "sources_metadata", "sources_ingestion", "sources"
1717
),
18+
has_columns("maps", "sources", "is_finalized", "scale_denominator"),
1819
]

‎cli/macrostrat/cli/database/migrations/maps_ingest_metadata/__init__.py

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
from macrostrat.core.migrations import Migration, has_columns, view_exists
2+
3+
4+
class IngestMetadataMigration(Migration):
5+
name = "ingest-metadata"
6+
7+
depends_on = ["api-v3", "macrostrat-api"]
8+
9+
postconditions = [
10+
has_columns(
11+
"maps_metadata",
12+
"ingest_process",
13+
"polygon_state",
14+
"line_state",
15+
"point_state",
16+
),
17+
# We should eventually move this to another _api schema
18+
view_exists("macrostrat_api", "map_ingest_metadata"),
19+
]

‎cli/macrostrat/cli/database/migrations/maps_ingest_metadata/maps-ingest.sql

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
DROP VIEW IF EXISTS macrostrat_api.map_ingest_metadata;
2+
3+
ALTER TABLE maps_metadata.ingest_process ADD COLUMN polygon_state jsonb;
4+
ALTER TABLE maps_metadata.ingest_process ADD COLUMN line_state jsonb;
5+
ALTER TABLE maps_metadata.ingest_process ADD COLUMN point_state jsonb;
6+
7+
CREATE OR REPLACE VIEW macrostrat_api.map_ingest_metadata AS
8+
SELECT * FROM maps_metadata.ingest_process;
9+
10+
-- Make it writeable by users
11+
GRANT SELECT, UPDATE ON maps_metadata.ingest_process TO web_user;
12+
GRANT SELECT, UPDATE ON macrostrat_api.map_ingest_metadata TO web_user;

0 commit comments

Comments
 (0)