Skip to content

Commit 371f6c3

Browse files
hrodmnhrodmn
authored
feat: add stactools-item-generator and stac-item-loader constructs (#150)
* chore: use uv + deploy dependency group for deploy.yml * chore: update npmignore to exclude unnecessary files from packages
1 parent 23f2844 commit 371f6c3

File tree

27 files changed

+4229
-84
lines changed

27 files changed

+4229
-84
lines changed

‎.github/workflows/deploy.yaml

Lines changed: 10 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -30,18 +30,20 @@ jobs:
3030
- name: Generate distribution packages
3131
run: npm run package
3232

33+
- name: Install uv
34+
uses: astral-sh/setup-uv@v3
35+
with:
36+
version: "0.7.*"
37+
enable-cache: true
3338

3439
- name: Install deployment environment
3540
id: install_deploy_env
3641
run: |
3742
# install deployment environment with eoapi-cdk from build
38-
python -m venv .deployment_venv
39-
source .deployment_venv/bin/activate
40-
pip install dist/python/*.gz
43+
uv sync --group deploy
44+
uv pip install dist/python/*.gz
4145
cd integration_tests/cdk
42-
pip install -r requirements.txt
4346
npm install
44-
deactivate
4547
cd -
4648
4749
# use short commit SHA to name stacks
@@ -55,15 +57,12 @@ jobs:
5557
env:
5658
PROJECT_ID: ${{ steps.short-sha.outputs.sha }}
5759
run: |
58-
source .deployment_venv/bin/activate
59-
6060
# synthesize the stack
6161
cd integration_tests/cdk
62-
npx cdk synth --debug --all --require-approval never
62+
uv run npx cdk synth --debug --all --require-approval never
6363
6464
# deploy the stack
65-
npx cdk deploy --ci --all --require-approval never
66-
deactivate
65+
uv run npx cdk deploy --ci --all --require-approval never
6766
cd -
6867
6968
- name: Tear down any infrastructure
@@ -75,10 +74,9 @@ jobs:
7574
# run this only if we find a 'cdk.out' directory, which means there might be things to tear down
7675
if [ -d "cdk.out" ]; then
7776
cd -
78-
source .deployment_venv/bin/activate
7977
cd integration_tests/cdk
8078
# see https://github.com/aws/aws-cdk/issues/24946
8179
# this didn't work : rm -f cdk.out/synth.lock
8280
# so we just duplicate the cdk output to cdk-destroy.out
83-
npx cdk destroy --output cdk-destroy.out --ci --all --force
81+
uv run npx cdk destroy --output cdk-destroy.out --ci --all --force
8482
fi

‎.github/workflows/unit-tests.yaml

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
name: Unit tests
2+
3+
on:
4+
workflow_dispatch:
5+
push:
6+
branches:
7+
- main
8+
tags:
9+
- 'v*'
10+
pull_request:
11+
12+
jobs:
13+
tests:
14+
runs-on: ubuntu-latest
15+
steps:
16+
- uses: actions/checkout@v4
17+
18+
- name: Install uv
19+
uses: astral-sh/setup-uv@v3
20+
with:
21+
version: "0.7.*"
22+
enable-cache: true
23+
24+
- name: Install PostgreSQL and PostGIS
25+
run: |
26+
sudo apt-get update
27+
sudo apt-get install -y postgresql postgresql-contrib postgresql-16-postgis-3
28+
sudo service postgresql start
29+
pg_isready
30+
31+
- name: Install dependencies
32+
run: |
33+
uv sync
34+
35+
- name: Run tests
36+
run: uv run pytest

‎.npmignore

Lines changed: 115 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,122 @@
11

2-
# Exclude typescript source and config
2+
# ====================
3+
# Source Files (exclude TypeScript sources, keep compiled JS and declarations)
4+
# ====================
35
*.ts
4-
tsconfig.json
5-
6-
# Include javascript files and typescript declarations
7-
!*.js
86
!*.d.ts
7+
!*.js
8+
tsconfig.json
9+
tsconfig.*.json
10+
tsconfig.tsbuildinfo
911

10-
# Exclude jsii outdir
11-
dist
12-
13-
# Include .jsii and .jsii.gz
12+
# ====================
13+
# jsii-specific files (include these in package)
14+
# ====================
1415
!.jsii
1516
!.jsii.gz
1617

17-
# Exclude any dev environments
18-
.venv
18+
# ====================
19+
# Build and Distribution
20+
# ====================
21+
dist/
22+
lib/**/*.ts
23+
!lib/**/*.js
24+
!lib/**/*.d.ts
25+
26+
# ====================
27+
# Development and Testing
28+
# ====================
29+
.git/
30+
.github/
31+
.gitignore
32+
.pre-commit-config.yaml
33+
.husky/
34+
.devcontainer/
35+
integration_tests/
36+
test/
37+
tests/
38+
__tests__/
39+
lib/**/tests/
40+
lib/**/test/
41+
lib/**/__tests__/
42+
*.test.js
43+
*.test.ts
44+
*.test.py
45+
*.spec.js
46+
*.spec.ts
47+
*.spec.py
48+
conftest.py
49+
lib/**/conftest.py
50+
.coverage/
51+
coverage/
52+
.nyc_output/
53+
54+
# ====================
55+
# Node.js and npm
56+
# ====================
57+
node_modules/
58+
npm-debug.log*
59+
yarn-debug.log*
60+
yarn-error.log*
61+
.npm
62+
.node_repl_history
63+
package-lock.json
64+
yarn.lock
65+
66+
# ====================
67+
# Python (for this hybrid project)
68+
# ====================
69+
.venv/
70+
__pycache__/
71+
*.py[cod]
72+
*$py.class
73+
.pytest_cache/
74+
.tox/
75+
toxenv/
76+
.coverage
77+
.coverage.*
78+
htmlcov/
79+
tox.ini
80+
ruff.toml
81+
.ruff_cache/
82+
83+
# ====================
84+
# Documentation and Demos
85+
# ====================
86+
docs/
87+
diagrams/
88+
*.md
89+
!README.md
90+
91+
# ====================
92+
# IDE and Editor
93+
# ====================
94+
.vscode/
95+
.idea/
96+
*.swp
97+
*.swo
98+
*~
99+
.DS_Store
100+
Thumbs.db
101+
102+
# ====================
103+
# Semantic Release and CI/CD
104+
# ====================
105+
.semantic-release/
106+
.travis.yml
107+
.circleci/
108+
.github/
109+
CHANGELOG.md
110+
111+
# ====================
112+
# Other Development Files
113+
# ====================
114+
.nvmrc
115+
.editorconfig
116+
.eslintrc*
117+
.prettierrc*
118+
nodemon.json
119+
120+
121+
# Exclude jsii outdir
122+
dist

‎integration_tests/cdk/app.py

Lines changed: 43 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,21 @@
1-
from aws_cdk import App, RemovalPolicy, Stack, aws_ec2, aws_iam, aws_rds
1+
from aws_cdk import (
2+
App,
3+
RemovalPolicy,
4+
Stack,
5+
aws_ec2,
6+
aws_iam,
7+
aws_rds,
8+
aws_s3,
9+
aws_s3_notifications,
10+
)
211
from config import AppConfig, build_app_config
312
from constructs import Construct
413
from eoapi_cdk import (
514
PgStacApiLambda,
615
PgStacDatabase,
716
StacIngestor,
17+
StacItemLoader,
18+
StactoolsItemGenerator,
819
TiPgApiLambda,
920
TitilerPgstacApiLambda,
1021
)
@@ -168,6 +179,37 @@ def __init__(
168179
},
169180
)
170181

182+
self.stac_item_loader = StacItemLoader(
183+
self,
184+
"stac-item-loader",
185+
pgstac_db=pgstac_db,
186+
batch_size=500,
187+
lambda_timeout_seconds=300,
188+
)
189+
190+
self.stac_item_generator = StactoolsItemGenerator(
191+
self,
192+
"stactools-item-generator",
193+
item_load_topic_arn=self.stac_item_loader.topic.topic_arn,
194+
)
195+
196+
self.stac_item_loader.topic.grant_publish(
197+
self.stac_item_generator.lambda_function
198+
)
199+
200+
stac_bucket = aws_s3.Bucket(
201+
self,
202+
"stac-item-bucket",
203+
)
204+
205+
stac_bucket.add_event_notification(
206+
aws_s3.EventType.OBJECT_CREATED,
207+
aws_s3_notifications.SnsDestination(self.stac_item_loader.topic),
208+
aws_s3.NotificationKeyFilter(suffix=".json"),
209+
)
210+
211+
stac_bucket.grant_read(self.stac_item_loader.lambda_function)
212+
171213

172214
app = App()
173215

‎integration_tests/cdk/package-lock.json

Lines changed: 40 additions & 39 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎integration_tests/cdk/package.json

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
{
2-
"name": "eoapi-template",
3-
"version": "0.1.0",
4-
"dependencies": {
5-
"aws-cdk": "2.130.0"
6-
}
2+
"name": "eoapi-template",
3+
"version": "0.1.0",
4+
"dependencies": {
5+
"aws-cdk": "2.1016.1"
76
}
7+
}

‎integration_tests/cdk/requirements.txt

Lines changed: 0 additions & 7 deletions
This file was deleted.

0 commit comments

Comments
 (0)