Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Validation Notebook #114

Draft
wants to merge 2 commits into
base: main
Choose a base branch
from
Draft

Validation Notebook #114

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0af34b3
Add validation notebook
Apr 8, 2024
287622e
Add stac-validator package
Apr 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions requirements.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,3 +4,4 @@ pre-commit
pystac[validation]
pytest
ruff
stac-validator
19 changes: 17 additions & 2 deletions requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
# This file is autogenerated by pip-compile with Python 3.11
# by the following command:
#
# pip-compile
# pip-compile requirements.in
#
appnope==0.1.3
# via ipython
Expand All @@ -18,12 +18,17 @@ black[jupyter]==23.10.0
# via -r requirements.in
build==1.0.3
# via pip-tools
certifi==2024.2.2
# via requests
cfgv==3.4.0
# via pre-commit
charset-normalizer==3.3.2
# via requests
click==8.1.7
# via
# black
# pip-tools
# stac-validator
decorator==5.1.1
# via ipython
distlib==0.3.7
Expand All @@ -34,14 +39,18 @@ filelock==3.12.4
# via virtualenv
identify==2.5.30
# via pre-commit
idna==3.6
# via requests
iniconfig==2.0.0
# via pytest
ipython==8.16.1
# via black
jedi==0.19.1
# via ipython
jsonschema==4.19.1
# via pystac
# via
# pystac
# stac-validator
jsonschema-specifications==2023.7.1
# via jsonschema
matplotlib-inline==0.1.6
Expand Down Expand Up @@ -95,6 +104,8 @@ referencing==0.30.2
# via
# jsonschema
# jsonschema-specifications
requests==2.31.0
# via stac-validator
rpds-py==0.10.6
# via
# jsonschema
Expand All @@ -105,6 +116,8 @@ six==1.16.0
# via
# asttokens
# python-dateutil
stac-validator==3.3.2
# via -r requirements.in
stack-data==0.6.3
# via ipython
tokenize-rt==5.2.0
Expand All @@ -113,6 +126,8 @@ traitlets==5.11.2
# via
# ipython
# matplotlib-inline
urllib3==2.2.1
# via requests
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I generated this file using pip-compile requirements.in though there are several more changes than I was expecting 🤔

virtualenv==20.24.5
# via pre-commit
wcwidth==0.2.8
Expand Down
251 changes: 251 additions & 0 deletions transformation-scripts/validate-updated-collections.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,251 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 30,
"metadata": {},
"outputs": [],
"source": [
"import boto3\n",
"import os\n",
"import glob\n",
"import shutil\n",
"\n",
"from stac_validator import stac_validator"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"AWS_ACCESS_KEY_ID = \"[CHANGE ME]\"\n",
"AWS_SECRET_ACCESS_KEY = \"[CHANGE ME]\"\n",
"AWS_SESSION_TOKEN = \"[CHANGE ME]\""
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
"s3_client = boto3.client(\n",
" \"s3\",\n",
" aws_access_key_id=AWS_ACCESS_KEY_ID,\n",
" aws_secret_access_key=AWS_SECRET_ACCESS_KEY,\n",
" aws_session_token=AWS_SESSION_TOKEN,\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 34,
"metadata": {},
"outputs": [],
"source": [
"# Define the directory path\n",
"local_dir = \"./collections-to-validate\""
]
},
{
"cell_type": "code",
"execution_count": 68,
"metadata": {},
"outputs": [],
"source": [
"def download_collections(bucket_name, s3_prefixes):\n",
" s3 = s3_client\n",
" s3_responses = []\n",
" json_keys = []\n",
"\n",
" for s3_prefix in s3_prefixes:\n",
" response = s3.list_objects_v2(Bucket=bucket_name, Prefix=s3_prefix)\n",
" s3_responses.append(response)\n",
"\n",
" for item in s3_responses:\n",
" for item in item[\"Contents\"]:\n",
" if item[\"Key\"].endswith(\".json\"):\n",
" json_keys.append(item[\"Key\"])\n",
"\n",
" # Create the directory if it doesn't exist\n",
" if not os.path.exists(local_dir):\n",
" os.makedirs(local_dir)\n",
" print(\"Directory created successfully at\", local_dir)\n",
" else:\n",
" print(\"Directory already exists at\", local_dir)\n",
"\n",
" # Download json files to local directory\n",
" for key in json_keys:\n",
" local_file_path = os.path.join(local_dir, os.path.basename(key))\n",
" s3.download_file(bucket_name, key, local_file_path)\n",
"\n",
"\n",
"def validate_collections():\n",
" json_files = glob.glob(os.path.join(local_dir, \"*.json\"))\n",
" for json_file in json_files:\n",
" print(f\"Validating file: {json_file}\")\n",
" stac = stac_validator.StacValidate(json_file)\n",
" stac.run()\n",
" valid_stac = stac.message[0][\"valid_stac\"]\n",
" print(valid_stac)\n",
" if not valid_stac:\n",
" print(f\"Invalid STAC. Error message {stac.message[0]['error_message']}\")\n",
"\n",
"\n",
"def cleanup():\n",
" shutil.rmtree(local_dir)"
]
},
{
"cell_type": "code",
"execution_count": 69,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Directory created successfully at ./collections-to-validate\n"
]
}
],
"source": [
"bucket_name = \"veda-data-store\"\n",
"s3_prefixes = [\"hlsl30-ej-reprocessed\", \"hlss30-ej-reprocessed\"]\n",
"download_collections(bucket_name, s3_prefixes)"
]
},
{
"cell_type": "code",
"execution_count": 70,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Validating file: ./collections-to-validate/HLS.L30.T19QHA.2017221T144403.v2.0_stac-ej-reprocessed.json\n",
"True\n",
"Validating file: ./collections-to-validate/HLS.S30.T15RYP.2021250T163901.v2.0_stac-ej-reprocessed.json\n",
"True\n",
"Validating file: ./collections-to-validate/HLS.S30.T15RYP.2021210T163901.v2.0_stac-ej-reprocessed.json\n",
"True\n",
"Validating file: ./collections-to-validate/HLS.S30.T15RYP.2021200T163901.v2.0_stac-ej-reprocessed.json\n",
"True\n",
"Validating file: ./collections-to-validate/HLS.L30.T19QHA.2017173T144347.v2.0_stac-ej-reprocessed.json\n",
"True\n",
"Validating file: ./collections-to-validate/HLS.S30.T15RYP.2021220T163901.v2.0_stac-ej-reprocessed.json\n",
"True\n",
"Validating file: ./collections-to-validate/HLS.S30.T19QHA.2017293T150709.v2.0_stac-ej-reprocessed.json\n",
"True\n",
"Validating file: ./collections-to-validate/HLS.L30.T19QHA.2017205T144356.v2.0_stac-ej-reprocessed.json\n",
"True\n",
"Validating file: ./collections-to-validate/HLS.S30.T15RYP.2021290T164311.v2.0_stac-ej-reprocessed.json\n",
"True\n",
"Validating file: ./collections-to-validate/HLS.L30.T15RYP.2021230T163215.v2.0_stac-ej-reprocessed.json\n",
"True\n",
"Validating file: ./collections-to-validate/HLS.S30.T19QHA.2017218T150721.v2.0_stac-ej-reprocessed.json\n",
"True\n",
"Validating file: ./collections-to-validate/HLS.S30.T19QHA.2017333T150709.v2.0_stac-ej-reprocessed.json\n",
"True\n",
"Validating file: ./collections-to-validate/HLS.S30.T15RYP.2021185T163839.v2.0_stac-ej-reprocessed.json\n",
"True\n",
"Validating file: ./collections-to-validate/HLS.S30.T19QHA.2017323T150709.v2.0_stac-ej-reprocessed.json\n",
"True\n",
"Validating file: ./collections-to-validate/HLS.L30.T15RYP.2021246T163220.v2.0_stac-ej-reprocessed.json\n",
"True\n",
"Validating file: ./collections-to-validate/HLS.S30.T15RYP.2021195T163839.v2.0_stac-ej-reprocessed.json\n",
"True\n",
"Validating file: ./collections-to-validate/HLS.L30.T15RYP.2021182T163159.v2.0_stac-ej-reprocessed.json\n",
"True\n",
"Validating file: ./collections-to-validate/HLS.S30.T15RYP.2021270T164051.v2.0_stac-ej-reprocessed.json\n",
"True\n",
"Validating file: ./collections-to-validate/HLS.L30.T19QHA.2017317T144417.v2.0_stac-ej-reprocessed.json\n",
"True\n",
"Validating file: ./collections-to-validate/HLS.S30.T19QHA.2017278T150721.v2.0_stac-ej-reprocessed.json\n",
"True\n",
"Validating file: ./collections-to-validate/HLS.L30.T15RYP.2021262T163223.v2.0_stac-ej-reprocessed.json\n",
"True\n",
"Validating file: ./collections-to-validate/HLS.S30.T15RYP.2021295T164339.v2.0_stac-ej-reprocessed.json\n",
"True\n",
"Validating file: ./collections-to-validate/HLS.S30.T19QHA.2017193T150719.v2.0_stac-ej-reprocessed.json\n",
"True\n",
"Validating file: ./collections-to-validate/HLS.L30.T19QHA.2017301T144420.v2.0_stac-ej-reprocessed.json\n",
"True\n",
"Validating file: ./collections-to-validate/HLS.L30.T15RYP.2021278T163229.v2.0_stac-ej-reprocessed.json\n",
"True\n",
"Validating file: ./collections-to-validate/HLS.L30.T19QHA.2017285T144419.v2.0_stac-ej-reprocessed.json\n",
"True\n",
"Validating file: ./collections-to-validate/HLS.L30.T15RYP.2021198T163201.v2.0_stac-ej-reprocessed.json\n",
"True\n",
"Validating file: ./collections-to-validate/HLS.L30.T19QHA.2017157T144341.v2.0_stac-ej-reprocessed.json\n",
"True\n",
"Validating file: ./collections-to-validate/HLS.S30.T15RYP.2021280T164201.v2.0_stac-ej-reprocessed.json\n",
"True\n",
"Validating file: ./collections-to-validate/HLS.S30.T19QHA.2017318T150721.v2.0_stac-ej-reprocessed.json\n",
"True\n",
"Validating file: ./collections-to-validate/HLS.S30.T15RYP.2021205T163839.v2.0_stac-ej-reprocessed.json\n",
"True\n",
"Validating file: ./collections-to-validate/HLS.S30.T15RYP.2021235T163839.v2.0_stac-ej-reprocessed.json\n",
"True\n",
"Validating file: ./collections-to-validate/HLS.S30.T15RYP.2021225T163839.v2.0_stac-ej-reprocessed.json\n",
"True\n",
"Validating file: ./collections-to-validate/HLS.S30.T15RYP.2021245T163839.v2.0_stac-ej-reprocessed.json\n",
"True\n",
"Validating file: ./collections-to-validate/HLS.S30.T15RYP.2021285T164139.v2.0_stac-ej-reprocessed.json\n",
"True\n",
"Validating file: ./collections-to-validate/HLS.S30.T15RYP.2021265T163909.v2.0_stac-ej-reprocessed.json\n",
"True\n",
"Validating file: ./collections-to-validate/HLS.L30.T15RYP.2021294T163232.v2.0_stac-ej-reprocessed.json\n",
"True\n",
"Validating file: ./collections-to-validate/HLS.S30.T15RYP.2021300T164411.v2.0_stac-ej-reprocessed.json\n",
"True\n",
"Validating file: ./collections-to-validate/HLS.L30.T19QHA.2017333T144411.v2.0_stac-ej-reprocessed.json\n",
"True\n",
"Validating file: ./collections-to-validate/HLS.L30.T19QHA.2017237T144407.v2.0_stac-ej-reprocessed.json\n",
"True\n",
"Validating file: ./collections-to-validate/HLS.L30.T19QHA.2017269T144414.v2.0_stac-ej-reprocessed.json\n",
"True\n",
"Validating file: ./collections-to-validate/HLS.S30.T19QHA.2017233T150719.v2.0_stac-ej-reprocessed.json\n",
"True\n"
]
}
],
"source": [
"validate_collections()"
]
},
{
"cell_type": "code",
"execution_count": 71,
"metadata": {},
"outputs": [],
"source": [
"cleanup()"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "venv",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.7"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
Loading