diff --git a/docs/mkdocs-base.yml b/docs/mkdocs-base.yml index d107534c..c608605f 100644 --- a/docs/mkdocs-base.yml +++ b/docs/mkdocs-base.yml @@ -53,5 +53,6 @@ nav: - Edit Record in Table: administrator/manual/django-change-record.md - Developers: - developer/index.md + - Architecture: developer/architecture.md - Data model: developer/data-model.md - Hyrax Configuration: developer/hyrax-config.md diff --git a/docs/src/developer/architecture.md b/docs/src/developer/architecture.md new file mode 100644 index 00000000..7d9b2708 --- /dev/null +++ b/docs/src/developer/architecture.md @@ -0,0 +1,120 @@ +--- +title: TomorrowNow - Global Access Platform (GAP) +summary: TomorrowNow - GAP is next-gen weather and climate data and services. + - Danang Tri Massandy + - Irwan Fathurrahman + - Ketan Bamniya +date: 2024-08-01 +some_url: https://github.com/kartoza/tomorrownow_gap +copyright: Copyright 2023, TomorrowNow +contact: danang@kartoza.com +license: This program is free software; you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation; either version 3 of the License, or (at your option) any later version. +--- + +# Architecture diagram + +![gap architecture diagram](./diagram/gap-architecture.png) + + +# Data flow diagram + +![gap data flow diagram](./diagram/gap-data-flow.png) + + +# Data ingestors + +## CBAM historical reanalysis + +![gap cbam ingestor](./diagram/gap-ingestor-cbam.png) + +Zarr structure for CBAM dataset: +``` + Size: 160GB +Dimensions: (date: 4384, lat: 1205, lon: 841) +Coordinates: + * date (date) datetime64[ns] 35kB 2012-01-01 ... ... + * lat (lat) float64 10kB -27.0 -26.97 ... 16.03 + * lon (lon) float64 7kB 21.78 21.82 ... 51.99 52.03 +Data variables: + average_solar_irradiance (date, lat, lon) float32 18GB dask.array + max_day_temperature (date, lat, lon) float32 18GB dask.array + max_night_temperature (date, lat, lon) float32 18GB dask.array + max_total_temperature (date, lat, lon) float32 18GB dask.array + min_day_temperature (date, lat, lon) float32 18GB dask.array + min_night_temperature (date, lat, lon) float32 18GB dask.array + min_total_temperature (date, lat, lon) float32 18GB dask.array + total_evapotranspiration_flux (date, lat, lon) float32 18GB dask.array + total_solar_irradiance (date, lat, lon) float32 18GB dask.array +``` + +## Salient seasonal forecast + +![gap salient ingestor](./diagram/gap-ingestor-salient.png) + +Zarr structure for Salient dataset: +``` + Size: 11GB +Dimensions: (ensemble: 50, forecast_day_idx: 91, forecast_date: 2, + lat: 174, lon: 123) +Coordinates: + analog (ensemble, forecast_day_idx) datetime64[ns] 36kB dask.array + * ensemble (ensemble) int64 400B 0 1 2 3 4 5 6 ... 44 45 46 47 48 49 + * forecast_date (forecast_date) datetime64[ns] 16B 2024-09-04 2024-09-09 + * forecast_day_idx (forecast_day_idx) int64 728B 0 1 2 3 4 ... 86 87 88 89 90 + * lat (lat) float64 1kB -27.12 -26.88 -26.62 ... 15.88 16.12 + * lon (lon) float64 984B 21.63 21.88 22.13 ... 51.63 51.88 52.13 +Data variables: (12/21) + precip (forecast_date, ensemble, forecast_day_idx, lat, lon) float32 779MB dask.array + precip_anom (forecast_date, ensemble, forecast_day_idx, lat, lon) float32 779MB dask.array + precip_clim (forecast_date, forecast_day_idx, lat, lon) float32 16MB dask.array + rh (forecast_date, ensemble, forecast_day_idx, lat, lon) float32 779MB dask.array + rh_anom (forecast_date, ensemble, forecast_day_idx, lat, lon) float32 779MB dask.array + rh_clim (forecast_date, forecast_day_idx, lat, lon) float32 16MB dask.array + ... ... + tsi (forecast_date, ensemble, forecast_day_idx, lat, lon) float32 779MB dask.array + tsi_anom (forecast_date, ensemble, forecast_day_idx, lat, lon) float32 779MB dask.array + tsi_clim (forecast_date, forecast_day_idx, lat, lon) float32 16MB dask.array + wspd (forecast_date, ensemble, forecast_day_idx, lat, lon) float32 779MB dask.array + wspd_anom (forecast_date, ensemble, forecast_day_idx, lat, lon) float32 779MB dask.array + wspd_clim (forecast_date, forecast_day_idx, lat, lon) float32 16MB dask.array +``` + +## Tomorrow.io short-term forecast + +![gap tomorrow.io ingestor](./diagram/gap-ingestor-tomorrowio.png) + +Zarr structure for Tomorrow.io dataset: +``` + Size: - GB +Dimensions: (forecast_date: 2, forecast_day_idx: 15, lat: 174, lon: 123) +Coordinates: + * forecast_date (forecast_date) datetime64[ns] 16B 2024-09-04 2024-09-05 + * forecast_day_idx (forecast_day_idx) int64 728B 0 1 2 3 4 ... 11 12 13 14 + * lat (lat) float64 1kB -27.12 -26.88 -26.62 ... 15.88 16.12 + * lon (lon) float64 984B 21.63 21.88 22.13 ... 51.63 51.88 52.13 +Data variables: (12/21) + total_rainfall (forecast_date, forecast_day_idx, lat, lon) float32 -MB dask.array + total_evapotranspiration_flux (forecast_date, forecast_day_idx, lat, lon) float32 -MB dask.array + max_total_temperature (forecast_date, forecast_day_idx, lat, lon) float32 -MB dask.array + min_total_temperature (forecast_date, forecast_day_idx, lat, lon) float32 -MB dask.array + precipitation_probability (forecast_date, forecast_day_idx, lat, lon) float32 -MB dask.array + humidity_maximum (forecast_date, forecast_day_idx, lat, lon) float32 -MB dask.array + humidity_minimum (forecast_date, forecast_day_idx, lat, lon) float32 -MB dask.array + wind_speed_avg (forecast_date, forecast_day_idx, lat, lon) float32 -MB dask.array +``` + +## Tahmo ground observations + +![gap tahmo ingestor](./diagram/gap-ingestor-tahmo.png) + +## Arable ground observations + +![gap arable ingestor](./diagram/gap-ingestor-arable.png) + + +# GAP Modules + +## SPW data flow diagram + +![gap spw](./diagram/gap-module-spw.png) + diff --git a/docs/src/developer/diagram/gap-architecture.drawio b/docs/src/developer/diagram/gap-architecture.drawio new file mode 100644 index 00000000..ac6555bc --- /dev/null +++ b/docs/src/developer/diagram/gap-architecture.drawio @@ -0,0 +1,1254 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/src/developer/diagram/gap-architecture.png b/docs/src/developer/diagram/gap-architecture.png new file mode 100644 index 00000000..ab2acd60 Binary files /dev/null and b/docs/src/developer/diagram/gap-architecture.png differ diff --git a/docs/src/developer/diagram/gap-data-flow.png b/docs/src/developer/diagram/gap-data-flow.png new file mode 100644 index 00000000..bb380bdf Binary files /dev/null and b/docs/src/developer/diagram/gap-data-flow.png differ diff --git a/docs/src/developer/diagram/gap-ingestor-arable.png b/docs/src/developer/diagram/gap-ingestor-arable.png new file mode 100644 index 00000000..582f8bef Binary files /dev/null and b/docs/src/developer/diagram/gap-ingestor-arable.png differ diff --git a/docs/src/developer/diagram/gap-ingestor-cbam.png b/docs/src/developer/diagram/gap-ingestor-cbam.png new file mode 100644 index 00000000..979f7b38 Binary files /dev/null and b/docs/src/developer/diagram/gap-ingestor-cbam.png differ diff --git a/docs/src/developer/diagram/gap-ingestor-salient.png b/docs/src/developer/diagram/gap-ingestor-salient.png new file mode 100644 index 00000000..874bdf48 Binary files /dev/null and b/docs/src/developer/diagram/gap-ingestor-salient.png differ diff --git a/docs/src/developer/diagram/gap-ingestor-tahmo.png b/docs/src/developer/diagram/gap-ingestor-tahmo.png new file mode 100644 index 00000000..9cc02f68 Binary files /dev/null and b/docs/src/developer/diagram/gap-ingestor-tahmo.png differ diff --git a/docs/src/developer/diagram/gap-ingestor-tomorrowio.png b/docs/src/developer/diagram/gap-ingestor-tomorrowio.png new file mode 100644 index 00000000..e5ea9abb Binary files /dev/null and b/docs/src/developer/diagram/gap-ingestor-tomorrowio.png differ diff --git a/docs/src/developer/diagram/gap-module-spw.png b/docs/src/developer/diagram/gap-module-spw.png new file mode 100644 index 00000000..99554d1f Binary files /dev/null and b/docs/src/developer/diagram/gap-module-spw.png differ