From 07d4678fdef8c5803074b4037518b778f2a6963c Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Thu, 19 Oct 2023 09:57:47 +0100 Subject: [PATCH 01/69] refactor: move scripts & josm to contrib dir --- contrib/README.md | 4 ++++ {josm => contrib/josm}/Dockerfile | 0 contrib/josm/README.md | 12 ++++++++++++ {josm => contrib/josm}/container-entrypoint.sh | 0 {josm => contrib/josm}/docker-compose.yml | 0 {josm => contrib/josm}/nginx-josm.conf | 0 {josm => contrib/josm}/novnc/Dockerfile | 0 {josm => contrib/josm}/novnc/index.html | 0 {josm => contrib/josm}/preferences.xml | 0 contrib/scripts/odk_fieldmap_original/README.md | 5 +++++ .../odk_fieldmap_original/parse_pip_error.py | 0 .../scripts}/odk_fieldmap_original/setup.sh | 0 .../scripts}/odk_fieldmap_original/utils/aoi2odk.py | 0 .../odk_fieldmap_original/utils/dl_submissions.py | 0 .../utils/example_odk_requests.py | 0 .../odk_fieldmap_original/utils/forms2server.py | 0 .../odk_fieldmap_original/utils/geo_utils.py | 0 .../odk_fieldmap_original/utils/odk_requests.py | 0 .../odk_fieldmap_original/utils/old_qr_stuff.py | 0 .../scripts}/odk_fieldmap_original/utils/overpass.py | 0 .../scripts}/odk_fieldmap_original/utils/renamer.py | 0 .../odk_fieldmap_original/utils/tasks2forms.py | 0 .../Examples_and_tests/building_clusters_of_5.sql | 0 .../Examples_and_tests/centroids.sql | 0 .../Examples_and_tests/clean_and_simplify.sql | 0 .../Examples_and_tests/clustering.sql | 0 .../concave_hull_tasks_from_clustered_buildings.sql | 0 .../Examples_and_tests/count_building_tags.sql | 0 ...erge_polygons_with_no_features_into_neighbors.sql | 0 .../Examples_and_tests/points_in_polygon.sql | 0 .../Examples_and_tests/polygonize.sql | 0 .../select_features_in_polygon.sql | 0 .../Examples_and_tests/split_aoi_by_osm_lines.sql | 0 .../Examples_and_tests/split_area_by_osm_lines.sql | 0 .../Examples_and_tests/task_splitting_optimized.sql | 0 .../postgis_snippets/Examples_and_tests/voronoi.sql | 0 contrib/scripts/postgis_snippets/README.md | 3 +++ .../import_geojson_as_postgis_with_jsonb.md | 0 .../scripts}/postgis_snippets/postgis_resources.md | 0 ...-split_01_split_AOI_by_existing_line_features.sql | 0 ...mtm-split_02_count_buildings_for_subsplitting.sql | 0 .../fmtm-split_03_cluster_buildings.sql | 0 ...04_create_polygons_around_clustered_buildings.sql | 0 .../task_splitting_for_osm_buildings.sql | 0 .../task_splitting/task_splitting_for_osm_roads.sql | 0 .../task_splitting/task_splitting_readme.md | 0 46 files changed, 24 insertions(+) create mode 100644 contrib/README.md rename {josm => contrib/josm}/Dockerfile (100%) create mode 100644 contrib/josm/README.md rename {josm => contrib/josm}/container-entrypoint.sh (100%) rename {josm => contrib/josm}/docker-compose.yml (100%) rename {josm => contrib/josm}/nginx-josm.conf (100%) rename {josm => contrib/josm}/novnc/Dockerfile (100%) rename {josm => contrib/josm}/novnc/index.html (100%) rename {josm => contrib/josm}/preferences.xml (100%) create mode 100644 contrib/scripts/odk_fieldmap_original/README.md rename {scripts => contrib/scripts}/odk_fieldmap_original/parse_pip_error.py (100%) rename {scripts => contrib/scripts}/odk_fieldmap_original/setup.sh (100%) rename {scripts => contrib/scripts}/odk_fieldmap_original/utils/aoi2odk.py (100%) rename {scripts => contrib/scripts}/odk_fieldmap_original/utils/dl_submissions.py (100%) rename {scripts => contrib/scripts}/odk_fieldmap_original/utils/example_odk_requests.py (100%) rename {scripts => contrib/scripts}/odk_fieldmap_original/utils/forms2server.py (100%) rename {scripts => contrib/scripts}/odk_fieldmap_original/utils/geo_utils.py (100%) rename {scripts => contrib/scripts}/odk_fieldmap_original/utils/odk_requests.py (100%) rename {scripts => contrib/scripts}/odk_fieldmap_original/utils/old_qr_stuff.py (100%) rename {scripts => contrib/scripts}/odk_fieldmap_original/utils/overpass.py (100%) rename {scripts => contrib/scripts}/odk_fieldmap_original/utils/renamer.py (100%) rename {scripts => contrib/scripts}/odk_fieldmap_original/utils/tasks2forms.py (100%) rename {scripts => contrib/scripts}/postgis_snippets/Examples_and_tests/building_clusters_of_5.sql (100%) rename {scripts => contrib/scripts}/postgis_snippets/Examples_and_tests/centroids.sql (100%) rename {scripts => contrib/scripts}/postgis_snippets/Examples_and_tests/clean_and_simplify.sql (100%) rename {scripts => contrib/scripts}/postgis_snippets/Examples_and_tests/clustering.sql (100%) rename {scripts => contrib/scripts}/postgis_snippets/Examples_and_tests/concave_hull_tasks_from_clustered_buildings.sql (100%) rename {scripts => contrib/scripts}/postgis_snippets/Examples_and_tests/count_building_tags.sql (100%) rename {scripts => contrib/scripts}/postgis_snippets/Examples_and_tests/merge_polygons_with_no_features_into_neighbors.sql (100%) rename {scripts => contrib/scripts}/postgis_snippets/Examples_and_tests/points_in_polygon.sql (100%) rename {scripts => contrib/scripts}/postgis_snippets/Examples_and_tests/polygonize.sql (100%) rename {scripts => contrib/scripts}/postgis_snippets/Examples_and_tests/select_features_in_polygon.sql (100%) rename {scripts => contrib/scripts}/postgis_snippets/Examples_and_tests/split_aoi_by_osm_lines.sql (100%) rename {scripts => contrib/scripts}/postgis_snippets/Examples_and_tests/split_area_by_osm_lines.sql (100%) rename {scripts => contrib/scripts}/postgis_snippets/Examples_and_tests/task_splitting_optimized.sql (100%) rename {scripts => contrib/scripts}/postgis_snippets/Examples_and_tests/voronoi.sql (100%) create mode 100644 contrib/scripts/postgis_snippets/README.md rename {scripts => contrib/scripts}/postgis_snippets/import_geojson_as_postgis_with_jsonb.md (100%) rename {scripts => contrib/scripts}/postgis_snippets/postgis_resources.md (100%) rename {scripts => contrib/scripts}/postgis_snippets/task_splitting/fmtm_task_splitting_for_buildings/fmtm-split_01_split_AOI_by_existing_line_features.sql (100%) rename {scripts => contrib/scripts}/postgis_snippets/task_splitting/fmtm_task_splitting_for_buildings/fmtm-split_02_count_buildings_for_subsplitting.sql (100%) rename {scripts => contrib/scripts}/postgis_snippets/task_splitting/fmtm_task_splitting_for_buildings/fmtm-split_03_cluster_buildings.sql (100%) rename {scripts => contrib/scripts}/postgis_snippets/task_splitting/fmtm_task_splitting_for_buildings/fmtm-split_04_create_polygons_around_clustered_buildings.sql (100%) rename {scripts => contrib/scripts}/postgis_snippets/task_splitting/task_splitting_for_osm_buildings.sql (100%) rename {scripts => contrib/scripts}/postgis_snippets/task_splitting/task_splitting_for_osm_roads.sql (100%) rename {scripts => contrib/scripts}/postgis_snippets/task_splitting/task_splitting_readme.md (100%) diff --git a/contrib/README.md b/contrib/README.md new file mode 100644 index 0000000000..bb52ce2cf8 --- /dev/null +++ b/contrib/README.md @@ -0,0 +1,4 @@ +# Contrib + +- Additional content not related to the main functionality of FMTM. +- External contributions not merged into core repo. diff --git a/josm/Dockerfile b/contrib/josm/Dockerfile similarity index 100% rename from josm/Dockerfile rename to contrib/josm/Dockerfile diff --git a/contrib/josm/README.md b/contrib/josm/README.md new file mode 100644 index 0000000000..e5cde779c4 --- /dev/null +++ b/contrib/josm/README.md @@ -0,0 +1,12 @@ +# JOSM + +- Container images for running JOSM in a remote container. + +## Use Cases + +- Control JOSM remotely via it's API. +- Accessing JOSM remotely via a web browser, without having to install it. + +## Usage + +See `docs/dev/Backend.md`. diff --git a/josm/container-entrypoint.sh b/contrib/josm/container-entrypoint.sh similarity index 100% rename from josm/container-entrypoint.sh rename to contrib/josm/container-entrypoint.sh diff --git a/josm/docker-compose.yml b/contrib/josm/docker-compose.yml similarity index 100% rename from josm/docker-compose.yml rename to contrib/josm/docker-compose.yml diff --git a/josm/nginx-josm.conf b/contrib/josm/nginx-josm.conf similarity index 100% rename from josm/nginx-josm.conf rename to contrib/josm/nginx-josm.conf diff --git a/josm/novnc/Dockerfile b/contrib/josm/novnc/Dockerfile similarity index 100% rename from josm/novnc/Dockerfile rename to contrib/josm/novnc/Dockerfile diff --git a/josm/novnc/index.html b/contrib/josm/novnc/index.html similarity index 100% rename from josm/novnc/index.html rename to contrib/josm/novnc/index.html diff --git a/josm/preferences.xml b/contrib/josm/preferences.xml similarity index 100% rename from josm/preferences.xml rename to contrib/josm/preferences.xml diff --git a/contrib/scripts/odk_fieldmap_original/README.md b/contrib/scripts/odk_fieldmap_original/README.md new file mode 100644 index 0000000000..659819b805 --- /dev/null +++ b/contrib/scripts/odk_fieldmap_original/README.md @@ -0,0 +1,5 @@ +# ODK Fieldmap + +Various scripts for interacting with ODK and +FMTM to automate tasks & produce useful outputs for +GIS visualisation. diff --git a/scripts/odk_fieldmap_original/parse_pip_error.py b/contrib/scripts/odk_fieldmap_original/parse_pip_error.py similarity index 100% rename from scripts/odk_fieldmap_original/parse_pip_error.py rename to contrib/scripts/odk_fieldmap_original/parse_pip_error.py diff --git a/scripts/odk_fieldmap_original/setup.sh b/contrib/scripts/odk_fieldmap_original/setup.sh similarity index 100% rename from scripts/odk_fieldmap_original/setup.sh rename to contrib/scripts/odk_fieldmap_original/setup.sh diff --git a/scripts/odk_fieldmap_original/utils/aoi2odk.py b/contrib/scripts/odk_fieldmap_original/utils/aoi2odk.py similarity index 100% rename from scripts/odk_fieldmap_original/utils/aoi2odk.py rename to contrib/scripts/odk_fieldmap_original/utils/aoi2odk.py diff --git a/scripts/odk_fieldmap_original/utils/dl_submissions.py b/contrib/scripts/odk_fieldmap_original/utils/dl_submissions.py similarity index 100% rename from scripts/odk_fieldmap_original/utils/dl_submissions.py rename to contrib/scripts/odk_fieldmap_original/utils/dl_submissions.py diff --git a/scripts/odk_fieldmap_original/utils/example_odk_requests.py b/contrib/scripts/odk_fieldmap_original/utils/example_odk_requests.py similarity index 100% rename from scripts/odk_fieldmap_original/utils/example_odk_requests.py rename to contrib/scripts/odk_fieldmap_original/utils/example_odk_requests.py diff --git a/scripts/odk_fieldmap_original/utils/forms2server.py b/contrib/scripts/odk_fieldmap_original/utils/forms2server.py similarity index 100% rename from scripts/odk_fieldmap_original/utils/forms2server.py rename to contrib/scripts/odk_fieldmap_original/utils/forms2server.py diff --git a/scripts/odk_fieldmap_original/utils/geo_utils.py b/contrib/scripts/odk_fieldmap_original/utils/geo_utils.py similarity index 100% rename from scripts/odk_fieldmap_original/utils/geo_utils.py rename to contrib/scripts/odk_fieldmap_original/utils/geo_utils.py diff --git a/scripts/odk_fieldmap_original/utils/odk_requests.py b/contrib/scripts/odk_fieldmap_original/utils/odk_requests.py similarity index 100% rename from scripts/odk_fieldmap_original/utils/odk_requests.py rename to contrib/scripts/odk_fieldmap_original/utils/odk_requests.py diff --git a/scripts/odk_fieldmap_original/utils/old_qr_stuff.py b/contrib/scripts/odk_fieldmap_original/utils/old_qr_stuff.py similarity index 100% rename from scripts/odk_fieldmap_original/utils/old_qr_stuff.py rename to contrib/scripts/odk_fieldmap_original/utils/old_qr_stuff.py diff --git a/scripts/odk_fieldmap_original/utils/overpass.py b/contrib/scripts/odk_fieldmap_original/utils/overpass.py similarity index 100% rename from scripts/odk_fieldmap_original/utils/overpass.py rename to contrib/scripts/odk_fieldmap_original/utils/overpass.py diff --git a/scripts/odk_fieldmap_original/utils/renamer.py b/contrib/scripts/odk_fieldmap_original/utils/renamer.py similarity index 100% rename from scripts/odk_fieldmap_original/utils/renamer.py rename to contrib/scripts/odk_fieldmap_original/utils/renamer.py diff --git a/scripts/odk_fieldmap_original/utils/tasks2forms.py b/contrib/scripts/odk_fieldmap_original/utils/tasks2forms.py similarity index 100% rename from scripts/odk_fieldmap_original/utils/tasks2forms.py rename to contrib/scripts/odk_fieldmap_original/utils/tasks2forms.py diff --git a/scripts/postgis_snippets/Examples_and_tests/building_clusters_of_5.sql b/contrib/scripts/postgis_snippets/Examples_and_tests/building_clusters_of_5.sql similarity index 100% rename from scripts/postgis_snippets/Examples_and_tests/building_clusters_of_5.sql rename to contrib/scripts/postgis_snippets/Examples_and_tests/building_clusters_of_5.sql diff --git a/scripts/postgis_snippets/Examples_and_tests/centroids.sql b/contrib/scripts/postgis_snippets/Examples_and_tests/centroids.sql similarity index 100% rename from scripts/postgis_snippets/Examples_and_tests/centroids.sql rename to contrib/scripts/postgis_snippets/Examples_and_tests/centroids.sql diff --git a/scripts/postgis_snippets/Examples_and_tests/clean_and_simplify.sql b/contrib/scripts/postgis_snippets/Examples_and_tests/clean_and_simplify.sql similarity index 100% rename from scripts/postgis_snippets/Examples_and_tests/clean_and_simplify.sql rename to contrib/scripts/postgis_snippets/Examples_and_tests/clean_and_simplify.sql diff --git a/scripts/postgis_snippets/Examples_and_tests/clustering.sql b/contrib/scripts/postgis_snippets/Examples_and_tests/clustering.sql similarity index 100% rename from scripts/postgis_snippets/Examples_and_tests/clustering.sql rename to contrib/scripts/postgis_snippets/Examples_and_tests/clustering.sql diff --git a/scripts/postgis_snippets/Examples_and_tests/concave_hull_tasks_from_clustered_buildings.sql b/contrib/scripts/postgis_snippets/Examples_and_tests/concave_hull_tasks_from_clustered_buildings.sql similarity index 100% rename from scripts/postgis_snippets/Examples_and_tests/concave_hull_tasks_from_clustered_buildings.sql rename to contrib/scripts/postgis_snippets/Examples_and_tests/concave_hull_tasks_from_clustered_buildings.sql diff --git a/scripts/postgis_snippets/Examples_and_tests/count_building_tags.sql b/contrib/scripts/postgis_snippets/Examples_and_tests/count_building_tags.sql similarity index 100% rename from scripts/postgis_snippets/Examples_and_tests/count_building_tags.sql rename to contrib/scripts/postgis_snippets/Examples_and_tests/count_building_tags.sql diff --git a/scripts/postgis_snippets/Examples_and_tests/merge_polygons_with_no_features_into_neighbors.sql b/contrib/scripts/postgis_snippets/Examples_and_tests/merge_polygons_with_no_features_into_neighbors.sql similarity index 100% rename from scripts/postgis_snippets/Examples_and_tests/merge_polygons_with_no_features_into_neighbors.sql rename to contrib/scripts/postgis_snippets/Examples_and_tests/merge_polygons_with_no_features_into_neighbors.sql diff --git a/scripts/postgis_snippets/Examples_and_tests/points_in_polygon.sql b/contrib/scripts/postgis_snippets/Examples_and_tests/points_in_polygon.sql similarity index 100% rename from scripts/postgis_snippets/Examples_and_tests/points_in_polygon.sql rename to contrib/scripts/postgis_snippets/Examples_and_tests/points_in_polygon.sql diff --git a/scripts/postgis_snippets/Examples_and_tests/polygonize.sql b/contrib/scripts/postgis_snippets/Examples_and_tests/polygonize.sql similarity index 100% rename from scripts/postgis_snippets/Examples_and_tests/polygonize.sql rename to contrib/scripts/postgis_snippets/Examples_and_tests/polygonize.sql diff --git a/scripts/postgis_snippets/Examples_and_tests/select_features_in_polygon.sql b/contrib/scripts/postgis_snippets/Examples_and_tests/select_features_in_polygon.sql similarity index 100% rename from scripts/postgis_snippets/Examples_and_tests/select_features_in_polygon.sql rename to contrib/scripts/postgis_snippets/Examples_and_tests/select_features_in_polygon.sql diff --git a/scripts/postgis_snippets/Examples_and_tests/split_aoi_by_osm_lines.sql b/contrib/scripts/postgis_snippets/Examples_and_tests/split_aoi_by_osm_lines.sql similarity index 100% rename from scripts/postgis_snippets/Examples_and_tests/split_aoi_by_osm_lines.sql rename to contrib/scripts/postgis_snippets/Examples_and_tests/split_aoi_by_osm_lines.sql diff --git a/scripts/postgis_snippets/Examples_and_tests/split_area_by_osm_lines.sql b/contrib/scripts/postgis_snippets/Examples_and_tests/split_area_by_osm_lines.sql similarity index 100% rename from scripts/postgis_snippets/Examples_and_tests/split_area_by_osm_lines.sql rename to contrib/scripts/postgis_snippets/Examples_and_tests/split_area_by_osm_lines.sql diff --git a/scripts/postgis_snippets/Examples_and_tests/task_splitting_optimized.sql b/contrib/scripts/postgis_snippets/Examples_and_tests/task_splitting_optimized.sql similarity index 100% rename from scripts/postgis_snippets/Examples_and_tests/task_splitting_optimized.sql rename to contrib/scripts/postgis_snippets/Examples_and_tests/task_splitting_optimized.sql diff --git a/scripts/postgis_snippets/Examples_and_tests/voronoi.sql b/contrib/scripts/postgis_snippets/Examples_and_tests/voronoi.sql similarity index 100% rename from scripts/postgis_snippets/Examples_and_tests/voronoi.sql rename to contrib/scripts/postgis_snippets/Examples_and_tests/voronoi.sql diff --git a/contrib/scripts/postgis_snippets/README.md b/contrib/scripts/postgis_snippets/README.md new file mode 100644 index 0000000000..bd0339bf3f --- /dev/null +++ b/contrib/scripts/postgis_snippets/README.md @@ -0,0 +1,3 @@ +# PostGIS Snippets + +Various useful PostGIS SQL commands contributed by @ivangayton. diff --git a/scripts/postgis_snippets/import_geojson_as_postgis_with_jsonb.md b/contrib/scripts/postgis_snippets/import_geojson_as_postgis_with_jsonb.md similarity index 100% rename from scripts/postgis_snippets/import_geojson_as_postgis_with_jsonb.md rename to contrib/scripts/postgis_snippets/import_geojson_as_postgis_with_jsonb.md diff --git a/scripts/postgis_snippets/postgis_resources.md b/contrib/scripts/postgis_snippets/postgis_resources.md similarity index 100% rename from scripts/postgis_snippets/postgis_resources.md rename to contrib/scripts/postgis_snippets/postgis_resources.md diff --git a/scripts/postgis_snippets/task_splitting/fmtm_task_splitting_for_buildings/fmtm-split_01_split_AOI_by_existing_line_features.sql b/contrib/scripts/postgis_snippets/task_splitting/fmtm_task_splitting_for_buildings/fmtm-split_01_split_AOI_by_existing_line_features.sql similarity index 100% rename from scripts/postgis_snippets/task_splitting/fmtm_task_splitting_for_buildings/fmtm-split_01_split_AOI_by_existing_line_features.sql rename to contrib/scripts/postgis_snippets/task_splitting/fmtm_task_splitting_for_buildings/fmtm-split_01_split_AOI_by_existing_line_features.sql diff --git a/scripts/postgis_snippets/task_splitting/fmtm_task_splitting_for_buildings/fmtm-split_02_count_buildings_for_subsplitting.sql b/contrib/scripts/postgis_snippets/task_splitting/fmtm_task_splitting_for_buildings/fmtm-split_02_count_buildings_for_subsplitting.sql similarity index 100% rename from scripts/postgis_snippets/task_splitting/fmtm_task_splitting_for_buildings/fmtm-split_02_count_buildings_for_subsplitting.sql rename to contrib/scripts/postgis_snippets/task_splitting/fmtm_task_splitting_for_buildings/fmtm-split_02_count_buildings_for_subsplitting.sql diff --git a/scripts/postgis_snippets/task_splitting/fmtm_task_splitting_for_buildings/fmtm-split_03_cluster_buildings.sql b/contrib/scripts/postgis_snippets/task_splitting/fmtm_task_splitting_for_buildings/fmtm-split_03_cluster_buildings.sql similarity index 100% rename from scripts/postgis_snippets/task_splitting/fmtm_task_splitting_for_buildings/fmtm-split_03_cluster_buildings.sql rename to contrib/scripts/postgis_snippets/task_splitting/fmtm_task_splitting_for_buildings/fmtm-split_03_cluster_buildings.sql diff --git a/scripts/postgis_snippets/task_splitting/fmtm_task_splitting_for_buildings/fmtm-split_04_create_polygons_around_clustered_buildings.sql b/contrib/scripts/postgis_snippets/task_splitting/fmtm_task_splitting_for_buildings/fmtm-split_04_create_polygons_around_clustered_buildings.sql similarity index 100% rename from scripts/postgis_snippets/task_splitting/fmtm_task_splitting_for_buildings/fmtm-split_04_create_polygons_around_clustered_buildings.sql rename to contrib/scripts/postgis_snippets/task_splitting/fmtm_task_splitting_for_buildings/fmtm-split_04_create_polygons_around_clustered_buildings.sql diff --git a/scripts/postgis_snippets/task_splitting/task_splitting_for_osm_buildings.sql b/contrib/scripts/postgis_snippets/task_splitting/task_splitting_for_osm_buildings.sql similarity index 100% rename from scripts/postgis_snippets/task_splitting/task_splitting_for_osm_buildings.sql rename to contrib/scripts/postgis_snippets/task_splitting/task_splitting_for_osm_buildings.sql diff --git a/scripts/postgis_snippets/task_splitting/task_splitting_for_osm_roads.sql b/contrib/scripts/postgis_snippets/task_splitting/task_splitting_for_osm_roads.sql similarity index 100% rename from scripts/postgis_snippets/task_splitting/task_splitting_for_osm_roads.sql rename to contrib/scripts/postgis_snippets/task_splitting/task_splitting_for_osm_roads.sql diff --git a/scripts/postgis_snippets/task_splitting/task_splitting_readme.md b/contrib/scripts/postgis_snippets/task_splitting/task_splitting_readme.md similarity index 100% rename from scripts/postgis_snippets/task_splitting/task_splitting_readme.md rename to contrib/scripts/postgis_snippets/task_splitting/task_splitting_readme.md From b263ef4b3336a6f6931ee1f7123fc4d6b92b753d Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Thu, 19 Oct 2023 09:58:13 +0100 Subject: [PATCH 02/69] ci: add contrib label to labeler.yml --- .github/labeler.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/labeler.yml b/.github/labeler.yml index 90d3afaeeb..3407c15cf5 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -9,7 +9,6 @@ - "**/Dockerfile" - "**/*.dockerfile" - "**/*entrypoint.sh" - - "josm/**/*" "migration": - "src/backend/migrations/**/*" "documentation": @@ -20,3 +19,5 @@ - "INSTALL.md" "ODK": - "odkcentral/**/*" +"contrib": + - "contrib/**/*" From e316941c6b320de6f7e008c26785fd968fc53be8 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Thu, 19 Oct 2023 12:02:21 +0100 Subject: [PATCH 03/69] ci: separate frontend and backend tests during pr --- .github/workflows/pr_test_backend.yml | 20 ++++++++++++++++++++ .github/workflows/pr_test_frontend.yml | 17 +++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 .github/workflows/pr_test_backend.yml create mode 100644 .github/workflows/pr_test_frontend.yml diff --git a/.github/workflows/pr_test_backend.yml b/.github/workflows/pr_test_backend.yml new file mode 100644 index 0000000000..e8a5cbb277 --- /dev/null +++ b/.github/workflows/pr_test_backend.yml @@ -0,0 +1,20 @@ +name: PR Tests + +on: + pull_request: + branches: + - main + - staging + - development + # Workflow is triggered only if src/backend changes + paths: + - src/backend/** + # Allow manual trigger (workflow_dispatch) + workflow_dispatch: + +jobs: + pytest: + uses: ./.github/workflows/r-pytest.yml + with: + image_tag: ci-${{ github.base_ref }} + secrets: inherit diff --git a/.github/workflows/pr_test_frontend.yml b/.github/workflows/pr_test_frontend.yml new file mode 100644 index 0000000000..43c2d7b858 --- /dev/null +++ b/.github/workflows/pr_test_frontend.yml @@ -0,0 +1,17 @@ +name: PR Tests Frontend + +on: + pull_request: + branches: + - main + - staging + - development + # Workflow is triggered only if src/frontend changes + paths: + - src/frontend/** + # Allow manual trigger (workflow_dispatch) + workflow_dispatch: + +jobs: + frontend-tests: + uses: ./.github/workflows/r-frontend_tests.yml From de429452ebd4eadb47c1cc59c812f4a7ec05d659 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Thu, 19 Oct 2023 12:02:49 +0100 Subject: [PATCH 04/69] ci: remove extract-vars (use auto-tagging) --- .github/workflows/build_and_deploy.yml | 43 ++++++--------------- .github/workflows/build_ci_img.yml | 9 ++--- .github/workflows/pr_test.yml | 23 ----------- .github/workflows/r-extract_vars.yml | 53 -------------------------- 4 files changed, 14 insertions(+), 114 deletions(-) delete mode 100644 .github/workflows/pr_test.yml delete mode 100644 .github/workflows/r-extract_vars.yml diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml index 3e1f0a2247..ffab5c38d1 100644 --- a/.github/workflows/build_and_deploy.yml +++ b/.github/workflows/build_and_deploy.yml @@ -1,3 +1,5 @@ +# Workflow for CI/CD of branches, build and auto-deploy + name: Build and Deploy on: @@ -23,44 +25,31 @@ jobs: frontend-tests: uses: ./.github/workflows/r-frontend_tests.yml - extract-vars: - needs: - - pytest - - frontend-tests - uses: ./.github/workflows/r-extract_vars.yml - with: - environment: ${{ github.ref_name }} - backend-build: uses: hotosm/gh-workflows/.github/workflows/image_build.yml@main - needs: [extract-vars] with: context: src/backend build_target: prod - image_tags: | - "ghcr.io/hotosm/fmtm/backend:${{ needs.extract-vars.outputs.api_version }}-${{ github.ref_name }}" - "ghcr.io/hotosm/fmtm/backend:latest" + image_name: ghcr.io/${{ github.repository }}/backend build_args: | - APP_VERSION=${{ needs.extract-vars.outputs.api_version }} + APP_VERSION=${{ github.ref_name }} + COMMIT_REF=${{ github.sha }} frontend-main-build: uses: hotosm/gh-workflows/.github/workflows/image_build.yml@main - needs: [extract-vars] with: context: src/frontend dockerfile: prod.dockerfile build_target: prod - image_tags: | - "ghcr.io/hotosm/fmtm/frontend:${{ needs.extract-vars.outputs.frontend_main_version }}-${{ github.ref_name }}" - "ghcr.io/hotosm/fmtm/frontend:latest" + image_name: ghcr.io/${{ github.repository }}/frontend build_args: | - APP_VERSION=${{ needs.extract-vars.outputs.frontend_main_version }} - VITE_API_URL=${{ needs.extract-vars.outputs.api_url }} + APP_VERSION=${{ github.ref_name }} + COMMIT_REF=${{ github.sha }} + VITE_API_URL=${{ vars.URL_SCHEME }}://${{ vars.API_URL }}" smoke-test-backend: runs-on: ubuntu-latest needs: - - extract-vars - backend-build environment: name: ${{ github.ref_name }} @@ -72,13 +61,9 @@ jobs: - name: Environment to .env env: GIT_BRANCH: ${{ github.ref_name }} - API_VERSION: ${{ needs.extract-vars.outputs.api_version }} - FRONTEND_MAIN_VERSION: ${{ needs.extract-vars.outputs.frontend_main_version }} run: | echo "${{ secrets.DOTENV }}" > .env echo "GIT_BRANCH=${GIT_BRANCH}" >> .env - echo "API_VERSION=${API_VERSION}" >> .env - echo "FRONTEND_MAIN_VERSION=${FRONTEND_MAIN_VERSION}" >> .env - name: Backend smoke test run: | @@ -95,7 +80,7 @@ jobs: # -e POSTGRES_DB=fmtm \ # "postgis/postgis:14-3.3-alpine" - # docker pull "ghcr.io/hotosm/fmtm/backend:${API_VERSION}-${GIT_BRANCH}" + # docker pull "ghcr.io/hotosm/fmtm/backend:${GIT_BRANCH}" # docker run --rm -d \ # --network=fmtm \ # -p 8080:8080 \ @@ -104,7 +89,7 @@ jobs: # -e OSM_SECRET_KEY="test" \ # -e S3_ACCESS_KEY="fmtm" \ # -e S3_SECRET_KEY="somelongpassword" \ - # "ghcr.io/hotosm/fmtm/backend:${API_VERSION}-${GIT_BRANCH}" + # "ghcr.io/hotosm/fmtm/backend:${GIT_BRANCH}" # # First wait 10 seconds for API # sleep 10 @@ -117,7 +102,6 @@ jobs: smoke-test-frontend: runs-on: ubuntu-latest needs: - - extract-vars - frontend-main-build environment: name: ${{ github.ref_name }} @@ -132,7 +116,6 @@ jobs: deploy-containers: runs-on: ubuntu-latest needs: - - extract-vars - smoke-test-backend - smoke-test-frontend environment: @@ -145,13 +128,9 @@ jobs: - name: Environment to .env env: GIT_BRANCH: ${{ github.ref_name }} - API_VERSION: ${{ needs.extract-vars.outputs.api_version }} - FRONTEND_MAIN_VERSION: ${{ needs.extract-vars.outputs.frontend_main_version }} run: | echo "${{ secrets.DOTENV }}" > .env echo "GIT_BRANCH=${GIT_BRANCH}" >> .env - echo "API_VERSION=${API_VERSION}" >> .env - echo "FRONTEND_MAIN_VERSION=${FRONTEND_MAIN_VERSION}" >> .env - uses: webfactory/ssh-agent@v0.7.0 with: diff --git a/.github/workflows/build_ci_img.yml b/.github/workflows/build_ci_img.yml index cae03d55a9..0b4d2fde0d 100644 --- a/.github/workflows/build_ci_img.yml +++ b/.github/workflows/build_ci_img.yml @@ -15,17 +15,14 @@ on: workflow_dispatch: jobs: - extract-vars: - uses: ./.github/workflows/r-extract_vars.yml - backend-ci-build: uses: hotosm/gh-workflows/.github/workflows/image_build.yml@main - needs: [extract-vars] with: context: src/backend build_target: ci + image_name: ghcr.io/${{ github.repository }}/backend image_tags: | - "ghcr.io/hotosm/fmtm/backend:${{ needs.extract-vars.outputs.api_version }}-ci-${{ github.ref_name }}" "ghcr.io/hotosm/fmtm/backend:ci-${{ github.ref_name }}" build_args: | - APP_VERSION=${{ needs.extract-vars.outputs.api_version }} + APP_VERSION=${{ github.ref_name }} + COMMIT_REF=${{ github.sha }} diff --git a/.github/workflows/pr_test.yml b/.github/workflows/pr_test.yml deleted file mode 100644 index 420287420c..0000000000 --- a/.github/workflows/pr_test.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: PR Tests - -on: - pull_request: - branches: - - main - - staging - - development - # Workflow is triggered only if src changes - paths: - - src/** - # Allow manual trigger (workflow_dispatch) - workflow_dispatch: - -jobs: - pytest: - uses: ./.github/workflows/r-pytest.yml - with: - image_tag: ci-${{ github.base_ref }} - secrets: inherit - - frontend-tests: - uses: ./.github/workflows/r-frontend_tests.yml diff --git a/.github/workflows/r-extract_vars.yml b/.github/workflows/r-extract_vars.yml deleted file mode 100644 index e4151ae9fd..0000000000 --- a/.github/workflows/r-extract_vars.yml +++ /dev/null @@ -1,53 +0,0 @@ -name: Extract Project Variables - -on: - workflow_call: - inputs: - environment: - description: "The GitHub environment to extract vars from." - required: false - type: string - default: "" - outputs: - api_version: - description: "Backend API Version." - value: ${{ jobs.extract-vars.outputs.api_version }} - frontend_main_version: - description: "Frontend Version." - value: ${{ jobs.extract-vars.outputs.frontend_main_version }} - api_url: - description: "URL to access the backend API." - value: ${{ jobs.extract-vars.outputs.api_url }} - -jobs: - extract-vars: - runs-on: ubuntu-latest - environment: ${{ inputs.environment }} - outputs: - api_version: ${{ steps.extract_api_version.outputs.api_version }} - frontend_main_version: ${{ steps.extract_frontend_version.outputs.frontend_main_version }} - api_url: ${{ steps.get_env_vars.outputs.api_url }} - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Extract api version - id: extract_api_version - working-directory: src/backend - run: | - API_VERSION=$(python -c 'from app.__version__ import __version__; print(__version__)') - echo "api_version=${API_VERSION}" >> $GITHUB_OUTPUT - - - name: Extract frontend versions - id: extract_frontend_version - working-directory: src/frontend - run: | - FRONTEND_MAIN_VERSION=$(jq -r '.version' package.json) - echo "frontend_main_version=${FRONTEND_MAIN_VERSION}" >> $GITHUB_OUTPUT - - - name: Get environment vars - id: get_env_vars - run: | - echo "api_url: ${{ vars.URL_SCHEME }}://${{ vars.API_URL }}" - echo "api_url=${{ vars.URL_SCHEME }}://${{ vars.API_URL }}" >> $GITHUB_OUTPUT From 0fc06ee64159ab0dba0039c793c768a52c5660bb Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Thu, 19 Oct 2023 12:03:14 +0100 Subject: [PATCH 05/69] build: update dockerfiles and compose to use COMMIT_REFs --- docker-compose.development.yml | 11 +++-------- docker-compose.main.yml | 11 +++-------- docker-compose.noodk.yml | 4 ++-- src/backend/Dockerfile | 2 ++ src/frontend/prod.dockerfile | 2 ++ 5 files changed, 12 insertions(+), 18 deletions(-) diff --git a/docker-compose.development.yml b/docker-compose.development.yml index a05cfcf5c4..9a80f198c0 100644 --- a/docker-compose.development.yml +++ b/docker-compose.development.yml @@ -79,12 +79,7 @@ services: restart: "unless-stopped" api: - image: "ghcr.io/hotosm/fmtm/backend:${API_VERSION}-${GIT_BRANCH}" - build: - context: src/backend - target: prod - args: - APP_VERSION: ${API_VERSION} + image: "ghcr.io/hotosm/fmtm/backend:${GIT_BRANCH}" container_name: fmtm_api volumes: - fmtm_logs:/opt/logs @@ -108,7 +103,7 @@ services: - "traefik.http.routers.api.service=api-svc" migrations: - image: "ghcr.io/hotosm/fmtm/backend:${API_VERSION}-${GIT_BRANCH}" + image: "ghcr.io/hotosm/fmtm/backend:${GIT_BRANCH}" container_name: fmtm_migrations depends_on: - fmtm-db @@ -120,7 +115,7 @@ services: restart: "on-failure:3" ui: - image: "ghcr.io/hotosm/fmtm/frontend:${FRONTEND_MAIN_VERSION}-${GIT_BRANCH}" + image: "ghcr.io/hotosm/fmtm/frontend:${GIT_BRANCH}" build: context: src/frontend dockerfile: prod.dockerfile diff --git a/docker-compose.main.yml b/docker-compose.main.yml index caec3d5f85..49d47dd797 100644 --- a/docker-compose.main.yml +++ b/docker-compose.main.yml @@ -78,12 +78,7 @@ services: restart: "unless-stopped" api: - image: "ghcr.io/hotosm/fmtm/backend:${API_VERSION}-${GIT_BRANCH}" - build: - context: src/backend - target: prod - args: - APP_VERSION: ${API_VERSION} + image: "ghcr.io/hotosm/fmtm/backend:${GIT_BRANCH}" container_name: fmtm_api volumes: - fmtm_logs:/opt/logs @@ -107,7 +102,7 @@ services: - "traefik.http.routers.api.service=api-svc" migrations: - image: "ghcr.io/hotosm/fmtm/backend:${API_VERSION}-${GIT_BRANCH}" + image: "ghcr.io/hotosm/fmtm/backend:${GIT_BRANCH}" container_name: fmtm_migrations depends_on: - fmtm-db @@ -119,7 +114,7 @@ services: restart: "on-failure:3" ui: - image: "ghcr.io/hotosm/fmtm/frontend:${FRONTEND_MAIN_VERSION}-${GIT_BRANCH}" + image: "ghcr.io/hotosm/fmtm/frontend:${GIT_BRANCH}" build: context: src/frontend dockerfile: prod.dockerfile diff --git a/docker-compose.noodk.yml b/docker-compose.noodk.yml index 4919e7c5a0..98ed155317 100644 --- a/docker-compose.noodk.yml +++ b/docker-compose.noodk.yml @@ -70,14 +70,14 @@ services: restart: "unless-stopped" migrations: - image: "ghcr.io/hotosm/fmtm/backend:${API_VERSION}-${GIT_BRANCH}" + image: "ghcr.io/hotosm/fmtm/backend:debug" container_name: fmtm_migrations depends_on: - fmtm-db env_file: - .env networks: - - fmtm-net + - fmtm-dev entrypoint: ["/migrate-entrypoint.sh"] restart: "on-failure:3" diff --git a/src/backend/Dockerfile b/src/backend/Dockerfile index 69d4f73844..51c9f154de 100644 --- a/src/backend/Dockerfile +++ b/src/backend/Dockerfile @@ -18,9 +18,11 @@ ARG PYTHON_IMG_TAG=3.10 FROM docker.io/python:${PYTHON_IMG_TAG}-slim-bookworm as base ARG APP_VERSION +ARG COMMIT_REF ARG PYTHON_IMG_TAG ARG MAINTAINER=admin@hotosm.org LABEL org.hotosm.fmtm.app-version="${APP_VERSION}" \ + org.hotosm.fmtm.git-commit-ref="${COMMIT_REF:-none}" \ org.hotosm.fmtm.python-img-tag="${PYTHON_IMG_TAG}" \ org.hotosm.fmtm.maintainer="${MAINTAINER}" \ org.hotosm.fmtm.api-port="8000" diff --git a/src/frontend/prod.dockerfile b/src/frontend/prod.dockerfile index 1d30372ce4..6794ff4151 100644 --- a/src/frontend/prod.dockerfile +++ b/src/frontend/prod.dockerfile @@ -2,11 +2,13 @@ FROM docker.io/node:18 as builder ARG MAINTAINER=admin@hotosm.org ARG APP_VERSION +ARG COMMIT_REF ARG VITE_API_URL ENV VITE_API_URL="${VITE_API_URL}" LABEL org.hotosm.fmtm.app-name="fmtm-frontend" \ org.hotosm.fmtm.app-version="${APP_VERSION}" \ + org.hotosm.fmtm.git-commit-ref="${COMMIT_REF:-none}" \ org.hotosm.fmtm.maintainer="${MAINTAINER}" \ org.hotosm.fmtm.api-url="${VITE_API_URL}" From 4156fd698901df7826109bcac9d6c24aed143a09 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Thu, 19 Oct 2023 12:03:53 +0100 Subject: [PATCH 06/69] ci: build backend images on tag and release events --- .github/workflows/tag_build.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 .github/workflows/tag_build.yml diff --git a/.github/workflows/tag_build.yml b/.github/workflows/tag_build.yml new file mode 100644 index 0000000000..6e5b35a8dd --- /dev/null +++ b/.github/workflows/tag_build.yml @@ -0,0 +1,25 @@ +name: Build Tag & Release Images + +on: + # Any tag push + push: + tags: + - "*" + # All releases + release: + types: [published] + # Allow manual trigger + workflow_dispatch: + +jobs: + backend-build: + uses: hotosm/gh-workflows/.github/workflows/image_build.yml@main + with: + context: src/backend + build_target: prod + image_name: ghcr.io/${{ github.repository }}/backend + build_args: | + APP_VERSION=${{ github.ref_name }} + COMMIT_REF=${{ github.sha }} + +# Frontend is not built as build variables are required From 1d9c022562cbb0a10b7f4198bf880a5b0ff96cc4 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Thu, 19 Oct 2023 12:06:06 +0100 Subject: [PATCH 07/69] ci: publish docs on changes to development branch --- .github/workflows/build_and_deploy.yml | 2 +- .github/workflows/docs.yml | 2 +- .github/workflows/wiki.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml index ffab5c38d1..e0519c1b31 100644 --- a/.github/workflows/build_and_deploy.yml +++ b/.github/workflows/build_and_deploy.yml @@ -1,4 +1,4 @@ -# Workflow for CI/CD of branches, build and auto-deploy +# Workflow for build and auto-deploy of branches name: Build and Deploy diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 1b5461d8e6..22c189d3d0 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -6,7 +6,7 @@ on: - docs/** - src/** - mkdocs.yml - branches: [main] + branches: [development] # Allow manual trigger (workflow_dispatch) workflow_dispatch: diff --git a/.github/workflows/wiki.yml b/.github/workflows/wiki.yml index d52f0a2266..f6674ca498 100644 --- a/.github/workflows/wiki.yml +++ b/.github/workflows/wiki.yml @@ -4,7 +4,7 @@ on: push: paths: - docs/** - branches: [main] + branches: [development] # Allow manual trigger (workflow_dispatch) workflow_dispatch: From 962752e8c35d5284434807c58a459aa3e5d0f9ad Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Thu, 19 Oct 2023 12:20:59 +0100 Subject: [PATCH 08/69] build: set default S3_ENDPOINT for entrypoint (if not set) --- src/backend/app-entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/app-entrypoint.sh b/src/backend/app-entrypoint.sh index f6ecf5af64..8f4c4e20b6 100644 --- a/src/backend/app-entrypoint.sh +++ b/src/backend/app-entrypoint.sh @@ -24,7 +24,7 @@ wait_for_s3() { retry_interval=5 for ((i = 0; i < max_retries; i++)); do - if curl --silent -I ${S3_ENDPOINT} >/dev/null; then + if curl --silent -I ${S3_ENDPOINT:-http://s3:9000} >/dev/null; then echo "S3 is available." return 0 # S3 is available, exit successfully fi From 65acea77236ad1e42d322c4f3986703ff9324786 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Thu, 19 Oct 2023 14:43:34 +0100 Subject: [PATCH 09/69] build: add healthchecks to dockerfiles --- odkcentral/api/Dockerfile | 4 +++- odkcentral/proxy/Dockerfile | 3 +++ src/backend/Dockerfile | 3 +++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/odkcentral/api/Dockerfile b/odkcentral/api/Dockerfile index 1aaff7e750..fa60f3943d 100644 --- a/odkcentral/api/Dockerfile +++ b/odkcentral/api/Dockerfile @@ -66,5 +66,7 @@ COPY ./init-user-and-start.sh ./ RUN chmod +x ./init-user-and-start.sh CMD ["./wait-for-it.sh", "central-db:5432", \ "--", "./init-user-and-start.sh"] - EXPOSE 8383 +# Add Healthcheck +HEALTHCHECK --start-period=10s --interval=5s --retries=10 \ + CMD nc -z localhost 8383 || exit 1 diff --git a/odkcentral/proxy/Dockerfile b/odkcentral/proxy/Dockerfile index 5c1fd8fb54..efe2803637 100644 --- a/odkcentral/proxy/Dockerfile +++ b/odkcentral/proxy/Dockerfile @@ -23,3 +23,6 @@ RUN rm -rf ./* /etc/nginx/conf.d/default.conf /etc/nginx/nginx.conf COPY . /etc/nginx RUN cat /etc/nginx/central.crt /etc/nginx/ca.crt \ >> /etc/nginx/central-fullchain.crt +# Add Healthcheck +HEALTHCHECK --start-period=5s --interval=5s --retries=8 \ + CMD nc -z localhost 443 || exit 1 diff --git a/src/backend/Dockerfile b/src/backend/Dockerfile index 51c9f154de..c96a28f7c0 100644 --- a/src/backend/Dockerfile +++ b/src/backend/Dockerfile @@ -122,6 +122,9 @@ VOLUME /opt/tiles VOLUME /opt/app/images # Change to non-root user USER appuser +# Add Healthcheck +HEALTHCHECK --start-period=10s --interval=5s --retries=8 --timeout=5s \ + CMD curl --fail http://localhost:8000 || exit 1 From d0fb4c68d3d4e069008a4a73c56625770cfe8abd Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Thu, 19 Oct 2023 14:44:02 +0100 Subject: [PATCH 10/69] ci: enable backend smoketest using compose --wait --- .github/workflows/build_and_deploy.yml | 32 +------------------------- 1 file changed, 1 insertion(+), 31 deletions(-) diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml index e0519c1b31..1ec29c939d 100644 --- a/.github/workflows/build_and_deploy.yml +++ b/.github/workflows/build_and_deploy.yml @@ -67,37 +67,7 @@ jobs: - name: Backend smoke test run: | - echo "Not implemented" - # source .env - # docker network create fmtm - - # docker pull "postgis/postgis:14-3.3-alpine" - # docker run --rm -d \ - # --name=fmtm-db \ - # --network=fmtm \ - # -e POSTGRES_USER=fmtm \ - # -e POSTGRES_PASSWORD=fmtm \ - # -e POSTGRES_DB=fmtm \ - # "postgis/postgis:14-3.3-alpine" - - # docker pull "ghcr.io/hotosm/fmtm/backend:${GIT_BRANCH}" - # docker run --rm -d \ - # --network=fmtm \ - # -p 8080:8080 \ - # -e OSM_CLIENT_ID="test" \ - # -e OSM_CLIENT_SECRET="test" \ - # -e OSM_SECRET_KEY="test" \ - # -e S3_ACCESS_KEY="fmtm" \ - # -e S3_SECRET_KEY="somelongpassword" \ - # "ghcr.io/hotosm/fmtm/backend:${GIT_BRANCH}" - - # # First wait 10 seconds for API - # sleep 10 - # # Check the exit status of curl and exit the job if it fails - # if ! curl -f http://localhost:8080/docs; then - # echo "curl failed to access http://localhost:8080/docs" - # exit 1 - # fi + docker compose up -d api --wait --wait-timeout 60 smoke-test-frontend: runs-on: ubuntu-latest From f95787f97cd062f14c11ff1f02aa555b56b0554e Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Thu, 19 Oct 2023 15:22:46 +0100 Subject: [PATCH 11/69] build: update odkcentral --> v2023.4.0 --- .env.example | 2 +- gen-env.sh | 2 +- odkcentral/api/Dockerfile | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.env.example b/.env.example index e7a20bbd2b..17bd1c6f75 100644 --- a/.env.example +++ b/.env.example @@ -1,7 +1,7 @@ ### copy to .env and set variables ### ODK Central ### -ODK_CENTRAL_VERSION=v2023.2.1 +ODK_CENTRAL_VERSION=v2023.4.0 ODK_CENTRAL_URL=https://central-proxy ODK_CENTRAL_USER=dev@fmtm.hotosm.org ODK_CENTRAL_PASSWD=testuserpassword diff --git a/gen-env.sh b/gen-env.sh index 067b27b9dd..a7c1804a82 100644 --- a/gen-env.sh +++ b/gen-env.sh @@ -191,7 +191,7 @@ echo "Generating dotenv file ${DOTENV_NAME}" echo "### ODK Central ###" # FIXME set central version via command line -echo "ODK_CENTRAL_VERSION=v2023.2.1" >> "${DOTENV_NAME}" +echo "ODK_CENTRAL_VERSION=v2023.4.0" >> "${DOTENV_NAME}" echo "ODK_CENTRAL_URL=${ODK_CENTRAL_URL}" >> "${DOTENV_NAME}" echo "ODK_CENTRAL_USER=${ODK_CENTRAL_USER}" >> "${DOTENV_NAME}" echo "ODK_CENTRAL_PASSWD=${ODK_CENTRAL_PASSWD}" >> "${DOTENV_NAME}" diff --git a/odkcentral/api/Dockerfile b/odkcentral/api/Dockerfile index fa60f3943d..a08583029b 100644 --- a/odkcentral/api/Dockerfile +++ b/odkcentral/api/Dockerfile @@ -15,7 +15,7 @@ # along with FMTM. If not, see . # -ARG node_version=16.19.1 +ARG node_version=18 @@ -24,7 +24,7 @@ RUN apt-get update \ && apt-get install -y --no-install-recommends \ "jq" \ && rm -rf /var/lib/apt/lists/* -ARG ODK_CENTRAL_VERSION=${ODK_CENTRAL_VERSION} +ARG ODK_CENTRAL_VERSION RUN git clone --depth 1 --branch ${ODK_CENTRAL_VERSION} \ "https://github.com/getodk/central.git" \ && cd central && git submodule update --init From 8b8064503731cc5f1ba80c30edb9299f53ba78aa Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Thu, 19 Oct 2023 16:07:40 +0100 Subject: [PATCH 12/69] build: healthcheck for minio service --- docker-compose.development.yml | 6 ++++++ docker-compose.noodk.yml | 6 ++++++ docker-compose.yml | 14 +++++++------- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/docker-compose.development.yml b/docker-compose.development.yml index 9a80f198c0..9db7b52205 100644 --- a/docker-compose.development.yml +++ b/docker-compose.development.yml @@ -154,3 +154,9 @@ services: - fmtm-net command: minio server # --console-address ":9090" restart: "unless-stopped" + healthcheck: + test: curl --fail http://localhost:9000/minio/health/live || exit 1 + start_period: 5s + interval: 5s + timeout: 5s + retries: 3 diff --git a/docker-compose.noodk.yml b/docker-compose.noodk.yml index 98ed155317..cf60c06c01 100644 --- a/docker-compose.noodk.yml +++ b/docker-compose.noodk.yml @@ -121,3 +121,9 @@ services: - fmtm-dev command: minio server # --console-address ":9090" restart: "unless-stopped" + healthcheck: + test: curl --fail http://localhost:9000/minio/health/live || exit 1 + start_period: 5s + interval: 5s + timeout: 5s + retries: 3 diff --git a/docker-compose.yml b/docker-compose.yml index 8023185011..87579ef3b4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -156,6 +156,7 @@ services: - EMAIL_IGNORE_TLS=${EMAIL_IGNORE_TLS:-true} - EMAIL_USER=${EMAIL_USER:-''} - EMAIL_PASSWORD=${EMAIL_PASSWORD:-''} + - OIDC_ENABLED=${OIDC_ENABLED:-false} - SENTRY_ORG_SUBDOMAIN=${SENTRY_ORG_SUBDOMAIN:-o130137} - SENTRY_KEY=${SENTRY_KEY:-3cf75f54983e473da6bd07daddf0d2ee} - SENTRY_PROJECT=${SENTRY_PROJECT:-1298632} @@ -163,13 +164,6 @@ services: - "8383:8383" networks: - fmtm-dev - command: - [ - "./wait-for-it.sh", - "${CENTRAL_DB_HOST:-central-db}:5432", - "--", - "./init-user-and-start.sh", - ] restart: "unless-stopped" central-proxy: @@ -199,3 +193,9 @@ services: - fmtm-dev command: minio server # --console-address ":9090" restart: "unless-stopped" + healthcheck: + test: curl --fail http://localhost:9000/minio/health/live || exit 1 + start_period: 5s + interval: 5s + timeout: 5s + retries: 3 From ab8ce1569a138a8bec55e9cc08c5aec472b61514 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Thu, 19 Oct 2023 16:08:01 +0100 Subject: [PATCH 13/69] ci(pytest): update pytest workflow to use img artifacts --- .github/workflows/pr_test_backend.yml | 2 +- .github/workflows/r-pytest.yml | 35 +++++++++++++++------------ 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/.github/workflows/pr_test_backend.yml b/.github/workflows/pr_test_backend.yml index e8a5cbb277..e294cce13e 100644 --- a/.github/workflows/pr_test_backend.yml +++ b/.github/workflows/pr_test_backend.yml @@ -1,4 +1,4 @@ -name: PR Tests +name: PR Test Backend on: pull_request: diff --git a/.github/workflows/r-pytest.yml b/.github/workflows/r-pytest.yml index bd5bf87b85..7b8e62dbd2 100644 --- a/.github/workflows/r-pytest.yml +++ b/.github/workflows/r-pytest.yml @@ -45,26 +45,29 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 - - name: Restore Img Caches - id: restore-imgs - uses: actions/cache@v3 + - name: Download Postgis Img + uses: actions/download-artifact@v3 with: - path: | - ${{ needs.cache-img-postgis.outputs.cache_path }} - ${{ needs.cache-img-odk.outputs.cache_path }} - ${{ needs.cache-img-odk-proxy-cache.outputs.cache_path }} - key: img- - restore-keys: | - ${{ needs.cache-img-postgis.outputs.cache_key }} - ${{ needs.cache-img-odk.outputs.cache_key }} - ${{ needs.cache-img-odk-proxy-cache.outputs.cache_key }} + name: ${{ needs.cache-img-postgis.outputs.cache_key }} + path: /tmp/${{ needs.cache-img-postgis.outputs.cache_key }}.tar + + - name: Download ODK Img + uses: actions/download-artifact@v3 + with: + name: ${{ needs.cache-img-odk.outputs.cache_key }} + path: /tmp/${{ needs.cache-img-odk.outputs.cache_key }}.tar + + - name: Download ODK Proxy Img + uses: actions/download-artifact@v3 + with: + name: ${{ needs.cache-img-odk-proxy.outputs.cache_key }} + path: /tmp/${{ needs.cache-img-odk-proxy.outputs.cache_key }}.tar - name: Load Cached Imgs - if: steps.restore-imgs.outputs.cache-hit == 'true' run: | - docker image load --input ${{ needs.cache-img-postgis.outputs.cache_path }} || true - docker image load --input ${{ needs.cache-img-odk.outputs.cache_path }} || true - docker image load --input ${{ needs.cache-img-odk-proxy-cache.outputs.cache_path }} || true + docker image load --input ${{ needs.cache-img-postgis.outputs.cache_key }} || true + docker image load --input ${{ needs.cache-img-odk.outputs.cache_key }} || true + docker image load --input ${{ needs.cache-img-odk-proxy-cache.outputs.cache_key }} || true - name: Environment to .env env: From 37c17962cbab5ece58e824e5695e6d0b5cc72026 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Thu, 19 Oct 2023 16:09:08 +0100 Subject: [PATCH 14/69] build: update odkcentral img, add entrypoint + healthcheck --- odkcentral/api/Dockerfile | 44 +++++++++++++-------------- odkcentral/api/init-user-and-start.sh | 11 ++++--- 2 files changed, 28 insertions(+), 27 deletions(-) diff --git a/odkcentral/api/Dockerfile b/odkcentral/api/Dockerfile index a08583029b..581b6d1805 100644 --- a/odkcentral/api/Dockerfile +++ b/odkcentral/api/Dockerfile @@ -35,38 +35,38 @@ FROM docker.io/node:${node_version} WORKDIR /usr/odk -RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ $(grep -oP 'VERSION_CODENAME=\K\w+' /etc/os-release)-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list && \ - curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor > /etc/apt/trusted.gpg.d/apt.postgresql.org.gpg && \ - apt-get update && \ - apt-get install -y cron gettext postgresql-client-14 - COPY --from=repo central/files/service/crontab /etc/cron.d/odk - -COPY --from=repo central/server/package*.json ./ - -RUN npm clean-install --omit=dev --legacy-peer-deps --no-audit --fund=false --update-notifier=false -RUN npm install pm2@5.2.2 -g - -COPY --from=repo central/server/ ./ COPY --from=repo central/files/service/scripts/ ./ -COPY --from=repo central/files/service/pm2.config.js ./ - COPY --from=repo central/files/service/config.json.template /usr/share/odk/ COPY --from=repo central/files/service/odk-cmd /usr/bin/ +# Add entrypoint script to init user +COPY ./init-user-and-start.sh ./ +# package.json must be added and installed prior to final COPY +COPY --from=repo central/server/package*.json ./ -# Required to start via entrypoint -RUN mkdir /etc/secrets sentry-versions \ +# Install system deps +RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ $(grep -oP 'VERSION_CODENAME=\K\w+' /etc/os-release)-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list && \ + curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor > /etc/apt/trusted.gpg.d/apt.postgresql.org.gpg && \ + apt-get update && \ + apt-get install -y cron wait-for-it gettext postgresql-client-14 netcat-traditional \ + # Install node_modules + && npm clean-install --omit=dev --legacy-peer-deps --no-audit \ + --fund=false --update-notifier=false \ + # Required to start via entrypoint + && mkdir /etc/secrets sentry-versions \ && echo 'jhs9udhy987gyds98gfyds98f' > /etc/secrets/enketo-api-key \ && echo '1' > sentry-versions/server \ && echo '1' > sentry-versions/central \ - && echo '1' > sentry-versions/client + && echo '1' > sentry-versions/client \ + # Set entrypoint executable + && chmod +x ./init-user-and-start.sh -# Add entrypoint script to init user -COPY ./init-user-and-start.sh ./ -RUN chmod +x ./init-user-and-start.sh -CMD ["./wait-for-it.sh", "central-db:5432", \ - "--", "./init-user-and-start.sh"] +# Add remaining files after deps installed +COPY --from=repo central/server/ ./ + +ENTRYPOINT ["./init-user-and-start.sh"] EXPOSE 8383 + # Add Healthcheck HEALTHCHECK --start-period=10s --interval=5s --retries=10 \ CMD nc -z localhost 8383 || exit 1 diff --git a/odkcentral/api/init-user-and-start.sh b/odkcentral/api/init-user-and-start.sh index 513ec63729..8ebd4be5be 100644 --- a/odkcentral/api/init-user-and-start.sh +++ b/odkcentral/api/init-user-and-start.sh @@ -1,5 +1,8 @@ set -eo pipefail +# Wait for database to be available +wait-for-it "${CENTRAL_DB_HOST:-central-db}:5432" + ### Init, generate config, migrate db ### echo "Stripping pm2 exec command from start-odk.sh script (last 2 lines)" head -n -2 ./start-odk.sh > ./init-odk-db.sh @@ -9,7 +12,6 @@ echo "Running ODKCentral start script to init environment and migrate DB" echo "The server will not start on this run" ./init-odk-db.sh - ### Create admin user ### echo "Creating test user ${SYSADMIN_EMAIL} with password ${SYSADMIN_PASSWD}" echo "${SYSADMIN_PASSWD}" | odk-cmd --email "${SYSADMIN_EMAIL}" user-create || true @@ -17,8 +19,7 @@ echo "${SYSADMIN_PASSWD}" | odk-cmd --email "${SYSADMIN_EMAIL}" user-create || t echo "Elevating user to admin" odk-cmd --email "${SYSADMIN_EMAIL}" user-promote || true - -### Run server ### +### Run server (copied from start-odk.sh) ### MEMTOT=$(vmstat -s | grep 'total memory' | awk '{ print $1 }') if [ "$MEMTOT" -gt "1100000" ] then @@ -28,5 +29,5 @@ else fi echo "using $WORKER_COUNT worker(s) based on available memory ($MEMTOT).." -echo "Starting server" -exec pm2-runtime ./pm2.config.js +echo "Starting server." +exec npx pm2-runtime ./pm2.config.js From ae16d40437bf947be5b747580551a56356837103 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Thu, 19 Oct 2023 16:19:48 +0100 Subject: [PATCH 15/69] ci: fix pytest img tar downloading --- .github/workflows/r-pytest.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/r-pytest.yml b/.github/workflows/r-pytest.yml index 7b8e62dbd2..c54726f282 100644 --- a/.github/workflows/r-pytest.yml +++ b/.github/workflows/r-pytest.yml @@ -65,9 +65,9 @@ jobs: - name: Load Cached Imgs run: | - docker image load --input ${{ needs.cache-img-postgis.outputs.cache_key }} || true - docker image load --input ${{ needs.cache-img-odk.outputs.cache_key }} || true - docker image load --input ${{ needs.cache-img-odk-proxy-cache.outputs.cache_key }} || true + docker image load --input ${{ needs.cache-img-postgis.outputs.cache_key }}.tar || true + docker image load --input ${{ needs.cache-img-odk.outputs.cache_key }}.tar || true + docker image load --input ${{ needs.cache-img-odk-proxy.outputs.cache_key.tar }} || true - name: Environment to .env env: From 293f2df83e08a7ef59ace679fd7b49883ccd2212 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Thu, 19 Oct 2023 16:31:21 +0100 Subject: [PATCH 16/69] ci(pytest): fix cached image loading from /tmp --- .github/workflows/r-pytest.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/r-pytest.yml b/.github/workflows/r-pytest.yml index c54726f282..fa06862602 100644 --- a/.github/workflows/r-pytest.yml +++ b/.github/workflows/r-pytest.yml @@ -50,24 +50,27 @@ jobs: with: name: ${{ needs.cache-img-postgis.outputs.cache_key }} path: /tmp/${{ needs.cache-img-postgis.outputs.cache_key }}.tar + # continue-on-error: true - name: Download ODK Img uses: actions/download-artifact@v3 with: name: ${{ needs.cache-img-odk.outputs.cache_key }} path: /tmp/${{ needs.cache-img-odk.outputs.cache_key }}.tar + # continue-on-error: true - name: Download ODK Proxy Img uses: actions/download-artifact@v3 with: name: ${{ needs.cache-img-odk-proxy.outputs.cache_key }} path: /tmp/${{ needs.cache-img-odk-proxy.outputs.cache_key }}.tar + # continue-on-error: true - name: Load Cached Imgs run: | - docker image load --input ${{ needs.cache-img-postgis.outputs.cache_key }}.tar || true - docker image load --input ${{ needs.cache-img-odk.outputs.cache_key }}.tar || true - docker image load --input ${{ needs.cache-img-odk-proxy.outputs.cache_key.tar }} || true + docker image load --input /tmp/${{ needs.cache-img-postgis.outputs.cache_key }}.tar || true + docker image load --input /tmp/${{ needs.cache-img-odk.outputs.cache_key }}.tar || true + docker image load --input /tmp/${{ needs.cache-img-odk-proxy.outputs.cache_key }}.tar || true - name: Environment to .env env: From 995f7f7395fe471ec95f567d1f10d2aa9b3c0beb Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Thu, 19 Oct 2023 17:02:33 +0100 Subject: [PATCH 17/69] ci(pytest): simplify image caching and loading --- .github/workflows/r-pytest.yml | 61 ++++++++++------------------------ 1 file changed, 18 insertions(+), 43 deletions(-) diff --git a/.github/workflows/r-pytest.yml b/.github/workflows/r-pytest.yml index fa06862602..1e20f5d442 100644 --- a/.github/workflows/r-pytest.yml +++ b/.github/workflows/r-pytest.yml @@ -14,63 +14,38 @@ permissions: contents: read jobs: - cache-img-postgis: - uses: hotosm/gh-workflows/.github/workflows/image_cache.yml@main - with: - image_name: postgis/postgis:14-3.3-alpine - cache_key: img-postgis - - cache-img-odk: - uses: hotosm/gh-workflows/.github/workflows/image_cache.yml@main - with: - image_name: ghcr.io/hotosm/fmtm/odkcentral:v2023.2.1 - cache_key: img-odk - - cache-img-odk-proxy: - uses: hotosm/gh-workflows/.github/workflows/image_cache.yml@main - with: - image_name: ghcr.io/hotosm/fmtm/odkcentral-proxy:latest - cache_key: img-odk-proxy - run-pytest: runs-on: ubuntu-latest environment: name: ${{ inputs.environment || 'test' }} - needs: - - cache-img-postgis - - cache-img-odk - - cache-img-odk-proxy steps: - name: Checkout repository uses: actions/checkout@v4 - - name: Download Postgis Img - uses: actions/download-artifact@v3 - with: - name: ${{ needs.cache-img-postgis.outputs.cache_key }} - path: /tmp/${{ needs.cache-img-postgis.outputs.cache_key }}.tar - # continue-on-error: true - - - name: Download ODK Img + - id: images + name: Download image cache uses: actions/download-artifact@v3 with: - name: ${{ needs.cache-img-odk.outputs.cache_key }} - path: /tmp/${{ needs.cache-img-odk.outputs.cache_key }}.tar - # continue-on-error: true - - - name: Download ODK Proxy Img - uses: actions/download-artifact@v3 - with: - name: ${{ needs.cache-img-odk-proxy.outputs.cache_key }} - path: /tmp/${{ needs.cache-img-odk-proxy.outputs.cache_key }}.tar - # continue-on-error: true + name: image-cache + path: /tmp/images + continue-on-error: true - name: Load Cached Imgs + if: images.outputs.download_path == '/tmp/images' run: | - docker image load --input /tmp/${{ needs.cache-img-postgis.outputs.cache_key }}.tar || true - docker image load --input /tmp/${{ needs.cache-img-odk.outputs.cache_key }}.tar || true - docker image load --input /tmp/${{ needs.cache-img-odk-proxy.outputs.cache_key }}.tar || true + for image_tar in /tmp/images/*; do + docker image load --input $image_tar || true + done + + - if: images.outputs.download_path != '/tmp/images' + uses: hotosm/gh-workflows/.github/workflows/image_cache.yml@main + with: + image_names: | + postgis/postgis:14-3.3-alpine + ghcr.io/hotosm/fmtm/odkcentral:v2023.2.1 + ghcr.io/hotosm/fmtm/odkcentral-proxy:latest + cache_key: image-cache - name: Environment to .env env: From fdacf0a59b1001765472bed9ff7ff9dc69596bbc Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Thu, 19 Oct 2023 17:05:05 +0100 Subject: [PATCH 18/69] ci(pytest): remove image cache upload --- .github/workflows/r-pytest.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.github/workflows/r-pytest.yml b/.github/workflows/r-pytest.yml index 1e20f5d442..c4335fd0f9 100644 --- a/.github/workflows/r-pytest.yml +++ b/.github/workflows/r-pytest.yml @@ -38,15 +38,6 @@ jobs: docker image load --input $image_tar || true done - - if: images.outputs.download_path != '/tmp/images' - uses: hotosm/gh-workflows/.github/workflows/image_cache.yml@main - with: - image_names: | - postgis/postgis:14-3.3-alpine - ghcr.io/hotosm/fmtm/odkcentral:v2023.2.1 - ghcr.io/hotosm/fmtm/odkcentral-proxy:latest - cache_key: image-cache - - name: Environment to .env env: DEBUG: True From fd0053f9dc261648dc960f2bc9ce7ee0509b83e6 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Thu, 19 Oct 2023 17:10:20 +0100 Subject: [PATCH 19/69] ci(pytest): add image cache job to end of workflow --- .github/workflows/r-pytest.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/r-pytest.yml b/.github/workflows/r-pytest.yml index c4335fd0f9..d2ebed2eae 100644 --- a/.github/workflows/r-pytest.yml +++ b/.github/workflows/r-pytest.yml @@ -23,7 +23,7 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 - - id: images + - id: download-images name: Download image cache uses: actions/download-artifact@v3 with: @@ -32,7 +32,7 @@ jobs: continue-on-error: true - name: Load Cached Imgs - if: images.outputs.download_path == '/tmp/images' + if: ${{ steps.download-images.outputs.download_path == '/tmp/images' }} run: | for image_tar in /tmp/images/*; do docker image load --input $image_tar || true @@ -72,3 +72,13 @@ jobs: wait-for-it fmtm-db:5432 --strict \ -- wait-for-it central:8383 --strict \ -- pytest + + cache-images: + if: ${{ jobs.run-pytest.steps.download-images.outputs.download_path != '/tmp/images' }} + uses: hotosm/gh-workflows/.github/workflows/image_cache.yml@main + with: + image_names: | + postgis/postgis:14-3.3-alpine + ghcr.io/hotosm/fmtm/odkcentral:v2023.2.1 + ghcr.io/hotosm/fmtm/odkcentral-proxy:latest + cache_key: image-cache From 349316c4050d89cf00c2d96a364047ca3a2fe59a Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Thu, 19 Oct 2023 17:14:29 +0100 Subject: [PATCH 20/69] ci(pytest): set cache loading based on needs outputs --- .github/workflows/r-pytest.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/r-pytest.yml b/.github/workflows/r-pytest.yml index d2ebed2eae..c2f39bdb2f 100644 --- a/.github/workflows/r-pytest.yml +++ b/.github/workflows/r-pytest.yml @@ -19,6 +19,9 @@ jobs: environment: name: ${{ inputs.environment || 'test' }} + outputs: + img_cache_loaded: ${{ steps.load-cache.outputs.img_cache_loaded }} + steps: - name: Checkout repository uses: actions/checkout@v4 @@ -32,11 +35,13 @@ jobs: continue-on-error: true - name: Load Cached Imgs + id: load-cache if: ${{ steps.download-images.outputs.download_path == '/tmp/images' }} run: | for image_tar in /tmp/images/*; do docker image load --input $image_tar || true done + echo "img_cache_loaded=true" >> $GITHUB_OUTPUT - name: Environment to .env env: @@ -74,7 +79,8 @@ jobs: -- pytest cache-images: - if: ${{ jobs.run-pytest.steps.download-images.outputs.download_path != '/tmp/images' }} + needs: [run-pytest] + if: ${{ needs.run-pytest.outputs.img_cache_loaded }} uses: hotosm/gh-workflows/.github/workflows/image_cache.yml@main with: image_names: | From 426a84f9692ffb549c8ed5d3737cf25ecf2cad43 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Thu, 19 Oct 2023 17:19:02 +0100 Subject: [PATCH 21/69] ci(pytest): cache images if download was unsuccessful --- .github/workflows/r-pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/r-pytest.yml b/.github/workflows/r-pytest.yml index c2f39bdb2f..be3cd9cfea 100644 --- a/.github/workflows/r-pytest.yml +++ b/.github/workflows/r-pytest.yml @@ -80,7 +80,7 @@ jobs: cache-images: needs: [run-pytest] - if: ${{ needs.run-pytest.outputs.img_cache_loaded }} + if: ${{ needs.run-pytest.outputs.img_cache_loaded =! true}} uses: hotosm/gh-workflows/.github/workflows/image_cache.yml@main with: image_names: | From 2b59aef46c1f29a0aedf4a27a0f3c702d951ca10 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Thu, 19 Oct 2023 17:19:50 +0100 Subject: [PATCH 22/69] ci(pytest): fix operator != --- .github/workflows/r-pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/r-pytest.yml b/.github/workflows/r-pytest.yml index be3cd9cfea..a099695400 100644 --- a/.github/workflows/r-pytest.yml +++ b/.github/workflows/r-pytest.yml @@ -80,7 +80,7 @@ jobs: cache-images: needs: [run-pytest] - if: ${{ needs.run-pytest.outputs.img_cache_loaded =! true}} + if: ${{ needs.run-pytest.outputs.img_cache_loaded != true}} uses: hotosm/gh-workflows/.github/workflows/image_cache.yml@main with: image_names: | From d2c870fdff92648f51112121ad48aca15cd3fd40 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Fri, 20 Oct 2023 06:54:44 +0100 Subject: [PATCH 23/69] build: use env vars for all image versions --- .env.example | 1 - docker-compose.development.yml | 4 ++-- docker-compose.main.yml | 2 +- docker-compose.noodk.yml | 4 ++-- docker-compose.yml | 12 ++++++------ gen-env.sh | 2 -- 6 files changed, 11 insertions(+), 14 deletions(-) diff --git a/.env.example b/.env.example index 17bd1c6f75..1159a419c4 100644 --- a/.env.example +++ b/.env.example @@ -1,7 +1,6 @@ ### copy to .env and set variables ### ODK Central ### -ODK_CENTRAL_VERSION=v2023.4.0 ODK_CENTRAL_URL=https://central-proxy ODK_CENTRAL_USER=dev@fmtm.hotosm.org ODK_CENTRAL_PASSWD=testuserpassword diff --git a/docker-compose.development.yml b/docker-compose.development.yml index 9db7b52205..1e5710611b 100644 --- a/docker-compose.development.yml +++ b/docker-compose.development.yml @@ -64,7 +64,7 @@ services: - "traefik.http.routers.http_catchall.middlewares=https_redirect" fmtm-db: - image: "postgis/postgis:14-3.3-alpine" + image: "postgis/postgis:${POSTGIS_TAG:-14-3.3-alpine}" container_name: fmtm_db volumes: - fmtm_db_data:/var/lib/postgresql/data/ @@ -141,7 +141,7 @@ services: - "traefik.http.routers.ui-main.service=ui-main-svc" s3: - image: "docker.io/minio/minio:RELEASE.2023-10-07T15-07-38Z" + image: "docker.io/minio/minio:${MINIO_TAG:-RELEASE.2023-10-07T15-07-38Z}" container_name: fmtm_s3 environment: MINIO_ROOT_USER: ${S3_ACCESS_KEY} diff --git a/docker-compose.main.yml b/docker-compose.main.yml index 49d47dd797..7f10f510f3 100644 --- a/docker-compose.main.yml +++ b/docker-compose.main.yml @@ -63,7 +63,7 @@ services: - "traefik.http.routers.http_catchall.middlewares=https_redirect" fmtm-db: - image: "postgis/postgis:14-3.3-alpine" + image: "postgis/postgis:${POSTGIS_TAG:-14-3.3-alpine}" container_name: fmtm_db volumes: - fmtm_db_data:/var/lib/postgresql/data/ diff --git a/docker-compose.noodk.yml b/docker-compose.noodk.yml index cf60c06c01..49e9246335 100644 --- a/docker-compose.noodk.yml +++ b/docker-compose.noodk.yml @@ -30,7 +30,7 @@ networks: services: fmtm-db: - image: "postgis/postgis:14-3.3-alpine" + image: "postgis/postgis:${POSTGIS_TAG:-14-3.3-alpine}" container_name: fmtm_db volumes: - fmtm_db_data:/var/lib/postgresql/data/ @@ -106,7 +106,7 @@ services: restart: "unless-stopped" s3: - image: "docker.io/minio/minio:RELEASE.2023-10-07T15-07-38Z" + image: "docker.io/minio/minio:${MINIO_TAG:-RELEASE.2023-10-07T15-07-38Z}" container_name: fmtm_s3 environment: MINIO_ROOT_USER: ${S3_ACCESS_KEY:-fmtm} diff --git a/docker-compose.yml b/docker-compose.yml index 87579ef3b4..1bddf6a05f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -31,7 +31,7 @@ networks: services: fmtm-db: - image: "postgis/postgis:14-3.3-alpine" + image: "postgis/postgis:${POSTGIS_TAG:-14-3.3-alpine}" container_name: fmtm_db volumes: - fmtm_db_data:/var/lib/postgresql/data/ @@ -116,7 +116,7 @@ services: restart: "unless-stopped" central-db: - image: "postgis/postgis:14-3.3-alpine" + image: "postgis/postgis:${POSTGIS_TAG:-14-3.3-alpine}" container_name: central_db volumes: - central_db_data:/var/lib/postgresql/data/ @@ -131,11 +131,11 @@ services: restart: "unless-stopped" central: - image: "ghcr.io/hotosm/fmtm/odkcentral:v2023.2.1" + image: "ghcr.io/hotosm/fmtm/odkcentral:${ODK_CENTRAL_VERSION:-v2023.4.0}" build: context: odkcentral/api args: - ODK_CENTRAL_VERSION: v2023.2.1 + ODK_CENTRAL_VERSION: v2023.4.0 container_name: central_api depends_on: - central-db @@ -167,7 +167,7 @@ services: restart: "unless-stopped" central-proxy: - image: "ghcr.io/hotosm/fmtm/odkcentral-proxy:latest" + image: "ghcr.io/hotosm/fmtm/odkcentral-proxy:${ODK_CENTRAL_VERSION:-v2023.4.0}" build: context: odkcentral/proxy container_name: central_proxy @@ -178,7 +178,7 @@ services: restart: "unless-stopped" s3: - image: "docker.io/minio/minio:RELEASE.2023-10-07T15-07-38Z" + image: "docker.io/minio/minio:${MINIO_TAG:-RELEASE.2023-10-07T15-07-38Z}" container_name: fmtm_s3 environment: MINIO_ROOT_USER: ${S3_ACCESS_KEY:-fmtm} diff --git a/gen-env.sh b/gen-env.sh index a7c1804a82..f876c85131 100644 --- a/gen-env.sh +++ b/gen-env.sh @@ -190,8 +190,6 @@ echo echo "Generating dotenv file ${DOTENV_NAME}" echo "### ODK Central ###" -# FIXME set central version via command line -echo "ODK_CENTRAL_VERSION=v2023.4.0" >> "${DOTENV_NAME}" echo "ODK_CENTRAL_URL=${ODK_CENTRAL_URL}" >> "${DOTENV_NAME}" echo "ODK_CENTRAL_USER=${ODK_CENTRAL_USER}" >> "${DOTENV_NAME}" echo "ODK_CENTRAL_PASSWD=${ODK_CENTRAL_PASSWD}" >> "${DOTENV_NAME}" From bca417ce519d05a1802ec9e09924209c2a5c10f6 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Fri, 20 Oct 2023 06:55:49 +0100 Subject: [PATCH 24/69] build: ODK_CENTRAL_VERSION --> ODK_CENTRAL_TAG --- .github/workflows/build_odk_imgs.yml | 6 +++--- docker-compose.yml | 6 +++--- odkcentral/api/Dockerfile | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build_odk_imgs.yml b/.github/workflows/build_odk_imgs.yml index 14832bfd00..75a9b08660 100644 --- a/.github/workflows/build_odk_imgs.yml +++ b/.github/workflows/build_odk_imgs.yml @@ -17,15 +17,15 @@ jobs: with: context: odkcentral/api image_tags: | - "ghcr.io/hotosm/fmtm/odkcentral:${{ vars.ODK_CENTRAL_VERSION }}" + "ghcr.io/hotosm/fmtm/odkcentral:${{ vars.ODK_CENTRAL_TAG }}" "ghcr.io/hotosm/fmtm/odkcentral:latest" build_args: | - ODK_CENTRAL_VERSION=${{ vars.ODK_CENTRAL_VERSION }} + ODK_CENTRAL_TAG=${{ vars.ODK_CENTRAL_TAG }} build-proxy: uses: hotosm/gh-workflows/.github/workflows/image_build.yml@main with: context: odkcentral/proxy image_tags: | - "ghcr.io/hotosm/fmtm/odkcentral-proxy:${{ vars.ODK_CENTRAL_VERSION }}" + "ghcr.io/hotosm/fmtm/odkcentral-proxy:${{ vars.ODK_CENTRAL_TAG }}" "ghcr.io/hotosm/fmtm/odkcentral-proxy:latest" diff --git a/docker-compose.yml b/docker-compose.yml index 1bddf6a05f..06162bf591 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -131,11 +131,11 @@ services: restart: "unless-stopped" central: - image: "ghcr.io/hotosm/fmtm/odkcentral:${ODK_CENTRAL_VERSION:-v2023.4.0}" + image: "ghcr.io/hotosm/fmtm/odkcentral:${ODK_CENTRAL_TAG:-v2023.4.0}" build: context: odkcentral/api args: - ODK_CENTRAL_VERSION: v2023.4.0 + ODK_CENTRAL_TAG: v2023.4.0 container_name: central_api depends_on: - central-db @@ -167,7 +167,7 @@ services: restart: "unless-stopped" central-proxy: - image: "ghcr.io/hotosm/fmtm/odkcentral-proxy:${ODK_CENTRAL_VERSION:-v2023.4.0}" + image: "ghcr.io/hotosm/fmtm/odkcentral-proxy:${ODK_CENTRAL_TAG:-v2023.4.0}" build: context: odkcentral/proxy container_name: central_proxy diff --git a/odkcentral/api/Dockerfile b/odkcentral/api/Dockerfile index 581b6d1805..e52384e66b 100644 --- a/odkcentral/api/Dockerfile +++ b/odkcentral/api/Dockerfile @@ -24,8 +24,8 @@ RUN apt-get update \ && apt-get install -y --no-install-recommends \ "jq" \ && rm -rf /var/lib/apt/lists/* -ARG ODK_CENTRAL_VERSION -RUN git clone --depth 1 --branch ${ODK_CENTRAL_VERSION} \ +ARG ODK_CENTRAL_TAG +RUN git clone --depth 1 --branch ${ODK_CENTRAL_TAG} \ "https://github.com/getodk/central.git" \ && cd central && git submodule update --init From 6923ca0c700ff2d0df53bb8695c88db0b6daa51f Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Fri, 20 Oct 2023 06:56:18 +0100 Subject: [PATCH 25/69] ci(pytest): set image tags for caching from github vars --- .github/workflows/r-pytest.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/r-pytest.yml b/.github/workflows/r-pytest.yml index a099695400..6cf682a7dd 100644 --- a/.github/workflows/r-pytest.yml +++ b/.github/workflows/r-pytest.yml @@ -84,7 +84,8 @@ jobs: uses: hotosm/gh-workflows/.github/workflows/image_cache.yml@main with: image_names: | - postgis/postgis:14-3.3-alpine - ghcr.io/hotosm/fmtm/odkcentral:v2023.2.1 - ghcr.io/hotosm/fmtm/odkcentral-proxy:latest + docker.io/postgis/postgis:${{ vars.POSTGIS_TAG }}-alpine + ghcr.io/hotosm/fmtm/odkcentral:${{ vars.ODK_CENTRAL_TAG }} + ghcr.io/hotosm/fmtm/odkcentral-proxy:${{ vars.ODK_CENTRAL_TAG }} + docker.io/minio/minio:${{ vars.MINIO_TAG }} cache_key: image-cache From 871f07b40a0b28318ebb9d2521452a22487542dc Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Fri, 20 Oct 2023 06:56:56 +0100 Subject: [PATCH 26/69] ci: replace deploy workflow with reusable --- .github/workflows/build_and_deploy.yml | 34 ++++---------------------- 1 file changed, 5 insertions(+), 29 deletions(-) diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml index 1ec29c939d..2252309dc8 100644 --- a/.github/workflows/build_and_deploy.yml +++ b/.github/workflows/build_and_deploy.yml @@ -84,35 +84,11 @@ jobs: run: echo "Not implemented" deploy-containers: - runs-on: ubuntu-latest needs: - smoke-test-backend - smoke-test-frontend - environment: - name: ${{ github.ref_name }} - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Environment to .env - env: - GIT_BRANCH: ${{ github.ref_name }} - run: | - echo "${{ secrets.DOTENV }}" > .env - echo "GIT_BRANCH=${GIT_BRANCH}" >> .env - - - uses: webfactory/ssh-agent@v0.7.0 - with: - ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} - - - name: Disable Host key verification - # Hack to prevent "Host key verification failed". Should be replaced with a ssh-keyscan based solution - run: echo "StrictHostKeyChecking no" >> ~/.ssh/config - - - name: Deploy - run: | - docker compose --file "docker-compose.${{ github.ref_name }}.yml" pull - docker compose --file "docker-compose.${{ github.ref_name }}.yml" up --detach --remove-orphans - env: - DOCKER_HOST: ${{ vars.DOCKER_HOST }} + uses: hotosm/gh-workflows/.github/workflows/remote_deploy.yml@main + with: + environment: ${{ github.ref_name }} + docker_compose_file: "docker-compose.${{ github.ref_name }}.yml" + secrets: inherit From 884a78f3d6beaadd3380c3e166997f1fdb60d48a Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Fri, 20 Oct 2023 07:14:02 +0100 Subject: [PATCH 27/69] ci(pytest): image caching cache_name --> artifact_name --- .github/workflows/r-pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/r-pytest.yml b/.github/workflows/r-pytest.yml index 6cf682a7dd..06c5bd1d70 100644 --- a/.github/workflows/r-pytest.yml +++ b/.github/workflows/r-pytest.yml @@ -88,4 +88,4 @@ jobs: ghcr.io/hotosm/fmtm/odkcentral:${{ vars.ODK_CENTRAL_TAG }} ghcr.io/hotosm/fmtm/odkcentral-proxy:${{ vars.ODK_CENTRAL_TAG }} docker.io/minio/minio:${{ vars.MINIO_TAG }} - cache_key: image-cache + artifact_name: image-cache From e91ea5c8f94fb7cc5fc292f0c267c62ab8376df7 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Fri, 20 Oct 2023 07:14:33 +0100 Subject: [PATCH 28/69] ci(deploy): set .env from var and secrets context (not manual) --- .github/workflows/build_and_deploy.yml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml index 2252309dc8..6d9a22a5db 100644 --- a/.github/workflows/build_and_deploy.yml +++ b/.github/workflows/build_and_deploy.yml @@ -58,12 +58,21 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 - - name: Environment to .env + - name: Vars and Secrets to Env env: GIT_BRANCH: ${{ github.ref_name }} + VARS_CONTEXT: ${{ toJson(vars) }} + SECRETS_CONTEXT: ${{ toJson(secrets) }} run: | - echo "${{ secrets.DOTENV }}" > .env - echo "GIT_BRANCH=${GIT_BRANCH}" >> .env + EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) + to_envs() { jq -r "to_entries[] | \"\(.key)<<$EOF\n\(.value)\n$EOF\n\""; } + + echo "$GIT_BRANCH" >> $GITHUB_ENV + echo "${{ env.VARS_CONTEXT }}" | to_envs >> $GITHUB_ENV + echo "${{ env.SECRETS_CONTEXT }}" | to_envs >> $GITHUB_ENV + + - name: Create .env file + run: env > .env - name: Backend smoke test run: | From 6f76e2a9d858550ba2a9e3b00b072ad10c0a3108 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Fri, 20 Oct 2023 07:20:12 +0100 Subject: [PATCH 29/69] ci(pytest): increase timeout for central wait-for-it 30s --- .github/workflows/r-pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/r-pytest.yml b/.github/workflows/r-pytest.yml index 06c5bd1d70..be74e72d29 100644 --- a/.github/workflows/r-pytest.yml +++ b/.github/workflows/r-pytest.yml @@ -75,7 +75,7 @@ jobs: run: | docker compose run api \ wait-for-it fmtm-db:5432 --strict \ - -- wait-for-it central:8383 --strict \ + -- wait-for-it central:8383 --strict --timeout 30 \ -- pytest cache-images: From ba1d99540ac41ebb5a3171416885f37b95cc3376 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Fri, 20 Oct 2023 10:03:57 +0100 Subject: [PATCH 30/69] ci(pytest): fix timeout syntax --- .github/workflows/r-pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/r-pytest.yml b/.github/workflows/r-pytest.yml index be74e72d29..ae95544db7 100644 --- a/.github/workflows/r-pytest.yml +++ b/.github/workflows/r-pytest.yml @@ -75,7 +75,7 @@ jobs: run: | docker compose run api \ wait-for-it fmtm-db:5432 --strict \ - -- wait-for-it central:8383 --strict --timeout 30 \ + -- wait-for-it central:8383 --strict --timeout=30 \ -- pytest cache-images: From c83a3b3558454385c657b6ef72e4218fec2e7b5c Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Fri, 20 Oct 2023 10:17:58 +0100 Subject: [PATCH 31/69] build: add shebang to odkcentral entrypoint script --- odkcentral/api/init-user-and-start.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/odkcentral/api/init-user-and-start.sh b/odkcentral/api/init-user-and-start.sh index 8ebd4be5be..7918d93077 100644 --- a/odkcentral/api/init-user-and-start.sh +++ b/odkcentral/api/init-user-and-start.sh @@ -1,3 +1,5 @@ +#!/bin/bash + set -eo pipefail # Wait for database to be available From 542ff1869afd529f66f2e34b605b522751ece53c Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Fri, 20 Oct 2023 10:28:25 +0100 Subject: [PATCH 32/69] ci: update env to .env for pytest --- .github/workflows/build_and_deploy.yml | 2 +- .github/workflows/r-pytest.yml | 40 ++++++++++---------------- odkcentral/api/Dockerfile | 6 ++-- 3 files changed, 19 insertions(+), 29 deletions(-) diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml index 6d9a22a5db..bb5627821a 100644 --- a/.github/workflows/build_and_deploy.yml +++ b/.github/workflows/build_and_deploy.yml @@ -67,7 +67,7 @@ jobs: EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) to_envs() { jq -r "to_entries[] | \"\(.key)<<$EOF\n\(.value)\n$EOF\n\""; } - echo "$GIT_BRANCH" >> $GITHUB_ENV + echo "GIT_BRANCH=${GIT_BRANCH}" >> $GITHUB_ENV echo "${{ env.VARS_CONTEXT }}" | to_envs >> $GITHUB_ENV echo "${{ env.SECRETS_CONTEXT }}" | to_envs >> $GITHUB_ENV diff --git a/.github/workflows/r-pytest.yml b/.github/workflows/r-pytest.yml index ae95544db7..1f9e8e58f3 100644 --- a/.github/workflows/r-pytest.yml +++ b/.github/workflows/r-pytest.yml @@ -43,33 +43,23 @@ jobs: done echo "img_cache_loaded=true" >> $GITHUB_OUTPUT - - name: Environment to .env + - name: Vars and Secrets to Env env: - DEBUG: True - LOG_LEVEL: DEBUG - API_TAG_OVERRIDE: "${{ inputs.image_tag }}" - FRONTEND_MAIN_URL: "${{ vars.FRONTEND_MAIN_URL }}" - ODK_CENTRAL_URL: "${{ vars.ODK_CENTRAL_URL }}" - ODK_CENTRAL_USER: "${{ vars.ODK_CENTRAL_USER }}" - ODK_CENTRAL_PASSWD: "${{ secrets.ODK_CENTRAL_PASSWD }}" - OSM_CLIENT_ID: "${{ secrets.OSM_CLIENT_ID }}" - OSM_CLIENT_SECRET: "${{ secrets.OSM_CLIENT_SECRET }}" - OSM_SECRET_KEY: "${{ secrets.OSM_SECRET_KEY }}" - S3_ACCESS_KEY: "fmtm" - S3_SECRET_KEY: "somelongpassword" + API_TAG_OVERRIDE: ${{ inputs.image_tag }} + GIT_BRANCH: ${{ github.ref_name }} + VARS_CONTEXT: ${{ toJson(vars) }} + SECRETS_CONTEXT: ${{ toJson(secrets) }} run: | - echo "DEBUG=${DEBUG}" >> .env - echo "LOG_LEVEL=${LOG_LEVEL}" >> .env - echo "API_TAG_OVERRIDE=${API_TAG_OVERRIDE}" >> .env - echo "FRONTEND_MAIN_URL=${FRONTEND_MAIN_URL}" >> .env - echo "ODK_CENTRAL_URL=${ODK_CENTRAL_URL}" >> .env - echo "ODK_CENTRAL_USER=${ODK_CENTRAL_USER}" >> .env - echo "ODK_CENTRAL_PASSWD=${ODK_CENTRAL_PASSWD}" >> .env - echo "OSM_CLIENT_ID=${OSM_CLIENT_ID}" >> .env - echo "OSM_CLIENT_SECRET=${OSM_CLIENT_SECRET}" >> .env - echo "OSM_SECRET_KEY=${OSM_SECRET_KEY}" >> .env - echo "S3_ACCESS_KEY=${S3_ACCESS_KEY}" >> .env - echo "S3_SECRET_KEY=${S3_SECRET_KEY}" >> .env + EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) + to_envs() { jq -r "to_entries[] | \"\(.key)<<$EOF\n\(.value)\n$EOF\n\""; } + + echo "API_TAG_OVERRIDE=${API_TAG_OVERRIDE}" >> $GITHUB_ENV + echo "GIT_BRANCH=${GIT_BRANCH}" >> $GITHUB_ENV + echo "${{ env.VARS_CONTEXT }}" | to_envs >> $GITHUB_ENV + echo "${{ env.SECRETS_CONTEXT }}" | to_envs >> $GITHUB_ENV + + - name: Create .env file + run: env > .env - name: Run PyTest run: | diff --git a/odkcentral/api/Dockerfile b/odkcentral/api/Dockerfile index e52384e66b..c8898da9de 100644 --- a/odkcentral/api/Dockerfile +++ b/odkcentral/api/Dockerfile @@ -40,7 +40,7 @@ COPY --from=repo central/files/service/scripts/ ./ COPY --from=repo central/files/service/config.json.template /usr/share/odk/ COPY --from=repo central/files/service/odk-cmd /usr/bin/ # Add entrypoint script to init user -COPY ./init-user-and-start.sh ./ +COPY init-user-and-start.sh / # package.json must be added and installed prior to final COPY COPY --from=repo central/server/package*.json ./ @@ -59,12 +59,12 @@ RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ $(grep -oP 'VERSION_CODEN && echo '1' > sentry-versions/central \ && echo '1' > sentry-versions/client \ # Set entrypoint executable - && chmod +x ./init-user-and-start.sh + && chmod +x /init-user-and-start.sh # Add remaining files after deps installed COPY --from=repo central/server/ ./ -ENTRYPOINT ["./init-user-and-start.sh"] +ENTRYPOINT ["/init-user-and-start.sh"] EXPOSE 8383 # Add Healthcheck From 6d2802ea1c677cd8d58c6176618076d8daf0f06d Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Fri, 20 Oct 2023 10:30:52 +0100 Subject: [PATCH 33/69] ci(pytest): remove additional -alpine from postgis img --- .github/workflows/r-pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/r-pytest.yml b/.github/workflows/r-pytest.yml index 1f9e8e58f3..ebad9b0e0f 100644 --- a/.github/workflows/r-pytest.yml +++ b/.github/workflows/r-pytest.yml @@ -74,7 +74,7 @@ jobs: uses: hotosm/gh-workflows/.github/workflows/image_cache.yml@main with: image_names: | - docker.io/postgis/postgis:${{ vars.POSTGIS_TAG }}-alpine + docker.io/postgis/postgis:${{ vars.POSTGIS_TAG }} ghcr.io/hotosm/fmtm/odkcentral:${{ vars.ODK_CENTRAL_TAG }} ghcr.io/hotosm/fmtm/odkcentral-proxy:${{ vars.ODK_CENTRAL_TAG }} docker.io/minio/minio:${{ vars.MINIO_TAG }} From cd8da2043813dcd6d23edf0d5a00f8ad0045010f Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Fri, 20 Oct 2023 10:33:41 +0100 Subject: [PATCH 34/69] build: redact password in odk-central entrypoint --- odkcentral/api/init-user-and-start.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/odkcentral/api/init-user-and-start.sh b/odkcentral/api/init-user-and-start.sh index 7918d93077..7cf586017d 100644 --- a/odkcentral/api/init-user-and-start.sh +++ b/odkcentral/api/init-user-and-start.sh @@ -15,7 +15,7 @@ echo "The server will not start on this run" ./init-odk-db.sh ### Create admin user ### -echo "Creating test user ${SYSADMIN_EMAIL} with password ${SYSADMIN_PASSWD}" +echo "Creating test user ${SYSADMIN_EMAIL} with password ***${SYSADMIN_PASSWD: -3}" echo "${SYSADMIN_PASSWD}" | odk-cmd --email "${SYSADMIN_EMAIL}" user-create || true echo "Elevating user to admin" From 222aa3965e54032540030c592f5e9f8118d4886e Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Fri, 20 Oct 2023 10:50:55 +0100 Subject: [PATCH 35/69] ci: fix to_envs single .env file parsing --- .github/workflows/build_and_deploy.yml | 3 +-- .github/workflows/r-pytest.yml | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml index bb5627821a..235eea4e15 100644 --- a/.github/workflows/build_and_deploy.yml +++ b/.github/workflows/build_and_deploy.yml @@ -64,8 +64,7 @@ jobs: VARS_CONTEXT: ${{ toJson(vars) }} SECRETS_CONTEXT: ${{ toJson(secrets) }} run: | - EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) - to_envs() { jq -r "to_entries[] | \"\(.key)<<$EOF\n\(.value)\n$EOF\n\""; } + to_envs() { jq -r "to_entries[] | \"\(.key)=\(.value)\""; } echo "GIT_BRANCH=${GIT_BRANCH}" >> $GITHUB_ENV echo "${{ env.VARS_CONTEXT }}" | to_envs >> $GITHUB_ENV diff --git a/.github/workflows/r-pytest.yml b/.github/workflows/r-pytest.yml index ebad9b0e0f..87ffb80d60 100644 --- a/.github/workflows/r-pytest.yml +++ b/.github/workflows/r-pytest.yml @@ -50,8 +50,7 @@ jobs: VARS_CONTEXT: ${{ toJson(vars) }} SECRETS_CONTEXT: ${{ toJson(secrets) }} run: | - EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) - to_envs() { jq -r "to_entries[] | \"\(.key)<<$EOF\n\(.value)\n$EOF\n\""; } + to_envs() { jq -r "to_entries[] | \"\(.key)=\(.value)\""; } echo "API_TAG_OVERRIDE=${API_TAG_OVERRIDE}" >> $GITHUB_ENV echo "GIT_BRANCH=${GIT_BRANCH}" >> $GITHUB_ENV From d64ba4cef54ece408f553107f9eaeeb87c9e82ce Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Fri, 20 Oct 2023 11:00:22 +0100 Subject: [PATCH 36/69] ci: pass env var directly to jq, not github var --- .github/workflows/build_and_deploy.yml | 4 ++-- .github/workflows/r-pytest.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml index 235eea4e15..89e4f0afb1 100644 --- a/.github/workflows/build_and_deploy.yml +++ b/.github/workflows/build_and_deploy.yml @@ -67,8 +67,8 @@ jobs: to_envs() { jq -r "to_entries[] | \"\(.key)=\(.value)\""; } echo "GIT_BRANCH=${GIT_BRANCH}" >> $GITHUB_ENV - echo "${{ env.VARS_CONTEXT }}" | to_envs >> $GITHUB_ENV - echo "${{ env.SECRETS_CONTEXT }}" | to_envs >> $GITHUB_ENV + echo "${VARS_CONTEXT}" | to_envs >> $GITHUB_ENV + echo "${SECRETS_CONTEXT}" | to_envs >> $GITHUB_ENV - name: Create .env file run: env > .env diff --git a/.github/workflows/r-pytest.yml b/.github/workflows/r-pytest.yml index 87ffb80d60..355c2bb5c6 100644 --- a/.github/workflows/r-pytest.yml +++ b/.github/workflows/r-pytest.yml @@ -54,8 +54,8 @@ jobs: echo "API_TAG_OVERRIDE=${API_TAG_OVERRIDE}" >> $GITHUB_ENV echo "GIT_BRANCH=${GIT_BRANCH}" >> $GITHUB_ENV - echo "${{ env.VARS_CONTEXT }}" | to_envs >> $GITHUB_ENV - echo "${{ env.SECRETS_CONTEXT }}" | to_envs >> $GITHUB_ENV + echo "${VARS_CONTEXT}" | to_envs >> $GITHUB_ENV + echo "${SECRETS_CONTEXT}" | to_envs >> $GITHUB_ENV - name: Create .env file run: env > .env From 2b7c3b9cbef4f53b8e26ce12d406b01778df87cd Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Fri, 20 Oct 2023 11:36:35 +0100 Subject: [PATCH 37/69] ci(pytest): fix by using scalar over literal syntax --- .github/workflows/r-pytest.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/r-pytest.yml b/.github/workflows/r-pytest.yml index 355c2bb5c6..611d8ea6b8 100644 --- a/.github/workflows/r-pytest.yml +++ b/.github/workflows/r-pytest.yml @@ -26,8 +26,8 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 - - id: download-images - name: Download image cache + - name: Download Image Cache + id: download-images uses: actions/download-artifact@v3 with: name: image-cache @@ -67,12 +67,13 @@ jobs: -- wait-for-it central:8383 --strict --timeout=30 \ -- pytest - cache-images: + # This stage only runs if the images were not found in the cache + upload-img-cache: needs: [run-pytest] if: ${{ needs.run-pytest.outputs.img_cache_loaded != true}} uses: hotosm/gh-workflows/.github/workflows/image_cache.yml@main with: - image_names: | + image_names: > docker.io/postgis/postgis:${{ vars.POSTGIS_TAG }} ghcr.io/hotosm/fmtm/odkcentral:${{ vars.ODK_CENTRAL_TAG }} ghcr.io/hotosm/fmtm/odkcentral-proxy:${{ vars.ODK_CENTRAL_TAG }} From 79b2d26b5ee39bcc0be607e801b70adaf98ba51b Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Fri, 20 Oct 2023 13:07:59 +0100 Subject: [PATCH 38/69] build: hardcode WORKER_COUNT=1 for odk-central start --- odkcentral/api/init-user-and-start.sh | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/odkcentral/api/init-user-and-start.sh b/odkcentral/api/init-user-and-start.sh index 7cf586017d..bb421ed25d 100644 --- a/odkcentral/api/init-user-and-start.sh +++ b/odkcentral/api/init-user-and-start.sh @@ -21,15 +21,7 @@ echo "${SYSADMIN_PASSWD}" | odk-cmd --email "${SYSADMIN_EMAIL}" user-create || t echo "Elevating user to admin" odk-cmd --email "${SYSADMIN_EMAIL}" user-promote || true -### Run server (copied from start-odk.sh) ### -MEMTOT=$(vmstat -s | grep 'total memory' | awk '{ print $1 }') -if [ "$MEMTOT" -gt "1100000" ] -then - export WORKER_COUNT=4 -else - export WORKER_COUNT=1 -fi -echo "using $WORKER_COUNT worker(s) based on available memory ($MEMTOT).." - +### Run server (hardcode WORKER_COUNT=1 for dev) ### +export WORKER_COUNT=1 echo "Starting server." exec npx pm2-runtime ./pm2.config.js From 894cfcedc6bd9f10f506c37a36389301c0b06c69 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Fri, 20 Oct 2023 14:39:22 +0100 Subject: [PATCH 39/69] ci: update docs workflow to use new cahcing --- .github/workflows/docs.yml | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 22c189d3d0..872fc8a607 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -14,20 +14,14 @@ jobs: build_doxygen: uses: hotosm/gh-workflows/.github/workflows/doxygen_build.yml@main with: - cache_paths: | - docs/apidocs - docs/openapi.json - cache_key: docs-build + output_path: docs/apidocs build_openapi_json: uses: hotosm/gh-workflows/.github/workflows/openapi_build.yml@main with: image: ghcr.io/hotosm/fmtm/backend:ci-${{ github.ref_name }} example_env_file_path: ".env.example" - cache_paths: | - docs/apidocs - docs/openapi.json - cache_key: docs-build + output_path: docs/openapi.json publish_docs: uses: hotosm/gh-workflows/.github/workflows/mkdocs_build.yml@main @@ -36,7 +30,5 @@ jobs: - build_openapi_json with: image: ghcr.io/hotosm/fmtm/backend:ci-${{ github.ref_name }} - cache_paths: | - docs/apidocs - docs/openapi.json - cache_key: docs-build + doxygen: true + openapi: true From be175c3c6228f862f98c56970407b9c9d19f9dfc Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Fri, 20 Oct 2023 14:39:38 +0100 Subject: [PATCH 40/69] docs: update info for testing workflows via act --- .github/workflows/tests/push_payload.json | 3 ++- .github/workflows/tests/test_ci.sh | 9 +++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests/push_payload.json b/.github/workflows/tests/push_payload.json index eb2701ab2c..d476f82346 100644 --- a/.github/workflows/tests/push_payload.json +++ b/.github/workflows/tests/push_payload.json @@ -1,3 +1,4 @@ { - "base_ref ": "development" + "base_ref ": "development", + "ref": "refs/heads/development" } diff --git a/.github/workflows/tests/test_ci.sh b/.github/workflows/tests/test_ci.sh index a743734438..60a319231d 100644 --- a/.github/workflows/tests/test_ci.sh +++ b/.github/workflows/tests/test_ci.sh @@ -11,8 +11,13 @@ set -e # GITHUB_TOKEN=input # Feed to act using -s flag: -s GITHUB_TOKEN=input_personal_access_token -# PR -act pull_request -W .github/workflows/pr_test.yml \ +# PR Test Backend +act pull_request -W .github/workflows/pr_test_backend.yml \ + -e .github/workflows/tests/pr_payload.json \ + --var-file=.env --secret-file=.env + +# PR Test Frontend +act pull_request -W .github/workflows/pr_test_frontend.yml \ -e .github/workflows/tests/pr_payload.json \ --var-file=.env --secret-file=.env From 70ff49db9f165c0e4e798c1cc36a67acdac7c177 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Fri, 20 Oct 2023 14:40:35 +0100 Subject: [PATCH 41/69] ci(pytest): add caching img steps directly to workflow (no reuse) --- .github/workflows/r-pytest.yml | 57 +++++++++++++++++++--------------- 1 file changed, 32 insertions(+), 25 deletions(-) diff --git a/.github/workflows/r-pytest.yml b/.github/workflows/r-pytest.yml index 611d8ea6b8..a64a4478c8 100644 --- a/.github/workflows/r-pytest.yml +++ b/.github/workflows/r-pytest.yml @@ -19,29 +19,49 @@ jobs: environment: name: ${{ inputs.environment || 'test' }} - outputs: - img_cache_loaded: ${{ steps.load-cache.outputs.img_cache_loaded }} - steps: - name: Checkout repository uses: actions/checkout@v4 - - name: Download Image Cache - id: download-images - uses: actions/download-artifact@v3 + - id: image-cache + uses: actions/cache@v3 with: - name: image-cache - path: /tmp/images - continue-on-error: true + path: /tmp/image + key: image-cache-${{ runner.os }} + + - name: Save Local Images + if: steps.image-cache.outputs.cache-hit != 'true' + run: | + # Make artifact dir + mkdir -p /tmp/images + + images_array=() + images_array+=("docker.io/postgis/postgis:${{ vars.POSTGIS_TAG }}") + images_array+=("ghcr.io/hotosm/fmtm/odkcentral:${{ vars.ODK_CENTRAL_TAG }}") + images_array+=("ghcr.io/hotosm/fmtm/odkcentral-proxy:${{ vars.ODK_CENTRAL_TAG }}") + images_array+=("docker.io/minio/minio:${{ vars.MINIO_TAG }}") + + # Iterate images + for image in "${images_array[@]}"; do + echo "Processing image ${image}" + docker pull "${image}" + + if [ $? -eq 0 ]; then + img_underscores=${image//[:\/.]/_} + echo "Packaging image to /tmp/images/${img_underscores}.tar" + docker image save "${image}" \ + --output /tmp/images/${img_underscores}.tar + else + echo "Failed to pull the image: ${image}" + fi + done - name: Load Cached Imgs - id: load-cache - if: ${{ steps.download-images.outputs.download_path == '/tmp/images' }} + if: steps.image-cache.outputs.cache-hit == 'true' run: | for image_tar in /tmp/images/*; do docker image load --input $image_tar || true done - echo "img_cache_loaded=true" >> $GITHUB_OUTPUT - name: Vars and Secrets to Env env: @@ -66,16 +86,3 @@ jobs: wait-for-it fmtm-db:5432 --strict \ -- wait-for-it central:8383 --strict --timeout=30 \ -- pytest - - # This stage only runs if the images were not found in the cache - upload-img-cache: - needs: [run-pytest] - if: ${{ needs.run-pytest.outputs.img_cache_loaded != true}} - uses: hotosm/gh-workflows/.github/workflows/image_cache.yml@main - with: - image_names: > - docker.io/postgis/postgis:${{ vars.POSTGIS_TAG }} - ghcr.io/hotosm/fmtm/odkcentral:${{ vars.ODK_CENTRAL_TAG }} - ghcr.io/hotosm/fmtm/odkcentral-proxy:${{ vars.ODK_CENTRAL_TAG }} - docker.io/minio/minio:${{ vars.MINIO_TAG }} - artifact_name: image-cache From 3b1ae5eaee616f188f326641a005c02def03a378 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Fri, 20 Oct 2023 14:48:13 +0100 Subject: [PATCH 42/69] ci(pytest): typo image --> images for cache key --- .github/workflows/r-pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/r-pytest.yml b/.github/workflows/r-pytest.yml index a64a4478c8..008544263e 100644 --- a/.github/workflows/r-pytest.yml +++ b/.github/workflows/r-pytest.yml @@ -26,7 +26,7 @@ jobs: - id: image-cache uses: actions/cache@v3 with: - path: /tmp/image + path: /tmp/images key: image-cache-${{ runner.os }} - name: Save Local Images From 8e57e493ea425c0d99ccaf2376f2bfc876d335cb Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Fri, 20 Oct 2023 14:58:45 +0100 Subject: [PATCH 43/69] refactor: pydantic deprecation FIeldValidationInfo --> ValidationInfo --- src/backend/app/config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/backend/app/config.py b/src/backend/app/config.py index 7ceb234468..47334aa225 100644 --- a/src/backend/app/config.py +++ b/src/backend/app/config.py @@ -20,7 +20,7 @@ from functools import lru_cache from typing import Any, Optional, Union -from pydantic import FieldValidationInfo, PostgresDsn, field_validator +from pydantic import PostgresDsn, ValidationInfo, field_validator from pydantic_settings import BaseSettings, SettingsConfigDict @@ -41,7 +41,7 @@ class Settings(BaseSettings): def assemble_cors_origins( cls, val: Union[str, list[str]], - info: FieldValidationInfo, + info: ValidationInfo, ) -> Union[list[str], str]: """Build and validate CORS origins list. @@ -82,7 +82,7 @@ def assemble_cors_origins( @field_validator("FMTM_DB_URL", mode="after") @classmethod - def assemble_db_connection(cls, v: Optional[str], info: FieldValidationInfo) -> Any: + def assemble_db_connection(cls, v: Optional[str], info: ValidationInfo) -> Any: """Build Postgres connection from environment variables.""" if isinstance(v, str): return v From 2c0b2db972921dcce67eaa7bf1a176356e4e265f Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Fri, 20 Oct 2023 15:23:26 +0100 Subject: [PATCH 44/69] ci: rewrite pytest to build pr image prior to test --- .github/workflows/build_and_deploy.yml | 2 + .github/workflows/pr_test_backend.yml | 2 +- .github/workflows/r-pytest.yml | 73 +++++++++++++++++++------- 3 files changed, 56 insertions(+), 21 deletions(-) diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml index 89e4f0afb1..3453a27f3a 100644 --- a/.github/workflows/build_and_deploy.yml +++ b/.github/workflows/build_and_deploy.yml @@ -19,6 +19,8 @@ jobs: pytest: uses: ./.github/workflows/r-pytest.yml with: + # TODO update to build image prior to test, like r-pytest.yml? + # Would prevent failures if the CI image does not contain new deps yet image_tag: ci-${{ github.ref_name }} secrets: inherit diff --git a/.github/workflows/pr_test_backend.yml b/.github/workflows/pr_test_backend.yml index e294cce13e..57ab2db4dd 100644 --- a/.github/workflows/pr_test_backend.yml +++ b/.github/workflows/pr_test_backend.yml @@ -16,5 +16,5 @@ jobs: pytest: uses: ./.github/workflows/r-pytest.yml with: - image_tag: ci-${{ github.base_ref }} + build_test_img: true secrets: inherit diff --git a/.github/workflows/r-pytest.yml b/.github/workflows/r-pytest.yml index 008544263e..cffeb9f67b 100644 --- a/.github/workflows/r-pytest.yml +++ b/.github/workflows/r-pytest.yml @@ -4,20 +4,43 @@ on: workflow_call: inputs: image_tag: - required: true + description: "Optional image tag override." + required: false type: string + default: ci-development + build_test_img: + description: "Build an image prior to test, e.g. if new deps added." + required: false + type: boolean + default: false environment: + description: "The environment to use for testing." required: false type: string + default: "test" permissions: contents: read jobs: + backend-test-build: + uses: hotosm/gh-workflows/.github/workflows/image_build.yml@main + if: ${{ inputs.build_test_img }} + with: + context: src/backend + build_target: ci + image_name: ghcr.io/${{ github.repository }}/backend + build_args: | + APP_VERSION=${{ github.ref_name }} + COMMIT_REF=${{ github.sha }} + run-pytest: runs-on: ubuntu-latest + needs: [backend-test-build] + # Ensure it runs, even if build_test_img=false + if: always() environment: - name: ${{ inputs.environment || 'test' }} + name: ${{ inputs.environment}} steps: - name: Checkout repository @@ -32,30 +55,40 @@ jobs: - name: Save Local Images if: steps.image-cache.outputs.cache-hit != 'true' run: | + # Function to pull and package Docker image + function pull_and_package_image() { + local image=$1 + echo "Processing image ${image}" + docker pull "${image}" + + if [ $? -eq 0 ]; then + img_underscores=${image//[:\/.]/_} + echo "Packaging image to /tmp/images/${img_underscores}.tar" + docker image save "${image}" --output "/tmp/images/${img_underscores}.tar" + else + echo "Failed to pull the image: ${image}" + fi + } + # Make artifact dir mkdir -p /tmp/images - images_array=() - images_array+=("docker.io/postgis/postgis:${{ vars.POSTGIS_TAG }}") - images_array+=("ghcr.io/hotosm/fmtm/odkcentral:${{ vars.ODK_CENTRAL_TAG }}") - images_array+=("ghcr.io/hotosm/fmtm/odkcentral-proxy:${{ vars.ODK_CENTRAL_TAG }}") - images_array+=("docker.io/minio/minio:${{ vars.MINIO_TAG }}") + images_array=( + "docker.io/postgis/postgis:${{ vars.POSTGIS_TAG }}" + "ghcr.io/hotosm/fmtm/odkcentral:${{ vars.ODK_CENTRAL_TAG }}" + "ghcr.io/hotosm/fmtm/odkcentral-proxy:${{ vars.ODK_CENTRAL_TAG }}" + "docker.io/minio/minio:${{ vars.MINIO_TAG }}" + ) - # Iterate images + # Iterate through dependency images for image in "${images_array[@]}"; do - echo "Processing image ${image}" - docker pull "${image}" - - if [ $? -eq 0 ]; then - img_underscores=${image//[:\/.]/_} - echo "Packaging image to /tmp/images/${img_underscores}.tar" - docker image save "${image}" \ - --output /tmp/images/${img_underscores}.tar - else - echo "Failed to pull the image: ${image}" - fi + pull_and_package_image "$image" done + # Cache backend build + image=${{ needs.backend-test-build.outputs.image_name || inputs.image_tag }} + pull_and_package_image "$image" + - name: Load Cached Imgs if: steps.image-cache.outputs.cache-hit == 'true' run: | @@ -65,7 +98,7 @@ jobs: - name: Vars and Secrets to Env env: - API_TAG_OVERRIDE: ${{ inputs.image_tag }} + API_TAG_OVERRIDE: ${{ needs.backend-test-build.outputs.image_name || inputs.image_tag }} GIT_BRANCH: ${{ github.ref_name }} VARS_CONTEXT: ${{ toJson(vars) }} SECRETS_CONTEXT: ${{ toJson(secrets) }} From 83b5c2c8d49244df677cb1edfe69fbbe46e482ac Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Fri, 20 Oct 2023 15:27:24 +0100 Subject: [PATCH 45/69] ci: don't push,cache,multi_arch PR image builds --- .github/workflows/r-pytest.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/r-pytest.yml b/.github/workflows/r-pytest.yml index cffeb9f67b..ed5ca0c7c1 100644 --- a/.github/workflows/r-pytest.yml +++ b/.github/workflows/r-pytest.yml @@ -33,6 +33,10 @@ jobs: build_args: | APP_VERSION=${{ github.ref_name }} COMMIT_REF=${{ github.sha }} + # Build temp (workflow) cached image for PR, not in registry + push: false + cache: false + multi_arch: false run-pytest: runs-on: ubuntu-latest From d1aa451cdeff133a3b4f7b3ef0d8200eea17d6ae Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Fri, 20 Oct 2023 15:32:10 +0100 Subject: [PATCH 46/69] ci: add permissions: write to pytest build + scan=false --- .github/workflows/r-pytest.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/r-pytest.yml b/.github/workflows/r-pytest.yml index ed5ca0c7c1..f5420573f8 100644 --- a/.github/workflows/r-pytest.yml +++ b/.github/workflows/r-pytest.yml @@ -19,11 +19,11 @@ on: type: string default: "test" -permissions: - contents: read - jobs: backend-test-build: + # Write permissions required for build + permissions: + contents: write uses: hotosm/gh-workflows/.github/workflows/image_build.yml@main if: ${{ inputs.build_test_img }} with: @@ -37,6 +37,7 @@ jobs: push: false cache: false multi_arch: false + scan: false run-pytest: runs-on: ubuntu-latest From 78f054a1d08dc1c0c2865205570d92142cfe44ef Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Fri, 20 Oct 2023 15:34:03 +0100 Subject: [PATCH 47/69] ci: remove permissions from pytest (handle downstream) --- .github/workflows/r-pytest.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/r-pytest.yml b/.github/workflows/r-pytest.yml index f5420573f8..4158a5b182 100644 --- a/.github/workflows/r-pytest.yml +++ b/.github/workflows/r-pytest.yml @@ -21,9 +21,6 @@ on: jobs: backend-test-build: - # Write permissions required for build - permissions: - contents: write uses: hotosm/gh-workflows/.github/workflows/image_build.yml@main if: ${{ inputs.build_test_img }} with: From 663ad015dafa665940c5f5f090074275cd10171e Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Fri, 20 Oct 2023 16:03:49 +0100 Subject: [PATCH 48/69] ci: update conditionals for building pr images --- .github/workflows/r-pytest.yml | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/.github/workflows/r-pytest.yml b/.github/workflows/r-pytest.yml index 4158a5b182..d773451cc7 100644 --- a/.github/workflows/r-pytest.yml +++ b/.github/workflows/r-pytest.yml @@ -20,9 +20,22 @@ on: default: "test" jobs: + check-img-cache: + runs-on: ubuntu-latest + if: ${{ inputs.build_test_img }} + environment: + name: ${{ inputs.environment }} + steps: + - id: image-cache + uses: actions/cache@v3 + with: + path: /tmp/images + key: image-cache-${{ runner.os }} + backend-test-build: uses: hotosm/gh-workflows/.github/workflows/image_build.yml@main - if: ${{ inputs.build_test_img }} + needs: [check-img-cache] + if: ${{ needs.check-img-cache.outputs.cache-hit != 'true' }} with: context: src/backend build_target: ci @@ -30,11 +43,10 @@ jobs: build_args: | APP_VERSION=${{ github.ref_name }} COMMIT_REF=${{ github.sha }} - # Build temp (workflow) cached image for PR, not in registry - push: false - cache: false + # Build temp image for PR, no multi-arch or scan required multi_arch: false scan: false + cache: false run-pytest: runs-on: ubuntu-latest @@ -42,7 +54,7 @@ jobs: # Ensure it runs, even if build_test_img=false if: always() environment: - name: ${{ inputs.environment}} + name: ${{ inputs.environment }} steps: - name: Checkout repository From cfe9c1fd7571b479fdd9f255a5f8a89fe549c928 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Fri, 20 Oct 2023 16:11:11 +0100 Subject: [PATCH 49/69] ci: update pytest cache-hit output --- .github/workflows/r-pytest.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/r-pytest.yml b/.github/workflows/r-pytest.yml index d773451cc7..a7953e7920 100644 --- a/.github/workflows/r-pytest.yml +++ b/.github/workflows/r-pytest.yml @@ -25,6 +25,8 @@ jobs: if: ${{ inputs.build_test_img }} environment: name: ${{ inputs.environment }} + outputs: + cache-hit: ${{ steps.image-cache.cache-hit }} steps: - id: image-cache uses: actions/cache@v3 From 14ea4531c83fe03f5bc8b979270cb4256d72c020 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Fri, 20 Oct 2023 16:17:23 +0100 Subject: [PATCH 50/69] ci: get image_tag instead of image_name from output --- .github/workflows/r-pytest.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/r-pytest.yml b/.github/workflows/r-pytest.yml index a7953e7920..52dc607ea0 100644 --- a/.github/workflows/r-pytest.yml +++ b/.github/workflows/r-pytest.yml @@ -102,7 +102,7 @@ jobs: done # Cache backend build - image=${{ needs.backend-test-build.outputs.image_name || inputs.image_tag }} + image=${{ needs.backend-test-build.outputs.image_tag || inputs.image_tag }} pull_and_package_image "$image" - name: Load Cached Imgs @@ -114,7 +114,7 @@ jobs: - name: Vars and Secrets to Env env: - API_TAG_OVERRIDE: ${{ needs.backend-test-build.outputs.image_name || inputs.image_tag }} + API_TAG_OVERRIDE: ${{ needs.backend-test-build.outputs.image_tag || inputs.image_tag }} GIT_BRANCH: ${{ github.ref_name }} VARS_CONTEXT: ${{ toJson(vars) }} SECRETS_CONTEXT: ${{ toJson(secrets) }} From 149c154f44f63abb434b6cc3a7ffd68c52446397 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Fri, 20 Oct 2023 16:19:30 +0100 Subject: [PATCH 51/69] ci(pytest): fix case where cache hit, skip build --- .github/workflows/r-pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/r-pytest.yml b/.github/workflows/r-pytest.yml index 52dc607ea0..3f42051828 100644 --- a/.github/workflows/r-pytest.yml +++ b/.github/workflows/r-pytest.yml @@ -26,7 +26,7 @@ jobs: environment: name: ${{ inputs.environment }} outputs: - cache-hit: ${{ steps.image-cache.cache-hit }} + cache-hit: ${{ steps.image-cache.outputs.cache-hit }} steps: - id: image-cache uses: actions/cache@v3 From 2b3c540b5d70bda0ea245402c0290acffe7e861f Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Fri, 20 Oct 2023 16:33:19 +0100 Subject: [PATCH 52/69] ci(pytest): set BACKEND_IMG_TAG in all if cases --- .github/workflows/r-pytest.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/r-pytest.yml b/.github/workflows/r-pytest.yml index 3f42051828..abebd5e496 100644 --- a/.github/workflows/r-pytest.yml +++ b/.github/workflows/r-pytest.yml @@ -104,17 +104,24 @@ jobs: # Cache backend build image=${{ needs.backend-test-build.outputs.image_tag || inputs.image_tag }} pull_and_package_image "$image" + echo "BACKEND_IMG_TAG=${image}" >> $GITHUB_ENV - name: Load Cached Imgs - if: steps.image-cache.outputs.cache-hit == 'true' + if: ${{ steps.image-cache.outputs.cache-hit == 'true' }} run: | for image_tar in /tmp/images/*; do docker image load --input $image_tar || true done + # Set BACKEND_IMG_TAG if not set + image=$(docker image ls | grep backend) + image_tag="${image##*:}" + echo "Image Tag: $image_tag" + echo "BACKEND_IMG_TAG=${image_tag}" >> $GITHUB_ENV + - name: Vars and Secrets to Env env: - API_TAG_OVERRIDE: ${{ needs.backend-test-build.outputs.image_tag || inputs.image_tag }} + API_TAG_OVERRIDE: ${BACKEND_IMG_TAG} GIT_BRANCH: ${{ github.ref_name }} VARS_CONTEXT: ${{ toJson(vars) }} SECRETS_CONTEXT: ${{ toJson(secrets) }} From 5779aa62973acbaf15155e7a7cdd752340739c26 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Fri, 20 Oct 2023 16:54:54 +0100 Subject: [PATCH 53/69] ci(pytest): replace BACKEND_IMG_TAG --> API_TAG_OVERRIDE --- .github/workflows/r-pytest.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/r-pytest.yml b/.github/workflows/r-pytest.yml index abebd5e496..35c7833dfc 100644 --- a/.github/workflows/r-pytest.yml +++ b/.github/workflows/r-pytest.yml @@ -102,9 +102,10 @@ jobs: done # Cache backend build - image=${{ needs.backend-test-build.outputs.image_tag || inputs.image_tag }} + image_tag=${{ needs.backend-test-build.outputs.image_tag || inputs.image_tag }} + image="ghcr.io/${{ github.repository }}/backend:${image_tag}" pull_and_package_image "$image" - echo "BACKEND_IMG_TAG=${image}" >> $GITHUB_ENV + echo "API_TAG_OVERRIDE=${image_tag}" >> $GITHUB_ENV - name: Load Cached Imgs if: ${{ steps.image-cache.outputs.cache-hit == 'true' }} @@ -113,15 +114,16 @@ jobs: docker image load --input $image_tar || true done - # Set BACKEND_IMG_TAG if not set + # Set API_TAG_OVERRIDE if not set + echo "Getting backend image tag via docker" image=$(docker image ls | grep backend) image_tag="${image##*:}" echo "Image Tag: $image_tag" - echo "BACKEND_IMG_TAG=${image_tag}" >> $GITHUB_ENV + echo "API_TAG_OVERRIDE=${image_tag}" >> $GITHUB_ENV - name: Vars and Secrets to Env env: - API_TAG_OVERRIDE: ${BACKEND_IMG_TAG} + API_TAG_OVERRIDE: ${{ env.API_TAG_OVERRIDE }} GIT_BRANCH: ${{ github.ref_name }} VARS_CONTEXT: ${{ toJson(vars) }} SECRETS_CONTEXT: ${{ toJson(secrets) }} From 8c0118c1c7e8b338a7b602d9a9b06ebdd54acf68 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Sun, 22 Oct 2023 03:35:05 +0100 Subject: [PATCH 54/69] ci(pytest): fix getting backend image tag from docker img ls --- .github/workflows/r-pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/r-pytest.yml b/.github/workflows/r-pytest.yml index 35c7833dfc..d0d743467d 100644 --- a/.github/workflows/r-pytest.yml +++ b/.github/workflows/r-pytest.yml @@ -116,7 +116,7 @@ jobs: # Set API_TAG_OVERRIDE if not set echo "Getting backend image tag via docker" - image=$(docker image ls | grep backend) + image=$(docker image ls --filter=reference='ghcr.io/hotosm/fmtm/backend' --format='{{.Tag}}') image_tag="${image##*:}" echo "Image Tag: $image_tag" echo "API_TAG_OVERRIDE=${image_tag}" >> $GITHUB_ENV From f18ad1a03f923f00a2fd7e87f7d93af6b253e880 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Sun, 22 Oct 2023 04:41:06 +0100 Subject: [PATCH 55/69] ci(pytest): only run image caching steps if explicity cache=false --- .github/workflows/r-pytest.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/r-pytest.yml b/.github/workflows/r-pytest.yml index d0d743467d..92cef28ecb 100644 --- a/.github/workflows/r-pytest.yml +++ b/.github/workflows/r-pytest.yml @@ -37,7 +37,7 @@ jobs: backend-test-build: uses: hotosm/gh-workflows/.github/workflows/image_build.yml@main needs: [check-img-cache] - if: ${{ needs.check-img-cache.outputs.cache-hit != 'true' }} + if: ${{ needs.check-img-cache.outputs.cache-hit == 'false' }} with: context: src/backend build_target: ci @@ -69,7 +69,7 @@ jobs: key: image-cache-${{ runner.os }} - name: Save Local Images - if: steps.image-cache.outputs.cache-hit != 'true' + if: ${{ steps.image-cache.outputs.cache-hit == 'false' }} run: | # Function to pull and package Docker image function pull_and_package_image() { @@ -91,10 +91,11 @@ jobs: images_array=( "docker.io/postgis/postgis:${{ vars.POSTGIS_TAG }}" - "ghcr.io/hotosm/fmtm/odkcentral:${{ vars.ODK_CENTRAL_TAG }}" - "ghcr.io/hotosm/fmtm/odkcentral-proxy:${{ vars.ODK_CENTRAL_TAG }}" "docker.io/minio/minio:${{ vars.MINIO_TAG }}" ) + # For caching odk central images, add: + # "ghcr.io/hotosm/fmtm/odkcentral:${{ vars.ODK_CENTRAL_TAG }}" + # "ghcr.io/hotosm/fmtm/odkcentral-proxy:${{ vars.ODK_CENTRAL_TAG }}" # Iterate through dependency images for image in "${images_array[@]}"; do From 9f8f8b97af93863edfdeea50419ef2d5e6e39f5f Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Sun, 22 Oct 2023 05:02:52 +0100 Subject: [PATCH 56/69] ci(pytest): revert ==false to !=true negation --- .github/workflows/r-pytest.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/r-pytest.yml b/.github/workflows/r-pytest.yml index 92cef28ecb..120b5992cb 100644 --- a/.github/workflows/r-pytest.yml +++ b/.github/workflows/r-pytest.yml @@ -37,7 +37,7 @@ jobs: backend-test-build: uses: hotosm/gh-workflows/.github/workflows/image_build.yml@main needs: [check-img-cache] - if: ${{ needs.check-img-cache.outputs.cache-hit == 'false' }} + if: ${{ needs.check-img-cache.outputs.cache-hit != 'true' }} with: context: src/backend build_target: ci @@ -69,7 +69,7 @@ jobs: key: image-cache-${{ runner.os }} - name: Save Local Images - if: ${{ steps.image-cache.outputs.cache-hit == 'false' }} + if: ${{ steps.image-cache.outputs.cache-hit != 'true' }} run: | # Function to pull and package Docker image function pull_and_package_image() { @@ -94,8 +94,8 @@ jobs: "docker.io/minio/minio:${{ vars.MINIO_TAG }}" ) # For caching odk central images, add: - # "ghcr.io/hotosm/fmtm/odkcentral:${{ vars.ODK_CENTRAL_TAG }}" - # "ghcr.io/hotosm/fmtm/odkcentral-proxy:${{ vars.ODK_CENTRAL_TAG }}" + # "ghcr.io/${{ github.repository }}/odkcentral:${{ vars.ODK_CENTRAL_TAG }}" + # "ghcr.io/${{ github.repository }}/odkcentral-proxy:${{ vars.ODK_CENTRAL_TAG }}" # Iterate through dependency images for image in "${images_array[@]}"; do @@ -117,7 +117,11 @@ jobs: # Set API_TAG_OVERRIDE if not set echo "Getting backend image tag via docker" - image=$(docker image ls --filter=reference='ghcr.io/hotosm/fmtm/backend' --format='{{.Tag}}') + image=$(\ + docker image ls \ + --filter=reference='ghcr.io/${{ github.repository }}/backend' \ + --format='{{.Tag}}'\ + ) image_tag="${image##*:}" echo "Image Tag: $image_tag" echo "API_TAG_OVERRIDE=${image_tag}" >> $GITHUB_ENV From 33123ee7f5a4768a871d8a1025070c0dbf5b6f8c Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Sun, 22 Oct 2023 05:03:41 +0100 Subject: [PATCH 57/69] ci: replace all refs to repo with gh vars --- .github/workflows/build_ci_img.yml | 3 +-- .github/workflows/build_odk_imgs.yml | 8 ++++---- .github/workflows/docs.yml | 4 ++-- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build_ci_img.yml b/.github/workflows/build_ci_img.yml index 0b4d2fde0d..385ccce5e2 100644 --- a/.github/workflows/build_ci_img.yml +++ b/.github/workflows/build_ci_img.yml @@ -20,9 +20,8 @@ jobs: with: context: src/backend build_target: ci - image_name: ghcr.io/${{ github.repository }}/backend image_tags: | - "ghcr.io/hotosm/fmtm/backend:ci-${{ github.ref_name }}" + "ghcr.io/${{ github.repository }}/backend:ci-${{ github.ref_name }}" build_args: | APP_VERSION=${{ github.ref_name }} COMMIT_REF=${{ github.sha }} diff --git a/.github/workflows/build_odk_imgs.yml b/.github/workflows/build_odk_imgs.yml index 75a9b08660..09ca4f6562 100644 --- a/.github/workflows/build_odk_imgs.yml +++ b/.github/workflows/build_odk_imgs.yml @@ -17,8 +17,8 @@ jobs: with: context: odkcentral/api image_tags: | - "ghcr.io/hotosm/fmtm/odkcentral:${{ vars.ODK_CENTRAL_TAG }}" - "ghcr.io/hotosm/fmtm/odkcentral:latest" + "ghcr.io/${{ github.repository }}/odkcentral:${{ vars.ODK_CENTRAL_TAG }}" + "ghcr.io/${{ github.repository }}/odkcentral:latest" build_args: | ODK_CENTRAL_TAG=${{ vars.ODK_CENTRAL_TAG }} @@ -27,5 +27,5 @@ jobs: with: context: odkcentral/proxy image_tags: | - "ghcr.io/hotosm/fmtm/odkcentral-proxy:${{ vars.ODK_CENTRAL_TAG }}" - "ghcr.io/hotosm/fmtm/odkcentral-proxy:latest" + "ghcr.io/${{ github.repository }}/odkcentral-proxy:${{ vars.ODK_CENTRAL_TAG }}" + "ghcr.io/${{ github.repository }}/odkcentral-proxy:latest" diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 872fc8a607..34ec15e157 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -19,7 +19,7 @@ jobs: build_openapi_json: uses: hotosm/gh-workflows/.github/workflows/openapi_build.yml@main with: - image: ghcr.io/hotosm/fmtm/backend:ci-${{ github.ref_name }} + image: ghcr.io/${{ github.repository }}/backend:ci-${{ github.ref_name }} example_env_file_path: ".env.example" output_path: docs/openapi.json @@ -29,6 +29,6 @@ jobs: - build_doxygen - build_openapi_json with: - image: ghcr.io/hotosm/fmtm/backend:ci-${{ github.ref_name }} + image: ghcr.io/${{ github.repository }}/backend:ci-${{ github.ref_name }} doxygen: true openapi: true From 406dd72685f7b347f0fc5bd007d9403862554b0d Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Sun, 22 Oct 2023 05:29:51 +0100 Subject: [PATCH 58/69] ci(pytest): prep pytest for moving to gh-workflows --- .github/workflows/build_and_deploy.yml | 1 + .github/workflows/pr_test_backend.yml | 7 ++++++ .github/workflows/r-pytest.yml | 33 ++++++++++++++++---------- 3 files changed, 29 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml index 3453a27f3a..83f73daaaa 100644 --- a/.github/workflows/build_and_deploy.yml +++ b/.github/workflows/build_and_deploy.yml @@ -22,6 +22,7 @@ jobs: # TODO update to build image prior to test, like r-pytest.yml? # Would prevent failures if the CI image does not contain new deps yet image_tag: ci-${{ github.ref_name }} + docker_compose_service: api secrets: inherit frontend-tests: diff --git a/.github/workflows/pr_test_backend.yml b/.github/workflows/pr_test_backend.yml index 57ab2db4dd..125f1eabd8 100644 --- a/.github/workflows/pr_test_backend.yml +++ b/.github/workflows/pr_test_backend.yml @@ -16,5 +16,12 @@ jobs: pytest: uses: ./.github/workflows/r-pytest.yml with: + docker_compose_service: api build_test_img: true + cache_imgs: | + "docker.io/postgis/postgis:${{ vars.POSTGIS_TAG }}" + "docker.io/minio/minio:${{ vars.MINIO_TAG }}" + # For caching odk central images, add: + # "ghcr.io/${{ github.repository }}/odkcentral:${{ vars.ODK_CENTRAL_TAG }}" + # "ghcr.io/${{ github.repository }}/odkcentral-proxy:${{ vars.ODK_CENTRAL_TAG }}" secrets: inherit diff --git a/.github/workflows/r-pytest.yml b/.github/workflows/r-pytest.yml index 120b5992cb..ac3738b72f 100644 --- a/.github/workflows/r-pytest.yml +++ b/.github/workflows/r-pytest.yml @@ -13,6 +13,19 @@ on: required: false type: boolean default: false + cache_imgs: + description: "Space separated list of images to cache on each run." + required: false + type: string + docker_compose_file: + description: "The docker compose file used to run the test." + required: false + type: string + default: docker-compose.yml + docker_compose_service: + description: "The docker compose service to run the test against." + required: true + type: string environment: description: "The environment to use for testing." required: false @@ -22,7 +35,7 @@ on: jobs: check-img-cache: runs-on: ubuntu-latest - if: ${{ inputs.build_test_img }} + if: ${{ inputs.cache_imgs }} environment: name: ${{ inputs.environment }} outputs: @@ -37,7 +50,7 @@ jobs: backend-test-build: uses: hotosm/gh-workflows/.github/workflows/image_build.yml@main needs: [check-img-cache] - if: ${{ needs.check-img-cache.outputs.cache-hit != 'true' }} + if: ${{ inputs.build_test_img && needs.check-img-cache.outputs.cache-hit != 'true' }} with: context: src/backend build_target: ci @@ -63,13 +76,14 @@ jobs: uses: actions/checkout@v4 - id: image-cache + if: ${{ inputs.cache_imgs }} uses: actions/cache@v3 with: path: /tmp/images key: image-cache-${{ runner.os }} - name: Save Local Images - if: ${{ steps.image-cache.outputs.cache-hit != 'true' }} + if: ${{ inputs.cache_imgs && steps.image-cache.outputs.cache-hit != 'true' }} run: | # Function to pull and package Docker image function pull_and_package_image() { @@ -89,13 +103,7 @@ jobs: # Make artifact dir mkdir -p /tmp/images - images_array=( - "docker.io/postgis/postgis:${{ vars.POSTGIS_TAG }}" - "docker.io/minio/minio:${{ vars.MINIO_TAG }}" - ) - # For caching odk central images, add: - # "ghcr.io/${{ github.repository }}/odkcentral:${{ vars.ODK_CENTRAL_TAG }}" - # "ghcr.io/${{ github.repository }}/odkcentral-proxy:${{ vars.ODK_CENTRAL_TAG }}" + images_array=${{ inputs.cache_imgs }} # Iterate through dependency images for image in "${images_array[@]}"; do @@ -109,7 +117,7 @@ jobs: echo "API_TAG_OVERRIDE=${image_tag}" >> $GITHUB_ENV - name: Load Cached Imgs - if: ${{ steps.image-cache.outputs.cache-hit == 'true' }} + if: ${{ inputs.cache_imgs && steps.image-cache.outputs.cache-hit == 'true' }} run: | for image_tar in /tmp/images/*; do docker image load --input $image_tar || true @@ -145,7 +153,8 @@ jobs: - name: Run PyTest run: | - docker compose run api \ + docker compose --file ${{ inputs.docker_compose_file }} \ + run ${{ inputs.docker_compose_service }} \ wait-for-it fmtm-db:5432 --strict \ -- wait-for-it central:8383 --strict --timeout=30 \ -- pytest From 3ec3c0dc071b63691e799d63878199502ec8ca37 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Sun, 22 Oct 2023 05:33:31 +0100 Subject: [PATCH 59/69] ci(pytest): test with caching off --- .github/workflows/pr_test_backend.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr_test_backend.yml b/.github/workflows/pr_test_backend.yml index 125f1eabd8..c4ffc2b631 100644 --- a/.github/workflows/pr_test_backend.yml +++ b/.github/workflows/pr_test_backend.yml @@ -18,9 +18,9 @@ jobs: with: docker_compose_service: api build_test_img: true - cache_imgs: | - "docker.io/postgis/postgis:${{ vars.POSTGIS_TAG }}" - "docker.io/minio/minio:${{ vars.MINIO_TAG }}" + # cache_imgs: | + # "docker.io/postgis/postgis:${{ vars.POSTGIS_TAG }}" + # "docker.io/minio/minio:${{ vars.MINIO_TAG }}" # For caching odk central images, add: # "ghcr.io/${{ github.repository }}/odkcentral:${{ vars.ODK_CENTRAL_TAG }}" # "ghcr.io/${{ github.repository }}/odkcentral-proxy:${{ vars.ODK_CENTRAL_TAG }}" From 814b5e092e21d9d53f4040118016610eaef240de Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Sun, 22 Oct 2023 05:37:40 +0100 Subject: [PATCH 60/69] ci(pytest): remove redundant var build_test_img --- .github/workflows/pr_test_backend.yml | 1 - .github/workflows/r-pytest.yml | 14 ++++---------- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/.github/workflows/pr_test_backend.yml b/.github/workflows/pr_test_backend.yml index c4ffc2b631..45e9d3370e 100644 --- a/.github/workflows/pr_test_backend.yml +++ b/.github/workflows/pr_test_backend.yml @@ -17,7 +17,6 @@ jobs: uses: ./.github/workflows/r-pytest.yml with: docker_compose_service: api - build_test_img: true # cache_imgs: | # "docker.io/postgis/postgis:${{ vars.POSTGIS_TAG }}" # "docker.io/minio/minio:${{ vars.MINIO_TAG }}" diff --git a/.github/workflows/r-pytest.yml b/.github/workflows/r-pytest.yml index ac3738b72f..532ede7545 100644 --- a/.github/workflows/r-pytest.yml +++ b/.github/workflows/r-pytest.yml @@ -7,12 +7,6 @@ on: description: "Optional image tag override." required: false type: string - default: ci-development - build_test_img: - description: "Build an image prior to test, e.g. if new deps added." - required: false - type: boolean - default: false cache_imgs: description: "Space separated list of images to cache on each run." required: false @@ -50,7 +44,7 @@ jobs: backend-test-build: uses: hotosm/gh-workflows/.github/workflows/image_build.yml@main needs: [check-img-cache] - if: ${{ inputs.build_test_img && needs.check-img-cache.outputs.cache-hit != 'true' }} + if: ${{ inputs.image_tag == "" && needs.check-img-cache.outputs.cache-hit != 'true' }} with: context: src/backend build_target: ci @@ -66,7 +60,7 @@ jobs: run-pytest: runs-on: ubuntu-latest needs: [backend-test-build] - # Ensure it runs, even if build_test_img=false + # Ensure it runs, even if backend-test-build does not run if: always() environment: name: ${{ inputs.environment }} @@ -110,8 +104,8 @@ jobs: pull_and_package_image "$image" done - # Cache backend build - image_tag=${{ needs.backend-test-build.outputs.image_tag || inputs.image_tag }} + # Cache backend build (fallback to ci-development tag) + image_tag=${{ needs.backend-test-build.outputs.image_tag || ci-development }} image="ghcr.io/${{ github.repository }}/backend:${image_tag}" pull_and_package_image "$image" echo "API_TAG_OVERRIDE=${image_tag}" >> $GITHUB_ENV From afc31885dc5ae90d1bb52403f4b7cfba50f87c6d Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Sun, 22 Oct 2023 05:38:45 +0100 Subject: [PATCH 61/69] ci(pytest): fix "" workflow syntax --> '' --- .github/workflows/r-pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/r-pytest.yml b/.github/workflows/r-pytest.yml index 532ede7545..6a62b15372 100644 --- a/.github/workflows/r-pytest.yml +++ b/.github/workflows/r-pytest.yml @@ -44,7 +44,7 @@ jobs: backend-test-build: uses: hotosm/gh-workflows/.github/workflows/image_build.yml@main needs: [check-img-cache] - if: ${{ inputs.image_tag == "" && needs.check-img-cache.outputs.cache-hit != 'true' }} + if: ${{ inputs.image_tag == '' && needs.check-img-cache.outputs.cache-hit != 'true' }} with: context: src/backend build_target: ci From 0b5c008ea747bf1b6e625143877ea9e34840a181 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Sun, 22 Oct 2023 05:39:47 +0100 Subject: [PATCH 62/69] ci(pytest): quote ci-development fallback --- .github/workflows/r-pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/r-pytest.yml b/.github/workflows/r-pytest.yml index 6a62b15372..16e3ee3e95 100644 --- a/.github/workflows/r-pytest.yml +++ b/.github/workflows/r-pytest.yml @@ -105,7 +105,7 @@ jobs: done # Cache backend build (fallback to ci-development tag) - image_tag=${{ needs.backend-test-build.outputs.image_tag || ci-development }} + image_tag=${{ needs.backend-test-build.outputs.image_tag || 'ci-development' }} image="ghcr.io/${{ github.repository }}/backend:${image_tag}" pull_and_package_image "$image" echo "API_TAG_OVERRIDE=${image_tag}" >> $GITHUB_ENV From 4aa8451b7eab4dd68ff2491508ef27e02b700c70 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Sun, 22 Oct 2023 05:48:22 +0100 Subject: [PATCH 63/69] ci(pytest): move where default ci-development tag is set --- .github/workflows/r-pytest.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/r-pytest.yml b/.github/workflows/r-pytest.yml index 16e3ee3e95..10f0982b14 100644 --- a/.github/workflows/r-pytest.yml +++ b/.github/workflows/r-pytest.yml @@ -8,7 +8,7 @@ on: required: false type: string cache_imgs: - description: "Space separated list of images to cache on each run." + description: "Space separated list of images to cache on each run (e.g. to avoid rate limiting)." required: false type: string docker_compose_file: @@ -105,7 +105,7 @@ jobs: done # Cache backend build (fallback to ci-development tag) - image_tag=${{ needs.backend-test-build.outputs.image_tag || 'ci-development' }} + image_tag=${{ needs.backend-test-build.outputs.image_tag }} image="ghcr.io/${{ github.repository }}/backend:${image_tag}" pull_and_package_image "$image" echo "API_TAG_OVERRIDE=${image_tag}" >> $GITHUB_ENV @@ -130,7 +130,7 @@ jobs: - name: Vars and Secrets to Env env: - API_TAG_OVERRIDE: ${{ env.API_TAG_OVERRIDE }} + API_TAG_OVERRIDE: ${{ env.API_TAG_OVERRIDE || 'ci-development' }} GIT_BRANCH: ${{ github.ref_name }} VARS_CONTEXT: ${{ toJson(vars) }} SECRETS_CONTEXT: ${{ toJson(secrets) }} From 9ec6cee51a2d7607960869b1207875f328a4ce51 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Sun, 22 Oct 2023 05:50:26 +0100 Subject: [PATCH 64/69] ci(pytest): reenable caching for workflow images --- .github/workflows/pr_test_backend.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr_test_backend.yml b/.github/workflows/pr_test_backend.yml index 45e9d3370e..c229ad63c3 100644 --- a/.github/workflows/pr_test_backend.yml +++ b/.github/workflows/pr_test_backend.yml @@ -17,9 +17,9 @@ jobs: uses: ./.github/workflows/r-pytest.yml with: docker_compose_service: api - # cache_imgs: | - # "docker.io/postgis/postgis:${{ vars.POSTGIS_TAG }}" - # "docker.io/minio/minio:${{ vars.MINIO_TAG }}" + cache_imgs: | + "docker.io/postgis/postgis:${{ vars.POSTGIS_TAG }}" + "docker.io/minio/minio:${{ vars.MINIO_TAG }}" # For caching odk central images, add: # "ghcr.io/${{ github.repository }}/odkcentral:${{ vars.ODK_CENTRAL_TAG }}" # "ghcr.io/${{ github.repository }}/odkcentral-proxy:${{ vars.ODK_CENTRAL_TAG }}" From 99dd6cb92d42117f5b90b888fa3f72640ee12200 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Sun, 22 Oct 2023 06:05:22 +0100 Subject: [PATCH 65/69] ci(pytest): inputs.cache_imgs to array --- .github/workflows/r-pytest.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/r-pytest.yml b/.github/workflows/r-pytest.yml index 10f0982b14..6fb1cbeccd 100644 --- a/.github/workflows/r-pytest.yml +++ b/.github/workflows/r-pytest.yml @@ -97,7 +97,8 @@ jobs: # Make artifact dir mkdir -p /tmp/images - images_array=${{ inputs.cache_imgs }} + images_array=(${{ inputs.cache_imgs }}) + echo "Images to cache: ${images_array}" # Iterate through dependency images for image in "${images_array[@]}"; do From 9cc4c793eb338dda4f4ec1c65f770d8c4d979240 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Sun, 22 Oct 2023 06:22:45 +0100 Subject: [PATCH 66/69] ci(pytest): allow passing of image_tag to image build stage --- .github/workflows/r-pytest.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/r-pytest.yml b/.github/workflows/r-pytest.yml index 6fb1cbeccd..f1ef0e8cca 100644 --- a/.github/workflows/r-pytest.yml +++ b/.github/workflows/r-pytest.yml @@ -44,11 +44,12 @@ jobs: backend-test-build: uses: hotosm/gh-workflows/.github/workflows/image_build.yml@main needs: [check-img-cache] - if: ${{ inputs.image_tag == '' && needs.check-img-cache.outputs.cache-hit != 'true' }} + if: ${{ needs.check-img-cache.outputs.cache-hit != 'true' }} with: context: src/backend build_target: ci image_name: ghcr.io/${{ github.repository }}/backend + image_tags: ${{ inputs.image_tag && format('ghcr.io/{0}/backend:{1}', github.repository, inputs.image_tag) || '' }} build_args: | APP_VERSION=${{ github.ref_name }} COMMIT_REF=${{ github.sha }} From 35cdf24ad8b32bae9e4c36558a67a7344e4c9a2a Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Sun, 22 Oct 2023 06:49:45 +0100 Subject: [PATCH 67/69] ci: replace all reusable workflows with hotosm/gh-workflows --- .github/workflows/build_and_deploy.yml | 8 +- .github/workflows/pr_test_backend.yml | 2 +- .github/workflows/pr_test_frontend.yml | 4 +- .github/workflows/r-frontend_tests.yml | 32 ----- .github/workflows/r-pytest.yml | 156 ------------------------- 5 files changed, 8 insertions(+), 194 deletions(-) delete mode 100644 .github/workflows/r-frontend_tests.yml delete mode 100644 .github/workflows/r-pytest.yml diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml index 83f73daaaa..f4a053915b 100644 --- a/.github/workflows/build_and_deploy.yml +++ b/.github/workflows/build_and_deploy.yml @@ -17,16 +17,16 @@ on: jobs: pytest: - uses: ./.github/workflows/r-pytest.yml + uses: hotosm/gh-workflows/.github/workflows/test_pytest_compose.yml@main with: - # TODO update to build image prior to test, like r-pytest.yml? - # Would prevent failures if the CI image does not contain new deps yet image_tag: ci-${{ github.ref_name }} docker_compose_service: api secrets: inherit frontend-tests: - uses: ./.github/workflows/r-frontend_tests.yml + uses: hotosm/gh-workflows/.github/workflows/test_pnpm.yml@main + with: + working_dir: src/frontend backend-build: uses: hotosm/gh-workflows/.github/workflows/image_build.yml@main diff --git a/.github/workflows/pr_test_backend.yml b/.github/workflows/pr_test_backend.yml index c229ad63c3..e7eb79d9ed 100644 --- a/.github/workflows/pr_test_backend.yml +++ b/.github/workflows/pr_test_backend.yml @@ -14,7 +14,7 @@ on: jobs: pytest: - uses: ./.github/workflows/r-pytest.yml + uses: hotosm/gh-workflows/.github/workflows/test_pytest_compose.yml@main with: docker_compose_service: api cache_imgs: | diff --git a/.github/workflows/pr_test_frontend.yml b/.github/workflows/pr_test_frontend.yml index 43c2d7b858..0a2bb7bcdb 100644 --- a/.github/workflows/pr_test_frontend.yml +++ b/.github/workflows/pr_test_frontend.yml @@ -14,4 +14,6 @@ on: jobs: frontend-tests: - uses: ./.github/workflows/r-frontend_tests.yml + uses: hotosm/gh-workflows/.github/workflows/test_pnpm.yml@main + with: + working_dir: src/frontend diff --git a/.github/workflows/r-frontend_tests.yml b/.github/workflows/r-frontend_tests.yml deleted file mode 100644 index 0d62d57a0e..0000000000 --- a/.github/workflows/r-frontend_tests.yml +++ /dev/null @@ -1,32 +0,0 @@ -name: Frontend Tests - -on: - workflow_call: - -jobs: - test: - name: Run Frontend Tests - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - uses: pnpm/action-setup@v2 - with: - version: 8 - - - name: Setup Node.js - uses: actions/setup-node@v3 - with: - node-version: 18 - cache: "pnpm" - cache-dependency-path: "src/frontend/pnpm-lock.yaml" - - - name: Install dependencies - working-directory: src/frontend - run: pnpm install - - - name: Test Frontend - working-directory: src/frontend - run: pnpm run test diff --git a/.github/workflows/r-pytest.yml b/.github/workflows/r-pytest.yml deleted file mode 100644 index f1ef0e8cca..0000000000 --- a/.github/workflows/r-pytest.yml +++ /dev/null @@ -1,156 +0,0 @@ -name: pytest - -on: - workflow_call: - inputs: - image_tag: - description: "Optional image tag override." - required: false - type: string - cache_imgs: - description: "Space separated list of images to cache on each run (e.g. to avoid rate limiting)." - required: false - type: string - docker_compose_file: - description: "The docker compose file used to run the test." - required: false - type: string - default: docker-compose.yml - docker_compose_service: - description: "The docker compose service to run the test against." - required: true - type: string - environment: - description: "The environment to use for testing." - required: false - type: string - default: "test" - -jobs: - check-img-cache: - runs-on: ubuntu-latest - if: ${{ inputs.cache_imgs }} - environment: - name: ${{ inputs.environment }} - outputs: - cache-hit: ${{ steps.image-cache.outputs.cache-hit }} - steps: - - id: image-cache - uses: actions/cache@v3 - with: - path: /tmp/images - key: image-cache-${{ runner.os }} - - backend-test-build: - uses: hotosm/gh-workflows/.github/workflows/image_build.yml@main - needs: [check-img-cache] - if: ${{ needs.check-img-cache.outputs.cache-hit != 'true' }} - with: - context: src/backend - build_target: ci - image_name: ghcr.io/${{ github.repository }}/backend - image_tags: ${{ inputs.image_tag && format('ghcr.io/{0}/backend:{1}', github.repository, inputs.image_tag) || '' }} - build_args: | - APP_VERSION=${{ github.ref_name }} - COMMIT_REF=${{ github.sha }} - # Build temp image for PR, no multi-arch or scan required - multi_arch: false - scan: false - cache: false - - run-pytest: - runs-on: ubuntu-latest - needs: [backend-test-build] - # Ensure it runs, even if backend-test-build does not run - if: always() - environment: - name: ${{ inputs.environment }} - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - id: image-cache - if: ${{ inputs.cache_imgs }} - uses: actions/cache@v3 - with: - path: /tmp/images - key: image-cache-${{ runner.os }} - - - name: Save Local Images - if: ${{ inputs.cache_imgs && steps.image-cache.outputs.cache-hit != 'true' }} - run: | - # Function to pull and package Docker image - function pull_and_package_image() { - local image=$1 - echo "Processing image ${image}" - docker pull "${image}" - - if [ $? -eq 0 ]; then - img_underscores=${image//[:\/.]/_} - echo "Packaging image to /tmp/images/${img_underscores}.tar" - docker image save "${image}" --output "/tmp/images/${img_underscores}.tar" - else - echo "Failed to pull the image: ${image}" - fi - } - - # Make artifact dir - mkdir -p /tmp/images - - images_array=(${{ inputs.cache_imgs }}) - echo "Images to cache: ${images_array}" - - # Iterate through dependency images - for image in "${images_array[@]}"; do - pull_and_package_image "$image" - done - - # Cache backend build (fallback to ci-development tag) - image_tag=${{ needs.backend-test-build.outputs.image_tag }} - image="ghcr.io/${{ github.repository }}/backend:${image_tag}" - pull_and_package_image "$image" - echo "API_TAG_OVERRIDE=${image_tag}" >> $GITHUB_ENV - - - name: Load Cached Imgs - if: ${{ inputs.cache_imgs && steps.image-cache.outputs.cache-hit == 'true' }} - run: | - for image_tar in /tmp/images/*; do - docker image load --input $image_tar || true - done - - # Set API_TAG_OVERRIDE if not set - echo "Getting backend image tag via docker" - image=$(\ - docker image ls \ - --filter=reference='ghcr.io/${{ github.repository }}/backend' \ - --format='{{.Tag}}'\ - ) - image_tag="${image##*:}" - echo "Image Tag: $image_tag" - echo "API_TAG_OVERRIDE=${image_tag}" >> $GITHUB_ENV - - - name: Vars and Secrets to Env - env: - API_TAG_OVERRIDE: ${{ env.API_TAG_OVERRIDE || 'ci-development' }} - GIT_BRANCH: ${{ github.ref_name }} - VARS_CONTEXT: ${{ toJson(vars) }} - SECRETS_CONTEXT: ${{ toJson(secrets) }} - run: | - to_envs() { jq -r "to_entries[] | \"\(.key)=\(.value)\""; } - - echo "API_TAG_OVERRIDE=${API_TAG_OVERRIDE}" >> $GITHUB_ENV - echo "GIT_BRANCH=${GIT_BRANCH}" >> $GITHUB_ENV - echo "${VARS_CONTEXT}" | to_envs >> $GITHUB_ENV - echo "${SECRETS_CONTEXT}" | to_envs >> $GITHUB_ENV - - - name: Create .env file - run: env > .env - - - name: Run PyTest - run: | - docker compose --file ${{ inputs.docker_compose_file }} \ - run ${{ inputs.docker_compose_service }} \ - wait-for-it fmtm-db:5432 --strict \ - -- wait-for-it central:8383 --strict --timeout=30 \ - -- pytest From f8cd157e94348f0d77e7503e44cacfe5f7c92d23 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Sun, 22 Oct 2023 10:49:14 +0100 Subject: [PATCH 68/69] ci(pytest): use new format of reusable pytest workflow --- .github/workflows/build_and_deploy.yml | 7 ++++++- .github/workflows/pr_test_backend.yml | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml index f4a053915b..fb11d8897f 100644 --- a/.github/workflows/build_and_deploy.yml +++ b/.github/workflows/build_and_deploy.yml @@ -19,8 +19,13 @@ jobs: pytest: uses: hotosm/gh-workflows/.github/workflows/test_pytest_compose.yml@main with: - image_tag: ci-${{ github.ref_name }} + image_name: ghcr.io/${{ github.repository }}/backend + build_context: src/backend + build_args: | + APP_VERSION=${{ github.ref_name }} + COMMIT_REF=${{ github.sha }} docker_compose_service: api + tag_override: ci-${{ github.ref_name }} secrets: inherit frontend-tests: diff --git a/.github/workflows/pr_test_backend.yml b/.github/workflows/pr_test_backend.yml index e7eb79d9ed..9b9591c006 100644 --- a/.github/workflows/pr_test_backend.yml +++ b/.github/workflows/pr_test_backend.yml @@ -16,8 +16,13 @@ jobs: pytest: uses: hotosm/gh-workflows/.github/workflows/test_pytest_compose.yml@main with: + image_name: ghcr.io/${{ github.repository }}/backend + build_context: src/backend + build_args: | + APP_VERSION=${{ github.ref_name }} + COMMIT_REF=${{ github.sha }} docker_compose_service: api - cache_imgs: | + cache_extra_imgs: | "docker.io/postgis/postgis:${{ vars.POSTGIS_TAG }}" "docker.io/minio/minio:${{ vars.MINIO_TAG }}" # For caching odk central images, add: From d82cb133552452d3c7570413910dbaa9840299ad Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Sun, 22 Oct 2023 15:24:24 +0100 Subject: [PATCH 69/69] build: rename API_TAG_OVERRIDE --> TAG_OVERRIDE --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 06162bf591..75e3b57b8e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -46,7 +46,7 @@ services: restart: "unless-stopped" api: - image: "ghcr.io/hotosm/fmtm/backend:${API_TAG_OVERRIDE:-debug}" + image: "ghcr.io/hotosm/fmtm/backend:${TAG_OVERRIDE:-debug}" build: context: src/backend target: debug-with-odk @@ -80,7 +80,7 @@ services: restart: "unless-stopped" migrations: - image: "ghcr.io/hotosm/fmtm/backend:${API_TAG_OVERRIDE:-debug}" + image: "ghcr.io/hotosm/fmtm/backend:${TAG_OVERRIDE:-debug}" container_name: fmtm_migrations depends_on: - fmtm-db