From 092577339ff28d802d894d0ad3d4e0dc7b45a53d Mon Sep 17 00:00:00 2001 From: Sam <78538841+spwoodcock@users.noreply.github.com> Date: Tue, 23 Jan 2024 20:08:24 +0000 Subject: [PATCH] build: replace project.private field with project.visibility enum (#1132) * build: remove project_allowed_users table from db * build: remove odk_central_src field from project table * refactor: tidy/reorder project db model fields * build: remove project.task_creation_mode field + enum * build: replace project.private field with project.visibility enum * fix: fix visibility migration column add syntax * fix: apply migration to projects table * build: fix check for projectvisibility type in migration * ci: update workflows 1.4.5 for mkdocs --dirty --- .github/workflows/build_and_deploy.yml | 10 +-- .github/workflows/build_ci_img.yml | 2 +- .github/workflows/build_odk_imgs.yml | 4 +- .github/workflows/build_proxy_imgs.yml | 8 +-- .github/workflows/docs.yml | 6 +- .github/workflows/pr_test_backend.yml | 2 +- .github/workflows/pr_test_frontend.yml | 2 +- .github/workflows/tag_build.yml | 2 +- .github/workflows/tests/test_ci.sh | 2 +- .github/workflows/wiki.yml | 2 +- src/backend/app/db/db_models.py | 65 +++++++------------ src/backend/app/models/enums.py | 19 +++--- .../migrations/004-organisation-odk-creds.sql | 23 +++++++ .../migrations/init/fmtm_base_schema.sql | 32 +++------ .../revert/004-organisation-odk-creds.sql | 11 +++- 15 files changed, 96 insertions(+), 94 deletions(-) diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml index c9c1a3dc64..278a4b347f 100644 --- a/.github/workflows/build_and_deploy.yml +++ b/.github/workflows/build_and_deploy.yml @@ -17,7 +17,7 @@ on: jobs: pytest: - uses: hotosm/gh-workflows/.github/workflows/test_compose.yml@1.4.4 + uses: hotosm/gh-workflows/.github/workflows/test_compose.yml@1.4.5 with: image_name: ghcr.io/${{ github.repository }}/backend build_context: src/backend @@ -29,12 +29,12 @@ jobs: secrets: inherit frontend-tests: - uses: hotosm/gh-workflows/.github/workflows/test_pnpm.yml@1.4.4 + uses: hotosm/gh-workflows/.github/workflows/test_pnpm.yml@1.4.5 with: working_dir: src/frontend backend-build: - uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.4 + uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.5 needs: [pytest] with: context: src/backend @@ -42,7 +42,7 @@ jobs: image_name: ghcr.io/${{ github.repository }}/backend frontend-build: - uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.4 + uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.5 needs: [frontend-tests] with: context: src/frontend @@ -147,7 +147,7 @@ jobs: needs: - smoke-test-backend - smoke-test-frontend - uses: hotosm/gh-workflows/.github/workflows/remote_deploy.yml@1.4.4 + uses: hotosm/gh-workflows/.github/workflows/remote_deploy.yml@1.4.5 with: environment: ${{ github.ref_name }} docker_compose_file: "docker-compose.${{ github.ref_name }}.yml" diff --git a/.github/workflows/build_ci_img.yml b/.github/workflows/build_ci_img.yml index 79813c0792..6f0e5cf89c 100644 --- a/.github/workflows/build_ci_img.yml +++ b/.github/workflows/build_ci_img.yml @@ -16,7 +16,7 @@ on: jobs: backend-ci-build: - uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.4 + uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.5 with: context: src/backend build_target: ci diff --git a/.github/workflows/build_odk_imgs.yml b/.github/workflows/build_odk_imgs.yml index 6916e2c170..d39e65f8d8 100644 --- a/.github/workflows/build_odk_imgs.yml +++ b/.github/workflows/build_odk_imgs.yml @@ -13,7 +13,7 @@ on: jobs: build-odkcentral: - uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.4 + uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.5 with: context: odkcentral/api image_tags: | @@ -26,7 +26,7 @@ jobs: # multi_arch: true build-odkcentral-ui: - uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.4 + uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.5 with: context: odkcentral/ui image_tags: | diff --git a/.github/workflows/build_proxy_imgs.yml b/.github/workflows/build_proxy_imgs.yml index 476492873b..728c783c1c 100644 --- a/.github/workflows/build_proxy_imgs.yml +++ b/.github/workflows/build_proxy_imgs.yml @@ -10,7 +10,7 @@ on: jobs: build-cert-init-main: - uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.4 + uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.5 with: context: nginx build_target: certs-init-main @@ -21,7 +21,7 @@ jobs: multi_arch: true build-cert-init-dev: - uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.4 + uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.5 with: context: nginx build_target: certs-init-development @@ -33,7 +33,7 @@ jobs: multi_arch: true build-proxy-main: - uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.4 + uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.5 with: context: nginx build_target: main @@ -44,7 +44,7 @@ jobs: multi_arch: true build-proxy-dev: - uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.4 + uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.5 with: context: nginx build_target: development diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index bcecbc23b2..e46126692f 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -12,19 +12,19 @@ on: jobs: build_doxygen: - uses: hotosm/gh-workflows/.github/workflows/doxygen_build.yml@1.4.4 + uses: hotosm/gh-workflows/.github/workflows/doxygen_build.yml@1.4.5 with: output_path: docs/apidocs build_openapi_json: - uses: hotosm/gh-workflows/.github/workflows/openapi_build.yml@1.4.4 + uses: hotosm/gh-workflows/.github/workflows/openapi_build.yml@1.4.5 with: image: ghcr.io/${{ github.repository }}/backend:ci-${{ github.ref_name }} example_env_file_path: ".env.example" output_path: docs/openapi.json publish_docs: - uses: hotosm/gh-workflows/.github/workflows/mkdocs_build.yml@1.4.4 + uses: hotosm/gh-workflows/.github/workflows/mkdocs_build.yml@1.4.5 needs: - build_doxygen - build_openapi_json diff --git a/.github/workflows/pr_test_backend.yml b/.github/workflows/pr_test_backend.yml index 3f61f873a0..70aeae426d 100644 --- a/.github/workflows/pr_test_backend.yml +++ b/.github/workflows/pr_test_backend.yml @@ -14,7 +14,7 @@ on: jobs: pytest: - uses: hotosm/gh-workflows/.github/workflows/test_compose.yml@1.4.4 + uses: hotosm/gh-workflows/.github/workflows/test_compose.yml@1.4.5 with: image_name: ghcr.io/${{ github.repository }}/backend build_context: src/backend diff --git a/.github/workflows/pr_test_frontend.yml b/.github/workflows/pr_test_frontend.yml index 78d1827fbf..a791f88af4 100644 --- a/.github/workflows/pr_test_frontend.yml +++ b/.github/workflows/pr_test_frontend.yml @@ -14,6 +14,6 @@ on: jobs: frontend-tests: - uses: hotosm/gh-workflows/.github/workflows/test_pnpm.yml@1.4.4 + uses: hotosm/gh-workflows/.github/workflows/test_pnpm.yml@1.4.5 with: working_dir: src/frontend diff --git a/.github/workflows/tag_build.yml b/.github/workflows/tag_build.yml index 7ae5d2bb1c..f7171d353a 100644 --- a/.github/workflows/tag_build.yml +++ b/.github/workflows/tag_build.yml @@ -9,7 +9,7 @@ on: jobs: backend-build: - uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.4 + uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.5 with: context: src/backend build_target: prod diff --git a/.github/workflows/tests/test_ci.sh b/.github/workflows/tests/test_ci.sh index 45689ddeae..5ba7cd05a1 100644 --- a/.github/workflows/tests/test_ci.sh +++ b/.github/workflows/tests/test_ci.sh @@ -21,7 +21,7 @@ export TARGET_OVERRIDE=ci # --var-file=.env --secret-file=.env # Instead, run backend PyTest manually -docker compose build api +TAG_OVERRIDE=ci TARGET_OVERRIDE=ci docker compose build api act pull_request -W .github/workflows/tests/pytest.yml \ -e .github/workflows/tests/pr_payload.json \ --var-file=.env --secret-file=.env diff --git a/.github/workflows/wiki.yml b/.github/workflows/wiki.yml index 268fda60fa..d5dc1a430c 100644 --- a/.github/workflows/wiki.yml +++ b/.github/workflows/wiki.yml @@ -10,6 +10,6 @@ on: jobs: publish-docs-to-wiki: - uses: hotosm/gh-workflows/.github/workflows/wiki.yml@1.4.4 + uses: hotosm/gh-workflows/.github/workflows/wiki.yml@1.4.5 with: homepage_path: "wiki_redirect.md" diff --git a/src/backend/app/db/db_models.py b/src/backend/app/db/db_models.py index 26494c594c..dccea2e90c 100644 --- a/src/backend/app/db/db_models.py +++ b/src/backend/app/db/db_models.py @@ -53,8 +53,8 @@ ProjectPriority, ProjectRole, ProjectStatus, + ProjectVisibility, TaskAction, - TaskCreationMode, TaskSplitType, TaskStatus, TeamVisibility, @@ -178,16 +178,6 @@ class DbTeam(Base): organisation = relationship(DbOrganisation, backref="teams") -# Secondary table defining many-to-many join for -# private projects that only defined users can map on -project_allowed_users = Table( - "project_allowed_users", - FmtmMetadata, - Column("project_id", Integer, ForeignKey("projects.id")), - Column("user_id", BigInteger, ForeignKey("users.id")), -) - - class DbProjectTeams(Base): """Link table between teams and projects.""" @@ -432,9 +422,8 @@ class DbProject(Base): ) author = relationship(DbUser, uselist=False, backref="user") created = Column(DateTime, default=timestamp, nullable=False) - task_creation_mode = Column( - Enum(TaskCreationMode), default=TaskCreationMode.UPLOAD, nullable=False - ) + + task_split_type = Column(Enum(TaskSplitType), nullable=True) # split_strategy = Column(Integer) # grid_meters = Column(Integer) # task_type = Column(Integer) @@ -454,11 +443,14 @@ class DbProject(Base): # GEOMETRY outline = Column(Geometry("POLYGON", srid=4326)) # geometry = Column(Geometry("POLYGON", srid=4326, from_text='ST_GeomFromWkt')) - # TODO add outline_geojson as computed @property + centroid = Column(Geometry("POINT", srid=4326)) # PROJECT STATUS last_updated = Column(DateTime, default=timestamp) status = Column(Enum(ProjectStatus), default=ProjectStatus.DRAFT, nullable=False) + visibility = Column( + Enum(ProjectVisibility), default=ProjectVisibility.PUBLIC, nullable=False + ) total_tasks = Column(Integer) # tasks_mapped = Column(Integer, default=0, nullable=False) # tasks_validated = Column(Integer, default=0, nullable=False) @@ -508,9 +500,6 @@ def tasks_bad(self): ) # XFORM DETAILS - # TODO This field was probably replaced by odk_central_url - # TODO remove in a migration - odk_central_src = Column(String, default="") xform_title = Column(String, ForeignKey("xlsforms.title", name="fk_xform")) xform = relationship(DbXForm) @@ -519,10 +508,6 @@ def tasks_bad(self): {}, ) - ## ---------------------------------------------- ## - # FOR REFERENCE: OTHER ATTRIBUTES IN TASKING MANAGER - # PROJECT ACCESS - private = Column(Boolean, default=False) # Only allowed users can validate mapper_level = Column( Enum(MappingLevel), default=MappingLevel.INTERMEDIATE, @@ -537,31 +522,13 @@ def tasks_bad(self): validation_permission = Column( Enum(ValidationPermission), default=ValidationPermission.LEVEL ) # Means only users with validator role can validate - allowed_users = relationship(DbUser, secondary=project_allowed_users) organisation_id = Column( Integer, ForeignKey("organisations.id", name="fk_organisations"), index=True, ) organisation = relationship(DbOrganisation, backref="projects") - # PROJECT DETAILS - due_date = Column(DateTime) changeset_comment = Column(String) - osmcha_filter_id = Column( - String - ) # Optional custom filter id for filtering on OSMCha - imagery = Column(String) - osm_preset = Column(String) - odk_preset = Column(String) - josm_preset = Column(String) - id_presets = Column(ARRAY(String)) - extra_id_params = Column(String) - license_id = Column(Integer, ForeignKey("licenses.id", name="fk_licenses")) - # GEOMETRY - centroid = Column(Geometry("POINT", srid=4326)) - # country = Column(ARRAY(String), default=[]) - # FEEDBACK - project_chat = relationship(DbProjectChat, lazy="dynamic", cascade="all") ## Odk central server odk_central_url = Column(String) @@ -582,6 +549,24 @@ def tasks_bad(self): hashtags = Column(ARRAY(String)) # Project hashtag + ## ---------------------------------------------- ## + # FOR REFERENCE: OTHER ATTRIBUTES IN TASKING MANAGER + imagery = Column(String) + osm_preset = Column(String) + odk_preset = Column(String) + josm_preset = Column(String) + id_presets = Column(ARRAY(String)) + extra_id_params = Column(String) + license_id = Column(Integer, ForeignKey("licenses.id", name="fk_licenses")) + # GEOMETRY + # country = Column(ARRAY(String), default=[]) + # FEEDBACK + project_chat = relationship(DbProjectChat, lazy="dynamic", cascade="all") + osmcha_filter_id = Column( + String + ) # Optional custom filter id for filtering on OSMCha + due_date = Column(DateTime) + # TODO: Add index on project geometry, tried to add in __table args__ # Index("idx_geometry", DbProject.geometry, postgresql_using="gist") diff --git a/src/backend/app/models/enums.py b/src/backend/app/models/enums.py index 7cd31c3c2a..0b0525f5be 100644 --- a/src/backend/app/models/enums.py +++ b/src/backend/app/models/enums.py @@ -103,9 +103,8 @@ class UserRole(IntEnum, Enum): class ProjectRole(IntEnum, Enum): """Available roles assigned to a user for a specific project. - Invitation is required for a MAPPER to join a project. All roles must be assigned by someone higher in the hierarchy: - - MAPPER = default for all (invitation required) + - MAPPER = default for all - VALIDATOR = can validate the mappers output - FIELD_MANAGER = can invite mappers and organise people - ASSOCIATE_PROJECT_MANAGER = helps the project manager, cannot delete project @@ -145,14 +144,6 @@ class ValidationPermission(IntEnum, Enum): TEAMS_LEVEL = 3 -class TaskCreationMode(IntEnum, Enum): - """Enum to describe task creation mode.""" - - GRID = 0 - ROADS = 1 - UPLOAD = 2 - - class TaskStatus(IntEnum, Enum): """Enum describing available Task Statuses.""" @@ -287,3 +278,11 @@ class TaskSplitType(IntEnum, Enum): DIVIDE_ON_SQUARE = 0 CHOOSE_AREA_AS_TASK = 1 TASK_SPLITTING_ALGORITHM = 2 + + +class ProjectVisibility(IntEnum, Enum): + """Enum describing task splitting type.""" + + PUBLIC = 0 + PRIVATE = 1 + INVITE_ONLY = 2 diff --git a/src/backend/migrations/004-organisation-odk-creds.sql b/src/backend/migrations/004-organisation-odk-creds.sql index d65b61a8a1..1d2d30de15 100644 --- a/src/backend/migrations/004-organisation-odk-creds.sql +++ b/src/backend/migrations/004-organisation-odk-creds.sql @@ -1,14 +1,37 @@ -- ## Migration to: -- * Add odk central credentials (str) to organisations table. -- * Add the approved (bool) field to organisations table. +-- * Add the visibility type for project visibility level. +-- * Add the visibility field to projects table. -- Start a transaction BEGIN; +-- Add fields to organisations table ALTER TABLE IF EXISTS public.organisations ADD COLUMN IF NOT EXISTS approved BOOLEAN DEFAULT false, ADD COLUMN IF NOT EXISTS odk_central_url VARCHAR, ADD COLUMN IF NOT EXISTS odk_central_user VARCHAR, ADD COLUMN IF NOT EXISTS odk_central_password VARCHAR; + +-- Create visibility enum if it doesn't exist +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'projectvisibility') THEN + CREATE TYPE public.projectvisibility AS ENUM ( + 'PUBLIC', + 'PRIVATE', + 'INVITE_ONLY' + ); + END IF; +END $$; +ALTER TYPE public.projectvisibility OWNER TO fmtm; + +-- Add field to projects table +ALTER TABLE IF EXISTS public.projects + DROP COLUMN IF EXISTS private, + ADD COLUMN IF NOT EXISTS visibility public.projectvisibility + NOT NULL DEFAULT 'PUBLIC'; + -- Commit the transaction COMMIT; diff --git a/src/backend/migrations/init/fmtm_base_schema.sql b/src/backend/migrations/init/fmtm_base_schema.sql index d6c328aa47..4e4447a772 100644 --- a/src/backend/migrations/init/fmtm_base_schema.sql +++ b/src/backend/migrations/init/fmtm_base_schema.sql @@ -102,13 +102,6 @@ CREATE TYPE public.taskaction AS ENUM ( ); ALTER TYPE public.taskaction OWNER TO fmtm; -CREATE TYPE public.taskcreationmode AS ENUM ( - 'GRID', - 'ROADS', - 'UPLOAD' -); -ALTER TYPE public.taskcreationmode OWNER TO fmtm; - CREATE TYPE public.taskstatus AS ENUM ( 'READY', 'LOCKED_FOR_MAPPING', @@ -152,6 +145,14 @@ CREATE TYPE public.validationpermission AS ENUM ( ); ALTER TYPE public.validationpermission OWNER TO fmtm; +CREATE TYPE public.projectvisibility AS ENUM ( + 'PUBLIC', + 'PRIVATE', + 'INVITE_ONLY' +); +ALTER TYPE public.projectvisibility OWNER TO fmtm; + + -- Extra @@ -294,13 +295,6 @@ ALTER TABLE public.organisations_id_seq OWNER TO fmtm; ALTER SEQUENCE public.organisations_id_seq OWNED BY public.organisations.id; -CREATE TABLE public.project_allowed_users ( - project_id integer, - user_id bigint -); -ALTER TABLE public.project_allowed_users OWNER TO fmtm; - - CREATE TABLE public.project_chat ( id bigint NOT NULL, project_id integer NOT NULL, @@ -344,7 +338,6 @@ CREATE TABLE public.projects ( odkid integer, author_id bigint NOT NULL, created timestamp without time zone NOT NULL, - task_creation_mode public.taskcreationmode NOT NULL, project_name_prefix character varying, task_type_prefix character varying, location_str character varying, @@ -352,9 +345,8 @@ CREATE TABLE public.projects ( last_updated timestamp without time zone, status public.projectstatus NOT NULL, total_tasks integer, - odk_central_src character varying, xform_title character varying, - private boolean, + visibility public.projectvisibility NOT NULL DEFAULT 'PUBLIC', mapper_level public.mappinglevel NOT NULL, priority public.projectpriority, featured boolean, @@ -787,12 +779,6 @@ ALTER TABLE ONLY public.organisation_managers ALTER TABLE ONLY public.organisation_managers ADD CONSTRAINT organisation_managers_user_id_fkey FOREIGN KEY (user_id) REFERENCES public.users(id); -ALTER TABLE ONLY public.project_allowed_users - ADD CONSTRAINT project_allowed_users_project_id_fkey FOREIGN KEY (project_id) REFERENCES public.projects(id); - -ALTER TABLE ONLY public.project_allowed_users - ADD CONSTRAINT project_allowed_users_user_id_fkey FOREIGN KEY (user_id) REFERENCES public.users(id); - ALTER TABLE ONLY public.project_chat ADD CONSTRAINT project_chat_project_id_fkey FOREIGN KEY (project_id) REFERENCES public.projects(id); diff --git a/src/backend/migrations/revert/004-organisation-odk-creds.sql b/src/backend/migrations/revert/004-organisation-odk-creds.sql index 263595afa4..51ea069dc7 100644 --- a/src/backend/migrations/revert/004-organisation-odk-creds.sql +++ b/src/backend/migrations/revert/004-organisation-odk-creds.sql @@ -2,12 +2,21 @@ BEGIN; -- Remove the odk central credentials columns and approved column ---- from the public.organisations table +-- from the public.organisations table ALTER TABLE IF EXISTS public.organisations DROP COLUMN IF EXISTS approved, DROP COLUMN IF EXISTS odk_central_url CASCADE, DROP COLUMN IF EXISTS odk_central_user CASCADE, DROP COLUMN IF EXISTS odk_central_password CASCADE; +-- Remove the visibility column, add private column +-- from the public.projects table +ALTER TABLE IF EXISTS public.projects + DROP COLUMN IF EXISTS visibility, + ADD COLUMN IF NOT EXISTS private BOOLEAN; + +-- Remove enum +DROP TYPE IF EXISTS public.projectvisibility; + -- Commit the transaction COMMIT;