diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml index 8873efe613..295d38cdb8 100644 --- a/.github/workflows/build_and_deploy.yml +++ b/.github/workflows/build_and_deploy.yml @@ -17,7 +17,7 @@ on: jobs: pytest: - uses: hotosm/gh-workflows/.github/workflows/test_compose.yml@1.5.2 + uses: hotosm/gh-workflows/.github/workflows/test_compose.yml@1.6.0 with: image_name: ghcr.io/${{ github.repository }}/backend build_context: src/backend @@ -29,12 +29,12 @@ jobs: secrets: inherit frontend-tests: - uses: hotosm/gh-workflows/.github/workflows/test_pnpm.yml@1.5.2 + uses: hotosm/gh-workflows/.github/workflows/test_pnpm.yml@1.6.0 with: working_dir: src/frontend backend-build: - uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.5.2 + uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.6.0 needs: [pytest] with: context: src/backend @@ -42,7 +42,7 @@ jobs: image_name: ghcr.io/${{ github.repository }}/backend frontend-build: - uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.5.2 + uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.6.0 needs: [frontend-tests] with: context: src/frontend @@ -152,7 +152,7 @@ jobs: needs: - smoke-test-backend - smoke-test-frontend - uses: hotosm/gh-workflows/.github/workflows/remote_deploy.yml@1.5.2 + uses: hotosm/gh-workflows/.github/workflows/remote_deploy.yml@1.6.0 with: environment: ${{ github.ref_name }} docker_compose_file: "docker-compose.${{ github.ref_name }}.yml" diff --git a/.github/workflows/build_ci_img.yml b/.github/workflows/build_ci_img.yml index 9205316eba..901842aada 100644 --- a/.github/workflows/build_ci_img.yml +++ b/.github/workflows/build_ci_img.yml @@ -16,7 +16,7 @@ on: jobs: backend-ci-build: - uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.5.2 + uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.6.0 with: context: src/backend build_target: ci diff --git a/.github/workflows/build_odk_imgs.yml b/.github/workflows/build_odk_imgs.yml index 4ded2d2c3a..3ca23e271e 100644 --- a/.github/workflows/build_odk_imgs.yml +++ b/.github/workflows/build_odk_imgs.yml @@ -13,7 +13,7 @@ on: jobs: build-odkcentral: - uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.5.2 + uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.6.0 with: context: odkcentral/api image_tags: | @@ -26,7 +26,7 @@ jobs: # multi_arch: true build-odkcentral-ui: - uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.5.2 + uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.6.0 with: context: odkcentral/ui image_tags: | diff --git a/.github/workflows/build_proxy_imgs.yml b/.github/workflows/build_proxy_imgs.yml index 57bd08f748..ea4bffa818 100644 --- a/.github/workflows/build_proxy_imgs.yml +++ b/.github/workflows/build_proxy_imgs.yml @@ -10,7 +10,7 @@ on: jobs: build-cert-init-main: - uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.5.2 + uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.6.0 with: context: nginx build_target: certs-init-main @@ -21,7 +21,7 @@ jobs: multi_arch: true build-cert-init-dev: - uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.5.2 + uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.6.0 with: context: nginx build_target: certs-init-development @@ -33,7 +33,7 @@ jobs: multi_arch: true build-proxy-main: - uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.5.2 + uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.6.0 with: context: nginx build_target: main @@ -44,7 +44,7 @@ jobs: multi_arch: true build-proxy-dev: - uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.5.2 + uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.6.0 with: context: nginx build_target: development diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 573967bdd3..9aae625e92 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -12,19 +12,19 @@ on: jobs: build_doxygen: - uses: hotosm/gh-workflows/.github/workflows/doxygen_build.yml@1.5.2 + uses: hotosm/gh-workflows/.github/workflows/doxygen_build.yml@1.6.0 with: output_path: docs/apidocs build_openapi_json: - uses: hotosm/gh-workflows/.github/workflows/openapi_build.yml@1.5.2 + uses: hotosm/gh-workflows/.github/workflows/openapi_build.yml@1.6.0 with: image: ghcr.io/${{ github.repository }}/backend:ci-${{ github.ref_name }} example_env_file_path: ".env.example" output_path: docs/openapi.json publish_docs: - uses: hotosm/gh-workflows/.github/workflows/mkdocs_build.yml@1.5.2 + uses: hotosm/gh-workflows/.github/workflows/mkdocs_build.yml@1.6.0 needs: - build_doxygen - build_openapi_json diff --git a/.github/workflows/pr_test_backend.yml b/.github/workflows/pr_test_backend.yml index e445392150..6f199747d0 100644 --- a/.github/workflows/pr_test_backend.yml +++ b/.github/workflows/pr_test_backend.yml @@ -14,7 +14,7 @@ on: jobs: pytest: - uses: hotosm/gh-workflows/.github/workflows/test_compose.yml@1.5.2 + uses: hotosm/gh-workflows/.github/workflows/test_compose.yml@1.6.0 with: image_name: ghcr.io/${{ github.repository }}/backend build_context: src/backend diff --git a/.github/workflows/pr_test_frontend.yml b/.github/workflows/pr_test_frontend.yml index 14d958d9bb..adeb02e3f3 100644 --- a/.github/workflows/pr_test_frontend.yml +++ b/.github/workflows/pr_test_frontend.yml @@ -14,7 +14,7 @@ on: jobs: unit-tests: - uses: hotosm/gh-workflows/.github/workflows/test_pnpm.yml@1.5.2 + uses: hotosm/gh-workflows/.github/workflows/test_pnpm.yml@1.6.0 with: working_dir: src/frontend diff --git a/.github/workflows/tag_build.yml b/.github/workflows/tag_build.yml index 0d20bcbe61..45ab39a257 100644 --- a/.github/workflows/tag_build.yml +++ b/.github/workflows/tag_build.yml @@ -9,7 +9,7 @@ on: jobs: backend-build: - uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.5.2 + uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.6.0 with: context: src/backend build_target: prod diff --git a/.github/workflows/wiki.yml b/.github/workflows/wiki.yml index 47869a60c7..47ec464a13 100644 --- a/.github/workflows/wiki.yml +++ b/.github/workflows/wiki.yml @@ -10,6 +10,6 @@ on: jobs: publish-docs-to-wiki: - uses: hotosm/gh-workflows/.github/workflows/wiki.yml@1.5.2 + uses: hotosm/gh-workflows/.github/workflows/wiki.yml@1.6.0 with: homepage_path: "wiki_redirect.md" diff --git a/README.md b/README.md index 4af5b2a683..725de4a0e5 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,13 @@ Alternatively see the [docs](https://docs.fmtm.dev) for various deployment guide ## Contributors ✨ +Here's how you can contribute: + +- [Open an issue](https://github.com/hotosm/fmtm/issues) if you believe you've + encountered a bug. +- Make a [pull request](https://github.com/hotosm/fmtm/pull) to add new features + or fix bugs. + Thanks goes to these wonderful people: @@ -145,3 +152,7 @@ Thanks goes to these wonderful people: + +## Repo Activity + +![FMTM repo activity – generated by Axiom](https://repobeats.axiom.co/api/embed/xxx.svg) diff --git a/odkcentral/ui/Dockerfile b/odkcentral/ui/Dockerfile index fc9d80af5b..c164735b35 100644 --- a/odkcentral/ui/Dockerfile +++ b/odkcentral/ui/Dockerfile @@ -35,7 +35,7 @@ RUN VUE_APP_OIDC_ENABLED="false" npm run build -FROM docker.io/rclone/rclone:1.64 as prod +FROM docker.io/rclone/rclone:1 as prod VOLUME /frontend COPY container-entrypoint.sh / RUN chmod +x /container-entrypoint.sh diff --git a/src/backend/app/auth/auth_routes.py b/src/backend/app/auth/auth_routes.py index 471a7c2711..fa03db633c 100644 --- a/src/backend/app/auth/auth_routes.py +++ b/src/backend/app/auth/auth_routes.py @@ -257,7 +257,7 @@ async def refresh_token( except Exception as e: raise HTTPException( status_code=HTTPStatus.BAD_REQUEST, - detail=f"fail to refresh the access token: {e}", + detail=f"Failed to refresh the access token: {e}", ) from e diff --git a/src/backend/app/auth/osm.py b/src/backend/app/auth/osm.py index cb03b82900..8ba814252b 100644 --- a/src/backend/app/auth/osm.py +++ b/src/backend/app/auth/osm.py @@ -120,7 +120,7 @@ def create_tokens(jwt_data: dict) -> tuple[str, str]: def refresh_access_token(payload: dict) -> str: """Generate a new access token.""" - payload["exp"] = int(time.time()) + 60 # Access token valid for 15 minutes + payload["exp"] = int(time.time()) + 3600 # Access token valid for 1 hour return jwt.encode( payload, diff --git a/src/backend/app/db/db_models.py b/src/backend/app/db/db_models.py index 4dc87c6f80..a1ead34251 100644 --- a/src/backend/app/db/db_models.py +++ b/src/backend/app/db/db_models.py @@ -54,7 +54,6 @@ BackgroundTaskStatus, CommunityType, MappingLevel, - MappingPermission, OrganisationType, ProjectPriority, ProjectRole, @@ -63,9 +62,7 @@ TaskAction, TaskSplitType, TaskStatus, - TeamVisibility, UserRole, - ValidationPermission, ) @@ -127,9 +124,6 @@ class DbUser(Base): # projects_notifications = Column(Boolean, default=True, nullable=False) # tasks_notifications = Column(Boolean, default=True, nullable=False) # tasks_comments_notifications = Column(Boolean, default=False, nullable=False) - # teams_announcement_notifications = Column( - # Boolean, default=True, nullable=False - # ) date_registered = cast(datetime, Column(DateTime, default=timestamp)) # Represents the date the user last had one of their tasks validated @@ -184,48 +178,6 @@ class DbOrganisation(Base): ) -class DbTeam(Base): - """Describes a team.""" - - __tablename__ = "teams" - - # Columns - id = cast(int, Column(Integer, primary_key=True)) - organisation_id = cast( - int, - Column( - Integer, - ForeignKey("organisations.id", name="fk_organisations"), - nullable=False, - ), - ) - name = cast(str, Column(String(512), nullable=False)) - logo = cast(str, Column(String)) # URL of a logo - description = cast(str, Column(String)) - invite_only = cast(bool, Column(Boolean, default=False, nullable=False)) - visibility = cast( - TeamVisibility, - Column(Enum(TeamVisibility), default=TeamVisibility.PUBLIC, nullable=False), - ) - organisation = relationship(DbOrganisation, backref="teams") - - -class DbProjectTeams(Base): - """Link table between teams and projects.""" - - __tablename__ = "project_teams" - team_id = cast(int, Column(Integer, ForeignKey("teams.id"), primary_key=True)) - project_id = cast(int, Column(Integer, ForeignKey("projects.id"), primary_key=True)) - role = cast(int, Column(Integer, nullable=False)) - - project = relationship( - "DbProject", backref=backref("teams", cascade="all, delete-orphan") - ) - team = relationship( - DbTeam, backref=backref("projects", cascade="all, delete-orphan") - ) - - class DbProjectInfo(Base): """Contains all project info localized into supported languages.""" @@ -247,22 +199,6 @@ class DbProjectInfo(Base): ) -class DbProjectChat(Base): - """Contains all project info localized into supported languages.""" - - __tablename__ = "project_chat" - id = cast(int, Column(BigInteger, primary_key=True)) - project_id = cast( - int, Column(Integer, ForeignKey("projects.id"), index=True, nullable=False) - ) - user_id = cast(int, Column(Integer, ForeignKey("users.id"), nullable=False)) - time_stamp = cast(datetime, Column(DateTime, nullable=False, default=timestamp)) - message = cast(str, Column(String, nullable=False)) - - # Relationships - posted_by = relationship(DbUser, foreign_keys=[user_id]) - - class DbXLSForm(Base): """XLSForm templates and custom uploads.""" @@ -291,83 +227,6 @@ class DbXForm(Base): category = cast(str, Column(String)) -class DbTaskInvalidationHistory(Base): - """Information on task invalidation. - - Describes the most recent history of task invalidation and subsequent validation. - """ - - __tablename__ = "task_invalidation_history" - id = cast(int, Column(Integer, primary_key=True)) - project_id = cast(int, Column(Integer, ForeignKey("projects.id"), nullable=False)) - task_id = cast(int, Column(Integer, nullable=False)) - is_closed = cast(bool, Column(Boolean, default=False)) - mapper_id = cast(int, Column(BigInteger, ForeignKey("users.id", name="fk_mappers"))) - mapped_date = cast(datetime, Column(DateTime)) - invalidator_id = cast( - int, Column(BigInteger, ForeignKey("users.id", name="fk_invalidators")) - ) - invalidated_date = cast(datetime, Column(DateTime)) - invalidation_history_id = cast( - int, - Column(Integer, ForeignKey("task_history.id", name="fk_invalidation_history")), - ) - validator_id = cast( - int, Column(BigInteger, ForeignKey("users.id", name="fk_validators")) - ) - validated_date = cast(datetime, Column(DateTime)) - updated_date = cast(datetime, Column(DateTime, default=timestamp)) - - __table_args__ = ( - ForeignKeyConstraint( - [task_id, project_id], ["tasks.id", "tasks.project_id"], name="fk_tasks" - ), - Index("idx_task_validation_history_composite", "task_id", "project_id"), - Index( - "idx_task_validation_validator_status_composite", - "invalidator_id", - "is_closed", - ), - Index("idx_task_validation_mapper_status_composite", "mapper_id", "is_closed"), - {}, - ) - - -class DbTaskMappingIssue(Base): - """Describes mapping issues. - - An issue (along with an occurrence count) with a - task mapping that contributed to invalidation of the task. - """ - - __tablename__ = "task_mapping_issues" - id = cast(int, Column(Integer, primary_key=True)) - task_history_id = cast( - int, Column(Integer, ForeignKey("task_history.id"), nullable=False, index=True) - ) - issue = cast(str, Column(String, nullable=False)) - mapping_issue_category_id = cast( - int, - Column( - Integer, - ForeignKey("mapping_issue_categories.id", name="fk_issue_category"), - nullable=False, - ), - ) - count = cast(int, Column(Integer, nullable=False)) - - -class DbMappingIssueCategory(Base): - """Represents a category of task mapping issues identified during validation.""" - - __tablename__ = "mapping_issue_categories" - - id = cast(int, Column(Integer, primary_key=True)) - name = cast(str, Column(String, nullable=False, unique=True)) - description = cast(str, Column(String, nullable=True)) - archived = cast(bool, Column(Boolean, default=False, nullable=False)) - - class DbTaskHistory(Base): """Describes the history associated with a task.""" @@ -391,11 +250,7 @@ class DbTaskHistory(Base): # Define relationships user = relationship(DbUser, uselist=False, backref="task_history_user") - invalidation_history = relationship( - DbTaskInvalidationHistory, lazy="dynamic", cascade="all" - ) actioned_by = relationship(DbUser, overlaps="task_history_user,user") - task_mapping_issues = relationship(DbTaskMappingIssue, cascade="all") __table_args__ = ( ForeignKeyConstraint( @@ -623,14 +478,6 @@ def tasks_bad(self): featured = cast( bool, Column(Boolean, default=False) ) # Only admins can set a project as featured - mapping_permission = cast( - MappingPermission, - Column(Enum(MappingPermission), default=MappingPermission.ANY), - ) - validation_permission = cast( - ValidationPermission, - Column(Enum(ValidationPermission), default=ValidationPermission.LEVEL), - ) # Means only users with validator role can validate changeset_comment = cast(str, Column(String)) # Odk central server @@ -665,9 +512,6 @@ def tasks_bad(self): josm_preset = cast(str, Column(String)) id_presets = cast(list, Column(ARRAY(String))) extra_id_params = cast(str, Column(String)) - license_id = cast( - int, Column(Integer, ForeignKey("licenses.id", name="fk_licenses")) - ) # GEOMETRY # country = Column(ARRAY(String), default=[]) @@ -679,31 +523,6 @@ def tasks_bad(self): due_date = cast(datetime, Column(DateTime)) -# Secondary table defining the many-to-many join -user_licenses_table = Table( - "user_licenses", - FmtmMetadata, - Column("user", BigInteger, ForeignKey("users.id")), - Column("license", Integer, ForeignKey("licenses.id")), -) - - -class DbLicense(Base): - """Describes an individual license.""" - - __tablename__ = "licenses" - - id = cast(int, Column(Integer, primary_key=True)) - name = cast(str, Column(String, unique=True)) - description = cast(str, Column(String)) - plain_text = cast(str, Column(String)) - - projects = relationship(DbProject, backref="license") - users = relationship( - DbUser, secondary=user_licenses_table - ) # Many to Many relationship - - class BackgroundTasks(Base): """Table managing long running background tasks.""" diff --git a/src/backend/app/models/enums.py b/src/backend/app/models/enums.py index 18bf241406..3e6a7b1c6c 100644 --- a/src/backend/app/models/enums.py +++ b/src/backend/app/models/enums.py @@ -54,13 +54,6 @@ class HTTPStatus(IntEnum): NOT_IMPLEMENTED = 501 -class TeamVisibility(IntEnum, Enum): - """Describes the visibility associated with an Team.""" - - PUBLIC = 0 - PRIVATE = 1 - - class OrganisationType(IntEnum, Enum): """Describes an organisation's subscription type.""" @@ -126,24 +119,6 @@ class MappingLevel(IntEnum, Enum): ADVANCED = 3 -class MappingPermission(IntEnum, Enum): - """Describes a set of permissions for mapping on a project.""" - - ANY = 0 - LEVEL = 1 - TEAMS = 2 - TEAMS_LEVEL = 3 - - -class ValidationPermission(IntEnum, Enum): - """Describes a set of permissions for validating on a project.""" - - ANY = 0 - LEVEL = 1 - TEAMS = 2 - TEAMS_LEVEL = 3 - - class TaskStatus(IntEnum, Enum): """Enum describing available Task Statuses.""" diff --git a/src/backend/app/projects/project_crud.py b/src/backend/app/projects/project_crud.py index d333f23dde..de3192b815 100644 --- a/src/backend/app/projects/project_crud.py +++ b/src/backend/app/projects/project_crud.py @@ -229,7 +229,7 @@ async def delete_one_project(db: Session, db_project: db_models.DbProject) -> No log.info(f"Deleted project with ID: {project_id}") except Exception as e: log.exception(e) - raise HTTPException(e) from e + raise HTTPException(status_code=HTTPStatus.CONFLICT, detail=e) from e async def partial_update_project_info( @@ -395,7 +395,7 @@ async def create_tasks_from_geojson( return True except Exception as e: log.exception(e) - raise HTTPException(e) from e + raise HTTPException(HTTPStatus.UNPROCESSABLE_ENTITY, detail=e) from e async def preview_split_by_square(boundary: str, meters: int): @@ -1368,7 +1368,7 @@ def get_project_tiles( background_task_id: uuid.UUID, source: str, output_format: str = "mbtiles", - tms: str = None, + tms: Optional[str] = None, ): """Get the tiles for a project. @@ -1381,7 +1381,12 @@ def get_project_tiles( Other options: "pmtiles", "sqlite3". tms (str, optional): Default None. Custom TMS provider URL. """ - zooms = "12-19" + # TODO update this for user input or automatic + # maxzoom can be determined from OAM: https://tiles.openaerialmap.org/663 + # c76196049ef00013b8494/0/663c76196049ef00013b8495 + # TODO xy should also be user configurable + # NOTE mbtile max supported zoom level is 22 (in GDAL at least) + zooms = "12-22" if tms else "12-19" tiles_dir = f"{TILESDIR}/{project_id}" outfile = f"{tiles_dir}/{project_id}_{source}tiles.{output_format}" @@ -1415,7 +1420,9 @@ def get_project_tiles( if project_bbox: min_lon, min_lat, max_lon, max_lat = project_bbox else: - log.error(f"Failed to get bbox from project: {project_id}") + msg = f"Failed to get bbox from project: {project_id}" + log.error(msg) + raise HTTPException(status_code=HTTPStatus.UNPROCESSABLE_ENTITY, detail=msg) log.debug( "Creating basemap with params: " @@ -1424,7 +1431,7 @@ def get_project_tiles( f"zooms={zooms} | " f"outdir={tiles_dir} | " f"source={source} | " - f"xy={False} | " + f"xy={True if tms else False} | " f"tms={tms}" ) @@ -1434,7 +1441,7 @@ def get_project_tiles( zooms=zooms, outdir=tiles_dir, source=source, - xy=False, + xy=True if tms else False, tms=tms, ) diff --git a/src/backend/app/projects/project_deps.py b/src/backend/app/projects/project_deps.py index e92bc96eb3..8b62d0cb19 100644 --- a/src/backend/app/projects/project_deps.py +++ b/src/backend/app/projects/project_deps.py @@ -18,9 +18,9 @@ """Project dependencies for use in Depends.""" -from functools import lru_cache from typing import Optional +from async_lru import alru_cache from fastapi import Depends from fastapi.exceptions import HTTPException from loguru import logger as log @@ -58,7 +58,7 @@ async def get_project_by_id( return db_project -@lru_cache(maxsize=None) +@alru_cache(maxsize=32) async def get_odk_credentials(db: Session, project_id: int): """Get ODK credentials of a project, or default organization credentials.""" sql = text( diff --git a/src/backend/app/projects/project_routes.py b/src/backend/app/projects/project_routes.py index 8b1669aa18..e11daa3f9a 100644 --- a/src/backend/app/projects/project_routes.py +++ b/src/backend/app/projects/project_routes.py @@ -370,7 +370,7 @@ async def download_tiles( return FileResponse( dbtile_obj.path, headers={ - "Content-Disposition": f'attachment; filename="{filename}"', + "Content-Disposition": f"attachment; filename={filename}", "Content-Type": tiles_mime_type, }, ) diff --git a/src/backend/app/projects/project_schemas.py b/src/backend/app/projects/project_schemas.py index 1b9ea43883..ce11e3030d 100644 --- a/src/backend/app/projects/project_schemas.py +++ b/src/backend/app/projects/project_schemas.py @@ -321,6 +321,7 @@ class ProjectBase(BaseModel): author: User project_info: ProjectInfo status: ProjectStatus + created: datetime # location_str: str xform_category: Optional[XLSFormType] = None hashtags: Optional[List[str]] = None diff --git a/src/backend/migrations/004-update-default-values.sql b/src/backend/migrations/004-update-default-values.sql index ac3d9fc146..a21055139a 100644 --- a/src/backend/migrations/004-update-default-values.sql +++ b/src/backend/migrations/004-update-default-values.sql @@ -4,18 +4,10 @@ -- Start a transaction BEGIN; --- Update mapping_issue_categories table -ALTER TABLE public.mapping_issue_categories -ALTER COLUMN archived SET DEFAULT false; - -- Update mbtiles_path table ALTER TABLE public.mbtiles_path ALTER COLUMN created_at SET DEFAULT now(); --- Update project_chat table -ALTER TABLE public.project_chat -ALTER COLUMN time_stamp SET DEFAULT now(); - -- Update projects table ALTER TABLE public.projects ALTER COLUMN created SET DEFAULT now(), @@ -23,28 +15,16 @@ ALTER COLUMN last_updated SET DEFAULT now(), ALTER COLUMN status SET DEFAULT 'DRAFT', ALTER COLUMN mapper_level SET DEFAULT 'INTERMEDIATE', ALTER COLUMN priority SET DEFAULT 'MEDIUM', -ALTER COLUMN featured SET DEFAULT false, -ALTER COLUMN mapping_permission SET DEFAULT 'ANY', -ALTER COLUMN validation_permission SET DEFAULT 'LEVEL'; +ALTER COLUMN featured SET DEFAULT false; -- Update task_history table ALTER TABLE public.task_history ALTER COLUMN action_date SET DEFAULT now(); --- Update task_invalidation_history table -ALTER TABLE public.task_invalidation_history -ALTER COLUMN is_closed SET DEFAULT false, -ALTER COLUMN updated_date SET DEFAULT now(); - -- Update tasks table ALTER TABLE public.tasks ALTER COLUMN task_status SET DEFAULT 'READY'; --- Update teams table -ALTER TABLE public.teams -ALTER COLUMN invite_only SET DEFAULT false, -ALTER COLUMN visibility SET DEFAULT 'PUBLIC'; - -- Update user_roles table ALTER TABLE public.user_roles ALTER COLUMN role SET DEFAULT 'MAPPER'; diff --git a/src/backend/migrations/005-drop-unnecessary-tables.sql b/src/backend/migrations/005-drop-unnecessary-tables.sql new file mode 100644 index 0000000000..4920ddfd66 --- /dev/null +++ b/src/backend/migrations/005-drop-unnecessary-tables.sql @@ -0,0 +1,50 @@ +-- ## Migration to: +-- * Drop task_mapping_issues +-- * Drop task_invalidation_history +-- * Drop user_licenses +-- * Drop licenses +-- * Drop mapping_issue_categories +-- * Drop project_teams +-- * Drop project_chat +-- * Drop teams +-- * Drop related enums and linked columns + +-- Start a transaction +BEGIN; + +DROP INDEX IF EXISTS public.ix_task_mapping_issues_task_history_id; +DROP TABLE IF EXISTS public.task_mapping_issues; +DROP SEQUENCE IF EXISTS public.task_mapping_issues_id_seq; + +DROP INDEX IF EXISTS public.idx_task_validation_history_composite; +DROP INDEX IF EXISTS public.idx_task_validation_mapper_status_composite; +DROP INDEX IF EXISTS public.idx_task_validation_validator_status_composite; +DROP TABLE IF EXISTS public.task_invalidation_history; +DROP SEQUENCE IF EXISTS public.task_invalidation_history_id_seq; + +ALTER TABLE public.projects +DROP COLUMN IF EXISTS license_id; + +DROP TABLE IF EXISTS public.user_licenses; +DROP TABLE IF EXISTS public.licenses; +DROP SEQUENCE IF EXISTS public.licenses_id_seq; + +DROP TABLE IF EXISTS public.mapping_issue_categories; +DROP SEQUENCE IF EXISTS public.mapping_issue_categories_id_seq; + +DROP TABLE IF EXISTS public.project_teams; +DROP INDEX IF EXISTS ix_project_chat_project_id; +DROP TABLE IF EXISTS public.project_chat; +DROP SEQUENCE IF EXISTS public.project_chat_id_seq; +DROP TABLE IF EXISTS public.teams; +DROP SEQUENCE IF EXISTS public.teams_id_seq; + +ALTER TABLE public.projects +DROP COLUMN IF EXISTS mapping_permission, +DROP COLUMN IF EXISTS validation_permission; +DROP TYPE IF EXISTS public.validationpermission; +DROP TYPE IF EXISTS public.mappingpermission; +DROP TYPE IF EXISTS public.teamvisibility; + +-- Commit the transaction +COMMIT; diff --git a/src/backend/migrations/init/fmtm_base_schema.sql b/src/backend/migrations/init/fmtm_base_schema.sql index 0bb9f1aa17..38416af3d0 100644 --- a/src/backend/migrations/init/fmtm_base_schema.sql +++ b/src/backend/migrations/init/fmtm_base_schema.sql @@ -58,14 +58,6 @@ CREATE TYPE public.mappinglevel AS ENUM ( ); ALTER TYPE public.mappinglevel OWNER TO fmtm; -CREATE TYPE public.mappingpermission AS ENUM ( - 'ANY', - 'LEVEL', - 'TEAMS', - 'TEAMS_LEVEL' -); -ALTER TYPE public.mappingpermission OWNER TO fmtm; - CREATE TYPE public.organisationtype AS ENUM ( 'FREE', 'DISCOUNTED', @@ -115,12 +107,6 @@ CREATE TYPE public.taskstatus AS ENUM ( ); ALTER TYPE public.taskstatus OWNER TO fmtm; -CREATE TYPE public.teamvisibility AS ENUM ( - 'PUBLIC', - 'PRIVATE' -); -ALTER TYPE public.teamvisibility OWNER TO fmtm; - CREATE TYPE public.userrole AS ENUM ( 'READ_ONLY', 'MAPPER', @@ -137,14 +123,6 @@ CREATE TYPE public.projectrole AS ENUM ( ); ALTER TYPE public.projectrole OWNER TO fmtm; -CREATE TYPE public.validationpermission AS ENUM ( - 'ANY', - 'LEVEL', - 'TEAMS', - 'TEAMS_LEVEL' -); -ALTER TYPE public.validationpermission OWNER TO fmtm; - CREATE TYPE public.projectvisibility AS ENUM ( 'PUBLIC', 'PRIVATE', @@ -194,43 +172,6 @@ CREATE TABLE public.background_tasks ( ALTER TABLE public.background_tasks OWNER TO fmtm; -CREATE TABLE public.licenses ( - id integer NOT NULL, - name character varying, - description character varying, - plain_text character varying -); -ALTER TABLE public.licenses OWNER TO fmtm; -CREATE SEQUENCE public.licenses_id_seq -AS integer -START WITH 1 -INCREMENT BY 1 -NO MINVALUE -NO MAXVALUE -CACHE 1; -ALTER TABLE public.licenses_id_seq OWNER TO fmtm; -ALTER SEQUENCE public.licenses_id_seq OWNED BY public.licenses.id; - - -CREATE TABLE public.mapping_issue_categories ( - id integer NOT NULL, - name character varying NOT NULL, - description character varying, - archived boolean NOT NULL DEFAULT false -); -ALTER TABLE public.mapping_issue_categories OWNER TO fmtm; -CREATE SEQUENCE public.mapping_issue_categories_id_seq -AS integer -START WITH 1 -INCREMENT BY 1 -NO MINVALUE -NO MAXVALUE -CACHE 1; -ALTER TABLE public.mapping_issue_categories_id_seq OWNER TO fmtm; -ALTER SEQUENCE public.mapping_issue_categories_id_seq -OWNED BY public.mapping_issue_categories.id; - - CREATE TABLE public.mbtiles_path ( id integer NOT NULL, project_id integer, @@ -293,24 +234,6 @@ ALTER TABLE public.organisations_id_seq OWNER TO fmtm; ALTER SEQUENCE public.organisations_id_seq OWNED BY public.organisations.id; -CREATE TABLE public.project_chat ( - id bigint NOT NULL, - project_id integer NOT NULL, - user_id integer NOT NULL, - time_stamp timestamp without time zone NOT NULL DEFAULT now(), - message character varying NOT NULL -); -ALTER TABLE public.project_chat OWNER TO fmtm; -CREATE SEQUENCE public.project_chat_id_seq -START WITH 1 -INCREMENT BY 1 -NO MINVALUE -NO MAXVALUE -CACHE 1; -ALTER TABLE public.project_chat_id_seq OWNER TO fmtm; -ALTER SEQUENCE public.project_chat_id_seq OWNED BY public.project_chat.id; - - CREATE TABLE public.project_info ( project_id integer NOT NULL, project_id_str character varying, @@ -323,14 +246,6 @@ CREATE TABLE public.project_info ( ALTER TABLE public.project_info OWNER TO fmtm; -CREATE TABLE public.project_teams ( - team_id integer NOT NULL, - project_id integer NOT NULL, - role integer NOT NULL -); -ALTER TABLE public.project_teams OWNER TO fmtm; - - CREATE TABLE public.projects ( id integer NOT NULL, organisation_id integer, @@ -349,8 +264,6 @@ CREATE TABLE public.projects ( mapper_level public.mappinglevel NOT NULL DEFAULT 'INTERMEDIATE', priority public.projectpriority DEFAULT 'MEDIUM', featured boolean DEFAULT false, - mapping_permission public.mappingpermission DEFAULT 'ANY', - validation_permission public.validationpermission DEFAULT 'LEVEL', due_date timestamp without time zone, changeset_comment character varying, osmcha_filter_id character varying, @@ -360,7 +273,6 @@ CREATE TABLE public.projects ( josm_preset character varying, id_presets character varying [], extra_id_params character varying, - license_id integer, centroid public.GEOMETRY (POINT, 4326), odk_central_url character varying, odk_central_user character varying, @@ -408,53 +320,6 @@ ALTER TABLE public.task_history_id_seq OWNER TO fmtm; ALTER SEQUENCE public.task_history_id_seq OWNED BY public.task_history.id; -CREATE TABLE public.task_invalidation_history ( - id integer NOT NULL, - project_id integer NOT NULL, - task_id integer NOT NULL, - is_closed boolean DEFAULT false, - mapper_id bigint, - mapped_date timestamp without time zone, - invalidator_id bigint, - invalidated_date timestamp without time zone, - invalidation_history_id integer, - validator_id bigint, - validated_date timestamp without time zone, - updated_date timestamp without time zone DEFAULT now() -); -ALTER TABLE public.task_invalidation_history OWNER TO fmtm; -CREATE SEQUENCE public.task_invalidation_history_id_seq -AS integer -START WITH 1 -INCREMENT BY 1 -NO MINVALUE -NO MAXVALUE -CACHE 1; -ALTER TABLE public.task_invalidation_history_id_seq OWNER TO fmtm; -ALTER SEQUENCE public.task_invalidation_history_id_seq -OWNED BY public.task_invalidation_history.id; - - -CREATE TABLE public.task_mapping_issues ( - id integer NOT NULL, - task_history_id integer NOT NULL, - issue character varying NOT NULL, - mapping_issue_category_id integer NOT NULL, - count integer NOT NULL -); -ALTER TABLE public.task_mapping_issues OWNER TO fmtm; -CREATE SEQUENCE public.task_mapping_issues_id_seq -AS integer -START WITH 1 -INCREMENT BY 1 -NO MINVALUE -NO MAXVALUE -CACHE 1; -ALTER TABLE public.task_mapping_issues_id_seq OWNER TO fmtm; -ALTER SEQUENCE public.task_mapping_issues_id_seq -OWNED BY public.task_mapping_issues.id; - - CREATE TABLE public.tasks ( id integer NOT NULL, project_id integer NOT NULL, @@ -480,34 +345,6 @@ ALTER TABLE public.tasks_id_seq OWNER TO fmtm; ALTER SEQUENCE public.tasks_id_seq OWNED BY public.tasks.id; -CREATE TABLE public.teams ( - id integer NOT NULL, - organisation_id integer NOT NULL, - name character varying(512) NOT NULL, - logo character varying, - description character varying, - invite_only boolean NOT NULL DEFAULT false, - visibility public.teamvisibility NOT NULL DEFAULT 'PUBLIC' -); -ALTER TABLE public.teams OWNER TO fmtm; -CREATE SEQUENCE public.teams_id_seq -AS integer -START WITH 1 -INCREMENT BY 1 -NO MINVALUE -NO MAXVALUE -CACHE 1; -ALTER TABLE public.teams_id_seq OWNER TO fmtm; -ALTER SEQUENCE public.teams_id_seq OWNED BY public.teams.id; - - -CREATE TABLE public.user_licenses ( - "user" bigint, - license integer -); -ALTER TABLE public.user_licenses OWNER TO fmtm; - - CREATE TABLE public.user_roles ( user_id bigint NOT NULL, project_id integer NOT NULL, @@ -572,41 +409,21 @@ ALTER SEQUENCE public.xforms_id_seq OWNED BY public.xforms.id; -- nextval for primary keys (autoincrement) -ALTER TABLE ONLY public.licenses ALTER COLUMN id SET DEFAULT nextval( - 'public.licenses_id_seq'::regclass -); -ALTER TABLE ONLY public.mapping_issue_categories ALTER COLUMN id -SET DEFAULT nextval( - 'public.mapping_issue_categories_id_seq'::regclass -); ALTER TABLE ONLY public.mbtiles_path ALTER COLUMN id SET DEFAULT nextval( 'public.mbtiles_path_id_seq'::regclass ); ALTER TABLE ONLY public.organisations ALTER COLUMN id SET DEFAULT nextval( 'public.organisations_id_seq'::regclass ); -ALTER TABLE ONLY public.project_chat ALTER COLUMN id SET DEFAULT nextval( - 'public.project_chat_id_seq'::regclass -); ALTER TABLE ONLY public.projects ALTER COLUMN id SET DEFAULT nextval( 'public.projects_id_seq'::regclass ); ALTER TABLE ONLY public.task_history ALTER COLUMN id SET DEFAULT nextval( 'public.task_history_id_seq'::regclass ); -ALTER TABLE ONLY public.task_invalidation_history ALTER COLUMN id -SET DEFAULT nextval( - 'public.task_invalidation_history_id_seq'::regclass -); -ALTER TABLE ONLY public.task_mapping_issues ALTER COLUMN id SET DEFAULT nextval( - 'public.task_mapping_issues_id_seq'::regclass -); ALTER TABLE ONLY public.tasks ALTER COLUMN id SET DEFAULT nextval( 'public.tasks_id_seq'::regclass ); -ALTER TABLE ONLY public.teams ALTER COLUMN id SET DEFAULT nextval( - 'public.teams_id_seq'::regclass -); ALTER TABLE ONLY public.xlsforms ALTER COLUMN id SET DEFAULT nextval( 'public.xlsforms_id_seq'::regclass ); @@ -623,18 +440,6 @@ ADD CONSTRAINT "_migrations_pkey" PRIMARY KEY (script_name); ALTER TABLE ONLY public.background_tasks ADD CONSTRAINT background_tasks_pkey PRIMARY KEY (id); -ALTER TABLE ONLY public.licenses -ADD CONSTRAINT licenses_name_key UNIQUE (name); - -ALTER TABLE ONLY public.licenses -ADD CONSTRAINT licenses_pkey PRIMARY KEY (id); - -ALTER TABLE ONLY public.mapping_issue_categories -ADD CONSTRAINT mapping_issue_categories_name_key UNIQUE (name); - -ALTER TABLE ONLY public.mapping_issue_categories -ADD CONSTRAINT mapping_issue_categories_pkey PRIMARY KEY (id); - ALTER TABLE ONLY public.mbtiles_path ADD CONSTRAINT mbtiles_path_pkey PRIMARY KEY (id); @@ -650,33 +455,18 @@ ADD CONSTRAINT organisations_pkey PRIMARY KEY (id); ALTER TABLE ONLY public.organisations ADD CONSTRAINT organisations_slug_key UNIQUE (slug); -ALTER TABLE ONLY public.project_chat -ADD CONSTRAINT project_chat_pkey PRIMARY KEY (id); - ALTER TABLE ONLY public.project_info ADD CONSTRAINT project_info_pkey PRIMARY KEY (project_id); -ALTER TABLE ONLY public.project_teams -ADD CONSTRAINT project_teams_pkey PRIMARY KEY (team_id, project_id); - ALTER TABLE ONLY public.projects ADD CONSTRAINT projects_pkey PRIMARY KEY (id); ALTER TABLE ONLY public.task_history ADD CONSTRAINT task_history_pkey PRIMARY KEY (id); -ALTER TABLE ONLY public.task_invalidation_history -ADD CONSTRAINT task_invalidation_history_pkey PRIMARY KEY (id); - -ALTER TABLE ONLY public.task_mapping_issues -ADD CONSTRAINT task_mapping_issues_pkey PRIMARY KEY (id); - ALTER TABLE ONLY public.tasks ADD CONSTRAINT tasks_pkey PRIMARY KEY (id, project_id); -ALTER TABLE ONLY public.teams -ADD CONSTRAINT teams_pkey PRIMARY KEY (id); - ALTER TABLE ONLY public.user_roles ADD CONSTRAINT user_roles_pkey PRIMARY KEY (user_id, project_id); @@ -707,25 +497,7 @@ CREATE INDEX idx_task_history_project_id_user_id ON public.task_history USING btree ( user_id, project_id ); -CREATE INDEX idx_task_validation_history_composite -ON public.task_invalidation_history -USING btree ( - task_id, project_id -); -CREATE INDEX idx_task_validation_mapper_status_composite -ON public.task_invalidation_history -USING btree ( - mapper_id, is_closed -); -CREATE INDEX idx_task_validation_validator_status_composite -ON public.task_invalidation_history -USING btree ( - invalidator_id, is_closed -); CREATE INDEX idx_tasks_outline ON public.tasks USING gist (outline); -CREATE INDEX ix_project_chat_project_id ON public.project_chat USING btree ( - project_id -); CREATE INDEX ix_projects_mapper_level ON public.projects USING btree ( mapper_level ); @@ -738,11 +510,6 @@ CREATE INDEX ix_task_history_project_id ON public.task_history USING btree ( CREATE INDEX ix_task_history_user_id ON public.task_history USING btree ( user_id ); -CREATE INDEX ix_task_mapping_issues_task_history_id -ON public.task_mapping_issues -USING btree ( - task_history_id -); CREATE INDEX ix_tasks_locked_by ON public.tasks USING btree (locked_by); CREATE INDEX ix_tasks_mapped_by ON public.tasks USING btree (mapped_by); CREATE INDEX ix_tasks_project_id ON public.tasks USING btree (project_id); @@ -760,34 +527,6 @@ CREATE INDEX idx_org_managers ON public.organisation_managers USING btree ( -- Foreign keys -ALTER TABLE ONLY public.task_invalidation_history -ADD CONSTRAINT fk_invalidation_history FOREIGN KEY ( - invalidation_history_id -) REFERENCES public.task_history (id); - -ALTER TABLE ONLY public.task_invalidation_history -ADD CONSTRAINT fk_invalidators FOREIGN KEY ( - invalidator_id -) REFERENCES public.users (id); - -ALTER TABLE ONLY public.task_mapping_issues -ADD CONSTRAINT fk_issue_category FOREIGN KEY ( - mapping_issue_category_id -) REFERENCES public.mapping_issue_categories (id); - -ALTER TABLE ONLY public.projects -ADD CONSTRAINT fk_licenses FOREIGN KEY ( - license_id -) REFERENCES public.licenses (id); - -ALTER TABLE ONLY public.task_invalidation_history -ADD CONSTRAINT fk_mappers FOREIGN KEY (mapper_id) REFERENCES public.users (id); - -ALTER TABLE ONLY public.teams -ADD CONSTRAINT fk_organisations FOREIGN KEY ( - organisation_id -) REFERENCES public.organisations (id); - ALTER TABLE ONLY public.projects ADD CONSTRAINT fk_organisations FOREIGN KEY ( organisation_id @@ -798,11 +537,6 @@ ADD CONSTRAINT fk_tasks FOREIGN KEY ( task_id, project_id ) REFERENCES public.tasks (id, project_id); -ALTER TABLE ONLY public.task_invalidation_history -ADD CONSTRAINT fk_tasks FOREIGN KEY ( - task_id, project_id -) REFERENCES public.tasks (id, project_id); - ALTER TABLE ONLY public.projects ADD CONSTRAINT fk_users FOREIGN KEY (author_id) REFERENCES public.users (id); @@ -824,11 +558,6 @@ ADD CONSTRAINT fk_users_validator FOREIGN KEY ( validated_by ) REFERENCES public.users (id); -ALTER TABLE ONLY public.task_invalidation_history -ADD CONSTRAINT fk_validators FOREIGN KEY ( - validator_id -) REFERENCES public.users (id); - ALTER TABLE ONLY public.organisation_managers ADD CONSTRAINT organisation_managers_organisation_id_fkey FOREIGN KEY ( organisation_id @@ -839,61 +568,21 @@ ADD CONSTRAINT organisation_managers_user_id_fkey FOREIGN KEY ( user_id ) REFERENCES public.users (id); -ALTER TABLE ONLY public.project_chat -ADD CONSTRAINT project_chat_project_id_fkey FOREIGN KEY ( - project_id -) REFERENCES public.projects (id); - -ALTER TABLE ONLY public.project_chat -ADD CONSTRAINT project_chat_user_id_fkey FOREIGN KEY ( - user_id -) REFERENCES public.users (id); - ALTER TABLE ONLY public.project_info ADD CONSTRAINT project_info_project_id_fkey FOREIGN KEY ( project_id ) REFERENCES public.projects (id); -ALTER TABLE ONLY public.project_teams -ADD CONSTRAINT project_teams_project_id_fkey FOREIGN KEY ( - project_id -) REFERENCES public.projects (id); - -ALTER TABLE ONLY public.project_teams -ADD CONSTRAINT project_teams_team_id_fkey FOREIGN KEY ( - team_id -) REFERENCES public.teams (id); - ALTER TABLE ONLY public.task_history ADD CONSTRAINT task_history_project_id_fkey FOREIGN KEY ( project_id ) REFERENCES public.projects (id); -ALTER TABLE ONLY public.task_invalidation_history -ADD CONSTRAINT task_invalidation_history_project_id_fkey FOREIGN KEY ( - project_id -) REFERENCES public.projects (id); - -ALTER TABLE ONLY public.task_mapping_issues -ADD CONSTRAINT task_mapping_issues_task_history_id_fkey FOREIGN KEY ( - task_history_id -) REFERENCES public.task_history (id); - ALTER TABLE ONLY public.tasks ADD CONSTRAINT tasks_project_id_fkey FOREIGN KEY ( project_id ) REFERENCES public.projects (id); -ALTER TABLE ONLY public.user_licenses -ADD CONSTRAINT user_licenses_license_fkey FOREIGN KEY ( - license -) REFERENCES public.licenses (id); - -ALTER TABLE ONLY public.user_licenses -ADD CONSTRAINT user_licenses_user_fkey FOREIGN KEY ( - "user" -) REFERENCES public.users (id); - ALTER TABLE ONLY public.user_roles ADD CONSTRAINT user_roles_project_id_fkey FOREIGN KEY ( project_id diff --git a/src/backend/migrations/revert/004-update-default-values.sql b/src/backend/migrations/revert/004-update-default-values.sql index 75a0b25beb..cf614d1b6d 100644 --- a/src/backend/migrations/revert/004-update-default-values.sql +++ b/src/backend/migrations/revert/004-update-default-values.sql @@ -4,18 +4,10 @@ -- Start a transaction BEGIN; --- Revert changes in mapping_issue_categories table -ALTER TABLE public.mapping_issue_categories -ALTER COLUMN archived DROP DEFAULT; - -- Revert changes in mbtiles_path table ALTER TABLE public.mbtiles_path ALTER COLUMN created_at DROP DEFAULT; --- Revert changes in project_chat table -ALTER TABLE public.project_chat -ALTER COLUMN time_stamp DROP DEFAULT; - -- Revert changes in projects table ALTER TABLE public.projects ALTER COLUMN created DROP DEFAULT, @@ -23,28 +15,16 @@ ALTER COLUMN last_updated DROP DEFAULT, ALTER COLUMN status DROP DEFAULT, ALTER COLUMN mapper_level DROP DEFAULT, ALTER COLUMN priority DROP DEFAULT, -ALTER COLUMN featured DROP DEFAULT, -ALTER COLUMN mapping_permission DROP DEFAULT, -ALTER COLUMN validation_permission DROP DEFAULT; +ALTER COLUMN featured DROP DEFAULT; -- Revert changes in task_history table ALTER TABLE public.task_history ALTER COLUMN action_date DROP DEFAULT; --- Revert changes in task_invalidation_history table -ALTER TABLE public.task_invalidation_history -ALTER COLUMN is_closed DROP DEFAULT, -ALTER COLUMN updated_date DROP DEFAULT; - -- Revert changes in task_invalidation_history_id_seq table ALTER TABLE public.task_invalidation_history_id_seq ALTER COLUMN task_status DROP DEFAULT; --- Revert changes in teams table -ALTER TABLE public.teams -ALTER COLUMN invite_only DROP DEFAULT, -ALTER COLUMN visibility DROP DEFAULT; - -- Revert changes in user_roles table ALTER TABLE public.user_roles ALTER COLUMN role DROP DEFAULT; diff --git a/src/backend/pdm.lock b/src/backend/pdm.lock index cf702f2b6a..99e5439502 100644 --- a/src/backend/pdm.lock +++ b/src/backend/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "debug", "dev", "docs", "test", "monitoring"] strategy = ["cross_platform"] lock_version = "4.4.1" -content_hash = "sha256:dc843ea7774920cf4858affe48fec8310148cea089b2877e190e12732e754e90" +content_hash = "sha256:498635b56dc6a5c8266b8580ec0b4e7bc794c92de8693ff9028f413ac147fe2b" [[package]] name = "aiohttp" @@ -187,6 +187,19 @@ files = [ {file = "asttokens-2.4.1.tar.gz", hash = "sha256:b03869718ba9a6eb027e134bfdf69f38a236d681c83c160d510768af11254ba0"}, ] +[[package]] +name = "async-lru" +version = "2.0.4" +requires_python = ">=3.8" +summary = "Simple LRU cache for asyncio" +dependencies = [ + "typing-extensions>=4.0.0; python_version < \"3.11\"", +] +files = [ + {file = "async-lru-2.0.4.tar.gz", hash = "sha256:b8a59a5df60805ff63220b2a0c5b5393da5521b113cd5465a44eb037d81a5627"}, + {file = "async_lru-2.0.4-py3-none-any.whl", hash = "sha256:ff02944ce3c288c5be660c42dbcca0742b32c3b279d6dceda655190240b99224"}, +] + [[package]] name = "async-timeout" version = "4.0.3" diff --git a/src/backend/pyproject.toml b/src/backend/pyproject.toml index ecff63e1d1..a1cc81b3e9 100644 --- a/src/backend/pyproject.toml +++ b/src/backend/pyproject.toml @@ -37,7 +37,6 @@ dependencies = [ "geojson==3.1.0", "shapely==2.0.2", "pyxform==2.0.2", - "py-cpuinfo==9.0.0", "loguru==0.7.2", "minio==7.2.0", "pyproj==3.6.1", @@ -46,6 +45,7 @@ dependencies = [ "cryptography>=42.0.1", "defusedxml>=0.7.1", "pyjwt>=2.8.0", + "async-lru>=2.0.4", "osm-login-python==1.0.3", "osm-fieldwork==0.12.4", "osm-rawdata==0.3.0", diff --git a/src/backend/tests/conftest.py b/src/backend/tests/conftest.py index 9ee0eed9f4..fb36a68d2d 100644 --- a/src/backend/tests/conftest.py +++ b/src/backend/tests/conftest.py @@ -35,11 +35,12 @@ from app.central import central_crud from app.config import settings from app.db.database import Base, get_db -from app.db.db_models import DbOrganisation +from app.db.db_models import DbOrganisation, DbTaskHistory from app.main import get_application -from app.models.enums import CommunityType, UserRole +from app.models.enums import CommunityType, TaskStatus, UserRole from app.projects import project_crud from app.projects.project_schemas import ODKCentralDecrypted, ProjectInfo, ProjectUpload +from app.users.user_crud import get_user engine = create_engine(settings.FMTM_DB_URL.unicode_string()) TestingSessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) @@ -188,6 +189,65 @@ async def project(db, admin_user, organisation): return new_project +@pytest.fixture(scope="function") +async def task(project, db): + """A test task, using the test project.""" + boundaries = { + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [85.3012091, 27.7122369], + [85.3012129, 27.7121403], + [85.3013408, 27.7121442], + [85.3013371, 27.7122408], + [85.3012441, 27.712238], + [85.3012091, 27.7122369], + ] + ], + }, + "properties": { + "osm_id": 650958368, + "version": 2, + "tags": {"building": "yes"}, + "changeset": 99124278, + "timestamp": "2021-02-11T17:21:06", + }, + } + try: + tasks = await project_crud.create_tasks_from_geojson( + db=db, project_id=project.id, boundaries=boundaries + ) + + assert tasks is True + + # Refresh the project to include the tasks + db.refresh(project) + except Exception as e: + log.exception(e) + pytest.fail(f"Test failed with exception: {str(e)}") + return project.tasks[0] + + +@pytest.fixture(scope="function") +async def task_history(db, project, task, admin_user): + """A test task history using the test user, project and task.""" + user = await get_user(db, admin_user.id) + task_history_entry = DbTaskHistory( + project_id=project.id, + task_id=task.id, + action=TaskStatus.READY, + action_text=f"Task created with action {TaskStatus.READY} by {user.username}", + actioned_by=user, + user_id=user.id, + ) + db.add(task_history_entry) + db.commit() + db.refresh(task_history_entry) + return task_history_entry + + # @pytest.fixture(scope="function") # def get_ids(db, project): # user_id_query = text(f"SELECT id FROM {DbUser.__table__.name} LIMIT 1") diff --git a/src/backend/tests/test_task_routes.py b/src/backend/tests/test_task_routes.py new file mode 100644 index 0000000000..d0fb3215ea --- /dev/null +++ b/src/backend/tests/test_task_routes.py @@ -0,0 +1,39 @@ +# Copyright (c) 2022, 2023 Humanitarian OpenStreetMap Team +# +# This file is part of FMTM. +# +# FMTM is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# FMTM is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with FMTM. If not, see . +# +"""Tests for task routes.""" + +import pytest + + +def test_read_task_history(client, task_history): + """Test task history for a project.""" + task_id = task_history.task_id + + assert task_id is not None + + response = client.get(f"/tasks/{task_id}/history/") + data = response.json()[0] + + assert response.status_code == 200 + assert data["id"] == task_history.id + assert data["username"] == task_history.actioned_by.username + + +if __name__ == "__main__": + """Main func if file invoked directly.""" + pytest.main() diff --git a/src/frontend/prod.dockerfile b/src/frontend/prod.dockerfile index c0cc092adb..4eacb66678 100644 --- a/src/frontend/prod.dockerfile +++ b/src/frontend/prod.dockerfile @@ -17,7 +17,7 @@ RUN pnpm run build --mode ${NODE_ENV} -FROM docker.io/rclone/rclone:1.64 as prod +FROM docker.io/rclone/rclone:1 as prod ARG APP_VERSION ARG COMMIT_REF ARG VITE_API_URL diff --git a/src/frontend/src/api/CreateProjectService.ts b/src/frontend/src/api/CreateProjectService.ts index 7ed097d918..9b7e9702b2 100755 --- a/src/frontend/src/api/CreateProjectService.ts +++ b/src/frontend/src/api/CreateProjectService.ts @@ -8,6 +8,7 @@ import { } from '@/models/createproject/createProjectModel'; import { CommonActions } from '@/store/slices/CommonSlice'; import { ValidateCustomFormResponse } from '@/store/types/ICreateProject'; +import { isStatusSuccess } from '@/utilfunctions/commonUtils'; const CreateProjectService: Function = ( url: string, @@ -21,31 +22,42 @@ const CreateProjectService: Function = ( dispatch(CreateProjectActions.CreateProjectLoading(true)); dispatch(CommonActions.SetLoading(true)); + let projectId: null | number = null; try { - // Create project + // halt project creation if any api call fails + let hasAPISuccess = false; + const postNewProjectDetails = await API.post(url, projectData); + hasAPISuccess = isStatusSuccess(postNewProjectDetails.status); + const projectCreateResp: ProjectDetailsModel = postNewProjectDetails.data; await dispatch(CreateProjectActions.PostProjectDetails(projectCreateResp)); - if (projectCreateResp.status >= 300) { + if (!hasAPISuccess) { throw new Error(`Request failed with status ${projectCreateResp.status}`); } - const projectId = projectCreateResp.id; + projectId = projectCreateResp.id; // Submit task boundaries - await dispatch( + hasAPISuccess = await dispatch( UploadTaskAreasService( `${import.meta.env.VITE_API_URL}/projects/${projectId}/upload-task-boundaries`, taskAreaGeojson, ), ); + if (!hasAPISuccess) { + throw new Error(`Request failed`); + } + // Upload data extract - let extractResponse + let extractResponse; if (isOsmExtract) { // Generated extract from raw-data-api extractResponse = await API.get( - `${import.meta.env.VITE_API_URL}/projects/data-extract-url/?project_id=${projectId}&url=${projectData.data_extract_url}`, + `${import.meta.env.VITE_API_URL}/projects/data-extract-url/?project_id=${projectId}&url=${ + projectData.data_extract_url + }`, ); } else if (dataExtractFile) { // Custom data extract from user @@ -56,12 +68,14 @@ const CreateProjectService: Function = ( dataExtractFormData, ); } - if (extractResponse.status >= 300) { + hasAPISuccess = isStatusSuccess(extractResponse.status); + + if (!hasAPISuccess) { throw new Error(`Request failed with status ${extractResponse.status}`); } // Generate project files - await dispatch( + const generateProjectFile = await dispatch( GenerateProjectFilesService( `${import.meta.env.VITE_API_URL}/projects/${projectId}/generate-project-data`, projectData, @@ -69,13 +83,22 @@ const CreateProjectService: Function = ( ), ); - dispatch(CreateProjectActions.CreateProjectLoading(false)); + hasAPISuccess = generateProjectFile; + if (!hasAPISuccess) { + throw new Error(`Request failed`); + } + dispatch(CreateProjectActions.GenerateProjectError(false)); + // dispatch(CreateProjectActions.CreateProjectLoading(false)); } catch (error: any) { + if (projectId) { + await dispatch(DeleteProjectService(`${import.meta.env.VITE_API_URL}/projects/${projectId}`, false)); + } + await dispatch(CreateProjectActions.GenerateProjectError(true)); dispatch( CommonActions.SetSnackBar({ open: true, - message: JSON.stringify(error?.response?.data?.detail) || 'Something went wrong.', + message: JSON.stringify(error?.response?.data?.detail) || 'Something went wrong. Please try again.', variant: 'error', duration: 2000, }), @@ -109,6 +132,7 @@ const UploadTaskAreasService: Function = (url: string, filePayload: any, project return async (dispatch) => { dispatch(CreateProjectActions.UploadAreaLoading(true)); const postUploadArea = async (url, filePayload) => { + let isAPISuccess = true; try { const areaFormData = new FormData(); areaFormData.append('task_geojson', filePayload); @@ -117,14 +141,16 @@ const UploadTaskAreasService: Function = (url: string, filePayload: any, project 'Content-Type': 'multipart/form-data', }, }); + isAPISuccess = isStatusSuccess(postNewProjectDetails.status); - if (postNewProjectDetails.status >= 200 && postNewProjectDetails.status < 300) { + if (isAPISuccess) { await dispatch(CreateProjectActions.UploadAreaLoading(false)); await dispatch(CreateProjectActions.PostUploadAreaSuccess(postNewProjectDetails.data)); } else { throw new Error(`Request failed with status ${postNewProjectDetails.status}`); } } catch (error: any) { + isAPISuccess = false; await dispatch(CreateProjectActions.GenerateProjectError(true)); dispatch( CommonActions.SetSnackBar({ @@ -136,9 +162,10 @@ const UploadTaskAreasService: Function = (url: string, filePayload: any, project ); dispatch(CreateProjectActions.UploadAreaLoading(false)); } + return isAPISuccess; }; - await postUploadArea(url, filePayload); + return await postUploadArea(url, filePayload); }; }; @@ -148,6 +175,7 @@ const GenerateProjectFilesService: Function = (url: string, projectData: any, fo dispatch(CommonActions.SetLoading(true)); const postUploadArea = async (url, projectData: any, formUpload) => { + let isAPISuccess = true; try { let response; @@ -163,8 +191,8 @@ const GenerateProjectFilesService: Function = (url: string, projectData: any, fo } else { response = await axios.post(url, {}); } - - if (response.status > 300) { + isAPISuccess = isStatusSuccess(response.status); + if (!isAPISuccess) { throw new Error(`Request failed with status ${response.status}`); } @@ -173,6 +201,7 @@ const GenerateProjectFilesService: Function = (url: string, projectData: any, fo // Trigger the watcher and redirect after success await dispatch(CreateProjectActions.GenerateProjectSuccess(true)); } catch (error: any) { + isAPISuccess = false; dispatch(CommonActions.SetLoading(false)); await dispatch(CreateProjectActions.GenerateProjectError(true)); dispatch( @@ -185,9 +214,10 @@ const GenerateProjectFilesService: Function = (url: string, projectData: any, fo ); dispatch(CreateProjectActions.GenerateProjectLoading(false)); } + return isAPISuccess; }; - await postUploadArea(url, projectData, formUpload); + return await postUploadArea(url, projectData, formUpload); }; }; @@ -462,7 +492,7 @@ const ValidateCustomForm: Function = (url: string, formUpload: any) => { }; }; -const DeleteProjectService: Function = (url: string) => { +const DeleteProjectService: Function = (url: string, hasRedirect: boolean = true) => { return async (dispatch) => { const deleteProject = async (url: string) => { try { @@ -470,15 +500,17 @@ const DeleteProjectService: Function = (url: string) => { dispatch( CommonActions.SetSnackBar({ open: true, - message: 'Project deleted. Redirecting...', + message: `Project deleted. ${hasRedirect && 'Redirecting...'}`, variant: 'success', duration: 2000, }), ); // Redirect to homepage - setTimeout(() => { - window.location.href = '/'; - }, 2000); + if (hasRedirect) { + setTimeout(() => { + window.location.href = '/'; + }, 2000); + } } catch (error) { if (error.response.status === 404) { dispatch( diff --git a/src/frontend/src/api/Project.js b/src/frontend/src/api/Project.js index 2bc985e789..7285cac28f 100755 --- a/src/frontend/src/api/Project.js +++ b/src/frontend/src/api/Project.js @@ -182,9 +182,10 @@ export const DownloadTile = (url, payload, toOpfs = false) => { return; } - const filename = response.headers['Content-Disposition'].split('filename=')[1]; + const filename = response.headers['content-disposition'].split('filename=')[1]; + console.log(filename) // Create Blob from ArrayBuffer - const blob = new Blob([tileData], { type: response.headers['Content-Type'] }); + const blob = new Blob([tileData], { type: response.headers['content-type'] }); const downloadUrl = URL.createObjectURL(blob); const a = document.createElement('a'); diff --git a/src/frontend/src/components/createnewproject/SplitTasks.tsx b/src/frontend/src/components/createnewproject/SplitTasks.tsx index 8bb3ff7893..c6d7457dad 100644 --- a/src/frontend/src/components/createnewproject/SplitTasks.tsx +++ b/src/frontend/src/components/createnewproject/SplitTasks.tsx @@ -213,7 +213,7 @@ const SplitTasks = ({ flag, geojsonFile, setGeojsonFile, customDataExtractUpload }; handleQRGeneration(); - }, [generateProjectSuccess]); + }, [generateProjectSuccess, generateProjectError]); const renderTraceback = (errorText: string) => { if (!errorText) { diff --git a/src/frontend/src/store/slices/CreateProjectSlice.ts b/src/frontend/src/store/slices/CreateProjectSlice.ts index 2cd9745ba5..004fc887cd 100755 --- a/src/frontend/src/store/slices/CreateProjectSlice.ts +++ b/src/frontend/src/store/slices/CreateProjectSlice.ts @@ -95,6 +95,9 @@ const CreateProject = createSlice({ state.uploadAreaSelection = ''; state.dividedTaskGeojson = null; state.dividedTaskLoading = false; + state.generateProjectSuccess = false; + state.generateProjectError = false; + state.drawToggle = false; }, UploadAreaLoading(state, action) { state.projectAreaLoading = action.payload; diff --git a/src/frontend/src/utilfunctions/commonUtils.ts b/src/frontend/src/utilfunctions/commonUtils.ts index e019ef4b78..cf04d93932 100644 --- a/src/frontend/src/utilfunctions/commonUtils.ts +++ b/src/frontend/src/utilfunctions/commonUtils.ts @@ -9,3 +9,10 @@ export const isInputEmpty = (text: string): boolean => { export const camelToFlat = (word: string): string => ( (word = word.replace(/[A-Z]/g, ' $&')), word[0].toUpperCase() + word.slice(1) ); + +export const isStatusSuccess = (status: number) => { + if (status < 300) { + return true; + } + return false; +};