feat: init public beta org and svcfmtm user migration (#1206)

* build: remove odk vars from frontend build dev compose

* build: remove user password field from init schema

* build: fix only continue migration if sql script succeeds

* build: update migration entrypoint envsubst + error handling

* build: migration to create fmtm public beta org + svcfmtm

* build: add gettext for envsubst command in dockerfile

* build: use odk user email for svcfmtm user email

* fix: replace org init with startup code for pass encrypt

* fix: specify user id for svcfmtm admin init

* build: fix remove sequential user id (create manually)

* build: remove default user creation from migration script

* build: fix project-roles migration if enum exists

docker-compose.yml

@@ -122,9 +122,6 @@ services:
       - /app/node_modules/
     environment:
       - VITE_API_URL=http://api.${FMTM_DOMAIN}:${FMTM_DEV_PORT:-7050}
-      - VITE_ODK_CENTRAL_URL=${ODK_CENTRAL_URL}
-      - VITE_ODK_CENTRAL_USER=${ODK_CENTRAL_USER}
-      - VITE_ODK_CENTRAL_PASSWD=${ODK_CENTRAL_PASSWD}
     ports:
       - "7051:7051"
     networks:

src/backend/Dockerfile

@@ -96,6 +96,7 @@ RUN set -ex \
     -y --no-install-recommends \
         "nano" \
         "curl" \
+        "gettext-base" \
         "libpcre3" \
         "mime-support" \
         "postgresql-client" \

src/backend/app/config.py

@@ -176,7 +176,7 @@ def get_cipher_suite():
     return Fernet(settings.ENCRYPTION_KEY)
 
 
-def encrypt_value(password: str) -> str:
+def encrypt_value(password: Union[str, HttpUrlStr]) -> str:
     """Encrypt value before going to the DB."""
     cipher_suite = get_cipher_suite()
     encrypted_password = cipher_suite.encrypt(password.encode("utf-8"))

src/backend/app/db/db_models.py

@@ -93,7 +93,7 @@ class DbUser(Base):
 
     __tablename__ = "users"
 
-    id = cast(int, Column(BigInteger, primary_key=True, index=True))
+    id = cast(int, Column(BigInteger, primary_key=True))
     username = cast(str, Column(String, unique=True))
     profile_img = cast(str, Column(String))
     role = cast(UserRole, Column(Enum(UserRole), default=UserRole.MAPPER))

src/backend/app/main.py

@@ -36,6 +36,7 @@
 from app.db.database import get_db
 from app.models.enums import HTTPStatus
 from app.organisations import organisation_routes
+from app.organisations.organisation_crud import init_admin_org
 from app.projects import project_routes
 from app.projects.project_crud import read_xlsforms
 from app.submissions import submission_routes
@@ -54,8 +55,11 @@
 async def lifespan(app: FastAPI):
     """FastAPI startup/shutdown event."""
     log.debug("Starting up FastAPI server.")
+    db_conn = next(get_db())
+    log.debug("Initialising admin org and user in DB.")
+    await init_admin_org(db_conn)
     log.debug("Reading XLSForms from DB.")
-    await read_xlsforms(next(get_db()), xlsforms_path)
+    await read_xlsforms(db_conn, xlsforms_path)
 
     yield
 

src/backend/app/organisations/organisation_crud.py

@@ -22,11 +22,11 @@
 
 from fastapi import File, HTTPException, Response, UploadFile
 from loguru import logger as log
-from sqlalchemy import update
+from sqlalchemy import text, update
 from sqlalchemy.orm import Session
 
 from app.auth.osm import AuthUser
-from app.config import settings
+from app.config import encrypt_value, settings
 from app.db import db_models
 from app.models.enums import HTTPStatus, UserRole
 from app.organisations.organisation_deps import (
@@ -37,6 +37,93 @@
 from app.users.user_crud import get_user
 
 
+async def init_admin_org(db: Session):
+    """Init admin org and user at application startup."""
+    sql = text(
+        """
+        -- Start a transaction
+        BEGIN;
+
+        -- Insert FMTM Public Beta organisation
+        INSERT INTO public.organisations (
+            name,
+            slug,
+            logo,
+            description,
+            url,
+            type,
+            approved,
+            odk_central_url,
+            odk_central_user,
+            odk_central_password
+        )
+        VALUES (
+            'FMTM Public Beta',
+            'fmtm-public-beta',
+            'https://avatars.githubusercontent.com/u/458752?s=280&v=4',
+            'HOTOSM Public Beta for FMTM.',
+            'https://hotosm.org',
+            'FREE',
+            true,
+            :odk_url,
+            :odk_user,
+            :odk_pass
+        )
+        ON CONFLICT ("name") DO NOTHING;
+
+        -- Insert svcfmtm admin user
+        INSERT INTO public.users (
+            id,
+            username,
+            role,
+            name,
+            email_address,
+            is_email_verified,
+            mapping_level,
+            tasks_mapped,
+            tasks_validated,
+            tasks_invalidated
+        )
+        VALUES (
+            :user_id,
+            :username,
+            'ADMIN',
+            'Admin',
+            :odk_user,
+            true,
+            'ADVANCED',
+            0,
+            0,
+            0
+        )
+        ON CONFLICT ("username") DO NOTHING;
+
+        -- Set svcfmtm user as org admin
+        WITH org_cte AS (
+            SELECT id FROM public.organisations
+            WHERE name = 'FMTM Public Beta'
+        )
+        INSERT INTO public.organisation_managers (organisation_id, user_id)
+        SELECT (SELECT id FROM org_cte), :user_id
+        ON CONFLICT DO NOTHING;
+
+        -- Commit the transaction
+        COMMIT;
+    """
+    )
+
+    db.execute(
+        sql,
+        {
+            "user_id": 20386219,
+            "username": "svcfmtm",
+            "odk_url": settings.ODK_CENTRAL_URL,
+            "odk_user": settings.ODK_CENTRAL_USER,
+            "odk_pass": encrypt_value(settings.ODK_CENTRAL_PASSWD),
+        },
+    )
+
+
 async def get_organisations(
     db: Session,
     current_user: AuthUser,

src/backend/migrate-entrypoint.sh

@@ -128,7 +128,7 @@ backup_db() {
         --host "$FMTM_DB_HOST" --username "$FMTM_DB_USER" "$FMTM_DB_NAME"
 
     echo "gzipping file --> ${db_backup_file}.gz"
-    gzip "$db_backup_file"
+    gzip --force "$db_backup_file"
     db_backup_file="${db_backup_file}.gz"
 
     BUCKET_NAME="fmtm-db-backups"
@@ -145,10 +145,11 @@ execute_migrations() {
     for script_name in "${scripts_to_execute[@]}"; do
         script_file="/opt/migrations/$script_name"
         pretty_echo "Executing migration: $script_name"
-        psql "$db_url" -a -f "$script_file"
-
-        # Add an entry in the migrations table to indicate completion
-        psql "$db_url" <<SQL
+        # Apply migration with env vars substituted & if succeeds,
+        # add an entry in the migrations table to indicate completion
+        envsubst < "$script_file" | psql "$db_url" \
+            --set ON_ERROR_STOP=1 --echo-all \
+        && psql "$db_url" <<SQL
     DO \$\$
     BEGIN
         INSERT INTO public."_migrations" (date_executed, script_name)
@@ -162,20 +163,6 @@ SQL
     done
 }
 
-create_svc_user() {
-    pretty_echo "Creating default svcfmtm user."
-    psql "$db_url" <<SQL
-    DO \$\$
-    BEGIN
-        INSERT INTO users (id, username, role, mapping_level, tasks_mapped, tasks_validated, tasks_invalidated)
-        VALUES (20386219, 'svcfmtm', 'MAPPER', 'BEGINNER', 0, 0, 0);
-        RAISE NOTICE 'User "svcfmtm" (uid 20386219) successfully created.';
-    EXCEPTION   
-        WHEN OTHERS THEN
-            RAISE NOTICE 'User "svcfmtm" (uid 20386219) already exists.';
-    END\$\$;
-SQL
-}
 ### Functions END ###
 
 
@@ -206,12 +193,7 @@ if [ ${#scripts_to_execute[@]} -gt 0 ]; then
 else
     pretty_echo "No new migrations found."
 fi
-pretty_echo "Migrations complete."
-
-# Create service user account, if not exists
-create_svc_user
-
-pretty_echo "### Script End ###"
+pretty_echo "### Script End: Migrations Complete ###"
 
 ####################
 ###  Script END  ###

src/backend/migrations/003-project-roles.sql

@@ -1,20 +1,27 @@
 -- ## Migration to:
--- * Add public.projectrol enum.
+-- * Add public.projectrole enum.
 -- * Update the user_roles table to use the enum
 
 -- Start a transaction
 BEGIN;
 
-CREATE TYPE public.projectrole as ENUM (
+-- Create projectrole enum if it doesn't exist
+DO $$
+BEGIN
+  IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'projectrole') THEN
+    CREATE TYPE public.projectrole AS ENUM (
     'MAPPER',
     'VALIDATOR',
     'FIELD_MANAGER',
     'ASSOCIATE_PROJECT_MANAGER',
     'PROJECT_MANAGER'
-);
+    );
+  END IF;
+END $$;
+ALTER TYPE public.projectrole OWNER TO fmtm;
+
 ALTER TABLE public.user_roles ALTER COLUMN "role" TYPE VARCHAR(24);
 ALTER TABLE public.user_roles ALTER COLUMN "role" TYPE public.projectrole USING role::public.projectrole;
-ALTER TYPE public.projectrole OWNER TO fmtm;
 
 -- Commit the transaction
 COMMIT;

src/backend/migrations/init/fmtm_base_schema.sql

@@ -531,18 +531,9 @@ CREATE TABLE public.users (
     tasks_invalidated integer NOT NULL,
     projects_mapped integer[],
     date_registered timestamp without time zone,
-    last_validation_date timestamp without time zone,
-    password character varying
+    last_validation_date timestamp without time zone
 );
 ALTER TABLE public.users OWNER TO fmtm;
-CREATE SEQUENCE public.users_id_seq
-    START WITH 1
-    INCREMENT BY 1
-    NO MINVALUE
-    NO MAXVALUE
-    CACHE 1;
-ALTER TABLE public.users_id_seq OWNER TO fmtm;
-ALTER SEQUENCE public.users_id_seq OWNED BY public.users.id;
 
 CREATE TABLE public.xlsforms (
     id integer NOT NULL,
@@ -578,7 +569,6 @@ ALTER TABLE ONLY public.task_invalidation_history ALTER COLUMN id SET DEFAULT ne
 ALTER TABLE ONLY public.task_mapping_issues ALTER COLUMN id SET DEFAULT nextval('public.task_mapping_issues_id_seq'::regclass);
 ALTER TABLE ONLY public.tasks ALTER COLUMN id SET DEFAULT nextval('public.tasks_id_seq'::regclass);
 ALTER TABLE ONLY public.teams ALTER COLUMN id SET DEFAULT nextval('public.teams_id_seq'::regclass);
-ALTER TABLE ONLY public.users ALTER COLUMN id SET DEFAULT nextval('public.users_id_seq'::regclass);
 ALTER TABLE ONLY public.xlsforms ALTER COLUMN id SET DEFAULT nextval('public.xlsforms_id_seq'::regclass);