Alembic Setup (#869)

* alembic ini file

* alembic migrations env.py file, initial migration files created

* commented line to create models on startup

* added function to ignore migration if the table exists

* updated initial migration file

* created alembic helpers file to check if the tables already exists

* relative imports

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* build: add alembic to backend deps

* build: rebuild pdm.lock after alembic addition

* refactor: move alembic files to separate dir

* build: refactor: compose unless-stopped --> "unless-stopped"

* build: add migration container to compose stack

* build: reuse main backend img for migrations

* build: backend dockerfile copy all dirs under /opt/

* build: add migration container to prod compose configs

* refactor: add additional logging to migrations

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: sam <sam.woodcock@protonmail.com>

docker-compose.deploy.yml

@@ -40,7 +40,7 @@ services:
       - 443:443
     networks:
       - fmtm-net
-    restart: unless-stopped
+    restart: "unless-stopped"
     command:
       - "--entrypoints.web.address=:80"
       - "--entrypoints.websecure.address=:443"
@@ -75,7 +75,7 @@ services:
       - "5433:5432"
     networks:
       - fmtm-net
-    restart: unless-stopped
+    restart: "unless-stopped"
 
   api:
     image: "ghcr.io/hotosm/fmtm/backend:${API_VERSION}-${GIT_BRANCH}"
@@ -96,7 +96,7 @@ services:
       - .env
     networks:
       - fmtm-net
-    restart: unless-stopped
+    restart: "unless-stopped"
     labels:
       - "traefik.enable=true"
       - "traefik.http.routers.api.tls=true"
@@ -105,6 +105,18 @@ services:
       - "traefik.http.services.api-svc.loadbalancer.server.port=8000"
       - "traefik.http.routers.api.service=api-svc"
 
+  migrations:
+    image: "ghcr.io/hotosm/fmtm/backend:${API_VERSION}-${GIT_BRANCH}"
+    container_name: fmtm_migrations
+    depends_on:
+      - api
+    env_file:
+      - .env
+    networks:
+      - fmtm-net
+    command: ["alembic", "upgrade", "head"]
+    restart: "no"
+
   ui-main:
     image: "ghcr.io/hotosm/fmtm/frontend:${FRONTEND_MAIN_VERSION}-${GIT_BRANCH}"
     build:
@@ -124,7 +136,7 @@ services:
       - BROTLI=true
       - API_URL=${URL_SCHEME}://${API_URL}
       - FRONTEND_MAIN_URL=${URL_SCHEME}://${FRONTEND_MAIN_URL}
-    restart: unless-stopped
+    restart: "unless-stopped"
     labels:
       - "traefik.enable=true"
       - "traefik.http.routers.ui-main.tls=true"

docker-compose.noodk.yml

@@ -41,7 +41,7 @@ services:
       - "5433:5432"
     networks:
       - fmtm-dev
-    restart: unless-stopped
+    restart: "unless-stopped"
 
   api:
     image: "ghcr.io/hotosm/fmtm/backend:debug"
@@ -65,7 +65,19 @@ services:
       - "5678:5678"
     networks:
       - fmtm-dev
-    restart: unless-stopped
+    restart: "unless-stopped"
+
+  migrations:
+    image: "ghcr.io/hotosm/fmtm/backend:${API_VERSION}-${GIT_BRANCH}"
+    container_name: fmtm_migrations
+    depends_on:
+      - api
+    env_file:
+      - .env
+    networks:
+      - fmtm-net
+    command: ["alembic", "upgrade", "head"]
+    restart: "no"
 
   ui-main:
     image: "ghcr.io/hotosm/fmtm/frontend:debug"
@@ -89,4 +101,4 @@ services:
       - "8081:8081"
     networks:
       - fmtm-dev
-    restart: unless-stopped
+    restart: "unless-stopped"

docker-compose.yml

@@ -42,7 +42,7 @@ services:
       - "5438:5432"
     networks:
       - fmtm-dev
-    restart: unless-stopped
+    restart: "unless-stopped"
 
   api:
     image: "ghcr.io/hotosm/fmtm/backend:debug"
@@ -72,7 +72,19 @@ services:
       - "5678:5678"
     networks:
       - fmtm-dev
-    restart: unless-stopped
+    restart: "unless-stopped"
+
+  migrations:
+    image: "ghcr.io/hotosm/fmtm/backend:debug"
+    container_name: fmtm_migrations
+    depends_on:
+      - api
+    env_file:
+      - .env
+    networks:
+      - fmtm-dev
+    command: ["alembic", "upgrade", "head"]
+    restart: "no"
 
   ui-main:
     image: "ghcr.io/hotosm/fmtm/frontend:debug"
@@ -96,7 +108,7 @@ services:
       - "8080:8080"
     networks:
       - fmtm-dev
-    restart: unless-stopped
+    restart: "unless-stopped"
 
   central-db:
     image: "postgis/postgis:14-3.3-alpine"
@@ -111,7 +123,7 @@ services:
       - "5434:5432"
     networks:
       - fmtm-dev
-    restart: unless-stopped
+    restart: "unless-stopped"
 
   central:
     image: "ghcr.io/hotosm/fmtm/odkcentral:v2023.2.1"
@@ -122,7 +134,6 @@ services:
     container_name: central_api
     depends_on:
       - central-db
-      - pyxform
     environment:
       - DOMAIN=local
       - SYSADMIN_EMAIL=${ODK_CENTRAL_USER}
@@ -154,7 +165,7 @@ services:
         "--",
         "./init-user-and-start.sh",
       ]
-    restart: unless-stopped
+    restart: "unless-stopped"
 
   central-proxy:
     image: "ghcr.io/hotosm/fmtm/odkcentral-proxy:latest"
@@ -165,10 +176,4 @@ services:
       - central
     networks:
       - fmtm-dev
-    restart: unless-stopped
-
-  pyxform:
-    image: "ghcr.io/getodk/pyxform-http:v1.10.1.1"
-    networks:
-      - fmtm-dev
-    restart: always
+    restart: "unless-stopped"

josm/docker-compose.yml

@@ -35,7 +35,7 @@ services:
       - x11
     ports:
       - 8111:80
-    restart: unless-stopped
+    restart: "unless-stopped"
 
   josm-novnc:
     image: "ghcr.io/hotosm/fmtm/josm-novnc:latest"
@@ -51,4 +51,4 @@ services:
       - "8112:8080"
     networks:
       - x11
-    restart: unless-stopped
+    restart: "unless-stopped"

src/backend/.dockerignore

@@ -8,3 +8,5 @@
 !container-entrypoint.sh
 !pyproject.toml
 !pdm.lock
+!migrations
+!alembic.ini

src/backend/Dockerfile

@@ -107,6 +107,8 @@ COPY --from=build \
 WORKDIR /opt
 # Add app code
 COPY app/ /opt/app/
+COPY migrations/ /opt/migrations/
+COPY alembic.ini /opt/
 # Add non-root user, permissions
 RUN useradd -r -u 1001 -m -c "hotosm account" -d /home/appuser -s /bin/false appuser \
     && mkdir -p /opt/logs /opt/tiles \
@@ -122,7 +124,6 @@ USER appuser
 
 
 FROM runtime as debug-no-odk
-USER appuser
 COPY --from=extract-deps --chown=appuser \
     /opt/python/requirements-debug.txt /opt/python/
 RUN pip install --user --upgrade --no-warn-script-location \
@@ -146,7 +147,7 @@ USER appuser
 
 
 
-FROM debug-no-odk as ci
+FROM runtime as ci
 # Run all ci as root
 USER root
 ENV PATH="/root/.local/bin:$PATH"

src/backend/alembic.ini

@@ -0,0 +1,113 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+script_location = migrations
+
+# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
+# Uncomment the line below if you want the files to be prepended with date and time
+# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file
+# for all available tokens
+# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s
+
+# sys.path path, will be prepended to sys.path if present.
+# defaults to the current working directory.
+prepend_sys_path = .
+
+# timezone to use when rendering the date within the migration file
+# as well as the filename.
+# If specified, requires the python-dateutil library that can be
+# installed by adding `alembic[tz]` to the pip requirements
+# string value is passed to dateutil.tz.gettz()
+# leave blank for localtime
+# timezone =
+
+# max length of characters to apply to the
+# "slug" field
+# truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# version location specification; This defaults
+# to migrations/versions.  When using multiple version
+# directories, initial revisions must be specified with --version-path.
+# The path separator used here should be the separator specified by "version_path_separator" below.
+# version_locations = %(here)s/bar:%(here)s/bat:migrations/versions
+
+# version path separator; As mentioned above, this is the character used to split
+# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep.
+# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas.
+# Valid values for version_path_separator are:
+#
+# version_path_separator = :
+# version_path_separator = ;
+# version_path_separator = space
+version_path_separator = os  # Use os.pathsep. Default configuration used for new projects.
+
+# set to 'true' to search source files recursively
+# in each "version_locations" directory
+# new in Alembic version 1.10
+# recursive_version_locations = false
+
+# the output encoding used when revision files
+# are written from script.py.mako
+# output_encoding = utf-8
+
+sqlalchemy.url = 
+
+[post_write_hooks]
+# post_write_hooks defines scripts or Python functions that are run
+# on newly generated revision scripts.  See the documentation for further
+# detail and examples
+
+# format using "black" - use the console_scripts runner, against the "black" entrypoint
+# hooks = black
+# black.type = console_scripts
+# black.entrypoint = black
+# black.options = -l 79 REVISION_SCRIPT_FILENAME
+
+# Custom param that enables us to specify tables to ignore when determining migrations
+[alembic:exclude]
+tables = spatial_ref_sys
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = "%(asctime)s.%(msecs)03d [%(levelname)s] %(name)s | %(funcName)s:%(lineno)d | %(message)s"
+datefmt = "%y-%m-%d %H:%M:%S"

src/backend/app/db/database.py

@@ -22,7 +22,7 @@
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.orm import sessionmaker
 
-from ..config import settings
+from app.config import settings
 
 engine = create_engine(settings.FMTM_DB_URL)
 SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)

src/backend/app/db/db_models.py

@@ -41,7 +41,7 @@
     relationship,
 )
 
-from ..models.enums import (
+from app.models.enums import (
     BackgroundTaskStatus,
     MappingLevel,
     MappingPermission,
@@ -55,6 +55,7 @@
     UserRole,
     ValidationPermission,
 )
+
 from .database import Base, FmtmMetadata
 from .postgis_utils import timestamp
 

src/backend/app/main.py

@@ -33,7 +33,7 @@
 from .auth import auth_routes
 from .central import central_routes
 from .config import settings
-from .db.database import Base, engine, get_db
+from .db.database import get_db
 from .organization import organization_routes
 from .projects import project_routes
 from .projects.project_crud import read_xlsforms
@@ -172,10 +172,7 @@ async def validation_exception_handler(request: Request, exc: RequestValidationE
 async def startup_event():
     """Commands to run on server startup."""
     log.debug("Starting up FastAPI server.")
-    log.debug("Connecting to DB with SQLAlchemy")
-    Base.metadata.create_all(bind=engine)
-
-    # Read in XLSForms
+    log.debug("Reading XLSForms from DB.")
     read_xlsforms(next(get_db()), xlsforms_path)