Ft/deployment setup

.github/workflows/docker-release.yml

@@ -4,20 +4,19 @@ on:
   push:
     tags:
       - '**'
+env:
+    IMAGE_NAME: "mcsystems/news-search-api"
 
 jobs:
 
   build:
-
     runs-on: ubuntu-latest
-
     steps:
-
       - name: Checkout Version
         uses: actions/checkout@v4
 
       - name: Dump tag name
-        run: echo "Building tag ${{github.ref_name}}"
+        run: echo "Building ${{ github.ref_type }} ${{ github.ref_name }}"
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
@@ -31,10 +30,22 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: ${{ env.IMAGE_NAME }}
+          tags: |
+            type=pep440,pattern={{raw}}
+
       - name: Build and push Docker image
         uses: docker/build-push-action@v5
         with:
           context: .
           platforms: linux/amd64,linux/arm64
           push: true
-          tags: mcsystems/news-search-api:${{github.ref_name}}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          # registry defaults to dockerhub
+          cache-from: type=registry,ref=${{ env.IMAGE_NAME }}:buildcache
+          cache-to: type=registry,ref=${{ env.IMAGE_NAME }}:buildcache,mode=max

Makefile

@@ -0,0 +1,21 @@
+.PHONY: down deploy shell up
+
+#setup image tag to latest for dev
+IMAGE_TAG?=latest
+DEPLOY_TYPE?=dev
+
+# stop containers
+down:
+	docker compose down
+
+# run the deploy script, passing in deployment type
+deploy:
+	./deploy.sh -d $(DEPLOY_TYPE)
+
+# exec into api container
+shell:
+	docker compose exec api sh
+
+# (local) development
+up:
+	IMAGE_TAG=$(IMAGE_TAG) docker compose up --build -d

README.md

@@ -50,7 +50,7 @@ commas or spaces as separators. Configuration via a config file in the syntax of
 Then run the API and UI services using Docker Compose:
 
 ```
-$ docker compose up
+$ make up
 ```
 
 Access an interactive API documentation and a collection index explorer in a web browser:
@@ -71,6 +71,14 @@ Deployments are now configured to be automatically built and released via GitHub
 ## Version History
 * __v1.3.1__ - Bugfix for 1.3.0
 * __v1.3.0__ - Change to return aliases as well as indexes as legal values in Collections, and update article endpoint to work in the ILM context
-* __v1.2.0__ - Change related to ID update in backend ES, including refurbishing the article endpoint and tests 
+* __v1.2.0__ - Change related to ID update in backend ES, including refurbishing the article endpoint and tests
 * __v1.1.0__ - Change to return `None` when data is missing (including publication date), update dependencies
 * __v1.0.0__ - First official release
+
+### Tags for dev and release
+
+Append the suffix `a` for a dev/alpha release and `b` for a staging/beta release.
+e.g
+
+* __v1.3.2b__ - Version 1.3.2 beta release
+* __v1.3.2a__ - Version 1.3.2 alpha release

deploy.sh

@@ -0,0 +1,155 @@
+#!/bin/bash
+# deploy.sh - Deployment script for News Search API and UI
+
+# Environment variables
+APP_NAME="news-search-api"
+API_PORT_BASE=8000
+UI_PORT_BASE=8501
+
+# Check if running as root
+if [ $(whoami) != "root" ]; then
+    echo "This script must be run as root."
+    exit 1
+fi
+
+echo "Running as root"
+
+# Check if running on a checked-out tag
+if git describe --exact-match --tags HEAD >/dev/null 2>&1; then
+    echo "Running on a checked-out tag: $(git describe --tags --abbrev=0)"
+else
+    echo "This script must be run on a checked-out tag."
+    exit 1
+fi
+
+LOGIN_USER=$(who am i | awk '{ print $1 }')
+if [ "x$LOGIN_USER" = x ]; then
+    # XXX fall back to whoami (look by uid)
+    echo could not find login user 1>&2
+    exit 1
+fi
+
+run_as_login_user() {
+	su $LOGIN_USER -c "$*"
+}
+
+help()
+{
+    echo "Usage: ./deploy.sh [options]"
+    echo "Options:"
+    echo "-h, --help    Show this help message"
+    echo "-d, --deployment-type  Specify the deployment type (dev, staging or production)"
+    echo ""
+    echo "This script deploys the News Search API and UI. It must be run on a checked-out git tag."
+    echo "The script will use the checked-out git tag as the image tag for deployment."
+    echo "If the script is not run on a checked-out git tag, it will exit with an error message."
+}
+
+
+log()
+{
+    echo "$1"
+}
+
+zzz() {
+    echo $1 | tr 'A-Za-z' 'N-ZA-Mn-za-m'
+}
+
+# Parse command-line options
+while (( "$#" )); do
+    case "$1" in
+        -h|--help)
+            help
+            exit 0
+            ;;
+        -d|--deployment-type)
+            shift
+            DEPLOYMENT_TYPE="$1"
+            ;;
+        -*|--*=) # unsupported flags
+            echo "Error: Unsupported flag $1" >&2
+            help
+            exit 1
+            ;;
+    esac
+done
+
+case "$DEPLOYMENT_TYPE" in
+    dev)
+        API_PORT=$(expr $API_PORT_BASE + 100)
+        UI_PORT=$(expr $UI_PORT_BASE + 100)
+        PROJECT_NAME="$LOGIN_USER-dev"
+        ENV_FILE="dev"
+        ;;
+    staging)
+        API_PORT=$(expr $API_PORT_BASE + 200)
+        UI_PORT=$(expr $UI_PORT_BASE + 200)
+        PROJECT_NAME="staging"
+        ENV_FILE="staging"
+        ;;
+    production)
+        API_PORT=$API_PORT_BASE
+        UI_PORT=$UI_PORT_BASE
+        PROJECT_NAME="prod"
+        ENV_FILE="prod"
+        ;;
+    *)
+        echo "Error: Invalid deployment type. Specify either 'dev', 'staging' or 'prod'."
+        exit 1
+        ;;
+esac
+
+IMAGE_TAG=$(git describe --tags --abbrev=0)
+
+PRIVATE_CONF_DIR="news-search-private-conf"
+rm -rf $PRIVATE_CONF_DIR
+run_as_login_user mkdir -p $PRIVATE_CONF_DIR
+chmod go-rwx $PRIVATE_CONF_DIR
+
+cd $PRIVATE_CONF_DIR
+CONFIG_REPO_PREFIX=$(zzz tvg@tvguho.pbz:zrqvnpybhq)
+CONFIG_REPO_NAME=$(zzz arjf-frnepu-ncv-pbasvt)
+PRIVATE_CONF_REPO=$(pwd)/$CONFIG_REPO_NAME
+
+echo cloning $CONFIG_REPO_NAME repo 1>&2
+if ! run_as_login_user "git clone $CONFIG_REPO_PREFIX/$CONFIG_REPO_NAME.git" >/dev/null 2>&1; then
+    echo "FATAL: could not clone config repo" 1>&2
+    exit 1
+fi
+
+PRIVATE_CONF_FILE=$PRIVATE_CONF_REPO/$APP_NAME.$ENV_FILE.sh
+cd ..
+
+if [ ! -f $PRIVATE_CONF_FILE ]; then
+    echo "FATAL: could not access $PRIVATE_CONF_FILE" 1>&2
+    exit 1
+fi
+#source private conf to load SENTRY_DSN
+. $PRIVATE_CONF_FILE
+
+GH_REPO_PREFIX=$(zzz uggcf://tvguho.pbz/zrqvnpybhq)
+GH_REPO_NAME=$(zzz arjf-frnepu-ncv)
+DOCKER_COMPOSE_FILE="docker-compose.yml"
+mv -f docker-compose.yml docker-compose-old.yml
+echo "Fetching $DOCKER_COMPOSE_FILE from $GH_REPO_NAME repo..."
+if ! curl -sSfL "$GH_REPO_PREFIX/$GH_REPO_NAME/raw/$IMAGE_TAG/$DOCKER_COMPOSE_FILE" -o "$(pwd)/$DOCKER_COMPOSE_FILE"; then
+    echo "FATAL: Could not fetch $DOCKER_COMPOSE_FILE from config repo"
+    exit 1
+fi
+
+export INDEXES="mc_search"
+export ESOPTS='{"timeout": 60, "max_retries": 3}' # 'timeout' parameter is deprecated
+export ELASTICSEARCH_INDEX_NAME_PREFIX="mc_search-*"
+export TERMFIELDS="article_title,text_content"
+export TERMAGGRS="top,significant,rare"
+export ESHOSTS=${ESHOSTS}
+export SENTRY_DSN=${SENTRY_DSN}
+export API_PORT=${API_PORT}
+export UI_PORT=${UI_PORT}
+export IMAGE_TAG
+
+# Deploy services using Docker Compose
+echo "Deploying services with image, project name: $PROJECT_NAME &  tag: $IMAGE_TAG"
+docker compose -f "$(pwd)/$DOCKER_COMPOSE_FILE" -p "$PROJECT_NAME" up -d
+
+echo "Deployment completed successfully!"

docker-compose.yml

@@ -1,30 +1,26 @@
 services:
   api:
-    image: mcsystems/news-search-api
-    build: .
+    image: mcsystems/news-search-api:${IMAGE_TAG}
+    environment:
+      INDEXES: ${INDEXES}
+      ESHOSTS: ${ESHOSTS}
+      ESOPTS: ${ESOPTS}
+      ELASTICSEARCH_INDEX_NAME_PREFIX: ${ELASTICSEARCH_INDEX_NAME_PREFIX}
+      TERMFIELDS: ${TERMFIELDS}
+      TERMAGGRS: ${TERMAGGRS}
     ports:
-      - 8000:8000
-    networks:
-      - story-indexer
+      - ${API_PORT}:8000
     volumes:
       - .:/app
 
   ui:
-    image: mcsystems/news-search-api
-    build: .
+    image: mcsystems/news-search-api:${IMAGE_TAG}
     environment:
       APIURL: http://api:8000/v1
     ports:
-      - 8001:8501
-    networks:
-      - story-indexer
+      - ${UI_PORT}:8501
     volumes:
       - .:/app
     depends_on:
       - api
     command: streamlit run ui.py
-
-networks:
-  story-indexer:
-    name: story-indexer
-    external: true

docs/deployment.md

@@ -1,25 +1,57 @@
-### Prerequisites
+## Prerequisites
 
 Before you begin, ensure you have the following prerequisites in place:
 
 1. Docker: Docker must be installed on the host where you plan to set up the Swarm. You can download and install Docker from Docker's [official website](https://docs.docker.com/engine/install/ubuntu/#install-from-a-package).
 
 2. Docker Compose: Make sure you have Docker Compose installed, as it's essential for managing multi-container applications. You can install Docker Compose by following the instructions in the official [documentation](https://docs.docker.com/compose/install/).
 
-### Network Setup
+## Network Setup
 
-The Web Search API should connect to the Elasticsearch cluster running from the `indexer`.
-The `indexer` runs as a swarm cluster and therefore to expose the ES endpoint to the API, we need to deploy the Web Search API in the same overlay network
+### Local Development
+The New Search API expects to connect to an Elasticsearch cluster. When running this with an external cluster, we need to define the `news-search-api` and the Elasticsearch cluster to run from the same docker network.
+
+If relying on a shared network for the Search API and the `story-indexer`, ensure the `docker-compose.yml` is attached to th overlay network created above.
 
 To create the network (if non exists)
 
     `docker network create -d overlay --attachable story-indexer`
 
+### Dev, Staging & production
+
+A staging/production deployment expects to connect to an external Elasticsearch cluster, with the Elasticsearch urls provides as the env variable `ESHOSTS`
+
+
+## Deployment
+
+### Local Development
+
+Deployment in the dev environment can be done using the `Makefile` command `make up`.
+For development purposes all our images are build and tagged with the `latest` tags, as initialized in the Makefile.
+
+## Dev, Staging & Production
+
+All releases are built and pushed to Dockerhub using the github workflow [here](../.github/workflows/docker-release.yml). We pull the pre-built images when deploying, based on the checked ou deploy tag
+
+When deploying a staging or production environment, use the `deploy.sh` script available [here](./deploy.sh).
+The `deploy.sh` script expects us to deploy from a checked out tag. The docker images for each of the stages are tagged with the git tag, the tagging should conform to [PEP 440](https://peps.python.org/pep-0440/). Further instructions on release and tagging available [here](../README.md)
+
+The `deploy.sh` script expects a deploy option -d (that specifies the deployment stage (dev, staging/production))
+
+e.g
+
+```
+Staging deployment
+sudo ./deploy.sh -d staging
+
+Production
+sudo ./deploy.sh -d prod
+```
 
-### Docker compose
+A `dev` deployment creates a docker compose project prefixed with the logged_in user, as
 
-If relying on a shared network for the Search API and the `indexer`, ensure the `docker-compose.yml` is attached to th overlay network created above.
+``` sudo docker compose -p $"LOGIN-USER-dev" up -d```
 
-To build & run the services
+To cleanup a dev deployment, run the command
 
-    `docker compose up --build`
+``` sudo docker compose down -p $"LOGIN-USER-dev" ```