deploy(ci): add gitlab ci and deploy script

.gitignore

@@ -1 +1,2 @@
-.idea/
+.idea/
+.env

.gitlab-ci.yml

@@ -1,8 +1,22 @@
 image: tiangolo/docker-with-compose
 
+variables:
+  DOCKER_HOST: tcp://docker:2375
+  DOCKER_TLS_CERTDIR: ""
+  DOCKER_OPTS: "--insecure-registry $CI_REGISTRY"
+
+services:
+   - name: docker:dind
+     alias: docker
+     # Change insecure-registry to your registry
+     command: ['--insecure-registry=10.0.1.185:5000']
+
 before_script:
+  - docker info
+  - echo $CI_JOB_TOKEN
+  - echo $CI_REGISTRY
   - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
-  - pip install docker-auto-labels
+  - docker swarm init
 
 stages:
   - build
@@ -11,72 +25,51 @@ stages:
 build-dev:
   stage: build
   script:
-    - cat $ENV_FILE_DEV >> ./docker/.env
-    - TAG=dev sh ./docker/scripts/build-push.sh
+    - cat $ENV_FILE_DEV >> .env
+    - DOCKER_IMAGE_BACKEND=$DOCKER_IMAGE_BACKEND TAG=dev sh ./docker/scripts/build-push.sh
   only:
     - develop
   tags:
     - build
 
-build-stag:
-  stage: build
-  script:
-    - cat $ENV_FILE_STAG >> ./docker/.env
-    - TAG=stag sh ./docker/scripts/build-push.sh
-  only:
-    - master
-  tags:
-    - build
-
-build-prod:
-  stage: build
-  script:
-    - cat $ENV_FILE_PROD >> ./docker/.env
-    - TAG=prod sh ./docker/scripts/build-push.sh
-  only:
-    - production
-  tags:
-    - build
-
-deploy-dev:
+deploy-dev-server:
   stage: deploy
-  script:
-    - cat $ENV_FILE_DEV >> ./docker/.env
-    - >
-      TAG=dev
-      sh ./docker/scripts/deploy.sh
-  environment:
-    name: development
   only:
     - develop
-  tags:
-    - swarm
-    - dev
-
-deploy-stag:
-  stage: deploy
+  before_script:
+    - apk update && apk add openssh-client bash
   script:
-    - >
-      TAG=stag
-      sh ./docker/scripts/deploy.sh
-  environment:
-    name: staging
-  only:
-    - master
-  tags:
-    - swarm
-    - stag
+    - cat $ENV_FILE_DEV >> .env
+    - cat .env
+    - eval $(ssh-agent -s)
+
+    - bash -c 'ssh-add <(echo "$SSH_PRIVATE_KEY")'
+
+    - mkdir -p ~/.ssh
+
+    - ssh-keyscan -H $SSH_SERVER >> ~/.ssh/known_hosts
+
+    - chmod 644 ~/.ssh/known_hosts
+
+    - scp -r * $SSH_USER@$SSH_SERVER:~/temp/
+    - scp .env $SSH_USER@$SSH_SERVER:~/temp/.env
 
-deploy-prod:
-  stage: deploy
-  script:
     - >
-      TAG=prod
-      sh ./scripts/deploy.sh
-  environment:
-    name: production
-  only:
-    - production
+      ssh $SSH_USER@$SSH_SERVER
+      "docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY;
+      cd ~;
+      mkdir -p $CI_PROJECT_NAME;
+      cd $CI_PROJECT_NAME;
+      mkdir -p app_repo;
+      cd app_repo;
+      cp ~/temp/.env ./.env;
+      cp -r ~/temp/* . && rm -rf ~/temp/*;
+      docker-compose down;
+      docker pull ${DOCKER_IMAGE_BACKEND};
+      docker-compose up -d;
+      docker image prune -f;
+      exit;"
   tags:
     - swarm
-    - prod
+    - dev
+    - deploy

docker/docker-compose.yml → docker-compose.yml

@@ -14,7 +14,7 @@ services:
       - 5432:5432
     restart: always
     healthcheck:
-        test: ['pg_isready', '-U', '${POSTGRES_USER}']
+        test: ['CMD-SHELL', 'pg_isready', '-U', '${POSTGRES_USER}']
         interval: 10s
         timeout: 5s
         retries: 5
@@ -76,26 +76,48 @@ services:
       - SERVER_NAME=${DOMAIN?Variable not set}
       - SERVER_HOST=https://${DOMAIN?Variable not set}
     build:
-      context: ../
-      dockerfile: /docker/celeryworker.dockerfile
+      context: ./
+      dockerfile: ./docker/celeryworker.dockerfile
       args:
         INSTALL_DEV: ${INSTALL_DEV-false}
 
+  celeryflower:
+    image: ${DOCKER_IMAGE_FLOWER}:${TAG:-latest}
+    depends_on:
+      - db
+      - rabbitmq
+      - redis
+    ports:
+      - 5555:5555
+    env_file:
+      - .env
+    environment:
+      - SERVER_NAME=${DOMAIN?Variable not set}
+      - SERVER_HOST=https://${DOMAIN?Variable not set}
+    build:
+      context: ./
+      dockerfile: ./docker/celeryworker.dockerfile
+      args:
+        INSTALL_DEV: ${INSTALL_DEV-false}
+    command: ['celery', '-A', 'celeryApp.worker', 'flower', '-l', 'info']
+
   backend:
-    image: 10.0.1.185:5000/jfrog/jfc-mes-ai:latest
+    image: ${DOCKER_IMAGE_BACKEND}:${TAG:-latest}
     depends_on:
       - db
+      - rabbitmq
+      - redis
       - celeryworker
     env_file:
       - .env
     environment:
       - SERVER_NAME=${DOMAIN?Variable not set}
       - SERVER_HOST=https://${DOMAIN?Variable not set}
     ports:
-      - 8000:8000
+      - 8100:8100
     build:
-      context: ../
-      dockerfile: /docker/backend.dockerfile
+      context: ./
+      dockerfile: ./docker/backend.dockerfile
       args:
         INSTALL_DEV: ${INSTALL_DEV-false}
 

docker/backend.dockerfile

@@ -22,7 +22,7 @@ ARG INSTALL_JUPYTER=false
 RUN bash -c "if [ $INSTALL_JUPYTER == 'true' ] ; then pip install jupyterlab ; fi"
 
 COPY /src/ /
-COPY /docker/.env /.env
+COPY /.env /.env
 
 ENV PYTHONPATH=/
 

docker/celeryworker.dockerfile

@@ -24,7 +24,7 @@ RUN bash -c "if [ $INSTALL_JUPYTER == 'true' ] ; then pip install jupyterlab ; f
 ENV C_FORCE_ROOT=1
 
 COPY /src/ /
-COPY /docker/.env /.env
+COPY /.env /.env
 WORKDIR /
 
 ENV PYTHONPATH=/

docker/scripts/build-push.sh

@@ -3,7 +3,13 @@
 # Exit in case of error
 set -e
 
+[ ! -f ./.env ] || export $(grep -v '^#' ./.env | xargs)
+
 TAG=${TAG?Variable not set} \
 sh ./docker/scripts/build.sh
 
-docker-compose -f ./docker/docker-compose.yml push
+#docker-compose -f ./docker-compose.yml push
+echo "test scripts"
+echo $TAG
+
+docker-compose push backend celeryworker celeryflower

docker/scripts/build.sh

@@ -5,5 +5,5 @@ set -e
 
 TAG=${TAG?Variable not set} \
 docker-compose \
--f ./docker/docker-compose.yml \
+-f ./docker-compose.yml \
 build

docker/scripts/deploy.sh

@@ -4,10 +4,7 @@
 set -e
 
 TAG=${TAG?Variable not set} \
-docker-compose \
--f ./docker/docker-compose.yml \
-config > docker-stack.yml
 
-docker-auto-labels docker-stack.yml
+(echo -e "version: '3.9'\n";  docker compose -f docker-compose.yml config) > docker-stack.yml
 
-docker stack deploy -c docker-stack.yml --with-registry-auth "${STACK_NAME?Variable not set}"
+docker stack deploy -c docker-stack.yml --with-registry-auth "${STACK_NAME?Variable not set}"

src/backend-start.sh

@@ -1,6 +1,8 @@
 #! /usr/bin/env bash
 set -e
 
+bash /prestart.sh
+
 python /app/backend_pre_start.py
 
 python /app/main.py

src/prestart.sh

@@ -4,7 +4,9 @@
 python /app/backend_pre_start.py
 
 # Run migrations
+echo "Running migrations"
 alembic upgrade head
+echo "Migrations finished"
 
 # Create initial data in DB
 python /app/initial_data.py

src/worker-start.sh

@@ -3,4 +3,4 @@ set -e
 
 python /app/celeryworker_pre_start.py
 
-celery -A celeryApp.worker worker -l info -Q main-queue -c 1
+celery -A celeryApp.worker worker -l info -Q main-queue -c 1 -E