diff --git a/docker/.env.example b/.env.example similarity index 100% rename from docker/.env.example rename to .env.example diff --git a/.gitignore b/.gitignore index 62c8935..72bfe1d 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ -.idea/ \ No newline at end of file +.idea/ +.env \ No newline at end of file diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d5d3123..bad9062 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,8 +1,22 @@ image: tiangolo/docker-with-compose +variables: + DOCKER_HOST: tcp://docker:2375 + DOCKER_TLS_CERTDIR: "" + DOCKER_OPTS: "--insecure-registry $CI_REGISTRY" + +services: + - name: docker:dind + alias: docker + # Change insecure-registry to your registry + command: ['--insecure-registry=10.0.1.185:5000'] + before_script: + - docker info + - echo $CI_JOB_TOKEN + - echo $CI_REGISTRY - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY - - pip install docker-auto-labels + - docker swarm init stages: - build @@ -11,72 +25,51 @@ stages: build-dev: stage: build script: - - cat $ENV_FILE_DEV >> ./docker/.env - - TAG=dev sh ./docker/scripts/build-push.sh + - cat $ENV_FILE_DEV >> .env + - DOCKER_IMAGE_BACKEND=$DOCKER_IMAGE_BACKEND TAG=dev sh ./docker/scripts/build-push.sh only: - develop tags: - build -build-stag: - stage: build - script: - - cat $ENV_FILE_STAG >> ./docker/.env - - TAG=stag sh ./docker/scripts/build-push.sh - only: - - master - tags: - - build - -build-prod: - stage: build - script: - - cat $ENV_FILE_PROD >> ./docker/.env - - TAG=prod sh ./docker/scripts/build-push.sh - only: - - production - tags: - - build - -deploy-dev: +deploy-dev-server: stage: deploy - script: - - cat $ENV_FILE_DEV >> ./docker/.env - - > - TAG=dev - sh ./docker/scripts/deploy.sh - environment: - name: development only: - develop - tags: - - swarm - - dev - -deploy-stag: - stage: deploy + before_script: + - apk update && apk add openssh-client bash script: - - > - TAG=stag - sh ./docker/scripts/deploy.sh - environment: - name: staging - only: - - master - tags: - - swarm - - stag + - cat $ENV_FILE_DEV >> .env + - cat .env + - eval $(ssh-agent -s) + + - bash -c 'ssh-add <(echo "$SSH_PRIVATE_KEY")' + + - mkdir -p ~/.ssh + + - ssh-keyscan -H $SSH_SERVER >> ~/.ssh/known_hosts + + - chmod 644 ~/.ssh/known_hosts + + - scp -r * $SSH_USER@$SSH_SERVER:~/temp/ + - scp .env $SSH_USER@$SSH_SERVER:~/temp/.env -deploy-prod: - stage: deploy - script: - > - TAG=prod - sh ./scripts/deploy.sh - environment: - name: production - only: - - production + ssh $SSH_USER@$SSH_SERVER + "docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY; + cd ~; + mkdir -p $CI_PROJECT_NAME; + cd $CI_PROJECT_NAME; + mkdir -p app_repo; + cd app_repo; + cp ~/temp/.env ./.env; + cp -r ~/temp/* . && rm -rf ~/temp/*; + docker-compose down; + docker pull ${DOCKER_IMAGE_BACKEND}; + docker-compose up -d; + docker image prune -f; + exit;" tags: - swarm - - prod + - dev + - deploy \ No newline at end of file diff --git a/docker/docker-compose.yml b/docker-compose.yml similarity index 71% rename from docker/docker-compose.yml rename to docker-compose.yml index 2a8949a..0767690 100644 --- a/docker/docker-compose.yml +++ b/docker-compose.yml @@ -14,7 +14,7 @@ services: - 5432:5432 restart: always healthcheck: - test: ['pg_isready', '-U', '${POSTGRES_USER}'] + test: ['CMD-SHELL', 'pg_isready', '-U', '${POSTGRES_USER}'] interval: 10s timeout: 5s retries: 5 @@ -76,15 +76,37 @@ services: - SERVER_NAME=${DOMAIN?Variable not set} - SERVER_HOST=https://${DOMAIN?Variable not set} build: - context: ../ - dockerfile: /docker/celeryworker.dockerfile + context: ./ + dockerfile: ./docker/celeryworker.dockerfile args: INSTALL_DEV: ${INSTALL_DEV-false} + celeryflower: + image: ${DOCKER_IMAGE_FLOWER}:${TAG:-latest} + depends_on: + - db + - rabbitmq + - redis + ports: + - 5555:5555 + env_file: + - .env + environment: + - SERVER_NAME=${DOMAIN?Variable not set} + - SERVER_HOST=https://${DOMAIN?Variable not set} + build: + context: ./ + dockerfile: ./docker/celeryworker.dockerfile + args: + INSTALL_DEV: ${INSTALL_DEV-false} + command: ['celery', '-A', 'celeryApp.worker', 'flower', '-l', 'info'] + backend: - image: 10.0.1.185:5000/jfrog/jfc-mes-ai:latest + image: ${DOCKER_IMAGE_BACKEND}:${TAG:-latest} depends_on: - db + - rabbitmq + - redis - celeryworker env_file: - .env @@ -92,10 +114,10 @@ services: - SERVER_NAME=${DOMAIN?Variable not set} - SERVER_HOST=https://${DOMAIN?Variable not set} ports: - - 8000:8000 + - 8100:8100 build: - context: ../ - dockerfile: /docker/backend.dockerfile + context: ./ + dockerfile: ./docker/backend.dockerfile args: INSTALL_DEV: ${INSTALL_DEV-false} diff --git a/docker/backend.dockerfile b/docker/backend.dockerfile index ba478e1..5a7a433 100644 --- a/docker/backend.dockerfile +++ b/docker/backend.dockerfile @@ -22,7 +22,7 @@ ARG INSTALL_JUPYTER=false RUN bash -c "if [ $INSTALL_JUPYTER == 'true' ] ; then pip install jupyterlab ; fi" COPY /src/ / -COPY /docker/.env /.env +COPY /.env /.env ENV PYTHONPATH=/ diff --git a/docker/celeryworker.dockerfile b/docker/celeryworker.dockerfile index 09812ef..7d8276d 100644 --- a/docker/celeryworker.dockerfile +++ b/docker/celeryworker.dockerfile @@ -24,7 +24,7 @@ RUN bash -c "if [ $INSTALL_JUPYTER == 'true' ] ; then pip install jupyterlab ; f ENV C_FORCE_ROOT=1 COPY /src/ / -COPY /docker/.env /.env +COPY /.env /.env WORKDIR / ENV PYTHONPATH=/ diff --git a/docker/scripts/build-push.sh b/docker/scripts/build-push.sh index fa3fa22..7b5efa9 100644 --- a/docker/scripts/build-push.sh +++ b/docker/scripts/build-push.sh @@ -3,7 +3,13 @@ # Exit in case of error set -e +[ ! -f ./.env ] || export $(grep -v '^#' ./.env | xargs) + TAG=${TAG?Variable not set} \ sh ./docker/scripts/build.sh -docker-compose -f ./docker/docker-compose.yml push +#docker-compose -f ./docker-compose.yml push +echo "test scripts" +echo $TAG + +docker-compose push backend celeryworker celeryflower diff --git a/docker/scripts/build.sh b/docker/scripts/build.sh index 1d01db6..5bf2847 100644 --- a/docker/scripts/build.sh +++ b/docker/scripts/build.sh @@ -5,5 +5,5 @@ set -e TAG=${TAG?Variable not set} \ docker-compose \ --f ./docker/docker-compose.yml \ +-f ./docker-compose.yml \ build diff --git a/docker/scripts/deploy.sh b/docker/scripts/deploy.sh index 7d2719f..4b0ba44 100644 --- a/docker/scripts/deploy.sh +++ b/docker/scripts/deploy.sh @@ -4,10 +4,7 @@ set -e TAG=${TAG?Variable not set} \ -docker-compose \ --f ./docker/docker-compose.yml \ -config > docker-stack.yml -docker-auto-labels docker-stack.yml +(echo -e "version: '3.9'\n"; docker compose -f docker-compose.yml config) > docker-stack.yml -docker stack deploy -c docker-stack.yml --with-registry-auth "${STACK_NAME?Variable not set}" +docker stack deploy -c docker-stack.yml --with-registry-auth "${STACK_NAME?Variable not set}" \ No newline at end of file diff --git a/src/backend-start.sh b/src/backend-start.sh index 6e5585b..8e0b171 100644 --- a/src/backend-start.sh +++ b/src/backend-start.sh @@ -1,6 +1,8 @@ #! /usr/bin/env bash set -e +bash /prestart.sh + python /app/backend_pre_start.py python /app/main.py diff --git a/src/prestart.sh b/src/prestart.sh index 57f185b..228fd44 100644 --- a/src/prestart.sh +++ b/src/prestart.sh @@ -4,7 +4,9 @@ python /app/backend_pre_start.py # Run migrations +echo "Running migrations" alembic upgrade head +echo "Migrations finished" # Create initial data in DB python /app/initial_data.py diff --git a/src/worker-start.sh b/src/worker-start.sh index 9f65371..589681c 100644 --- a/src/worker-start.sh +++ b/src/worker-start.sh @@ -3,4 +3,4 @@ set -e python /app/celeryworker_pre_start.py -celery -A celeryApp.worker worker -l info -Q main-queue -c 1 +celery -A celeryApp.worker worker -l info -Q main-queue -c 1 -E