Skip to content

Commit

Permalink
Merge pull request #72 from unicef/develop
Browse files Browse the repository at this point in the history 
2.17 Release
  • Loading branch information
@reinbach
reinbach authored Jan 11, 2021
2 parents 0937822 + 99ffe6f commit 268b4f4
Show file tree
Hide file tree
Showing 238 changed files with 3,274 additions and 4,576 deletions.
272 changes: 88 additions & 184 deletions .circleci/config.yml
View file
Original file line number Diff line number Diff line change
@@ -1,223 +1,127 @@
version: 2

parameters:
base-tag:
type: string
default: "latest"

jobs:
build:
setup:
machine: true
steps:
- checkout
- run:
name: Make sure base is up to date and remote
command: |
docker login -u $DOCKER_USER -p $DOCKER_PASS
BASE_TAG="$(md5sum Pipfile.lock | cut -c1-6)$(md5sum Dockerfile-installed | cut -c1-6)"
TAG=${CIRCLE_BRANCH}
if (curl --silent -f -lSL https://index.docker.io/v1/repositories/unicef/datamart/tags/$BASE_TAG > /dev/null); then
echo "Up to date!"
else
docker build -t unicef/datamart:$BASE_TAG -f Dockerfile-installed .
docker push unicef/datamart:$BASE_TAG
if (echo "develop" | grep -q "$CIRCLE_BRANCH"); then
docker tag unicef/datamart:$BASE_TAG unicef/datamart:latest
docker push unicef/datamart:latest
elif (echo "master" | grep -q "$CIRCLE_BRANCH"); then
docker tag unicef/datamart:$BASE_TAG unicef/datamart:latest_prod
docker push unicef/datamart:latest_prod
else
echo "Not a followed branch not pushing latest"
fi
fi
- run:
name: Triggering test workflow
command: |
BASE_TAG="$(md5sum Pipfile.lock | cut -c1-6)$(md5sum Dockerfile-installed | cut -c1-6)"
curl --user ${CIRCLE_TOKEN}: \
--data build_parameters[CIRCLE_JOB]=test \
--data build_parameters[BASE_TAG]=$BASE_TAG \
--data revision=$CIRCLE_SHA1 \
https://circleci.com/api/v1.1/project/github/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME/tree/$CIRCLE_BRANCH
test:
working_directory: ~/code
docker:
- image: unicef/etools-base:latest
- image: unicef/datamart:$BASE_TAG
environment:
PGHOST: 127.0.0.1
PIPENV_VENV_IN_PROJECT: 1
DATABASE_URL: "postgis://postgres:postgres@127.0.0.1:5432/etools_datamart"
DATABASE_URL_ETOOLS: "postgis://postgres:postgres@127.0.0.1:5432/etools"
RELEASE_MATCH: "release/*"
CELERY_BROKER_URL: "amqp://guest:guest@127.0.0.1/datamart"
CELERY_RESULT_BACKEND: "amqp://guest:guest@127.0.0.1/datamart"

- image: redis:alpine
- image: rabbitmq:3.7-alpine
environment:
RABBITMQ_DEFAULT_VHOST: "datamart"
- image: mdillon/postgis:9.6
CELERY_BROKER_URL: "redis://localhost:6379/0"
CELERY_RESULT_BACKEND: "redis://localhost:6379/0"
DEPLOY_BRANCHES: "develop|staging|master"
- image: circleci/redis:6-alpine
- image: circleci/postgres:9.6-alpine-postgis
environment:
POSTGRES_USER: postgres
PGUSER: postgres
POSTGRES_DB: etools_datamart
POSTGRES_PASSWORD: postgres
steps:
- restore_cache:
keys:
- source-{{ .Branch }}-{{ .Revision }}
- checkout
- restore_cache:
keys:
- deps-v2-{{ checksum "poetry.lock" }}

- run:
name: install deps
command: |
pip install poetry
poetry install
key: source-{{ .Branch }}--{{ checksum "Pipfile.lock" }}-{{ checksum ".circleci/config.yml" }}
- run:
name: create db
name: run tests
command: |
export PATH=/home/circleci/.local/bin:$PATH
export PYTHONHASHSEED=${RANDOM}
poetry run pytest tests/ --create-db
- store_artifacts:
path: ~build/coverage
destination: coverage

pipenv install -d --system --ignore-pipfile
pytest tests/ --cov=etools_datamart --cov-config=tests/.coveragerc --cov-report=term --create-db
- save_cache:
key: source-{{ .Branch }}-{{ .Revision }}
key: source-{{ .Branch }}--{{ checksum "Pipfile.lock" }}-{{ checksum ".circleci/config.yml" }}
paths:
- ".git"

- save_cache:
key: deps-v2-{{ checksum "poetry.lock" }}
paths:
- ".tox"
- ".venv"
- "~/.cache/pip"

- deploy:
name: tag and release if release candidate
- store_artifacts:
path: build/coverage
destination: coverage
- run:
name: Triggering build_and_deploy job
command: |
if [[ $CIRCLE_BRANCH == $RELEASE_MATCH ]]; then
if (echo "$DEPLOY_BRANCHES" | grep -q "$CIRCLE_BRANCH"); then
curl --user ${CIRCLE_TOKEN}: \
--data build_parameters[CIRCLE_JOB]=tag \
--data build_parameters[CIRCLE_JOB]=build_and_upload \
--data revision=$CIRCLE_SHA1 \
https://circleci.com/api/v1.1/project/github/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME/tree/$CIRCLE_BRANCH
else
echo "Skipped as '$CIRCLE_BRANCH' does not match '$RELEASE_MATCH' branch"
echo "not triggered - '$CIRCLE_BRANCH' is not a deployable branch: '$DEPLOY_BRANCHES'"
fi
tag:
docker:
- image: circleci/python:3.6
environment:
API_URL: https://api.github.com/repos/unicef/etools-datamart
RELEASE_MATCH: "release/*"
steps:
- checkout
- add_ssh_keys:
fingerprints:
- "73:a6:f1:56:5a:28:c5:ea:50:ed:4d:c5:b3:00:54:9c"
- run:
name: close release branch and create tag
command: |
export PATH=/home/circleci/.local/bin:$PATH
export TAG=${TAG:=${CIRCLE_BRANCH#*/}}
git status
git show-branch "release/$TAG"
git reset --hard
git config user.email "ci@unicef.org"
git config user.name "CI"
echo
echo "Process master"
git checkout master
git pull
git merge --no-ff release/$TAG -m "merge release/$TAG"
echo
echo "Creating tag $TAG"
git tag -a -m "v$TAG" $TAG
echo
echo "Process develop"
git checkout develop
git pull
git merge --no-ff release/$TAG -m "merge release/$TAG"
echo
echo "Deleting branch release/$TAG"
git branch -d release/$TAG
echo
echo "Pushing everything to origin"
git push --verbose --all
git push --verbose --tags
git push origin --verbose --delete release/$TAG
- run:
name: create github release
command: |
export TAG=${TAG:=${CIRCLE_BRANCH#*/}}
echo
export TODAY=`date '+%d %B %Y at %H:%M'`
function is_pre()
{
[[ "$(echo "$TAG" | sed 's/[0-9.]//g')" == "" ]] && echo false || echo true
}
function data() {
cat <<EOF
{ "tag_name": "$TAG",
"name": "v$TAG",
"body": "version $TAG - Built on $TODAY",
"draft": false,
"prerelease": $(is_pre)
}
EOF
}
data=$(data)
curl -X POST -s \
--fail --verbose \
-H "Accept: application/json" \
-H "Content-Type:application/json" \
-H "Authorization: token ${GITHUB_TOKEN}" \
${API_URL}/releases \
-d "$data"
- deploy:
name: dockerize
command: |
export TAG=${TAG:=${CIRCLE_BRANCH#*/}}
curl --user ${CIRCLE_TOKEN}: \
--data build_parameters[TAG]=$TAG \
--data build_parameters[JOB]=dockerize \
https://circleci.com/api/v1.1/project/github/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME/tree/develop
dockerize:
working_directory: ~/code
docker:
- image: docker:17.05.0-ce-git
environment:
PGHOST: 127.0.0.1
DATABASE_URL: "postgres://postgres:postgres@127.0.0.1:5432/etools_datamart"
DATABASE_URL_ETOOLS: "postgres://postgres:postgres@127.0.0.1:5432/etools"
- image: circleci/postgres:9.6-alpine-postgis
environment:
POSTGRES_USER: postgres
PGUSER: postgres
POSTGRES_DB: test
POSTGRES_PASSWORD: postgres

environment:
DOCKER_IMAGE: unicef/datamart
build_and_upload:
machine: true
steps:
- checkout
- add_ssh_keys:
fingerprints:
- "70:aa:15:2a:d9:1c:74:f3:a1:f7:c4:cd:9b:d2:d1:f5"
- setup_remote_docker
- run:
name: Dump Environment
name: Building the image
command: |
env | sort
docker login -u $DOCKER_USER -p $DOCKER_PASS
BASE_TAG="$(md5sum Pipfile.lock | cut -c1-6)$(md5sum Dockerfile-installed | cut -c1-6)"
TAG=${CIRCLE_BRANCH}
(docker pull unicef/datamart:$BASE_TAG) ||
(docker build -t unicef/datamart:$BASE_TAG -f Dockerfile-installed . && docker push unicef/datamart:$BASE_TAG)
docker build --build-arg BASE_TAG=$BASE_TAG -t unicef/datamart:$TAG .
- run:
name: Build docker image
name: Pushing to Docker Hub
command: |
if [ -z "$TAG" ];then
echo "Tag not set. exit"
exit 1
TAG=${CIRCLE_BRANCH}
BASE_TAG="$(md5sum Pipfile.lock | cut -c1-6)$(md5sum Dockerfile-installed | cut -c1-6)"
docker login -u $DOCKER_USER -p $DOCKER_PASS
docker push unicef/datamart:$TAG
if (echo "develop" | grep -q "$CIRCLE_BRANCH"); then
docker tag unicef/datamart:$BASE_TAG unicef/datamart:latest
docker push unicef/datamart:latest
elif (echo "master" | grep -q "$CIRCLE_BRANCH"); then
docker tag unicef/datamart:$BASE_TAG unicef/datamart:latest_prod
docker push unicef/datamart:latest_prod
else
echo "Not a followed branch not pushing latest"
fi
docker build \
--build-arg GITHUB_TOKEN=${GITHUB_TOKEN} \
--build-arg VERSION=${TAG} \
-t ${DOCKER_IMAGE}:${TAG} \
-f docker/Dockerfile .
docker images "unicef/datamart*"
- run:
name: Test Backend docker image
command: |
echo "Testing ${DOCKER_IMAGE}:${TAG}"
docker run -p 8000:8000 \
--rm \
-e DATABASE_URL=${DATABASE_URL} \
-e DEBUG=0 \
-e SECURE_SSL_REDIRECT=0 \
-e SESSION_COOKIE_SECURE=0 \
-e SESSION_COOKIE_HTTPONLY=9 \
-e SESSION_COOKIE_HTTPONLY=0 \
-e CSRF_COOKIE_SECURE=0 \
-it ${DOCKER_IMAGE}:${TAG} \
django-admin check --deploy
- deploy:
name: Push Backend docker image
command: |
echo "Pushing ${DOCKER_IMAGE}:${TAG} to Docker Hub"
export TODAY=`date '+%d %B %Y at %H:%M'`
docker login -u $DOCKER_USER -p $DOCKER_PASS
docker tag ${DOCKER_IMAGE}:${TAG} ${DOCKER_IMAGE}:latest
docker push ${DOCKER_IMAGE}:latest
docker push ${DOCKER_IMAGE}:${TAG}
workflows:
version: 2
all:
jobs:
- setup
1 change: 1 addition & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,3 +43,4 @@ xml
**/dual-listbox-master/
#src/etools_datamart/apps/security/static/security/dual-listbox-master
build/
celerybeat.pid
6 changes: 3 additions & 3 deletions CONTRIBUTING.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,14 @@



setup local en0 alias
setup local en0 alias

$ sudo ifconfig en0 alias 192.168.66.66

### Initialize app

Create a `.env` file in the root folder based on the content for `.env.tpl`

$ pipenv sync
$ pipenv shell
$ ./manage.py init-setup --all
Expand Down
20 changes: 20 additions & 0 deletions Dockerfile
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
ARG BASE_TAG=installed
FROM unicef/datamart:$BASE_TAG

ADD src /code/
ADD manage.py /code/manage.py

WORKDIR /code/
RUN apk add bash

ENV PYTHONUNBUFFERED 1
ENV PYTHONPATH /code

# CLEANUP
# Disabled for now, till move to etools-base
# See https://github.com/unicef/etools/pull/2716
# RUN apk del .build-deps

RUN python manage.py collectstatic --noinput

EXPOSE 8000
8 changes: 8 additions & 0 deletions Dockerfile-installed
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
# TODO change this to etools-base, if etools Dockerfile-base changes accepted
# See https://github.com/unicef/etools/pull/2716
FROM unicef/etools-prp-base:latest

WORKDIR /code/
ADD Pipfile .
ADD Pipfile.lock .
RUN pipenv install --system --ignore-pipfile --deploy
14 changes: 4 additions & 10 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,17 +17,14 @@ help:


develop:
@poetry run
poetry run pre-commit install
poetry run pre-commit install --hook-type pre-push.
pipenv sync --dev

test:
poetry run py.test -v --create-db
pipenv run py.test -v --create-db

lint:
poetry run pre-commit run --all-files
poetry run pre-commit run --all-files --hook-stage push
poetry run pre-commit run --all-files --hook-stage manual
pipenv run flake8 src/ tests/; exit 0;
pipenv run isort . --check-only; exit 0;

clean:
rm -fr ${BUILDDIR} dist *.egg-info .coverage coverage.xml .eggs
Expand All @@ -46,9 +43,6 @@ docs:
.mkbuilddir
mkdir -p ${BUILDDIR}/docs
sphinx-build -aE docs/ ${BUILDDIR}/docs
ifdef BROWSE
firefox ${BUILDDIR}/docs/index.html
endif

run:
docker-compose up
Loading

0 comments on commit 268b4f4

Please sign in to comment.