Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: add Docker compose, use in Makefile, try in CI #1315

Open
wants to merge 1 commit into
base: dev
Choose a base branch
from
Open

feat: add Docker compose, use in Makefile, try in CI #1315

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 9 additions & 36 deletions .github/workflows/ci.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ jobs:
build:
runs-on: ubuntu-latest
steps:
# This imitates the dev/docker/local/install.sh script.
- run: mkdir -p repos/undefx
- name: Checkout undefx/py3tester
uses: actions/checkout@v2
Expand All @@ -18,7 +19,6 @@ jobs:
path: repos/undefx/undef-analysis

- run: mkdir -p repos/delphi

- name: Checkoutcmu-delphi/operations
uses: actions/checkout@v2
with:
Expand Down Expand Up @@ -49,47 +49,20 @@ jobs:
repository: cmu-delphi/nowcast
path: repos/delphi/nowcast

- name: Build docker images
run: |
docker build -t delphi_database_epidata -f ./repos/delphi/delphi-epidata/dev/docker/database/epidata/Dockerfile .
docker build -t delphi_web_python -f repos/delphi/delphi-epidata/dev/docker/python/Dockerfile .
sudo docker build -t delphi_redis -f repos/delphi/delphi-epidata/dev/docker/redis/Dockerfile .
cd ./repos/delphi/delphi-epidata
docker build -t delphi_web_epidata -f ./devops/Dockerfile .
cd ../../../

# MODULE_NAME specifies the location of the `app` variable, the actual WSGI application object to run.
# see https://github.com/tiangolo/meinheld-gunicorn-docker#module_name
- name: Start database and Redis services
run: |
docker network create --driver bridge delphi-net
docker run --rm -d -p 13306:3306 --network delphi-net --name delphi_database_epidata --cap-add=sys_nice delphi_database_epidata
docker run --rm -d -p 6379:6379 --network delphi-net --env "REDIS_PASSWORD=1234" --name delphi_redis delphi_redis


- run: |
wget https://raw.githubusercontent.com/eficode/wait-for/master/wait-for
chmod +x wait-for
./wait-for localhost:13306 -- echo 'ready'
sleep 10s

- name: Start delphi_web_epidata
run: |
docker run --rm -d -p 10080:80 --env "MODULE_NAME=delphi.epidata.server.main" --env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" --env "FLASK_SECRET=abc" --env "FLASK_PREFIX=/epidata" --env "REDIS_HOST=delphi_redis" --env "REDIS_PASSWORD=1234" --env "API_KEY_REGISTER_WEBHOOK_TOKEN=abc" --env "API_KEY_ADMIN_PASSWORD=test_admin_password" --env "TESTING_MODE=True" --network delphi-net --name delphi_web_epidata delphi_web_epidata
docker ps

- name: Run Unit Tests
run: |
docker run --rm --network delphi-net --env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" --env "FLASK_SECRET=abc" delphi_web_python python -m pytest --import-mode importlib repos/delphi/delphi-epidata/tests
ln -s repos/delphi/delphi-epidata/dev/local/Makefile
ln -s repos/delphi/delphi-epidata/dev/local/docker-compose.yaml
ln -s repos/delphi/delphi-epidata/dev/local/.dockerignore
ln -s repos/delphi/delphi-epidata/dev/local/pyproject.toml repos/
ln -s repos/delphi/delphi-epidata/dev/local/setup.cfg repos/

- name: Run Integration Tests
- name: Run tests
run: |
docker run --rm --network delphi-net delphi_web_python python -m pytest --import-mode importlib repos/delphi/delphi-epidata/integrations
make test

- name: Clean Up
run: |
docker stop delphi_database_epidata delphi_web_epidata delphi_redis
docker network remove delphi-net
docker compose down

build_js_client:
runs-on: ubuntu-latest
Expand Down
132 changes: 24 additions & 108 deletions dev/local/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
#
# all: Runs the commands 'web' 'db' and 'python'.
#
# test: Runs test and integrations in delphi-epidata. If test
# test: Runs test and integrations in delphi-epidata. If test
# optional arg is provided, then only the tests in that subdir
# are run.
#
Expand Down Expand Up @@ -65,140 +65,56 @@ NOW:=$(shell date "+%Y-%m-%d")
LOG_WEB:=delphi_web_epidata_$(NOW).log
LOG_DB:=delphi_database_epidata_$(NOW).log
LOG_REDIS:=delphi_redis_instance_$(NOW).log
WEB_CONTAINER_ID:=$(shell docker ps -q --filter 'name=delphi_web_epidata')
DATABASE_CONTAINER_ID:=$(shell docker ps -q --filter 'name=delphi_database_epidata')
REDIS_CONTAINER_ID:=$(shell docker ps -q --filter 'name=delphi_redis')

M1=
ifeq ($(shell uname -smp), Darwin arm64 arm)
$(info M1 system detected, changing docker platform to linux/amd64.)
override M1 =--platform linux/amd64
endif

.PHONY=web
web:
@# Stop container if running
@if [ $(WEB_CONTAINER_ID) ]; then\
docker stop $(WEB_CONTAINER_ID);\
fi

@# Setup virtual network if it doesn't exist
@docker network ls | grep delphi-net || docker network create --driver bridge delphi-net

@# Build the web_epidata image
@cd repos/delphi/delphi-epidata;\
docker build -t delphi_web_epidata\
$(M1) \
-f ./devops/Dockerfile .;\
cd -

@# Run the web server
@# MODULE_NAME specifies the location of the `app` variable, the actual WSGI application object to run.
@# see https://github.com/tiangolo/meinheld-gunicorn-docker#module_name
@docker run --rm -p 127.0.0.1:10080:80 \
$(M1) \
--env "MODULE_NAME=delphi.epidata.server.main" \
--env "SQLALCHEMY_DATABASE_URI=$(sqlalchemy_uri)" \
--env "FLASK_SECRET=abc" --env "FLASK_PREFIX=/epidata" --env "LOG_DEBUG" \
--env "REDIS_HOST=delphi_redis" \
--env "REDIS_PASSWORD=1234" \
--env "API_KEY_ADMIN_PASSWORD=test_admin_password" \
--env "API_KEY_REGISTER_WEBHOOK_TOKEN=abc" \
--env "TESTING_MODE=True" \
--network delphi-net --name delphi_web_epidata \
delphi_web_epidata >$(LOG_WEB) 2>&1 &

.PHONY=db
db:
@# Stop container if running
@if [ $(DATABASE_CONTAINER_ID) ]; then\
docker stop $(DATABASE_CONTAINER_ID);\
fi

@# Setup virtual network if it doesn't exist
@docker network ls | grep delphi-net || docker network create --driver bridge delphi-net

@# Build the database_epidata image
@docker build -t delphi_database_epidata \
$(M1) \
-f repos/delphi/delphi-epidata/dev/docker/database/epidata/Dockerfile .

@# Run the database
@docker run --rm -p 127.0.0.1:13306:3306 \
$(M1) \
--network delphi-net --name delphi_database_epidata \
--cap-add=sys_nice \
delphi_database_epidata >$(LOG_DB) 2>&1 &

@# Block until DB is ready
@while true; do \
sed -n '/mysqld: ready for connections/p' $(LOG_DB) | grep "ready for connections" && break; \
tail -1 $(LOG_DB); \
sleep 1; \
done

.PHONY=py
py:
@docker build -t delphi_web_python \
$(M1) \
-f repos/delphi/delphi-epidata/dev/docker/python/Dockerfile .
@docker compose up delphi_database_epidata $(M1) --detach
@docker logs -f delphi_database_epidata >$(LOG_DB) 2>&1 &

.PHONY=web
web:
@SQLALCHEMY_DATABASE_URI=$(sqlalchemy_uri) docker compose up delphi_web_epidata $(M1) --detach
@docker logs -f delphi_web_epidata >$(LOG_WEB) 2>&1 &

.PHONY=redis
redis:
@# Stop container if running
@if [ $(REDIS_CONTAINER_ID) ]; then\
docker stop $(REDIS_CONTAINER_ID);\
fi

@docker build -t delphi_redis \
$(M1) \
-f repos/delphi/delphi-epidata/dev/docker/redis/Dockerfile .

@docker run --rm -d -p 127.0.0.1:6379:6379 \
$(M1) \
--network delphi-net \
--env "REDIS_PASSWORD=1234" \
--name delphi_redis delphi_redis >$(LOG_REDIS) 2>&1 &

.PHONY=all
all: db web py redis
@docker compose up delphi_redis $(M1) --detach
@docker logs -f delphi_redis >$(LOG_REDIS) 2>&1 &

.PHONY=test
test:
@docker run -i --rm --network delphi-net \
$(M1) \
--mount type=bind,source=$(CWD)repos/delphi/delphi-epidata,target=/usr/src/app/repos/delphi/delphi-epidata,readonly \
--mount type=bind,source=$(CWD)repos/delphi/delphi-epidata/src,target=/usr/src/app/delphi/epidata,readonly \
--env "SQLALCHEMY_DATABASE_URI=$(sqlalchemy_uri)" \
--env "FLASK_SECRET=abc" \
delphi_web_python python -m pytest --import-mode importlib $(pdb) $(test) | tee test_output_$(NOW).log
@SQLALCHEMY_DATABASE_URI=$(sqlalchemy_uri) docker compose run delphi_web_python $(M1) \
python -m pytest --import-mode importlib $(pdb) $(test) | tee test_output_$(NOW).log

.PHONY=r-test
r-test:
@docker run -i --rm --network delphi-net \
$(M1) \
--mount type=bind,source=$(CWD)repos/delphi/delphi-epidata,target=/usr/src/app/repos/delphi/delphi-epidata,readonly \
--mount type=bind,source=$(CWD)repos/delphi/delphi-epidata/src,target=/usr/src/app/delphi/epidata,readonly \
--env "SQLALCHEMY_DATABASE_URI=$(sqlalchemy_uri)" \
--env "FLASK_SECRET=abc" \
delphi_web_python Rscript repos/delphi/delphi-epidata/integrations/client/test_delphi_epidata.R | tee r-test_output_$(NOW).log

@SQLALCHEMY_DATABASE_URI=$(sqlalchemy_uri) docker compose run delphi_web_python $(M1) \
Rscript repos/delphi/delphi-epidata/integrations/client/test_delphi_epidata.R | tee r-test_output_$(NOW).log

.PHONY=bash
bash:
@docker run -it --rm --network delphi-net \
$(M1) \
--mount type=bind,source=$(CWD)repos/delphi/delphi-epidata,target=/usr/src/app/repos/delphi/delphi-epidata,readonly \
--mount type=bind,source=$(CWD)repos/delphi/delphi-epidata/src,target=/usr/src/app/delphi/epidata,readonly \
--env "SQLALCHEMY_DATABASE_URI=$(sqlalchemy_uri)" \
--env "FLASK_SECRET=abc" \
delphi_web_python bash
@SQLALCHEMY_DATABASE_URI=$(sqlalchemy_uri) docker compose run delphi_web_python $(M1) bash

.PHONY=sql
sql:
@docker run --rm -it --network delphi-net --cap-add=sys_nice \
percona mysql --user=user --password=pass --port 3306 --host delphi_database_epidata epidata

.PHONY=down
down:
@docker compose down

.PHONY=down-force-rebuild
down-force-rebuild:
@docker compose down --rmi all --volumes
@docker compose up --build --force-recreate --detach

.PHONY=clean
clean:
@docker images -f "dangling=true" -q | xargs docker rmi >/dev/null 2>&1
90 changes: 90 additions & 0 deletions dev/local/docker-compose.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
networks:
delphi-net:
driver: bridge

services:
delphi_database_epidata:
container_name: delphi_database_epidata
build:
context: .
dockerfile: repos/delphi/delphi-epidata/dev/docker/database/epidata/Dockerfile
cap_add:
- SYS_NICE
ports:
- "127.0.0.1:13306:3306"
networks:
- delphi-net
healthcheck:
test: "mysql --user=user --password=pass --execute \"SHOW DATABASES;\""
start_period: 5s
timeout: 10s
retries: 5

delphi_web_epidata:
container_name: delphi_web_epidata
build:
context: repos/delphi/delphi-epidata
dockerfile: ./devops/Dockerfile
environment:
MODULE_NAME: delphi.epidata.server.main
SQLALCHEMY_DATABASE_URI: ${SQLALCHEMY_DATABASE_URI:-mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata}
FLASK_SECRET: abc
FLASK_PREFIX: /epidata
LOG_DEBUG: "1"
REDIS_HOST: delphi_redis
REDIS_PASSWORD: 1234
API_KEY_ADMIN_PASSWORD: test_admin_password
API_KEY_REGISTER_WEBHOOK_TOKEN: abc
TESTING_MODE: True
ports:
- "127.0.0.1:10080:80"
networks:
- delphi-net
healthcheck:
test: "curl -f http://localhost/epidata/covidcast || exit 1"
timeout: 10s
retries: 5

delphi_redis:
container_name: delphi_redis
build:
context: .
dockerfile: repos/delphi/delphi-epidata/dev/docker/redis/Dockerfile
environment:
REDIS_PASSWORD: 1234
ports:
- "127.0.0.1:6379:6379"
networks:
- delphi-net
depends_on:
delphi_database_epidata:
condition: service_healthy
delphi_web_epidata:
condition: service_healthy

delphi_web_python:
container_name: delphi_web_python
build:
context: .
dockerfile: repos/delphi/delphi-epidata/dev/docker/python/Dockerfile
environment:
SQLALCHEMY_DATABASE_URI: ${SQLALCHEMY_DATABASE_URI:-mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata}
FLASK_SECRET: abc
volumes:
- type: bind
source: repos/delphi/delphi-epidata
target: /usr/src/app/delphi/epidata
read_only: true
- type: bind
source: repos/delphi/delphi-epidata/src
target: /usr/src/app/delphi/epidata
read_only: true
networks:
- delphi-net
depends_on:
delphi_database_epidata:
condition: service_healthy
delphi_web_epidata:
condition: service_healthy
delphi_redis:
condition: service_started
9 changes: 4 additions & 5 deletions dev/local/install.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#!/bin/bash
# Bootstrap delphi-epidata development
#
# Downloads the repos needed for local delphi-epidata development into current dir
# Downloads the repos needed for local delphi-epidata development into current dir
# and provides a Makefile with Docker control commands
# as well as pyproject/setup.cfg files for IDE mappings.
#
Expand Down Expand Up @@ -44,9 +44,8 @@ git clone https://github.com/undefx/py3tester
git clone https://github.com/undefx/undef-analysis
cd ../../

ln -s repos/delphi/delphi-epidata/dev/local/docker-compose.yaml
ln -s repos/delphi/delphi-epidata/dev/local/Makefile
ln -s repos/delphi/delphi-epidata/dev/local/.dockerignore
cd repos
ln -s delphi/delphi-epidata/dev/local/pyproject.toml
ln -s delphi/delphi-epidata/dev/local/setup.cfg
cd -
ln -s repos/delphi/delphi-epidata/dev/local/pyproject.toml repos/
ln -s repos/delphi/delphi-epidata/dev/local/setup.cfg repos/
Loading