From aa5169746468aa1d995c3cf532513a3a5694c1f8 Mon Sep 17 00:00:00 2001 From: yuwol Date: Tue, 26 Mar 2024 17:47:25 +0900 Subject: [PATCH 1/6] fix: update env example --- .env.example | 38 ++++++++++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/.env.example b/.env.example index 4262afbe..32439e38 100644 --- a/.env.example +++ b/.env.example @@ -1,15 +1,37 @@ -DJANGO_ENV=development|production -SECRET_KEY= +# AWS S3 AWS_BUCKET_NAME= AWS_BUCKET_NAME_STATIC= AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= + +# SPARCS SSO SSO_CLIENT_ID= SSO_SECRET_KEY= -PORTAL_ID= -PORTAL_PASSWORD= -PORTAL_2FA_KEY=/[2-7A-Z]{16}/ -DOCKERHUB_USERNAME= -DOCKERHUB_PASSWORD= -SENTRY_DSN= + +# Django +DJANGO_ENV=development +SECRET_KEY= + +# MySQL +NEWARA_DB_HOST=127.0.0.1 +NEWARA_DB_PORT=3306 +NEWARA_DB_USER=root +NEWARA_DB_PASSWORD= +NEWARA_DB_NAME= + +# Redis +NEWARA_REDIS_ADDRESS=127.0.0.1 +NEWARA_REDIS_PORT=6379 + +# Celery +C_FORCE_ROOT=true + +# Elasticsearch +NEWARA_ELASTICSEARCH_HOST=127.0.0.1 +NEWARA_ELASTICSEARCH_PORT=9200 + +# Logging +LOG_FILE_PATH=logs/http_access.log + +# KAIST portal PORTAL_JSESSIONID= From 30feb3ccfc34abc5ef68dc9f3d35b28aa9bcf34f Mon Sep 17 00:00:00 2001 From: yuwol Date: Fri, 5 Apr 2024 17:36:03 +0900 Subject: [PATCH 2/6] refactor: define redis port as env variable --- .docker/run-celery.sh | 2 +- .docker/run.sh | 2 +- ara/settings/cacheops.py | 4 ++-- ara/settings/redis.py | 4 +++- ara/settings/test.py | 2 +- 5 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.docker/run-celery.sh b/.docker/run-celery.sh index 977efa77..cef494c9 100644 --- a/.docker/run-celery.sh +++ b/.docker/run-celery.sh @@ -9,7 +9,7 @@ done sleep 5 -while ! nc -vz $NEWARA_REDIS_ADDRESS 6379; do +while ! nc -vz $NEWARA_REDIS_ADDRESS $NEWARA_REDIS_PORT; do >&2 echo "Redis is unavailable - sleeping" sleep 1 done diff --git a/.docker/run.sh b/.docker/run.sh index 844bcc48..f6c9c989 100644 --- a/.docker/run.sh +++ b/.docker/run.sh @@ -7,7 +7,7 @@ while ! nc -vz $NEWARA_DB_HOST $NEWARA_DB_PORT; do sleep 1 done -while ! nc -vz $NEWARA_REDIS_ADDRESS 6379; do +while ! nc -vz $NEWARA_REDIS_ADDRESS $NEWARA_REDIS_PORT; do >&2 echo "Redis is unavailable - sleeping" sleep 1 done diff --git a/ara/settings/cacheops.py b/ara/settings/cacheops.py index 3e2d1e0c..d3bf3b1f 100644 --- a/ara/settings/cacheops.py +++ b/ara/settings/cacheops.py @@ -1,8 +1,8 @@ -from .redis import REDIS_HOST +from .redis import REDIS_HOST, REDIS_PORT CACHEOPS_REDIS = { "host": REDIS_HOST, - "port": 6379, + "port": REDIS_PORT, "db": 1, } diff --git a/ara/settings/redis.py b/ara/settings/redis.py index 7b663f71..41ea2c84 100644 --- a/ara/settings/redis.py +++ b/ara/settings/redis.py @@ -2,8 +2,10 @@ from os import environ as os_environ REDIS_HOST = os_environ.get("NEWARA_REDIS_ADDRESS", "localhost") +REDIS_PORT = os_environ.get("NEWARA_REDIS_PORT", 6379) REDIS_DATABASE = int(os_environ.get("NEWARA_REDIS_DATABASE", 0)) -REDIS_URL = f"redis://{REDIS_HOST}:6379/{REDIS_DATABASE}" + +REDIS_URL = f"redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_DATABASE}" CACHES_TIMEOUT = timedelta(days=14) diff --git a/ara/settings/test.py b/ara/settings/test.py index 31310c31..bcaf52cd 100644 --- a/ara/settings/test.py +++ b/ara/settings/test.py @@ -24,7 +24,7 @@ } REDIS_DATABASE = int(environ.get("NEWARA_REDIS_DATABASE", 2)) -REDIS_URL = f"redis://{REDIS_HOST}:6379/{REDIS_DATABASE}" +REDIS_URL = f"redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_DATABASE}" CELERY_TASK_ALWAYS_EAGER = True ELASTICSEARCH_INDEX_NAME = "articles_test" From fdb6614c77bae5f35ad41e0815d5994589788cd5 Mon Sep 17 00:00:00 2001 From: yuwol Date: Fri, 5 Apr 2024 17:59:43 +0900 Subject: [PATCH 3/6] feat: add docker compose for local development --- docker-compose.example.yml | 110 ------------------------------------- docker-compose.local.yml | 41 ++++++++++++++ docker-compose.test.yml | 42 +++++--------- 3 files changed, 56 insertions(+), 137 deletions(-) delete mode 100644 docker-compose.example.yml create mode 100644 docker-compose.local.yml diff --git a/docker-compose.example.yml b/docker-compose.example.yml deleted file mode 100644 index 15aec7c1..00000000 --- a/docker-compose.example.yml +++ /dev/null @@ -1,110 +0,0 @@ -# Check S3 'sparcs-credentials' bucket for the complete file -version: '3' - -services: - api: - container_name: newara - image: newara:local - tty: false - depends_on: - - db - - redis - - celery_worker - - elasticsearch - ports: - - 9000:9000 - environment: - - DJANGO_ENV=development - - SECRET_KEY= - - AWS_BUCKET_NAME=sparcs-newara - - AWS_BUCKET_NAME_STATIC=sparcs-kaist-ara-beta-static - - AWS_ACCESS_KEY_ID= - - AWS_SECRET_ACCESS_KEY= - - SSO_CLIENT_ID= - - SSO_SECRET_KEY= - - SENTRY_DSN= - - NEWARA_DB_HOST=db - - NEWARA_DB_PORT=3306 - - NEWARA_DB_USER=root - - NEWARA_DB_PASSWORD= - - NEWARA_DB_NAME=new_ara - - NEWARA_REDIS_ADDRESS=redis - - PORTAL_ID= - - PORTAL_PASSWORD= - - NEWARA_ELASTICSEARCH_HOST=elasticsearch - - NEWARA_ELASTICSEARCH_PORT=9200 - - HASH_SECRET_VALUE=1 - - PORTAL_2FA_KEY= - entrypoint: /newara/www/.docker/run.sh - volumes: - - /var/log/newara_api:/var/log/newara - - nori:/newara/www/ara/db/elasticsearch - - celery_worker: - image: newara:local - restart: always - depends_on: - - db - - redis - environment: - - DJANGO_ENV=development - - SECRET_KEY= - - AWS_BUCKET_NAME=sparcs-newara - - AWS_BUCKET_NAME_STATIC=sparcs-kaist-ara-beta-static - - AWS_ACCESS_KEY_ID= - - AWS_SECRET_ACCESS_KEY= - - SSO_CLIENT_ID= - - SSO_SECRET_KEY= - - SENTRY_DSN= - - NEWARA_DB_HOST=db - - NEWARA_DB_PORT=3306 - - NEWARA_DB_USER=root - - NEWARA_DB_PASSWORD= - - NEWARA_DB_NAME=new_ara - - NEWARA_REDIS_ADDRESS=redis - - PORTAL_ID= - - PORTAL_PASSWORD= - - NEWARA_ELASTICSEARCH_HOST=elasticsearch - - NEWARA_ELASTICSEARCH_PORT=9200 - - C_FORCE_ROOT=true - - HASH_SECRET_VALUE=1 - - PORTAL_2FA_KEY= - entrypoint: /newara/www/.docker/run-celery.sh - volumes: - - /var/log/newara_celery:/var/log/newara - - db: - container_name: db - image: mysql:8.0 - expose: - - 3306 - ports: - - 3306:3306 - environment: - - MYSQL_ROOT_PASSWORD= - - MYSQL_DATABASE=new_ara - command: - - --character-set-server=utf8mb4 - - --collation-server=utf8mb4_general_ci - - redis: - image: redis:6 - expose: - - 6379 - - elasticsearch: - image: elasticsearch:7.9.2 - expose: - - 9200 - - 9300 - environment: - - discovery.type=single-node - command: bash -c " - bin/elasticsearch-plugin list | grep -q analysis-nori || bin/elasticsearch-plugin install --batch analysis-nori - && /usr/local/bin/docker-entrypoint.sh - " - volumes: - - nori:/usr/share/elasticsearch/config/analysis - -volumes: - nori: diff --git a/docker-compose.local.yml b/docker-compose.local.yml new file mode 100644 index 00000000..c2e7244d --- /dev/null +++ b/docker-compose.local.yml @@ -0,0 +1,41 @@ +version: "3" + +services: + db: + container_name: ara-db + image: mysql:8 + ports: + - "${NEWARA_DB_PORT:?err}:3306" + environment: + - MYSQL_ROOT_PASSWORD=${NEWARA_DB_PASSWORD:?err} + - MYSQL_DATABASE=${NEWARA_DB_NAME:?err} + volumes: + - vol-db:/var/lib/mysql + command: --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci + + redis: + container_name: ara-redis + image: redis:6 + ports: + - "${NEWARA_REDIS_PORT:?err}:6379" + volumes: + - vol-redis:/data + + elastic: + container_name: ara-elastic + image: elasticsearch:7.9.2 + ports: + - "${NEWARA_ELASTICSEARCH_PORT:?err}:9200" + environment: + - discovery.type=single-node + command: > + bash -c "bin/elasticsearch-plugin list | grep -q analysis-nori || bin/elasticsearch-plugin install -b analysis-nori + && /usr/local/bin/docker-entrypoint.sh" + volumes: + - ./ara/db/elasticsearch/synonym.txt:/usr/share/elasticsearch/config/analysis/synonym.txt + - vol-elastic:/usr/share/elasticsearch + +volumes: + vol-db: + vol-redis: + vol-elastic: diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 7878c848..59367873 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -1,13 +1,13 @@ -# 테스트 환경에서만 사용 version: "3" + services: api: image: newara depends_on: + - celery - db - redis - - celery_worker - - elasticsearch + - elastic environment: - DJANGO_SETTINGS_MODULE=ara.settings.test - DJANGO_ENV=${DJANGO_ENV} @@ -24,19 +24,17 @@ services: - NEWARA_DB_PASSWORD=root-password - NEWARA_DB_NAME=new_ara - NEWARA_REDIS_ADDRESS=redis - - NEWARA_ELASTICSEARCH_HOST=elasticsearch + - NEWARA_REDIS_PORT=6379 + - NEWARA_ELASTICSEARCH_HOST=elastic - NEWARA_ELASTICSEARCH_PORT=9200 - - PORTAL_2FA_KEY=${PORTAL_2FA_KEY} entrypoint: /newara/www/.docker/run.sh - volumes: - - nori:/newara/www/ara/db/elasticsearch - celery_worker: + celery: image: newara depends_on: - db - redis - - elasticsearch + - elastic environment: - DJANGO_SETTINGS_MODULE=main.settings.test - DJANGO_ENV=${DJANGO_ENV} @@ -53,42 +51,32 @@ services: - NEWARA_DB_PASSWORD=root-password - NEWARA_DB_NAME=new_ara - NEWARA_REDIS_ADDRESS=redis - - NEWARA_ELASTICSEARCH_HOST=elasticsearch + - NEWARA_REDIS_PORT=6379 + - NEWARA_ELASTICSEARCH_HOST=elastic - NEWARA_ELASTICSEARCH_PORT=9200 - C_FORCE_ROOT=true - - PORTAL_2FA_KEY=${PORTAL_2FA_KEY} entrypoint: /newara/www/.docker/run-celery.sh db: - container_name: db - image: mysql:8.0 + image: mysql:8 expose: - 3306 environment: - MYSQL_ROOT_PASSWORD=root-password - MYSQL_DATABASE=new_ara - command: - - --character-set-server=utf8mb4 - - --collation-server=utf8mb4_unicode_ci + command: --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci redis: image: redis:6 expose: - 6379 - elasticsearch: + elastic: image: elasticsearch:7.9.2 expose: - 9200 - - 9300 environment: - discovery.type=single-node - command: bash -c " - bin/elasticsearch-plugin list | grep -q analysis-nori || bin/elasticsearch-plugin install --batch analysis-nori - && /usr/local/bin/docker-entrypoint.sh - " - volumes: - - nori:/usr/share/elasticsearch/config/analysis - -volumes: - nori: + command: > + bash -c "bin/elasticsearch-plugin list | grep -q analysis-nori || bin/elasticsearch-plugin install --batch analysis-nori + && /usr/local/bin/docker-entrypoint.sh" From 545368a1e71f0b532b527f81cb351ce9da1686ca Mon Sep 17 00:00:00 2001 From: yuwol Date: Fri, 5 Apr 2024 18:03:00 +0900 Subject: [PATCH 4/6] docs: add docker local setup description --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index a7acf807..652ca8a9 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ pre-commit install --hook-type commit-msg ## How to Run ```bash +docker compose -f docker-compose.local.yml up -d pipenv shell make run ``` From de03991da1e6ff882dfe3b69e21a37bb890ac038 Mon Sep 17 00:00:00 2001 From: yuwol Date: Fri, 5 Apr 2024 18:20:14 +0900 Subject: [PATCH 5/6] fix: use volume to share synonyms between api and elastic --- docker-compose.test.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 59367873..d9911c79 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -28,6 +28,8 @@ services: - NEWARA_ELASTICSEARCH_HOST=elastic - NEWARA_ELASTICSEARCH_PORT=9200 entrypoint: /newara/www/.docker/run.sh + volumes: + - synonym:/newara/www/ara/db/elasticsearch celery: image: newara @@ -80,3 +82,8 @@ services: command: > bash -c "bin/elasticsearch-plugin list | grep -q analysis-nori || bin/elasticsearch-plugin install --batch analysis-nori && /usr/local/bin/docker-entrypoint.sh" + volumes: + - synonym:/usr/share/elasticsearch/config/analysis + +volumes: + synonym: From edbab302d617a989c52a1953e00124a95d3e0175 Mon Sep 17 00:00:00 2001 From: yuwol Date: Sun, 7 Apr 2024 14:17:45 +0900 Subject: [PATCH 6/6] feat: add makefile commands for local setup --- Makefile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Makefile b/Makefile index b97d93fd..7747cda8 100644 --- a/Makefile +++ b/Makefile @@ -67,3 +67,9 @@ show_sql: schema: python3 manage.py spectacular --color --file schema.yml + +local_env: + docker compose -f docker-compose.local.yml up -d + +local_env_down: + docker compose -f docker-compose.local.yml down