From c7a40f3bead61626a65e9f8c15eaf818a8572647 Mon Sep 17 00:00:00 2001 From: Kiril-Poposki1998 Date: Wed, 4 Dec 2024 11:51:31 +0100 Subject: [PATCH 1/7] Refactor compose for 2.11 setup --- compose/2.11/composes/.composes-env | 1 + .../composes/backend-services/.datapusher-env | 9 ++ compose/2.11/composes/backend-services/.env | 8 ++ .../2.11/composes/backend-services/.solr-env | 1 + .../backend-services/backend-services.yaml | 58 ++++++++++ compose/2.11/composes/ckan/.env | 57 ++++++++++ compose/2.11/composes/ckan/ckan.yaml | 25 ++++ compose/2.11/docker-compose.yml | 107 +----------------- compose/2.11/postgresql/Dockerfile | 4 +- .../00_create_datastore.sh | 2 +- 10 files changed, 164 insertions(+), 108 deletions(-) create mode 100644 compose/2.11/composes/.composes-env create mode 100644 compose/2.11/composes/backend-services/.datapusher-env create mode 100644 compose/2.11/composes/backend-services/.env create mode 100644 compose/2.11/composes/backend-services/.solr-env create mode 100644 compose/2.11/composes/backend-services/backend-services.yaml create mode 100644 compose/2.11/composes/ckan/.env create mode 100644 compose/2.11/composes/ckan/ckan.yaml mode change 100644 => 100755 compose/2.11/postgresql/docker-entrypoint-initdb.d/00_create_datastore.sh diff --git a/compose/2.11/composes/.composes-env b/compose/2.11/composes/.composes-env new file mode 100644 index 0000000..0e82187 --- /dev/null +++ b/compose/2.11/composes/.composes-env @@ -0,0 +1 @@ +CKAN_VERSION=2.11.0 \ No newline at end of file diff --git a/compose/2.11/composes/backend-services/.datapusher-env b/compose/2.11/composes/backend-services/.datapusher-env new file mode 100644 index 0000000..19e13a7 --- /dev/null +++ b/compose/2.11/composes/backend-services/.datapusher-env @@ -0,0 +1,9 @@ +DATAPUSHER_VERSION=0.0.17 +DATAPUSHER_MAX_CONTENT_LENGTH=10485760 +DATAPUSHER_CHUNK_SIZE=16384 +DATAPUSHER_CHUNK_INSERT_ROWS=250 +DATAPUSHER_DOWNLOAD_TIMEOUT=30 +DATAPUSHER_SSL_VERIFY=False +DATAPUSHER_REWRITE_RESOURCES=True +DATAPUSHER_REWRITE_URL=http://ckan:5005 +DATAPUSHER_SSL_VERIFY=False \ No newline at end of file diff --git a/compose/2.11/composes/backend-services/.env b/compose/2.11/composes/backend-services/.env new file mode 100644 index 0000000..4a72ffd --- /dev/null +++ b/compose/2.11/composes/backend-services/.env @@ -0,0 +1,8 @@ +# Database +POSTGRES_PASSWORD=ckan +POSTGRES_PORT=5432 +DATASTORE_READONLY_PASSWORD=datastore +PGDATA=/var/lib/postgresql/data/db + +# Redis +REDIS_VERSION=6.0.7 \ No newline at end of file diff --git a/compose/2.11/composes/backend-services/.solr-env b/compose/2.11/composes/backend-services/.solr-env new file mode 100644 index 0000000..a1b833e --- /dev/null +++ b/compose/2.11/composes/backend-services/.solr-env @@ -0,0 +1 @@ +CKAN_CORE_NAME=ckan \ No newline at end of file diff --git a/compose/2.11/composes/backend-services/backend-services.yaml b/compose/2.11/composes/backend-services/backend-services.yaml new file mode 100644 index 0000000..bb49bd3 --- /dev/null +++ b/compose/2.11/composes/backend-services/backend-services.yaml @@ -0,0 +1,58 @@ +services: + datapusher: + container_name: datapusher + image: ghcr.io/keitaroinc/datapusher:${DATAPUSHER_VERSION} + networks: + - frontend + - backend + ports: + - "8000:8000" + env_file: + - ./.datapusher-env + + db: + container_name: db + build: + context: ../../postgresql + networks: + - backend + env_file: + - ./.env + volumes: + - pg_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD", "pg_isready", "-U", "ckan"] + + solr: + container_name: solr + image: solr:8.11.1 + networks: + - backend + env_file: + - ../.composes-env + - ./.solr-env + volumes: + - solr_data:/var/solr + - ../../solr8/ckan_init_solr.sh:/docker-entrypoint-initdb.d/ckan_init_solr.sh + healthcheck: + test: ["CMD-SHELL", "/opt/solr/bin/solr","status"] + interval: 5s + timeout: 5s + retries: 5 + start_period: 5s + + redis: + container_name: redis + image: redis:${REDIS_VERSION} + networks: + - backend + healthcheck: + test: ["CMD", "redis-cli", "--raw", "incr", "ping"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 30s + +volumes: + pg_data: + solr_data: \ No newline at end of file diff --git a/compose/2.11/composes/ckan/.env b/compose/2.11/composes/ckan/.env new file mode 100644 index 0000000..81b3e8e --- /dev/null +++ b/compose/2.11/composes/ckan/.env @@ -0,0 +1,57 @@ +# Runtime configuration of CKAN enabled through ckanext-envvars +# Information about how it works: https://github.com/okfn/ckanext-envvars +# Note that variables here take presedence over build/up time variables in .env + +#Compose variables +# CKAN +CKAN_SITE_ID=default +CKAN_SITE_URL=http://localhost:5000 +CKAN_PORT=5000 +CKAN_MAX_UPLOAD_SIZE_MB=10 +CKAN_SQLALCHEMY_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/ckan +CKAN_DATASTORE_WRITE_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/datastore +CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore +CKAN_SOLR_URL=http://solr:8983/solr/ckan +CKAN_REDIS_URL=redis://redis:6379/1 +CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB} +POSTGRES_PASSWORD=${POSTGRES_PASSWORD} +DS_RO_PASS=${DATASTORE_READONLY_PASSWORD} + +# Set to true to disable CKAN from starting and serve a maintenance page +MAINTENANCE_MODE=false + +# General Settings +CKAN_SITE_ID=default +CKAN_SITE_URL=http://localhost:5000 +CKAN_PORT=5000 +CKAN_MAX_UPLOAD_SIZE_MB=20 +CKAN___BEAKER__SESSION__SECRET=CHANGE_ME +# See https://docs.ckan.org/en/latest/maintaining/configuration.html#api-token-settings +CKAN___API_TOKEN__JWT__ENCODE__SECRET=string:CHANGE_ME +CKAN___API_TOKEN__JWT__DECODE__SECRET=string:CHANGE_ME +# CKAN Plugins +CKAN__PLUGINS=envvars activity image_view +# CKAN requires storage path to be set in order for filestore to be enabled +CKAN__STORAGE_PATH=/srv/app/data +CKAN__WEBASSETS__PATH=/srv/app/data/webassets +# SYSADMIN settings, a sysadmin user is created automatically with the below credentials +CKAN_SYSADMIN_NAME=sysadmin +CKAN_SYSADMIN_PASSWORD=password +CKAN_SYSADMIN_EMAIL=sysadmin@ckantest.com + +# Email settings +CKAN_SMTP_SERVER=smtp.corporateict.domain:25 +CKAN_SMTP_STARTTLS=True +CKAN_SMTP_USER=user +CKAN_SMTP_PASSWORD=pass +CKAN_SMTP_MAIL_FROM=ckan@localhost + +# Datapusher configuration +CKAN__DATAPUSHER__URL=http://datapusher:8000 +CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000/ +CKAN__DATAPUSHER__API_TOKEN=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJqdGkiOiJ6M0lGRGw4SVdFR3lVRzFYRHJ0cm56WmNmcDlzbmFuUHlUMjdqLXRVSW9nIiwiaWF0IjoxNjc3MDYwNjQ1fQ.ECCa0wU-FOOGpdE9_TTVj__2J9SfANRBD6gRClBvPk0 +# CKAN__DATAPUSHER__API_TOKEN=replace_this_with_api_token_once_ckan_starts + +# Solr configuration +CKAN_VERSION=2.11 +CKAN_CORE_NAME=ckan \ No newline at end of file diff --git a/compose/2.11/composes/ckan/ckan.yaml b/compose/2.11/composes/ckan/ckan.yaml new file mode 100644 index 0000000..ee53f8e --- /dev/null +++ b/compose/2.11/composes/ckan/ckan.yaml @@ -0,0 +1,25 @@ +services: + ckan: + container_name: ckan + image: ghcr.io/keitaroinc/ckan:${CKAN_VERSION} + networks: + - frontend + - backend + depends_on: + db: + condition: service_healthy + restart: true + solr: + condition: service_healthy + restart: true + ports: + - "0.0.0.0:${CKAN_PORT}:5000" + env_file: + - ../.composes-env + - ../backend-services/.env + - ./.env + volumes: + - ckan_data:/srv/app/data + +volumes: + ckan_data: \ No newline at end of file diff --git a/compose/2.11/docker-compose.yml b/compose/2.11/docker-compose.yml index 74c85e3..8316e6a 100644 --- a/compose/2.11/docker-compose.yml +++ b/compose/2.11/docker-compose.yml @@ -1,108 +1,7 @@ # docker-compose build && docker-compose up -d -volumes: - ckan_data: - pg_data: - solr_data: - -services: - ckan: - container_name: ckan - image: ghcr.io/keitaroinc/ckan:${CKAN_VERSION} - networks: - - frontend - - backend - depends_on: - db: - condition: service_healthy - restart: true - solr: - condition: service_healthy - restart: true - ports: - - "0.0.0.0:${CKAN_PORT}:5000" - env_file: - - ./.ckan-env - environment: - - CKAN_SQLALCHEMY_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/ckan - - CKAN_DATASTORE_WRITE_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/datastore - - CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore - - CKAN_SOLR_URL=http://solr:8983/solr/ckan - - CKAN_REDIS_URL=redis://redis:6379/1 - - CKAN_SITE_URL=${CKAN_SITE_URL} - - CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB} - - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} - - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD} - volumes: - - ckan_data:/srv/app/data - - datapusher: - container_name: datapusher - image: ghcr.io/keitaroinc/datapusher:${DATAPUSHER_VERSION} - networks: - - frontend - - backend - ports: - - "8000:8000" - environment: - - DATAPUSHER_MAX_CONTENT_LENGTH=${DATAPUSHER_MAX_CONTENT_LENGTH} - - DATAPUSHER_CHUNK_SIZE=${DATAPUSHER_CHUNK_SIZE} - - DATAPUSHER_CHUNK_INSERT_ROWS=${DATAPUSHER_CHUNK_INSERT_ROWS} - - DATAPUSHER_DOWNLOAD_TIMEOUT=${DATAPUSHER_DOWNLOAD_TIMEOUT} - - DATAPUSHER_SSL_VERIFY=${DATAPUSHER_SSL_VERIFY} - - DATAPUSHER_REWRITE_RESOURCES=${DATAPUSHER_REWRITE_RESOURCES} - - DATAPUSHER_REWRITE_URL=${DATAPUSHER_REWRITE_URL} - - - db: - container_name: db - build: - context: . - dockerfile: postgresql/Dockerfile - args: - - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD} - - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} - networks: - - backend - environment: - - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD} - - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} - - PGDATA=/var/lib/postgresql/data/db - volumes: - - pg_data:/var/lib/postgresql/data - healthcheck: - test: ["CMD", "pg_isready", "-U", "ckan"] - - solr: - container_name: solr - image: solr:8.11.1 - networks: - - backend - env_file: - - ./.ckan-env - environment: - - CKAN_CORE_NAME=${CKAN_CORE_NAME} - - CKAN_VERSION=${CKAN_VERSION} - volumes: - - solr_data:/var/solr - - ${PWD}/solr8/ckan_init_solr.sh:/docker-entrypoint-initdb.d/ckan_init_solr.sh - healthcheck: - test: ["CMD-SHELL", "/opt/solr/bin/solr","status"] - interval: 5s - timeout: 5s - retries: 5 - start_period: 5s - - redis: - container_name: redis - image: redis:${REDIS_VERSION} - networks: - - backend - healthcheck: - test: ["CMD", "redis-cli", "--raw", "incr", "ping"] - interval: 10s - timeout: 5s - retries: 5 - start_period: 30s +include: + - composes/ckan/ckan.yaml + - composes/backend-services/backend-services.yaml networks: frontend: diff --git a/compose/2.11/postgresql/Dockerfile b/compose/2.11/postgresql/Dockerfile index e5fcbb7..61538aa 100644 --- a/compose/2.11/postgresql/Dockerfile +++ b/compose/2.11/postgresql/Dockerfile @@ -6,8 +6,6 @@ RUN echo "host all all 0.0.0.0/0 md5" >> /var/lib/postgresql/data/pg_hba.co # Customize default user/pass/db ENV POSTGRES_DB ckan ENV POSTGRES_USER ckan -ARG POSTGRES_PASSWORD -ARG DS_RO_PASS # Include datastore setup scripts -COPY ./postgresql/docker-entrypoint-initdb.d /docker-entrypoint-initdb.d +COPY docker-entrypoint-initdb.d /docker-entrypoint-initdb.d \ No newline at end of file diff --git a/compose/2.11/postgresql/docker-entrypoint-initdb.d/00_create_datastore.sh b/compose/2.11/postgresql/docker-entrypoint-initdb.d/00_create_datastore.sh old mode 100644 new mode 100755 index ec1b3c3..1993c48 --- a/compose/2.11/postgresql/docker-entrypoint-initdb.d/00_create_datastore.sh +++ b/compose/2.11/postgresql/docker-entrypoint-initdb.d/00_create_datastore.sh @@ -2,7 +2,7 @@ set -e psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" <<-EOSQL - CREATE ROLE datastore_ro NOSUPERUSER NOCREATEDB NOCREATEROLE LOGIN PASSWORD '$DS_RO_PASS'; + CREATE ROLE datastore_ro NOSUPERUSER NOCREATEDB NOCREATEROLE LOGIN PASSWORD '$DATASTORE_READONLY_PASSWORD'; CREATE DATABASE datastore OWNER ckan ENCODING 'utf-8'; GRANT ALL PRIVILEGES ON DATABASE datastore TO ckan; EOSQL \ No newline at end of file From a0e4eb3f7c5a3723310d87b10c20a9027025483b Mon Sep 17 00:00:00 2001 From: Kiril-Poposki1998 Date: Wed, 4 Dec 2024 12:42:49 +0100 Subject: [PATCH 2/7] Add environment variables configuration to root docker compose --- compose/2.11/.ckan-env | 42 ------------------- compose/2.11/.env | 36 ---------------- .../composes/backend-services/.datapusher-env | 1 - compose/2.11/composes/backend-services/.env | 5 ++- .../backend-services/backend-services.yaml | 12 +++--- compose/2.11/composes/ckan/.env | 13 ++---- compose/2.11/composes/ckan/ckan.yaml | 6 +-- compose/2.11/docker-compose.yml | 10 ++++- 8 files changed, 25 insertions(+), 100 deletions(-) delete mode 100644 compose/2.11/.ckan-env delete mode 100644 compose/2.11/.env diff --git a/compose/2.11/.ckan-env b/compose/2.11/.ckan-env deleted file mode 100644 index a329522..0000000 --- a/compose/2.11/.ckan-env +++ /dev/null @@ -1,42 +0,0 @@ -# Runtime configuration of CKAN enabled through ckanext-envvars -# Information about how it works: https://github.com/okfn/ckanext-envvars -# Note that variables here take presedence over build/up time variables in .env - -# Set to true to disable CKAN from starting and serve a maintenance page -MAINTENANCE_MODE=false - -# General Settings -CKAN_SITE_ID=default -CKAN_SITE_URL=http://localhost:5000 -CKAN_PORT=5000 -CKAN_MAX_UPLOAD_SIZE_MB=20 -CKAN___BEAKER__SESSION__SECRET=CHANGE_ME -# See https://docs.ckan.org/en/latest/maintaining/configuration.html#api-token-settings -CKAN___API_TOKEN__JWT__ENCODE__SECRET=string:CHANGE_ME -CKAN___API_TOKEN__JWT__DECODE__SECRET=string:CHANGE_ME -# CKAN Plugins -CKAN__PLUGINS=envvars activity image_view -# CKAN requires storage path to be set in order for filestore to be enabled -CKAN__STORAGE_PATH=/srv/app/data -CKAN__WEBASSETS__PATH=/srv/app/data/webassets -# SYSADMIN settings, a sysadmin user is created automatically with the below credentials -CKAN_SYSADMIN_NAME=sysadmin -CKAN_SYSADMIN_PASSWORD=password -CKAN_SYSADMIN_EMAIL=sysadmin@ckantest.com - -# Email settings -CKAN_SMTP_SERVER=smtp.corporateict.domain:25 -CKAN_SMTP_STARTTLS=True -CKAN_SMTP_USER=user -CKAN_SMTP_PASSWORD=pass -CKAN_SMTP_MAIL_FROM=ckan@localhost - -# Datapusher configuration -CKAN__DATAPUSHER__URL=http://datapusher:8000 -CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000/ -CKAN__DATAPUSHER__API_TOKEN=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJqdGkiOiJ6M0lGRGw4SVdFR3lVRzFYRHJ0cm56WmNmcDlzbmFuUHlUMjdqLXRVSW9nIiwiaWF0IjoxNjc3MDYwNjQ1fQ.ECCa0wU-FOOGpdE9_TTVj__2J9SfANRBD6gRClBvPk0 -# CKAN__DATAPUSHER__API_TOKEN=replace_this_with_api_token_once_ckan_starts - -# Solr configuration -CKAN_VERSION=2.11 -CKAN_CORE_NAME=ckan \ No newline at end of file diff --git a/compose/2.11/.env b/compose/2.11/.env deleted file mode 100644 index a0c157d..0000000 --- a/compose/2.11/.env +++ /dev/null @@ -1,36 +0,0 @@ -# Variables in this file will be used as build arguments when running -# docker-compose build and docker-compose up -# Verify correct substitution with "docker-compose config" -# If variables are newly added or enabled, please delete and rebuild the images to pull in changes: -# docker-compose down -v -# docker-compose build -# docker-compose up -d - -# Database -POSTGRES_PASSWORD=ckan -POSTGRES_PORT=5432 -DATASTORE_READONLY_PASSWORD=datastore - -# CKAN -CKAN_VERSION=2.11.0 -CKAN_SITE_ID=default -CKAN_SITE_URL=http://localhost:5000 -CKAN_PORT=5000 -CKAN_MAX_UPLOAD_SIZE_MB=10 - -# Datapusher -DATAPUSHER_VERSION=0.0.17 -DATAPUSHER_MAX_CONTENT_LENGTH=10485760 -DATAPUSHER_CHUNK_SIZE=16384 -DATAPUSHER_CHUNK_INSERT_ROWS=250 -DATAPUSHER_DOWNLOAD_TIMEOUT=30 -DATAPUSHER_SSL_VERIFY=False -DATAPUSHER_REWRITE_RESOURCES=True -DATAPUSHER_REWRITE_URL=http://ckan:5005 -DATAPUSHER_SSL_VERIFY=False - -# SOLR -CKAN_CORE_NAME=ckan - -# Redis -REDIS_VERSION=6.0.7 diff --git a/compose/2.11/composes/backend-services/.datapusher-env b/compose/2.11/composes/backend-services/.datapusher-env index 19e13a7..8f89818 100644 --- a/compose/2.11/composes/backend-services/.datapusher-env +++ b/compose/2.11/composes/backend-services/.datapusher-env @@ -1,4 +1,3 @@ -DATAPUSHER_VERSION=0.0.17 DATAPUSHER_MAX_CONTENT_LENGTH=10485760 DATAPUSHER_CHUNK_SIZE=16384 DATAPUSHER_CHUNK_INSERT_ROWS=250 diff --git a/compose/2.11/composes/backend-services/.env b/compose/2.11/composes/backend-services/.env index 4a72ffd..c441b38 100644 --- a/compose/2.11/composes/backend-services/.env +++ b/compose/2.11/composes/backend-services/.env @@ -5,4 +5,7 @@ DATASTORE_READONLY_PASSWORD=datastore PGDATA=/var/lib/postgresql/data/db # Redis -REDIS_VERSION=6.0.7 \ No newline at end of file +REDIS_VERSION=6.0.7 + +# Datapusher +DATAPUSHER_VERSION=0.0.17 \ No newline at end of file diff --git a/compose/2.11/composes/backend-services/backend-services.yaml b/compose/2.11/composes/backend-services/backend-services.yaml index bb49bd3..6e3cf65 100644 --- a/compose/2.11/composes/backend-services/backend-services.yaml +++ b/compose/2.11/composes/backend-services/backend-services.yaml @@ -8,16 +8,16 @@ services: ports: - "8000:8000" env_file: - - ./.datapusher-env + - ${PWD}/composes/backend-services/.datapusher-env db: container_name: db build: - context: ../../postgresql + context: ${PWD}/postgresql networks: - backend env_file: - - ./.env + - ${PWD}/composes/backend-services/.env volumes: - pg_data:/var/lib/postgresql/data healthcheck: @@ -29,11 +29,11 @@ services: networks: - backend env_file: - - ../.composes-env - - ./.solr-env + - ${PWD}/composes/.composes-env + - ${PWD}/composes/backend-services/.solr-env volumes: - solr_data:/var/solr - - ../../solr8/ckan_init_solr.sh:/docker-entrypoint-initdb.d/ckan_init_solr.sh + - ${PWD}/solr8/ckan_init_solr.sh:/docker-entrypoint-initdb.d/ckan_init_solr.sh healthcheck: test: ["CMD-SHELL", "/opt/solr/bin/solr","status"] interval: 5s diff --git a/compose/2.11/composes/ckan/.env b/compose/2.11/composes/ckan/.env index 81b3e8e..c3ee0a9 100644 --- a/compose/2.11/composes/ckan/.env +++ b/compose/2.11/composes/ckan/.env @@ -8,14 +8,13 @@ CKAN_SITE_ID=default CKAN_SITE_URL=http://localhost:5000 CKAN_PORT=5000 CKAN_MAX_UPLOAD_SIZE_MB=10 -CKAN_SQLALCHEMY_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/ckan -CKAN_DATASTORE_WRITE_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/datastore -CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore +CKAN_SQLALCHEMY_URL=postgresql://ckan:${POSTGRES_PASSWORD:-ckan}@db/ckan +CKAN_DATASTORE_WRITE_URL=postgresql://ckan:${POSTGRES_PASSWORD:-ckan}@db/datastore +CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD:-datastore}@db/datastore CKAN_SOLR_URL=http://solr:8983/solr/ckan CKAN_REDIS_URL=redis://redis:6379/1 CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB} -POSTGRES_PASSWORD=${POSTGRES_PASSWORD} -DS_RO_PASS=${DATASTORE_READONLY_PASSWORD} + # Set to true to disable CKAN from starting and serve a maintenance page MAINTENANCE_MODE=false @@ -51,7 +50,3 @@ CKAN__DATAPUSHER__URL=http://datapusher:8000 CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000/ CKAN__DATAPUSHER__API_TOKEN=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJqdGkiOiJ6M0lGRGw4SVdFR3lVRzFYRHJ0cm56WmNmcDlzbmFuUHlUMjdqLXRVSW9nIiwiaWF0IjoxNjc3MDYwNjQ1fQ.ECCa0wU-FOOGpdE9_TTVj__2J9SfANRBD6gRClBvPk0 # CKAN__DATAPUSHER__API_TOKEN=replace_this_with_api_token_once_ckan_starts - -# Solr configuration -CKAN_VERSION=2.11 -CKAN_CORE_NAME=ckan \ No newline at end of file diff --git a/compose/2.11/composes/ckan/ckan.yaml b/compose/2.11/composes/ckan/ckan.yaml index ee53f8e..30c79b7 100644 --- a/compose/2.11/composes/ckan/ckan.yaml +++ b/compose/2.11/composes/ckan/ckan.yaml @@ -15,9 +15,9 @@ services: ports: - "0.0.0.0:${CKAN_PORT}:5000" env_file: - - ../.composes-env - - ../backend-services/.env - - ./.env + - ${PWD}/composes/.composes-env + - ${PWD}/composes/backend-services/.env + - ${PWD}/composes/ckan/.env volumes: - ckan_data:/srv/app/data diff --git a/compose/2.11/docker-compose.yml b/compose/2.11/docker-compose.yml index 8316e6a..9693f30 100644 --- a/compose/2.11/docker-compose.yml +++ b/compose/2.11/docker-compose.yml @@ -1,7 +1,13 @@ # docker-compose build && docker-compose up -d include: - - composes/ckan/ckan.yaml - - composes/backend-services/backend-services.yaml + - path: composes/ckan/ckan.yaml + env_file: + - composes/ckan/.env + - composes/.composes-env + project_directory: . + - path: composes/backend-services/backend-services.yaml + env_file: composes/backend-services/.env + project_directory: . networks: frontend: From 1722630fe9227ce0850f788080621f43c52f1d11 Mon Sep 17 00:00:00 2001 From: Kiril-Poposki1998 Date: Fri, 6 Dec 2024 14:48:04 +0100 Subject: [PATCH 3/7] Delete folders for all ckan version --- compose/2.10/.ckan-env | 42 --- compose/2.10/.env | 36 -- compose/2.10/docker-compose.yml | 109 ------ compose/2.10/postgresql/Dockerfile | 13 - .../00_create_datastore.sh | 8 - compose/2.11/composes/.composes-env | 1 - .../20_postgis_permissions.sql | 2 - compose/2.11/solr8/ckan_init_solr.sh | 41 --- compose/2.8/.ckan-env | 36 -- compose/2.8/.env | 32 -- compose/2.8/docker-compose.yml | 93 ----- compose/2.8/postgresql/Dockerfile | 13 - .../00_create_datastore.sh | 8 - .../20_postgis_permissions.sql | 2 - compose/2.8/solr/Dockerfile | 35 -- compose/2.8/solr/solrconfig-2.7.12.xml | 1 - compose/2.8/solr/solrconfig-2.8.12.xml | 343 ----------------- compose/2.8/solr/solrconfig-2.9.7.xml | 345 ------------------ compose/2.8/solr/solrconfig-2.9.8.xml | 203 ----------- compose/2.9/.ckan-env | 40 -- compose/2.9/.env | 35 -- compose/2.9/docker-compose.yml | 95 ----- compose/2.9/postgresql/Dockerfile | 13 - .../00_create_datastore.sh | 8 - .../20_postgis_permissions.sql | 2 - compose/composes/.composes-env | 1 + .../composes/backend-services/.datapusher-env | 0 .../{2.11 => }/composes/backend-services/.env | 0 .../composes/backend-services/.solr-env | 0 .../backend-services/backend-services.yaml | 2 +- compose/{2.11 => }/composes/ckan/.env | 0 compose/{2.11 => }/composes/ckan/ckan.yaml | 0 compose/{2.11 => }/docker-compose.yml | 0 compose/{2.11 => }/postgresql/Dockerfile | 0 .../00_create_datastore.sh | 0 .../20_postgis_permissions.sql | 0 compose/{2.10 => }/solr8/ckan_init_solr.sh | 0 .../ckan_init_solr_2_9.sh} | 0 38 files changed, 2 insertions(+), 1557 deletions(-) delete mode 100644 compose/2.10/.ckan-env delete mode 100644 compose/2.10/.env delete mode 100644 compose/2.10/docker-compose.yml delete mode 100644 compose/2.10/postgresql/Dockerfile delete mode 100644 compose/2.10/postgresql/docker-entrypoint-initdb.d/00_create_datastore.sh delete mode 100644 compose/2.11/composes/.composes-env delete mode 100644 compose/2.11/postgresql/docker-entrypoint-initdb.d/20_postgis_permissions.sql delete mode 100755 compose/2.11/solr8/ckan_init_solr.sh delete mode 100644 compose/2.8/.ckan-env delete mode 100644 compose/2.8/.env delete mode 100644 compose/2.8/docker-compose.yml delete mode 100644 compose/2.8/postgresql/Dockerfile delete mode 100644 compose/2.8/postgresql/docker-entrypoint-initdb.d/00_create_datastore.sh delete mode 100644 compose/2.8/postgresql/docker-entrypoint-initdb.d/20_postgis_permissions.sql delete mode 100644 compose/2.8/solr/Dockerfile delete mode 120000 compose/2.8/solr/solrconfig-2.7.12.xml delete mode 100644 compose/2.8/solr/solrconfig-2.8.12.xml delete mode 100644 compose/2.8/solr/solrconfig-2.9.7.xml delete mode 100644 compose/2.8/solr/solrconfig-2.9.8.xml delete mode 100644 compose/2.9/.ckan-env delete mode 100644 compose/2.9/.env delete mode 100644 compose/2.9/docker-compose.yml delete mode 100644 compose/2.9/postgresql/Dockerfile delete mode 100644 compose/2.9/postgresql/docker-entrypoint-initdb.d/00_create_datastore.sh delete mode 100644 compose/2.9/postgresql/docker-entrypoint-initdb.d/20_postgis_permissions.sql create mode 100644 compose/composes/.composes-env rename compose/{2.11 => }/composes/backend-services/.datapusher-env (100%) rename compose/{2.11 => }/composes/backend-services/.env (100%) rename compose/{2.11 => }/composes/backend-services/.solr-env (100%) rename compose/{2.11 => }/composes/backend-services/backend-services.yaml (99%) rename compose/{2.11 => }/composes/ckan/.env (100%) rename compose/{2.11 => }/composes/ckan/ckan.yaml (100%) rename compose/{2.11 => }/docker-compose.yml (100%) rename compose/{2.11 => }/postgresql/Dockerfile (100%) rename compose/{2.11 => }/postgresql/docker-entrypoint-initdb.d/00_create_datastore.sh (100%) rename compose/{2.10 => }/postgresql/docker-entrypoint-initdb.d/20_postgis_permissions.sql (100%) rename compose/{2.10 => }/solr8/ckan_init_solr.sh (100%) rename compose/{2.9/solr8/ckan_init_solr.sh => solr8/ckan_init_solr_2_9.sh} (100%) diff --git a/compose/2.10/.ckan-env b/compose/2.10/.ckan-env deleted file mode 100644 index b9f2598..0000000 --- a/compose/2.10/.ckan-env +++ /dev/null @@ -1,42 +0,0 @@ -# Runtime configuration of CKAN enabled through ckanext-envvars -# Information about how it works: https://github.com/okfn/ckanext-envvars -# Note that variables here take presedence over build/up time variables in .env - -# Set to true to disable CKAN from starting and serve a maintenance page -MAINTENANCE_MODE=false - -# General Settings -CKAN_SITE_ID=default -CKAN_SITE_URL=http://localhost:5000 -CKAN_PORT=5000 -CKAN_MAX_UPLOAD_SIZE_MB=20 -CKAN___BEAKER__SESSION__SECRET=CHANGE_ME -# See https://docs.ckan.org/en/latest/maintaining/configuration.html#api-token-settings -CKAN___API_TOKEN__JWT__ENCODE__SECRET=string:CHANGE_ME -CKAN___API_TOKEN__JWT__DECODE__SECRET=string:CHANGE_ME -# CKAN Plugins -CKAN__PLUGINS=envvars image_view text_view recline_view datastore datapusher -# CKAN requires storage path to be set in order for filestore to be enabled -CKAN__STORAGE_PATH=/srv/app/data -CKAN__WEBASSETS__PATH=/srv/app/data/webassets -# SYSADMIN settings, a sysadmin user is created automatically with the below credentials -CKAN_SYSADMIN_NAME=sysadmin -CKAN_SYSADMIN_PASSWORD=password -CKAN_SYSADMIN_EMAIL=sysadmin@ckantest.com - -# Email settings -CKAN_SMTP_SERVER=smtp.corporateict.domain:25 -CKAN_SMTP_STARTTLS=True -CKAN_SMTP_USER=user -CKAN_SMTP_PASSWORD=pass -CKAN_SMTP_MAIL_FROM=ckan@localhost - -# Datapusher configuration -CKAN__DATAPUSHER__URL=http://datapusher:8000 -CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000/ -CKAN__DATAPUSHER__API_TOKEN=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJqdGkiOiJ6M0lGRGw4SVdFR3lVRzFYRHJ0cm56WmNmcDlzbmFuUHlUMjdqLXRVSW9nIiwiaWF0IjoxNjc3MDYwNjQ1fQ.ECCa0wU-FOOGpdE9_TTVj__2J9SfANRBD6gRClBvPk0 -# CKAN__DATAPUSHER__API_TOKEN=replace_this_with_api_token_once_ckan_starts - -# Solr configuration -CKAN_VERSION=2.10 -CKAN_CORE_NAME=ckan \ No newline at end of file diff --git a/compose/2.10/.env b/compose/2.10/.env deleted file mode 100644 index 742df69..0000000 --- a/compose/2.10/.env +++ /dev/null @@ -1,36 +0,0 @@ -# Variables in this file will be used as build arguments when running -# docker-compose build and docker-compose up -# Verify correct substitution with "docker-compose config" -# If variables are newly added or enabled, please delete and rebuild the images to pull in changes: -# docker-compose down -v -# docker-compose build -# docker-compose up -d - -# Database -POSTGRES_PASSWORD=ckan -POSTGRES_PORT=5432 -DATASTORE_READONLY_PASSWORD=datastore - -# CKAN -CKAN_VERSION=2.10.5 -CKAN_SITE_ID=default -CKAN_SITE_URL=http://localhost:5000 -CKAN_PORT=5000 -CKAN_MAX_UPLOAD_SIZE_MB=10 - -# Datapusher -DATAPUSHER_VERSION=0.0.17 -DATAPUSHER_MAX_CONTENT_LENGTH=10485760 -DATAPUSHER_CHUNK_SIZE=16384 -DATAPUSHER_CHUNK_INSERT_ROWS=250 -DATAPUSHER_DOWNLOAD_TIMEOUT=30 -DATAPUSHER_SSL_VERIFY=False -DATAPUSHER_REWRITE_RESOURCES=True -DATAPUSHER_REWRITE_URL=http://ckan:5005 -DATAPUSHER_SSL_VERIFY=False - -# SOLR -CKAN_CORE_NAME=ckan - -# Redis -REDIS_VERSION=6.0.7 diff --git a/compose/2.10/docker-compose.yml b/compose/2.10/docker-compose.yml deleted file mode 100644 index 74c85e3..0000000 --- a/compose/2.10/docker-compose.yml +++ /dev/null @@ -1,109 +0,0 @@ -# docker-compose build && docker-compose up -d -volumes: - ckan_data: - pg_data: - solr_data: - -services: - ckan: - container_name: ckan - image: ghcr.io/keitaroinc/ckan:${CKAN_VERSION} - networks: - - frontend - - backend - depends_on: - db: - condition: service_healthy - restart: true - solr: - condition: service_healthy - restart: true - ports: - - "0.0.0.0:${CKAN_PORT}:5000" - env_file: - - ./.ckan-env - environment: - - CKAN_SQLALCHEMY_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/ckan - - CKAN_DATASTORE_WRITE_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/datastore - - CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore - - CKAN_SOLR_URL=http://solr:8983/solr/ckan - - CKAN_REDIS_URL=redis://redis:6379/1 - - CKAN_SITE_URL=${CKAN_SITE_URL} - - CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB} - - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} - - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD} - volumes: - - ckan_data:/srv/app/data - - datapusher: - container_name: datapusher - image: ghcr.io/keitaroinc/datapusher:${DATAPUSHER_VERSION} - networks: - - frontend - - backend - ports: - - "8000:8000" - environment: - - DATAPUSHER_MAX_CONTENT_LENGTH=${DATAPUSHER_MAX_CONTENT_LENGTH} - - DATAPUSHER_CHUNK_SIZE=${DATAPUSHER_CHUNK_SIZE} - - DATAPUSHER_CHUNK_INSERT_ROWS=${DATAPUSHER_CHUNK_INSERT_ROWS} - - DATAPUSHER_DOWNLOAD_TIMEOUT=${DATAPUSHER_DOWNLOAD_TIMEOUT} - - DATAPUSHER_SSL_VERIFY=${DATAPUSHER_SSL_VERIFY} - - DATAPUSHER_REWRITE_RESOURCES=${DATAPUSHER_REWRITE_RESOURCES} - - DATAPUSHER_REWRITE_URL=${DATAPUSHER_REWRITE_URL} - - - db: - container_name: db - build: - context: . - dockerfile: postgresql/Dockerfile - args: - - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD} - - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} - networks: - - backend - environment: - - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD} - - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} - - PGDATA=/var/lib/postgresql/data/db - volumes: - - pg_data:/var/lib/postgresql/data - healthcheck: - test: ["CMD", "pg_isready", "-U", "ckan"] - - solr: - container_name: solr - image: solr:8.11.1 - networks: - - backend - env_file: - - ./.ckan-env - environment: - - CKAN_CORE_NAME=${CKAN_CORE_NAME} - - CKAN_VERSION=${CKAN_VERSION} - volumes: - - solr_data:/var/solr - - ${PWD}/solr8/ckan_init_solr.sh:/docker-entrypoint-initdb.d/ckan_init_solr.sh - healthcheck: - test: ["CMD-SHELL", "/opt/solr/bin/solr","status"] - interval: 5s - timeout: 5s - retries: 5 - start_period: 5s - - redis: - container_name: redis - image: redis:${REDIS_VERSION} - networks: - - backend - healthcheck: - test: ["CMD", "redis-cli", "--raw", "incr", "ping"] - interval: 10s - timeout: 5s - retries: 5 - start_period: 30s - -networks: - frontend: - backend: diff --git a/compose/2.10/postgresql/Dockerfile b/compose/2.10/postgresql/Dockerfile deleted file mode 100644 index e5fcbb7..0000000 --- a/compose/2.10/postgresql/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -FROM postgis/postgis:14-3.2-alpine - -# Allow connections; we don't map out any ports so only linked docker containers can connect -RUN echo "host all all 0.0.0.0/0 md5" >> /var/lib/postgresql/data/pg_hba.conf - -# Customize default user/pass/db -ENV POSTGRES_DB ckan -ENV POSTGRES_USER ckan -ARG POSTGRES_PASSWORD -ARG DS_RO_PASS - -# Include datastore setup scripts -COPY ./postgresql/docker-entrypoint-initdb.d /docker-entrypoint-initdb.d diff --git a/compose/2.10/postgresql/docker-entrypoint-initdb.d/00_create_datastore.sh b/compose/2.10/postgresql/docker-entrypoint-initdb.d/00_create_datastore.sh deleted file mode 100644 index ec1b3c3..0000000 --- a/compose/2.10/postgresql/docker-entrypoint-initdb.d/00_create_datastore.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -set -e - -psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" <<-EOSQL - CREATE ROLE datastore_ro NOSUPERUSER NOCREATEDB NOCREATEROLE LOGIN PASSWORD '$DS_RO_PASS'; - CREATE DATABASE datastore OWNER ckan ENCODING 'utf-8'; - GRANT ALL PRIVILEGES ON DATABASE datastore TO ckan; -EOSQL \ No newline at end of file diff --git a/compose/2.11/composes/.composes-env b/compose/2.11/composes/.composes-env deleted file mode 100644 index 0e82187..0000000 --- a/compose/2.11/composes/.composes-env +++ /dev/null @@ -1 +0,0 @@ -CKAN_VERSION=2.11.0 \ No newline at end of file diff --git a/compose/2.11/postgresql/docker-entrypoint-initdb.d/20_postgis_permissions.sql b/compose/2.11/postgresql/docker-entrypoint-initdb.d/20_postgis_permissions.sql deleted file mode 100644 index 8b5348e..0000000 --- a/compose/2.11/postgresql/docker-entrypoint-initdb.d/20_postgis_permissions.sql +++ /dev/null @@ -1,2 +0,0 @@ -ALTER VIEW geometry_columns OWNER TO ckan; -ALTER TABLE spatial_ref_sys OWNER TO ckan; diff --git a/compose/2.11/solr8/ckan_init_solr.sh b/compose/2.11/solr8/ckan_init_solr.sh deleted file mode 100755 index 091450f..0000000 --- a/compose/2.11/solr8/ckan_init_solr.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -# -# Initialize SOLR for CKAN by creating a ckan core -# Arguments are supplied via environment variables: CKAN_CORE_NAME CKAN_VERSION -# Example: -# CKAN_CORE_NAME=ckan -# CKAN_VERSION=2.9.7 - -set -e - -MAJOR_VERSION=`echo $CKAN_VERSION | cut -d "." -f 1` -MINOR_VERSION=`echo $CKAN_VERSION | cut -d "." -f 2` - -CKAN_SOLR_SCHEMA_URL=https://raw.githubusercontent.com/ckan/ckan/`echo $MAJOR_VERSION.$MINOR_VERSION`/ckan/config/solr/schema.xml - -echo "Check whether managed schema exists for CKAN $CKAN_VERSION" -if ! curl --output /dev/null --silent --head --fail "$CKAN_SOLR_SCHEMA_URL"; then - echo "Can't find CKAN SOLR schema at URL: $CKAN_SOLR_SCHEMA_URL. Exiting..." - exit 1 -fi - -echo "Check whether SOLR is initialized for CKAN" -CORESDIR=/var/solr/data - -COREDIR="$CORESDIR/$CKAN_CORE_NAME" -if [ -d "$COREDIR" ]; then - echo "SOLR already initialized, skipping initialization" -else - echo "Initializing SOLR core $CKAN_CORE_NAME for CKAN $CKAN_VERSION" - # init script for handling an empty /var/solr - /opt/docker-solr/scripts/init-var-solr - - # Precreate CKAN core - /opt/docker-solr/scripts/precreate-core $CKAN_CORE_NAME - - # Replace the managed schema with CKANs schema - echo "Adding CKAN managed schema" - curl $CKAN_SOLR_SCHEMA_URL -o /var/solr/data/$CKAN_CORE_NAME/conf/managed-schema -s - - echo "SOLR initialized" -fi diff --git a/compose/2.8/.ckan-env b/compose/2.8/.ckan-env deleted file mode 100644 index 6e37970..0000000 --- a/compose/2.8/.ckan-env +++ /dev/null @@ -1,36 +0,0 @@ -# Runtime configuration of CKAN enabled through ckanext-envvars -# Information about how it works: https://github.com/okfn/ckanext-envvars -# Note that variables here take presedence over build/up time variables in .env - -# Set to true to disable CKAN from starting and serve a maintenance page -MAINTENANCE_MODE=false - -# General Settings -CKAN_SITE_ID=default -CKAN_SITE_URL=http://localhost:5000 -CKAN_PORT=5000 -CKAN_MAX_UPLOAD_SIZE_MB=10 -CKAN___BEAKER__SESSION__SECRET=CHANGE_ME -# See https://docs.ckan.org/en/latest/maintaining/configuration.html#api-token-settings -CKAN___API_TOKEN__JWT__ENCODE__SECRET=string:CHANGE_ME -CKAN___API_TOKEN__JWT__DECODE__SECRET=string:CHANGE_ME -# CKAN Plugins -CKAN__PLUGINS=envvars image_view text_view recline_view datastore datapusher -# CKAN requires storage path to be set in order for filestore to be enabled -CKAN__STORAGE_PATH=/srv/app/data -CKAN__WEBASSETS__PATH=/srv/app/data/webassets -# SYSADMIN settings, a sysadmin user is created automatically with the below credentials -CKAN_SYSADMIN_NAME=sysadmin -CKAN_SYSADMIN_PASSWORD=password -CKAN_SYSADMIN_EMAIL=sysadmin@ckantest.com - -# Email settings -CKAN_SMTP_SERVER=smtp.corporateict.domain:25 -CKAN_SMTP_STARTTLS=True -CKAN_SMTP_USER=user -CKAN_SMTP_PASSWORD=pass -CKAN_SMTP_MAIL_FROM=ckan@localhost - -# Datapusher configuration -CKAN__DATAPUSHER__URL=http://datapusher:8000 -CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000/ diff --git a/compose/2.8/.env b/compose/2.8/.env deleted file mode 100644 index 83b4c85..0000000 --- a/compose/2.8/.env +++ /dev/null @@ -1,32 +0,0 @@ -# Variables in this file will be used as build arguments when running -# docker-compose build and docker-compose up -# Verify correct substitution with "docker-compose config" -# If variables are newly added or enabled, please delete and rebuild the images to pull in changes: -# docker-compose down -v -# docker-compose build -# docker-compose up -d - -# Database -POSTGRES_PASSWORD=ckan -POSTGRES_PORT=5432 -DATASTORE_READONLY_PASSWORD=datastore - -# CKAN -CKAN_VERSION=2.8.12 -CKAN_SITE_ID=default -CKAN_SITE_URL=http://localhost:5000 -CKAN_PORT=5000 -CKAN_MAX_UPLOAD_SIZE_MB=10 - -# Datapusher -DATAPUSHER_VERSION=0.0.17 -DATAPUSHER_MAX_CONTENT_LENGTH=10485760 -DATAPUSHER_CHUNK_SIZE=16384 -DATAPUSHER_CHUNK_INSERT_ROWS=250 -DATAPUSHER_DOWNLOAD_TIMEOUT=30 -DATAPUSHER_SSL_VERIFY=False -DATAPUSHER_REWRITE_RESOURCES=True -DATAPUSHER_REWRITE_URL=http://ckan:5000 - -# Redis -REDIS_VERSION=6.0.7 diff --git a/compose/2.8/docker-compose.yml b/compose/2.8/docker-compose.yml deleted file mode 100644 index 18c7b36..0000000 --- a/compose/2.8/docker-compose.yml +++ /dev/null @@ -1,93 +0,0 @@ -# docker-compose build && docker-compose up -d -version: "3" - -volumes: - ckan_data: - pg_data: - solr_data: - -services: - ckan: - container_name: ckan - image: ghcr.io/keitaroinc/ckan:${CKAN_VERSION} - networks: - - frontend - - backend - depends_on: - - db - - solr - ports: - - "0.0.0.0:${CKAN_PORT}:5000" - env_file: - - ./.ckan-env - environment: - - CKAN_SQLALCHEMY_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/ckan - - CKAN_DATASTORE_WRITE_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/datastore - - CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore - - CKAN_SOLR_URL=http://solr:8983/solr/ckan - - CKAN_REDIS_URL=redis://redis:6379/1 - - CKAN_SITE_URL=${CKAN_SITE_URL} - - CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB} - - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} - - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD} - volumes: - - ckan_data:/srv/app/data - - datapusher: - container_name: datapusher - image: ghcr.io/keitaroinc/datapusher:${DATAPUSHER_VERSION} - networks: - - frontend - - backend - ports: - - "8000:8000" - environment: - - DATAPUSHER_MAX_CONTENT_LENGTH=${DATAPUSHER_MAX_CONTENT_LENGTH} - - DATAPUSHER_CHUNK_SIZE=${DATAPUSHER_CHUNK_SIZE} - - DATAPUSHER_CHUNK_INSERT_ROWS=${DATAPUSHER_CHUNK_INSERT_ROWS} - - DATAPUSHER_DOWNLOAD_TIMEOUT=${DATAPUSHER_DOWNLOAD_TIMEOUT} - - DATAPUSHER_SSL_VERIFY=${DATA_PUSHER_SSL_VERIFY} - - DATAPUSHER_REWRITE_RESOURCES=${DATAPUSHER_REWRITE_RESOURCES} - - DATAPUSHER_REWRITE_URL=${DATAPUSHER_REWRITE_URL} - - - db: - container_name: db - build: - context: . - dockerfile: postgresql/Dockerfile - args: - - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD} - - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} - networks: - - backend - environment: - - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD} - - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} - - PGDATA=/var/lib/postgresql/data/db - volumes: - - pg_data:/var/lib/postgresql/data - healthcheck: - test: ["CMD", "pg_isready", "-U", "ckan"] - - solr: - container_name: solr - build: - context: . - dockerfile: solr/Dockerfile - args: - - CKAN_VERSION=${CKAN_VERSION} - networks: - - backend - volumes: - - solr_data:/opt/solr/server/solr/ckan/data - - redis: - container_name: redis - image: redis:${REDIS_VERSION} - networks: - - backend - -networks: - frontend: - backend: diff --git a/compose/2.8/postgresql/Dockerfile b/compose/2.8/postgresql/Dockerfile deleted file mode 100644 index 95e8cf6..0000000 --- a/compose/2.8/postgresql/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -FROM postgis/postgis:11-3.3-alpine - -# Allow connections; we don't map out any ports so only linked docker containers can connect -RUN echo "host all all 0.0.0.0/0 md5" >> /var/lib/postgresql/data/pg_hba.conf - -# Customize default user/pass/db -ENV POSTGRES_DB ckan -ENV POSTGRES_USER ckan -ARG POSTGRES_PASSWORD -ARG DS_RO_PASS - -# Include datastore setup scripts -COPY ./postgresql/docker-entrypoint-initdb.d /docker-entrypoint-initdb.d diff --git a/compose/2.8/postgresql/docker-entrypoint-initdb.d/00_create_datastore.sh b/compose/2.8/postgresql/docker-entrypoint-initdb.d/00_create_datastore.sh deleted file mode 100644 index ec1b3c3..0000000 --- a/compose/2.8/postgresql/docker-entrypoint-initdb.d/00_create_datastore.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -set -e - -psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" <<-EOSQL - CREATE ROLE datastore_ro NOSUPERUSER NOCREATEDB NOCREATEROLE LOGIN PASSWORD '$DS_RO_PASS'; - CREATE DATABASE datastore OWNER ckan ENCODING 'utf-8'; - GRANT ALL PRIVILEGES ON DATABASE datastore TO ckan; -EOSQL \ No newline at end of file diff --git a/compose/2.8/postgresql/docker-entrypoint-initdb.d/20_postgis_permissions.sql b/compose/2.8/postgresql/docker-entrypoint-initdb.d/20_postgis_permissions.sql deleted file mode 100644 index 8b5348e..0000000 --- a/compose/2.8/postgresql/docker-entrypoint-initdb.d/20_postgis_permissions.sql +++ /dev/null @@ -1,2 +0,0 @@ -ALTER VIEW geometry_columns OWNER TO ckan; -ALTER TABLE spatial_ref_sys OWNER TO ckan; diff --git a/compose/2.8/solr/Dockerfile b/compose/2.8/solr/Dockerfile deleted file mode 100644 index 6cdbae2..0000000 --- a/compose/2.8/solr/Dockerfile +++ /dev/null @@ -1,35 +0,0 @@ -FROM solr:6.6.6 - -# Enviroment -ENV SOLR_CORE ckan -ENV SOLR_VERSION 6.6.6 - -# Build Arguments -ARG CKAN_VERSION - -# Create Directories -RUN mkdir -p /opt/solr/server/solr/$SOLR_CORE/conf -RUN mkdir -p /opt/solr/server/solr/$SOLR_CORE/data - -# Adding Files -COPY ./solr/solrconfig-$CKAN_VERSION.xml /opt/solr/server/solr/$SOLR_CORE/conf/ -ADD https://raw.githubusercontent.com/ckan/ckan/ckan-$CKAN_VERSION/ckan/config/solr/schema.xml \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/basic_configs/conf/currency.xml \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/basic_configs/conf/synonyms.txt \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/basic_configs/conf/stopwords.txt \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/basic_configs/conf/protwords.txt \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/data_driven_schema_configs/conf/elevate.xml \ -/opt/solr/server/solr/$SOLR_CORE/conf/ - -# Create Core.properties -RUN mv /opt/solr/server/solr/$SOLR_CORE/conf/solrconfig-$CKAN_VERSION.xml /opt/solr/server/solr/$SOLR_CORE/conf/solrconfig.xml && \ - echo name=$SOLR_CORE > /opt/solr/server/solr/$SOLR_CORE/core.properties - -# Giving ownership to Solr - -USER root - -RUN chown -R "$SOLR_USER:$SOLR_USER" /opt/solr/server/solr/$SOLR_CORE - -# User -USER $SOLR_USER:$SOLR_USER diff --git a/compose/2.8/solr/solrconfig-2.7.12.xml b/compose/2.8/solr/solrconfig-2.7.12.xml deleted file mode 120000 index 6e325cc..0000000 --- a/compose/2.8/solr/solrconfig-2.7.12.xml +++ /dev/null @@ -1 +0,0 @@ -solrconfig-2.8.10.xml \ No newline at end of file diff --git a/compose/2.8/solr/solrconfig-2.8.12.xml b/compose/2.8/solr/solrconfig-2.8.12.xml deleted file mode 100644 index 9ac620c..0000000 --- a/compose/2.8/solr/solrconfig-2.8.12.xml +++ /dev/null @@ -1,343 +0,0 @@ - - - - - - 6.0.0 - - - - - - - - - - - ${solr.data.dir:} - - - - - - - ${solr.lock.type:native} - - - - - - - - ${solr.ulog.dir:} - ${solr.ulog.numVersionBuckets:65536} - - - - ${solr.autoCommit.maxTime:15000} - false - - - - ${solr.autoSoftCommit.maxTime:-1} - - - - - - - 1024 - - - - - true - 20 - 200 - - - - - - - - - - false - 2 - - - - - - - - - - - - - - explicit - 10 - - - - - - - - explicit - json - true - - - - - - - - explicit - - - - - - - - _text_ - - - - - - - - add-unknown-fields-to-the-schema - - - - - - - - true - ignored_ - _text_ - - - - - - - - - - - explicit - true - - - - - - - text_general - - - default - _text_ - solr.DirectSolrSpellChecker - internal - 0.5 - 2 - 1 - 5 - 4 - 0.01 - - - - - - - - default - on - true - 10 - 5 - 5 - true - true - 10 - 5 - - - - spellcheck - - - - - - - - - - true - - - tvComponent - - - - - - - - - - true - false - - - - terms - - - - - - - string - elevate.xml - - - - - - - explicit - - - elevator - - - - - - - - - - - 100 - - - - - - 70 - 0.5 - [-\w ,/\n\"']{20,200} - - - - - - ]]> - ]]> - - - - - - - - - - - - ,, - ,, - ,, - ,, - ,]]> - ]]> - - - - - - - 10 - .,!? - - - - - - WORD - en - US - - - - - - - - - - - - - - - - [^\w-\.] - _ - - - - - - - yyyy-MM-dd'T'HH:mm:ss.SSSZ - yyyy-MM-dd'T'HH:mm:ss,SSSZ - yyyy-MM-dd'T'HH:mm:ss.SSS - yyyy-MM-dd'T'HH:mm:ss,SSS - yyyy-MM-dd'T'HH:mm:ssZ - yyyy-MM-dd'T'HH:mm:ss - yyyy-MM-dd'T'HH:mmZ - yyyy-MM-dd'T'HH:mm - yyyy-MM-dd HH:mm:ss.SSSZ - yyyy-MM-dd HH:mm:ss,SSSZ - yyyy-MM-dd HH:mm:ss.SSS - yyyy-MM-dd HH:mm:ss,SSS - yyyy-MM-dd HH:mm:ssZ - yyyy-MM-dd HH:mm:ss - yyyy-MM-dd HH:mmZ - yyyy-MM-dd HH:mm - yyyy-MM-dd - - - - - - - text/plain; charset=UTF-8 - - - - ${velocity.template.base.dir:} - ${velocity.solr.resource.loader.enabled:true} - ${velocity.params.resource.loader.enabled:false} - - - - 5 - - - diff --git a/compose/2.8/solr/solrconfig-2.9.7.xml b/compose/2.8/solr/solrconfig-2.9.7.xml deleted file mode 100644 index 8a3eade..0000000 --- a/compose/2.8/solr/solrconfig-2.9.7.xml +++ /dev/null @@ -1,345 +0,0 @@ - - - - - - 6.0.0 - - - - - - - - - - - ${solr.data.dir:} - - - - - - - ${solr.lock.type:native} - - - - - - - - ${solr.ulog.dir:} - ${solr.ulog.numVersionBuckets:65536} - - - - ${solr.autoCommit.maxTime:15000} - false - - - - ${solr.autoSoftCommit.maxTime:-1} - - - - - - - 1024 - - - - - true - 20 - 200 - - - - - - - - - - false - 2 - - - - - - - - - - - - - - explicit - 10 - - - - - - - - explicit - json - true - - - - - - - - explicit - - - - - - - - _text_ - - - - - - - - add-unknown-fields-to-the-schema - - - - - - - - true - ignored_ - _text_ - - - - - - - - - - - explicit - true - - - - - - - text_general - - - default - _text_ - solr.DirectSolrSpellChecker - internal - 0.5 - 2 - 1 - 5 - 4 - 0.01 - - - - - - - - default - on - true - 10 - 5 - 5 - true - true - 10 - 5 - - - - spellcheck - - - - - - - - - - true - - - tvComponent - - - - - - - - - - true - false - - - - terms - - - - - - - string - elevate.xml - - - - - - - explicit - - - elevator - - - - - - - - - - - 100 - - - - - - 70 - 0.5 - [-\w ,/\n\"']{20,200} - - - - - - ]]> - ]]> - - - - - - - - - - - - ,, - ,, - ,, - ,, - ,]]> - ]]> - - - - - - - 10 - .,!? - - - - - - WORD - en - US - - - - - - - - - true - - - - - - - - - [^\w-\.] - _ - - - - - - - yyyy-MM-dd'T'HH:mm:ss.SSSZ - yyyy-MM-dd'T'HH:mm:ss,SSSZ - yyyy-MM-dd'T'HH:mm:ss.SSS - yyyy-MM-dd'T'HH:mm:ss,SSS - yyyy-MM-dd'T'HH:mm:ssZ - yyyy-MM-dd'T'HH:mm:ss - yyyy-MM-dd'T'HH:mmZ - yyyy-MM-dd'T'HH:mm - yyyy-MM-dd HH:mm:ss.SSSZ - yyyy-MM-dd HH:mm:ss,SSSZ - yyyy-MM-dd HH:mm:ss.SSS - yyyy-MM-dd HH:mm:ss,SSS - yyyy-MM-dd HH:mm:ssZ - yyyy-MM-dd HH:mm:ss - yyyy-MM-dd HH:mmZ - yyyy-MM-dd HH:mm - yyyy-MM-dd - - - - - - - text/plain; charset=UTF-8 - - - - ${velocity.template.base.dir:} - ${velocity.solr.resource.loader.enabled:true} - ${velocity.params.resource.loader.enabled:false} - - - - 5 - - - diff --git a/compose/2.8/solr/solrconfig-2.9.8.xml b/compose/2.8/solr/solrconfig-2.9.8.xml deleted file mode 100644 index 9b90496..0000000 --- a/compose/2.8/solr/solrconfig-2.9.8.xml +++ /dev/null @@ -1,203 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -index_id - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/compose/2.9/.ckan-env b/compose/2.9/.ckan-env deleted file mode 100644 index c264a1c..0000000 --- a/compose/2.9/.ckan-env +++ /dev/null @@ -1,40 +0,0 @@ -# Runtime configuration of CKAN enabled through ckanext-envvars -# Information about how it works: https://github.com/okfn/ckanext-envvars -# Note that variables here take presedence over build/up time variables in .env - -# Set to true to disable CKAN from starting and serve a maintenance page -MAINTENANCE_MODE=false - -# General Settings -CKAN_SITE_ID=default -CKAN_SITE_URL=http://localhost:5000 -CKAN_PORT=5000 -CKAN_MAX_UPLOAD_SIZE_MB=10 -CKAN___BEAKER__SESSION__SECRET=CHANGE_ME -# See https://docs.ckan.org/en/latest/maintaining/configuration.html#api-token-settings -CKAN___API_TOKEN__JWT__ENCODE__SECRET=string:CHANGE_ME -CKAN___API_TOKEN__JWT__DECODE__SECRET=string:CHANGE_ME -# CKAN Plugins -CKAN__PLUGINS=envvars image_view text_view recline_view datastore datapusher -# CKAN requires storage path to be set in order for filestore to be enabled -CKAN__STORAGE_PATH=/srv/app/data -CKAN__WEBASSETS__PATH=/srv/app/data/webassets -# SYSADMIN settings, a sysadmin user is created automatically with the below credentials -CKAN_SYSADMIN_NAME=sysadmin -CKAN_SYSADMIN_PASSWORD=password -CKAN_SYSADMIN_EMAIL=sysadmin@ckantest.com - -# Email settings -CKAN_SMTP_SERVER=smtp.corporateict.domain:25 -CKAN_SMTP_STARTTLS=True -CKAN_SMTP_USER=user -CKAN_SMTP_PASSWORD=pass -CKAN_SMTP_MAIL_FROM=ckan@localhost - -# Datapusher configuration -CKAN__DATAPUSHER__URL=http://datapusher:8000 -CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000/ - -# Solr configuration -CKAN_VERSION=2.9.11 -CKAN_CORE_NAME=ckan diff --git a/compose/2.9/.env b/compose/2.9/.env deleted file mode 100644 index 2eba8d0..0000000 --- a/compose/2.9/.env +++ /dev/null @@ -1,35 +0,0 @@ -# Variables in this file will be used as build arguments when running -# docker-compose build and docker-compose up -# Verify correct substitution with "docker-compose config" -# If variables are newly added or enabled, please delete and rebuild the images to pull in changes: -# docker-compose down -v -# docker-compose build -# docker-compose up -d - -# Database -POSTGRES_PASSWORD=ckan -POSTGRES_PORT=5432 -DATASTORE_READONLY_PASSWORD=datastore - -# CKAN -CKAN_VERSION=2.9.11 -CKAN_SITE_ID=default -CKAN_SITE_URL=http://localhost:5000 -CKAN_PORT=5000 -CKAN_MAX_UPLOAD_SIZE_MB=10 - -# Datapusher -DATAPUSHER_VERSION=0.0.17 -DATAPUSHER_MAX_CONTENT_LENGTH=10485760 -DATAPUSHER_CHUNK_SIZE=16384 -DATAPUSHER_CHUNK_INSERT_ROWS=250 -DATAPUSHER_DOWNLOAD_TIMEOUT=30 -DATAPUSHER_SSL_VERIFY=False -DATAPUSHER_REWRITE_RESOURCES=True -DATAPUSHER_REWRITE_URL=http://ckan:5000 - -# SOLR -CKAN_CORE_NAME=ckan - -# Redis -REDIS_VERSION=6.0.7 diff --git a/compose/2.9/docker-compose.yml b/compose/2.9/docker-compose.yml deleted file mode 100644 index 95f693c..0000000 --- a/compose/2.9/docker-compose.yml +++ /dev/null @@ -1,95 +0,0 @@ -# docker-compose build && docker-compose up -d -version: "3" - -volumes: - ckan_data: - pg_data: - solr_data: - -services: - ckan: - container_name: ckan - image: ghcr.io/keitaroinc/ckan:${CKAN_VERSION} - networks: - - frontend - - backend - depends_on: - - db - - solr - ports: - - "0.0.0.0:${CKAN_PORT}:5000" - env_file: - - ./.ckan-env - environment: - - CKAN_SQLALCHEMY_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/ckan - - CKAN_DATASTORE_WRITE_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/datastore - - CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore - - CKAN_SOLR_URL=http://solr:8983/solr/ckan - - CKAN_REDIS_URL=redis://redis:6379/1 - - CKAN_SITE_URL=${CKAN_SITE_URL} - - CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB} - - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} - - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD} - volumes: - - ckan_data:/srv/app/data - - datapusher: - container_name: datapusher - image: ghcr.io/keitaroinc/datapusher:${DATAPUSHER_VERSION} - networks: - - frontend - - backend - ports: - - "8000:8000" - environment: - - DATAPUSHER_MAX_CONTENT_LENGTH=${DATAPUSHER_MAX_CONTENT_LENGTH} - - DATAPUSHER_CHUNK_SIZE=${DATAPUSHER_CHUNK_SIZE} - - DATAPUSHER_CHUNK_INSERT_ROWS=${DATAPUSHER_CHUNK_INSERT_ROWS} - - DATAPUSHER_DOWNLOAD_TIMEOUT=${DATAPUSHER_DOWNLOAD_TIMEOUT} - - DATAPUSHER_SSL_VERIFY=${DATA_PUSHER_SSL_VERIFY} - - DATAPUSHER_REWRITE_RESOURCES=${DATAPUSHER_REWRITE_RESOURCES} - - DATAPUSHER_REWRITE_URL=${DATAPUSHER_REWRITE_URL} - - - db: - container_name: db - build: - context: . - dockerfile: postgresql/Dockerfile - args: - - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD} - - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} - networks: - - backend - environment: - - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD} - - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} - - PGDATA=/var/lib/postgresql/data/db - volumes: - - pg_data:/var/lib/postgresql/data - healthcheck: - test: ["CMD", "pg_isready", "-U", "ckan"] - - solr: - container_name: solr - image: solr:8.11.1 - networks: - - backend - env_file: - - ./.ckan-env - environment: - - CKAN_CORE_NAME=${CKAN_CORE_NAME} - - CKAN_VERSION=${CKAN_VERSION} - volumes: - - solr_data:/var/solr - - ${PWD}/solr8/ckan_init_solr.sh:/docker-entrypoint-initdb.d/ckan_init_solr.sh - - redis: - container_name: redis - image: redis:${REDIS_VERSION} - networks: - - backend - -networks: - frontend: - backend: diff --git a/compose/2.9/postgresql/Dockerfile b/compose/2.9/postgresql/Dockerfile deleted file mode 100644 index e5fcbb7..0000000 --- a/compose/2.9/postgresql/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -FROM postgis/postgis:14-3.2-alpine - -# Allow connections; we don't map out any ports so only linked docker containers can connect -RUN echo "host all all 0.0.0.0/0 md5" >> /var/lib/postgresql/data/pg_hba.conf - -# Customize default user/pass/db -ENV POSTGRES_DB ckan -ENV POSTGRES_USER ckan -ARG POSTGRES_PASSWORD -ARG DS_RO_PASS - -# Include datastore setup scripts -COPY ./postgresql/docker-entrypoint-initdb.d /docker-entrypoint-initdb.d diff --git a/compose/2.9/postgresql/docker-entrypoint-initdb.d/00_create_datastore.sh b/compose/2.9/postgresql/docker-entrypoint-initdb.d/00_create_datastore.sh deleted file mode 100644 index ec1b3c3..0000000 --- a/compose/2.9/postgresql/docker-entrypoint-initdb.d/00_create_datastore.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -set -e - -psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" <<-EOSQL - CREATE ROLE datastore_ro NOSUPERUSER NOCREATEDB NOCREATEROLE LOGIN PASSWORD '$DS_RO_PASS'; - CREATE DATABASE datastore OWNER ckan ENCODING 'utf-8'; - GRANT ALL PRIVILEGES ON DATABASE datastore TO ckan; -EOSQL \ No newline at end of file diff --git a/compose/2.9/postgresql/docker-entrypoint-initdb.d/20_postgis_permissions.sql b/compose/2.9/postgresql/docker-entrypoint-initdb.d/20_postgis_permissions.sql deleted file mode 100644 index 8b5348e..0000000 --- a/compose/2.9/postgresql/docker-entrypoint-initdb.d/20_postgis_permissions.sql +++ /dev/null @@ -1,2 +0,0 @@ -ALTER VIEW geometry_columns OWNER TO ckan; -ALTER TABLE spatial_ref_sys OWNER TO ckan; diff --git a/compose/composes/.composes-env b/compose/composes/.composes-env new file mode 100644 index 0000000..d9b43f0 --- /dev/null +++ b/compose/composes/.composes-env @@ -0,0 +1 @@ +CKAN_VERSION=2.11.0 diff --git a/compose/2.11/composes/backend-services/.datapusher-env b/compose/composes/backend-services/.datapusher-env similarity index 100% rename from compose/2.11/composes/backend-services/.datapusher-env rename to compose/composes/backend-services/.datapusher-env diff --git a/compose/2.11/composes/backend-services/.env b/compose/composes/backend-services/.env similarity index 100% rename from compose/2.11/composes/backend-services/.env rename to compose/composes/backend-services/.env diff --git a/compose/2.11/composes/backend-services/.solr-env b/compose/composes/backend-services/.solr-env similarity index 100% rename from compose/2.11/composes/backend-services/.solr-env rename to compose/composes/backend-services/.solr-env diff --git a/compose/2.11/composes/backend-services/backend-services.yaml b/compose/composes/backend-services/backend-services.yaml similarity index 99% rename from compose/2.11/composes/backend-services/backend-services.yaml rename to compose/composes/backend-services/backend-services.yaml index 6e3cf65..1c7317b 100644 --- a/compose/2.11/composes/backend-services/backend-services.yaml +++ b/compose/composes/backend-services/backend-services.yaml @@ -55,4 +55,4 @@ services: volumes: pg_data: - solr_data: \ No newline at end of file + solr_data: diff --git a/compose/2.11/composes/ckan/.env b/compose/composes/ckan/.env similarity index 100% rename from compose/2.11/composes/ckan/.env rename to compose/composes/ckan/.env diff --git a/compose/2.11/composes/ckan/ckan.yaml b/compose/composes/ckan/ckan.yaml similarity index 100% rename from compose/2.11/composes/ckan/ckan.yaml rename to compose/composes/ckan/ckan.yaml diff --git a/compose/2.11/docker-compose.yml b/compose/docker-compose.yml similarity index 100% rename from compose/2.11/docker-compose.yml rename to compose/docker-compose.yml diff --git a/compose/2.11/postgresql/Dockerfile b/compose/postgresql/Dockerfile similarity index 100% rename from compose/2.11/postgresql/Dockerfile rename to compose/postgresql/Dockerfile diff --git a/compose/2.11/postgresql/docker-entrypoint-initdb.d/00_create_datastore.sh b/compose/postgresql/docker-entrypoint-initdb.d/00_create_datastore.sh similarity index 100% rename from compose/2.11/postgresql/docker-entrypoint-initdb.d/00_create_datastore.sh rename to compose/postgresql/docker-entrypoint-initdb.d/00_create_datastore.sh diff --git a/compose/2.10/postgresql/docker-entrypoint-initdb.d/20_postgis_permissions.sql b/compose/postgresql/docker-entrypoint-initdb.d/20_postgis_permissions.sql similarity index 100% rename from compose/2.10/postgresql/docker-entrypoint-initdb.d/20_postgis_permissions.sql rename to compose/postgresql/docker-entrypoint-initdb.d/20_postgis_permissions.sql diff --git a/compose/2.10/solr8/ckan_init_solr.sh b/compose/solr8/ckan_init_solr.sh similarity index 100% rename from compose/2.10/solr8/ckan_init_solr.sh rename to compose/solr8/ckan_init_solr.sh diff --git a/compose/2.9/solr8/ckan_init_solr.sh b/compose/solr8/ckan_init_solr_2_9.sh similarity index 100% rename from compose/2.9/solr8/ckan_init_solr.sh rename to compose/solr8/ckan_init_solr_2_9.sh From 935a8c64e8ddc689280065b3ef36d31ce7f42345 Mon Sep 17 00:00:00 2001 From: Kiril-Poposki1998 Date: Fri, 6 Dec 2024 15:30:02 +0100 Subject: [PATCH 4/7] Change compose global env --- compose/composes/{.composes-env => .global-env} | 0 compose/composes/backend-services/backend-services.yaml | 2 +- compose/composes/ckan/ckan.yaml | 1 - compose/docker-compose.yml | 2 +- 4 files changed, 2 insertions(+), 3 deletions(-) rename compose/composes/{.composes-env => .global-env} (100%) diff --git a/compose/composes/.composes-env b/compose/composes/.global-env similarity index 100% rename from compose/composes/.composes-env rename to compose/composes/.global-env diff --git a/compose/composes/backend-services/backend-services.yaml b/compose/composes/backend-services/backend-services.yaml index 1c7317b..d4d52ab 100644 --- a/compose/composes/backend-services/backend-services.yaml +++ b/compose/composes/backend-services/backend-services.yaml @@ -29,7 +29,7 @@ services: networks: - backend env_file: - - ${PWD}/composes/.composes-env + - ${PWD}/composes/.global-env - ${PWD}/composes/backend-services/.solr-env volumes: - solr_data:/var/solr diff --git a/compose/composes/ckan/ckan.yaml b/compose/composes/ckan/ckan.yaml index 30c79b7..2af9b2d 100644 --- a/compose/composes/ckan/ckan.yaml +++ b/compose/composes/ckan/ckan.yaml @@ -15,7 +15,6 @@ services: ports: - "0.0.0.0:${CKAN_PORT}:5000" env_file: - - ${PWD}/composes/.composes-env - ${PWD}/composes/backend-services/.env - ${PWD}/composes/ckan/.env volumes: diff --git a/compose/docker-compose.yml b/compose/docker-compose.yml index 9693f30..9cbb867 100644 --- a/compose/docker-compose.yml +++ b/compose/docker-compose.yml @@ -3,7 +3,7 @@ include: - path: composes/ckan/ckan.yaml env_file: - composes/ckan/.env - - composes/.composes-env + - composes/.global-env project_directory: . - path: composes/backend-services/backend-services.yaml env_file: composes/backend-services/.env From bb8753bef92a91d909a585e9da1d2818990ae9b4 Mon Sep 17 00:00:00 2001 From: Kiril-Poposki1998 Date: Tue, 17 Dec 2024 11:25:00 +0100 Subject: [PATCH 5/7] Change folder for compose configuration --- compose/composes/.global-env | 1 - compose/config/.global-env | 1 + .../backend-services/.datapusher-env | 0 compose/{composes => config}/backend-services/.env | 0 .../{composes => config}/backend-services/.solr-env | 0 .../backend-services/backend-services.yaml | 8 ++++---- compose/{composes => config}/ckan/.env | 0 compose/{composes => config}/ckan/ckan.yaml | 4 ++-- compose/docker-compose.yml | 10 +++++----- 9 files changed, 12 insertions(+), 12 deletions(-) delete mode 100644 compose/composes/.global-env create mode 100644 compose/config/.global-env rename compose/{composes => config}/backend-services/.datapusher-env (100%) rename compose/{composes => config}/backend-services/.env (100%) rename compose/{composes => config}/backend-services/.solr-env (100%) rename compose/{composes => config}/backend-services/backend-services.yaml (85%) rename compose/{composes => config}/ckan/.env (100%) rename compose/{composes => config}/ckan/ckan.yaml (84%) diff --git a/compose/composes/.global-env b/compose/composes/.global-env deleted file mode 100644 index d9b43f0..0000000 --- a/compose/composes/.global-env +++ /dev/null @@ -1 +0,0 @@ -CKAN_VERSION=2.11.0 diff --git a/compose/config/.global-env b/compose/config/.global-env new file mode 100644 index 0000000..9328408 --- /dev/null +++ b/compose/config/.global-env @@ -0,0 +1 @@ +CKAN_VERSION=2.11.1 diff --git a/compose/composes/backend-services/.datapusher-env b/compose/config/backend-services/.datapusher-env similarity index 100% rename from compose/composes/backend-services/.datapusher-env rename to compose/config/backend-services/.datapusher-env diff --git a/compose/composes/backend-services/.env b/compose/config/backend-services/.env similarity index 100% rename from compose/composes/backend-services/.env rename to compose/config/backend-services/.env diff --git a/compose/composes/backend-services/.solr-env b/compose/config/backend-services/.solr-env similarity index 100% rename from compose/composes/backend-services/.solr-env rename to compose/config/backend-services/.solr-env diff --git a/compose/composes/backend-services/backend-services.yaml b/compose/config/backend-services/backend-services.yaml similarity index 85% rename from compose/composes/backend-services/backend-services.yaml rename to compose/config/backend-services/backend-services.yaml index d4d52ab..58b799e 100644 --- a/compose/composes/backend-services/backend-services.yaml +++ b/compose/config/backend-services/backend-services.yaml @@ -8,7 +8,7 @@ services: ports: - "8000:8000" env_file: - - ${PWD}/composes/backend-services/.datapusher-env + - ${PWD}/config/backend-services/.datapusher-env db: container_name: db @@ -17,7 +17,7 @@ services: networks: - backend env_file: - - ${PWD}/composes/backend-services/.env + - ${PWD}/config/backend-services/.env volumes: - pg_data:/var/lib/postgresql/data healthcheck: @@ -29,8 +29,8 @@ services: networks: - backend env_file: - - ${PWD}/composes/.global-env - - ${PWD}/composes/backend-services/.solr-env + - ${PWD}/config/.global-env + - ${PWD}/config/backend-services/.solr-env volumes: - solr_data:/var/solr - ${PWD}/solr8/ckan_init_solr.sh:/docker-entrypoint-initdb.d/ckan_init_solr.sh diff --git a/compose/composes/ckan/.env b/compose/config/ckan/.env similarity index 100% rename from compose/composes/ckan/.env rename to compose/config/ckan/.env diff --git a/compose/composes/ckan/ckan.yaml b/compose/config/ckan/ckan.yaml similarity index 84% rename from compose/composes/ckan/ckan.yaml rename to compose/config/ckan/ckan.yaml index 2af9b2d..211bdb0 100644 --- a/compose/composes/ckan/ckan.yaml +++ b/compose/config/ckan/ckan.yaml @@ -15,8 +15,8 @@ services: ports: - "0.0.0.0:${CKAN_PORT}:5000" env_file: - - ${PWD}/composes/backend-services/.env - - ${PWD}/composes/ckan/.env + - ${PWD}/config/backend-services/.env + - ${PWD}/config/ckan/.env volumes: - ckan_data:/srv/app/data diff --git a/compose/docker-compose.yml b/compose/docker-compose.yml index 9cbb867..b2fabd1 100644 --- a/compose/docker-compose.yml +++ b/compose/docker-compose.yml @@ -1,12 +1,12 @@ # docker-compose build && docker-compose up -d include: - - path: composes/ckan/ckan.yaml + - path: config/ckan/ckan.yaml env_file: - - composes/ckan/.env - - composes/.global-env + - config/ckan/.env + - config/.global-env project_directory: . - - path: composes/backend-services/backend-services.yaml - env_file: composes/backend-services/.env + - path: config/backend-services/backend-services.yaml + env_file: config/backend-services/.env project_directory: . networks: From e64908804e5e2e63b203f3abc25756eb3e354911 Mon Sep 17 00:00:00 2001 From: Kiril-Poposki1998 Date: Tue, 17 Dec 2024 13:47:44 +0100 Subject: [PATCH 6/7] Restructure compose with global-env as version managing --- compose/config/.global-env | 10 +++++ .../backend-services/{.env => .postgres-env} | 5 --- .../backend-services/backend-services.yaml | 9 ++-- compose/config/ckan/.env | 5 --- compose/config/ckan/ckan.yaml | 2 +- compose/docker-compose.yml | 4 +- compose/makefile | 9 ++++ compose/solr8/ckan_init_solr.sh | 41 ------------------- compose/solr8/ckan_init_solr_2_9.sh | 38 ----------------- 9 files changed, 29 insertions(+), 94 deletions(-) rename compose/config/backend-services/{.env => .postgres-env} (65%) create mode 100644 compose/makefile delete mode 100755 compose/solr8/ckan_init_solr.sh delete mode 100755 compose/solr8/ckan_init_solr_2_9.sh diff --git a/compose/config/.global-env b/compose/config/.global-env index 9328408..1041231 100644 --- a/compose/config/.global-env +++ b/compose/config/.global-env @@ -1 +1,11 @@ +#CKAN CKAN_VERSION=2.11.1 + +# Redis +REDIS_VERSION=6.0.7 + +# Datapusher +DATAPUSHER_VERSION=0.0.17 + +# Solr +SOLR_VERSION=9 \ No newline at end of file diff --git a/compose/config/backend-services/.env b/compose/config/backend-services/.postgres-env similarity index 65% rename from compose/config/backend-services/.env rename to compose/config/backend-services/.postgres-env index c441b38..2111e3b 100644 --- a/compose/config/backend-services/.env +++ b/compose/config/backend-services/.postgres-env @@ -4,8 +4,3 @@ POSTGRES_PORT=5432 DATASTORE_READONLY_PASSWORD=datastore PGDATA=/var/lib/postgresql/data/db -# Redis -REDIS_VERSION=6.0.7 - -# Datapusher -DATAPUSHER_VERSION=0.0.17 \ No newline at end of file diff --git a/compose/config/backend-services/backend-services.yaml b/compose/config/backend-services/backend-services.yaml index 58b799e..4295091 100644 --- a/compose/config/backend-services/backend-services.yaml +++ b/compose/config/backend-services/backend-services.yaml @@ -17,7 +17,7 @@ services: networks: - backend env_file: - - ${PWD}/config/backend-services/.env + - ${PWD}/config/backend-services/.postgres-env volumes: - pg_data:/var/lib/postgresql/data healthcheck: @@ -25,7 +25,11 @@ services: solr: container_name: solr - image: solr:8.11.1 + build: + context: ${PWD}/solr + args: + CKAN_BRANCH: ckan-${CKAN_VERSION} + SOLR_VERSION: ${SOLR_VERSION} networks: - backend env_file: @@ -33,7 +37,6 @@ services: - ${PWD}/config/backend-services/.solr-env volumes: - solr_data:/var/solr - - ${PWD}/solr8/ckan_init_solr.sh:/docker-entrypoint-initdb.d/ckan_init_solr.sh healthcheck: test: ["CMD-SHELL", "/opt/solr/bin/solr","status"] interval: 5s diff --git a/compose/config/ckan/.env b/compose/config/ckan/.env index c3ee0a9..d1884b0 100644 --- a/compose/config/ckan/.env +++ b/compose/config/ckan/.env @@ -4,16 +4,12 @@ #Compose variables # CKAN -CKAN_SITE_ID=default -CKAN_SITE_URL=http://localhost:5000 -CKAN_PORT=5000 CKAN_MAX_UPLOAD_SIZE_MB=10 CKAN_SQLALCHEMY_URL=postgresql://ckan:${POSTGRES_PASSWORD:-ckan}@db/ckan CKAN_DATASTORE_WRITE_URL=postgresql://ckan:${POSTGRES_PASSWORD:-ckan}@db/datastore CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD:-datastore}@db/datastore CKAN_SOLR_URL=http://solr:8983/solr/ckan CKAN_REDIS_URL=redis://redis:6379/1 -CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB} # Set to true to disable CKAN from starting and serve a maintenance page @@ -23,7 +19,6 @@ MAINTENANCE_MODE=false CKAN_SITE_ID=default CKAN_SITE_URL=http://localhost:5000 CKAN_PORT=5000 -CKAN_MAX_UPLOAD_SIZE_MB=20 CKAN___BEAKER__SESSION__SECRET=CHANGE_ME # See https://docs.ckan.org/en/latest/maintaining/configuration.html#api-token-settings CKAN___API_TOKEN__JWT__ENCODE__SECRET=string:CHANGE_ME diff --git a/compose/config/ckan/ckan.yaml b/compose/config/ckan/ckan.yaml index 211bdb0..9765429 100644 --- a/compose/config/ckan/ckan.yaml +++ b/compose/config/ckan/ckan.yaml @@ -15,7 +15,7 @@ services: ports: - "0.0.0.0:${CKAN_PORT}:5000" env_file: - - ${PWD}/config/backend-services/.env + - ${PWD}/config/backend-services/.postgres-env - ${PWD}/config/ckan/.env volumes: - ckan_data:/srv/app/data diff --git a/compose/docker-compose.yml b/compose/docker-compose.yml index b2fabd1..846d549 100644 --- a/compose/docker-compose.yml +++ b/compose/docker-compose.yml @@ -6,7 +6,9 @@ include: - config/.global-env project_directory: . - path: config/backend-services/backend-services.yaml - env_file: config/backend-services/.env + env_file: + - config/backend-services/.postgres-env + - config/.global-env project_directory: . networks: diff --git a/compose/makefile b/compose/makefile new file mode 100644 index 0000000..8c5300d --- /dev/null +++ b/compose/makefile @@ -0,0 +1,9 @@ +start: + docker compose build + docker compose up -d + +prune: + docker compose down + docker volume rm compose_ckan_data + docker volume rm compose_solr_data + docker volume rm compose_pg_data \ No newline at end of file diff --git a/compose/solr8/ckan_init_solr.sh b/compose/solr8/ckan_init_solr.sh deleted file mode 100755 index 091450f..0000000 --- a/compose/solr8/ckan_init_solr.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -# -# Initialize SOLR for CKAN by creating a ckan core -# Arguments are supplied via environment variables: CKAN_CORE_NAME CKAN_VERSION -# Example: -# CKAN_CORE_NAME=ckan -# CKAN_VERSION=2.9.7 - -set -e - -MAJOR_VERSION=`echo $CKAN_VERSION | cut -d "." -f 1` -MINOR_VERSION=`echo $CKAN_VERSION | cut -d "." -f 2` - -CKAN_SOLR_SCHEMA_URL=https://raw.githubusercontent.com/ckan/ckan/`echo $MAJOR_VERSION.$MINOR_VERSION`/ckan/config/solr/schema.xml - -echo "Check whether managed schema exists for CKAN $CKAN_VERSION" -if ! curl --output /dev/null --silent --head --fail "$CKAN_SOLR_SCHEMA_URL"; then - echo "Can't find CKAN SOLR schema at URL: $CKAN_SOLR_SCHEMA_URL. Exiting..." - exit 1 -fi - -echo "Check whether SOLR is initialized for CKAN" -CORESDIR=/var/solr/data - -COREDIR="$CORESDIR/$CKAN_CORE_NAME" -if [ -d "$COREDIR" ]; then - echo "SOLR already initialized, skipping initialization" -else - echo "Initializing SOLR core $CKAN_CORE_NAME for CKAN $CKAN_VERSION" - # init script for handling an empty /var/solr - /opt/docker-solr/scripts/init-var-solr - - # Precreate CKAN core - /opt/docker-solr/scripts/precreate-core $CKAN_CORE_NAME - - # Replace the managed schema with CKANs schema - echo "Adding CKAN managed schema" - curl $CKAN_SOLR_SCHEMA_URL -o /var/solr/data/$CKAN_CORE_NAME/conf/managed-schema -s - - echo "SOLR initialized" -fi diff --git a/compose/solr8/ckan_init_solr_2_9.sh b/compose/solr8/ckan_init_solr_2_9.sh deleted file mode 100755 index 1f8d0c9..0000000 --- a/compose/solr8/ckan_init_solr_2_9.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash -# -# Initialize SOLR for CKAN by creating a ckan core -# Arguments are supplied via environment variables: CKAN_CORE_NAME CKAN_VERSION -# Example: -# CKAN_CORE_NAME=ckan -# CKAN_VERSION=2.9.11 - -set -e - -CKAN_SOLR_SCHEMA_URL=https://raw.githubusercontent.com/ckan/ckan/ckan-$CKAN_VERSION/ckan/config/solr/schema.solr8.xml - -echo "Check whether managed schema exists for CKAN $CKAN_VERSION" -if ! curl --output /dev/null --silent --head --fail "$CKAN_SOLR_SCHEMA_URL"; then - echo "Can't find CKAN SOLR schema at URL: $CKAN_SOLR_SCHEMA_URL. Exiting..." - exit 1 -fi - -echo "Check whether SOLR is initialized for CKAN" -CORESDIR=/var/solr/data - -COREDIR="$CORESDIR/$CKAN_CORE_NAME" -if [ -d "$COREDIR" ]; then - echo "SOLR already initialized, skipping initialization" -else - echo "Initializing SOLR core $CKAN_CORE_NAME for CKAN $CKAN_VERSION" - # init script for handling an empty /var/solr - /opt/docker-solr/scripts/init-var-solr - - # Precreate CKAN core - /opt/docker-solr/scripts/precreate-core $CKAN_CORE_NAME - - # Replace the managed schema with CKANs schema - echo "Adding CKAN managed schema" - curl $CKAN_SOLR_SCHEMA_URL -o /var/solr/data/$CKAN_CORE_NAME/conf/managed-schema -s - - echo "SOLR initialized" -fi From 5e066497287a1ce7eca41072cddfb0f3caec3d61 Mon Sep 17 00:00:00 2001 From: Kiril-Poposki1998 Date: Tue, 17 Dec 2024 13:47:56 +0100 Subject: [PATCH 7/7] Add solr dockerfile --- compose/solr/Dockerfile | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 compose/solr/Dockerfile diff --git a/compose/solr/Dockerfile b/compose/solr/Dockerfile new file mode 100644 index 0000000..a42ae8a --- /dev/null +++ b/compose/solr/Dockerfile @@ -0,0 +1,23 @@ +ARG SOLR_VERSION + +FROM solr:${SOLR_VERSION:-9} + +EXPOSE 8983 + +ARG CKAN_BRANCH + +ENV SOLR_CONFIG_DIR="/opt/solr/server/solr/configsets" +ENV SOLR_SCHEMA_FILE="$SOLR_CONFIG_DIR/ckan/conf/managed-schema" + +USER root + +# Create a CKAN configset by copying the default one +RUN cp -R $SOLR_CONFIG_DIR/_default $SOLR_CONFIG_DIR/ckan + +# Update the schema +ADD https://raw.githubusercontent.com/ckan/ckan/$CKAN_BRANCH/ckan/config/solr/schema.xml $SOLR_SCHEMA_FILE +RUN chmod 644 $SOLR_SCHEMA_FILE + +USER solr + +CMD ["sh", "-c", "solr-precreate ckan $SOLR_CONFIG_DIR/ckan"] \ No newline at end of file