From 674df187e58e5b9d2540ac861c75a6fe197a3f4b Mon Sep 17 00:00:00 2001 From: Xavier Frankline Date: Thu, 23 Mar 2023 08:57:22 +0300 Subject: [PATCH 01/20] bump ckan to 2.8.11 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 150ae78..c8b433e 100755 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM codeforafrica/ckan:2.8.3 +FROM codeforafrica/ckan:2.8.11 EXPOSE 5000/tcp From 5bf9419e11e1c406068a190b89cdb3b2b426df2d Mon Sep 17 00:00:00 2001 From: Xavier Frankline Date: Mon, 3 Apr 2023 09:21:20 +0300 Subject: [PATCH 02/20] UI changes --- Dockerfile | 2 +- Makefile | 2 +- contrib/ckan/Dockerfile | 2 +- docker-compose.yml | 31 ++++++++++++++++--------------- requirements.txt | 3 ++- 5 files changed, 21 insertions(+), 19 deletions(-) diff --git a/Dockerfile b/Dockerfile index c8b433e..150ae78 100755 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM codeforafrica/ckan:2.8.11 +FROM codeforafrica/ckan:2.8.3 EXPOSE 5000/tcp diff --git a/Makefile b/Makefile index 88c6b8d..2e9b985 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ issues-init: ckan: - docker build --no-cache --build-arg CKAN_VERSION=2.8.11 -t codeforafrica/ckan:latest -t codeforafrica/ckan:2.8.11 contrib/ckan + docker build --no-cache --build-arg CKAN_VERSION=2.8.3 -t codeforafrica/ckan:latest -t codeforafrica/ckan:2.8.11 contrib/ckan ckan-publish: docker push codeforafrica/ckan:latest diff --git a/contrib/ckan/Dockerfile b/contrib/ckan/Dockerfile index 3f64506..c3a18e9 100644 --- a/contrib/ckan/Dockerfile +++ b/contrib/ckan/Dockerfile @@ -2,7 +2,7 @@ FROM python:2.7 ENV DEBIAN_FRONTEND noninteractive -ARG CKAN_VERSION=2.8.11 +ARG CKAN_VERSION=2.8.3 RUN pip install -U -q pip setuptools diff --git a/docker-compose.yml b/docker-compose.yml index ef05cb1..e5383df 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,7 @@ services: web: build: . depends_on: - - db + # - db - solr - redis ports: @@ -20,7 +20,7 @@ services: ckan_gather_consumer: build: . depends_on: - - db + # - db - solr - redis env_file: @@ -36,19 +36,20 @@ services: depends_on: - web - db: - image: postgres:13.7 - ports: - - "54321:5432" - environment: - - POSTGRES_USER=ckan_default - - POSTGRES_PASSWORD=pass - - PGUSER=ckan_default - - PGPASSWORD=pass - - POSTGRES_ADD_DATABASES=datastore_default:datastore_default - volumes: - - postgres-data:/var/lib/postgresql/data - - ./contrib/postgres:/docker-entrypoint-initdb.d + # db: + # image: postgres:13.7 + # ports: + # - "54321:5432" + # environment: + # - POSTGRES_USER=ckan_default + # - POSTGRES_PASSWORD=pass + # - PGUSER=ckan_default + # - PGPASSWORD=pass + # - POSTGRES_ADD_DATABASES=datastore_default:datastore_default + # volumes: + # - postgres-data:/var/lib/postgresql/data + # - ./contrib/postgres:/docker-entrypoint-initdb.d + # "postgresql://ckan_default:ckan_default@ckan-test.cfgmtx8ishfx.eu-west-1.rds.amazonaws.com:5432/ckan" solr: image: codeforafrica/ckan-solr:2.8.11 diff --git a/requirements.txt b/requirements.txt index 0d049fa..10af81a 100755 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,8 @@ gevent==1.2.2 # CKAN Extensions --e git+https://github.com/CodeForAfrica/ckanext-openafrica.git#egg=ckanext-openafrica +# -e git+https://github.com/CodeForAfrica/ckanext-openafrica.git#egg=ckanext-openafrica +-e git+https://github.com/CodeForAfrica/ckanext-openafrica.git@ft/ui-changes#egg=ckanext-openafrica -e git+https://github.com/okfn/ckanext-envvars.git@0.0.1#egg=ckanext-envvars -e git+https://github.com/okfn/ckanext-s3filestore.git@v0.1.1#egg=ckanext-s3filestore From 533a3b16618b68bb1ba9246aa9fb72dfd94cd76a Mon Sep 17 00:00:00 2001 From: Xavier Frankline Date: Mon, 3 Apr 2023 09:30:03 +0300 Subject: [PATCH 03/20] update requirements --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 10af81a..002290d 100755 --- a/requirements.txt +++ b/requirements.txt @@ -3,8 +3,8 @@ gevent==1.2.2 # CKAN Extensions -# -e git+https://github.com/CodeForAfrica/ckanext-openafrica.git#egg=ckanext-openafrica -e git+https://github.com/CodeForAfrica/ckanext-openafrica.git@ft/ui-changes#egg=ckanext-openafrica +# -e git+https://github.com/CodeForAfrica/ckanext-openafrica.git#egg=ckanext-openafrica -e git+https://github.com/okfn/ckanext-envvars.git@0.0.1#egg=ckanext-envvars -e git+https://github.com/okfn/ckanext-s3filestore.git@v0.1.1#egg=ckanext-s3filestore From 98cd1b7f1a26443c3385bc9ea0a1feb1615f9edd Mon Sep 17 00:00:00 2001 From: Xavier Frankline Date: Thu, 13 Apr 2023 09:54:13 +0300 Subject: [PATCH 04/20] update requirements --- requirements.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 002290d..36b98d1 100755 --- a/requirements.txt +++ b/requirements.txt @@ -3,9 +3,11 @@ gevent==1.2.2 # CKAN Extensions --e git+https://github.com/CodeForAfrica/ckanext-openafrica.git@ft/ui-changes#egg=ckanext-openafrica +# -e git+https://github.com/CodeForAfrica/ckanext-openafrica.git@ft/ui-changes#egg=ckanext-openafrica # -e git+https://github.com/CodeForAfrica/ckanext-openafrica.git#egg=ckanext-openafrica +-e git+https://github.com/CodeForAfrica/ckanext-openafrica.git@ft/ui-changes#egg=ckanext-openafrica + -e git+https://github.com/okfn/ckanext-envvars.git@0.0.1#egg=ckanext-envvars -e git+https://github.com/okfn/ckanext-s3filestore.git@v0.1.1#egg=ckanext-s3filestore From 386c234faca9e8c813adf2d89784ec6bbd378cc2 Mon Sep 17 00:00:00 2001 From: Xavier Frankline Date: Thu, 13 Apr 2023 17:14:09 +0300 Subject: [PATCH 05/20] update requirements --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 36b98d1..d7eac24 100755 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ gevent==1.2.2 # -e git+https://github.com/CodeForAfrica/ckanext-openafrica.git@ft/ui-changes#egg=ckanext-openafrica # -e git+https://github.com/CodeForAfrica/ckanext-openafrica.git#egg=ckanext-openafrica --e git+https://github.com/CodeForAfrica/ckanext-openafrica.git@ft/ui-changes#egg=ckanext-openafrica +-e git+https://github.com/CodeForAfrica/ckanext-openafrica.git#egg=ckanext-openafrica -e git+https://github.com/okfn/ckanext-envvars.git@0.0.1#egg=ckanext-envvars -e git+https://github.com/okfn/ckanext-s3filestore.git@v0.1.1#egg=ckanext-s3filestore From e2871a3daf556885d73aeda0507a6b84b84c8fec Mon Sep 17 00:00:00 2001 From: Xavier Frankline Date: Mon, 17 Apr 2023 09:11:59 +0300 Subject: [PATCH 06/20] fix UI changes --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index d7eac24..93faa9d 100755 --- a/requirements.txt +++ b/requirements.txt @@ -3,10 +3,10 @@ gevent==1.2.2 # CKAN Extensions -# -e git+https://github.com/CodeForAfrica/ckanext-openafrica.git@ft/ui-changes#egg=ckanext-openafrica +-e git+https://github.com/CodeForAfrica/ckanext-openafrica.git@ft/ui-changes#egg=ckanext-openafrica # -e git+https://github.com/CodeForAfrica/ckanext-openafrica.git#egg=ckanext-openafrica --e git+https://github.com/CodeForAfrica/ckanext-openafrica.git#egg=ckanext-openafrica +# -e git+https://github.com/CodeForAfrica/ckanext-openafrica.git#egg=ckanext-openafrica -e git+https://github.com/okfn/ckanext-envvars.git@0.0.1#egg=ckanext-envvars -e git+https://github.com/okfn/ckanext-s3filestore.git@v0.1.1#egg=ckanext-s3filestore From 89fd5d029c662bf217accad8a3e0702125edc361 Mon Sep 17 00:00:00 2001 From: Xavier Frankline Date: Mon, 12 Jun 2023 12:40:20 +0300 Subject: [PATCH 07/20] upgrade ckan to 2.9.0 & python 3.8 --- Dockerfile | 2 +- Makefile | 6 +++--- Procfile | 8 +++++--- contrib/ckan/Dockerfile | 6 +++--- contrib/solr/Dockerfile | 2 +- docker-compose.yml | 4 ++-- requirements.txt | 4 ++-- 7 files changed, 17 insertions(+), 15 deletions(-) diff --git a/Dockerfile b/Dockerfile index 150ae78..e9b14d0 100755 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM codeforafrica/ckan:2.8.3 +FROM codeforafrica/ckan:2.9.0 EXPOSE 5000/tcp diff --git a/Makefile b/Makefile index 2e9b985..87ca0fd 100644 --- a/Makefile +++ b/Makefile @@ -24,14 +24,14 @@ issues-init: ckan: - docker build --no-cache --build-arg CKAN_VERSION=2.8.3 -t codeforafrica/ckan:latest -t codeforafrica/ckan:2.8.11 contrib/ckan + docker build --no-cache --build-arg CKAN_VERSION=2.9.0 -t codeforafrica/ckan:latest -t codeforafrica/ckan:2.9.0 contrib/ckan ckan-publish: docker push codeforafrica/ckan:latest - docker push codeforafrica/ckan:2.8.11 + docker push codeforafrica/ckan:2.9.0 solr: - docker build --no-cache --build-arg CKAN_VERSION=2.8.11 -t codeforafrica/ckan-solr:latest -t codeforafrica/ckan-solr:2.8.11 contrib/solr + docker build --no-cache --build-arg CKAN_VERSION=2.9.0 -t codeforafrica/ckan-solr:latest -t codeforafrica/ckan-solr:2.9.0 contrib/solr solr-publish: docker push codeforafrica/ckan-solr:latest diff --git a/Procfile b/Procfile index fc1331a..036ee7d 100644 --- a/Procfile +++ b/Procfile @@ -1,3 +1,5 @@ -web: gunicorn --workers 5 --worker-class gevent --paste ckan.ini -t 10800 -ckan_gather_consumer: paster --plugin=ckanext-harvest harvester gather_consumer --config=ckan.ini -ckan_fetch_consumer: paster --plugin=ckanext-harvest harvester fetch_consumer --config=ckan.ini +web: gunicorn --workers 5 --worker-class gevent --ckan ckan.ini -t 10800 + +ckan_gather_consumer: ckan --config=ckan.ini --plugin=ckanext-harvest harvester gather_consumer + +ckan_fetch_consumer: ckan --config=ckan.ini --plugin=ckanext-harvest harvester fetch_consumer diff --git a/contrib/ckan/Dockerfile b/contrib/ckan/Dockerfile index c3a18e9..4c52837 100644 --- a/contrib/ckan/Dockerfile +++ b/contrib/ckan/Dockerfile @@ -1,10 +1,10 @@ -FROM python:2.7 +FROM python:3.8 ENV DEBIAN_FRONTEND noninteractive -ARG CKAN_VERSION=2.8.3 +ARG CKAN_VERSION=2.9.0 -RUN pip install -U -q pip setuptools +RUN pip install -U -q pip "setuptools<46.0.0" RUN pip install -q -e "git+https://github.com/ckan/ckan.git@ckan-${CKAN_VERSION}#egg=ckan" diff --git a/contrib/solr/Dockerfile b/contrib/solr/Dockerfile index f73cfd9..3b59ac8 100644 --- a/contrib/solr/Dockerfile +++ b/contrib/solr/Dockerfile @@ -3,7 +3,7 @@ MAINTAINER Open Knowledge ## Taken from https://github.com/ckan/ckan/blob/ckan-2.7.0/contrib/docker/solr/Dockerfile # Updated the ARG values -ARG CKAN_VERSION=2.8.11 +ARG CKAN_VERSION=2.9.0 # Enviroment ENV SOLR_CORE ckan diff --git a/docker-compose.yml b/docker-compose.yml index e5383df..5d05cc0 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,7 +25,7 @@ services: - redis env_file: - .env - command: paster --plugin=ckanext-harvest harvester gather_consumer --config=ckan.ini + command: ckan --config=ckan.ini --plugin=ckanext-harvest harvester gather_consumer volumes: - ./ckan.ini:/ckan.ini - ckan-filestore:/var/lib/ckan/default @@ -52,7 +52,7 @@ services: # "postgresql://ckan_default:ckan_default@ckan-test.cfgmtx8ishfx.eu-west-1.rds.amazonaws.com:5432/ckan" solr: - image: codeforafrica/ckan-solr:2.8.11 + image: codeforafrica/ckan-solr:2.9.0 ports: - "8983:8983" volumes: diff --git a/requirements.txt b/requirements.txt index 93faa9d..6d3b3f6 100755 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ -gunicorn==19.7.1 -gevent==1.2.2 +gunicorn==19.7.0 +gevent==20.9.0 # CKAN Extensions From dc8ff00081d9c0fd8b759673deb5cf1869fab00d Mon Sep 17 00:00:00 2001 From: Xavier Frankline Date: Fri, 28 Jul 2023 15:44:28 +0300 Subject: [PATCH 08/20] update docker-compose and config file --- Dockerfile | 7 ++++++- Procfile | 2 -- contrib/ckan/Dockerfile | 2 +- docker-compose.yml | 29 ++++++++++++++--------------- 4 files changed, 21 insertions(+), 19 deletions(-) diff --git a/Dockerfile b/Dockerfile index e9b14d0..e75454f 100755 --- a/Dockerfile +++ b/Dockerfile @@ -15,4 +15,9 @@ ADD ckan.ini /ckan.ini ADD Procfile /Procfile -CMD ["gunicorn", "--workers", "3", "--worker-class", "gevent", "--paste", "ckan.ini", "-t", "600"] +ADD entrypoint.sh /entrypoint.sh + +RUN chmod +x /entrypoint.sh + +# CMD ["gunicorn", "--workers", "3", "--worker-class", "gevent", "--paste", "ckan.ini", "-t", "600"] +CMD ["./entrypoint.sh"] diff --git a/Procfile b/Procfile index 036ee7d..69fd1d6 100644 --- a/Procfile +++ b/Procfile @@ -1,5 +1,3 @@ -web: gunicorn --workers 5 --worker-class gevent --ckan ckan.ini -t 10800 - ckan_gather_consumer: ckan --config=ckan.ini --plugin=ckanext-harvest harvester gather_consumer ckan_fetch_consumer: ckan --config=ckan.ini --plugin=ckanext-harvest harvester fetch_consumer diff --git a/contrib/ckan/Dockerfile b/contrib/ckan/Dockerfile index 4c52837..ee8c024 100644 --- a/contrib/ckan/Dockerfile +++ b/contrib/ckan/Dockerfile @@ -8,7 +8,7 @@ RUN pip install -U -q pip "setuptools<46.0.0" RUN pip install -q -e "git+https://github.com/ckan/ckan.git@ckan-${CKAN_VERSION}#egg=ckan" -ADD meta.py /src/ckan/ckan/model/meta.py +# ADD meta.py /src/ckan/ckan/model/meta.py RUN pip install -q -r /src/ckan/requirements.txt RUN pip install -q vdm==0.15 sqlalchemy==1.2.19 diff --git a/docker-compose.yml b/docker-compose.yml index 5d05cc0..16d3b61 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,7 @@ services: web: build: . depends_on: - # - db + - db - solr - redis ports: @@ -36,20 +36,19 @@ services: depends_on: - web - # db: - # image: postgres:13.7 - # ports: - # - "54321:5432" - # environment: - # - POSTGRES_USER=ckan_default - # - POSTGRES_PASSWORD=pass - # - PGUSER=ckan_default - # - PGPASSWORD=pass - # - POSTGRES_ADD_DATABASES=datastore_default:datastore_default - # volumes: - # - postgres-data:/var/lib/postgresql/data - # - ./contrib/postgres:/docker-entrypoint-initdb.d - # "postgresql://ckan_default:ckan_default@ckan-test.cfgmtx8ishfx.eu-west-1.rds.amazonaws.com:5432/ckan" + db: + image: postgres:13.7 + ports: + - "54321:5432" + environment: + - POSTGRES_USER=ckan_default + - POSTGRES_PASSWORD=pass + - PGUSER=ckan_default + - PGPASSWORD=pass + - POSTGRES_ADD_DATABASES=datastore_default:datastore_default + volumes: + - postgres-data:/var/lib/postgresql/data + - ./contrib/postgres:/docker-entrypoint-initdb.d solr: image: codeforafrica/ckan-solr:2.9.0 From 3f15886fe508eb009af702ad64ddc5d2cc95a729 Mon Sep 17 00:00:00 2001 From: Xavier Frankline Date: Fri, 28 Jul 2023 15:46:43 +0300 Subject: [PATCH 09/20] update docker-compose and config file --- Procfile | 2 ++ docker-compose.yml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Procfile b/Procfile index 69fd1d6..036ee7d 100644 --- a/Procfile +++ b/Procfile @@ -1,3 +1,5 @@ +web: gunicorn --workers 5 --worker-class gevent --ckan ckan.ini -t 10800 + ckan_gather_consumer: ckan --config=ckan.ini --plugin=ckanext-harvest harvester gather_consumer ckan_fetch_consumer: ckan --config=ckan.ini --plugin=ckanext-harvest harvester fetch_consumer diff --git a/docker-compose.yml b/docker-compose.yml index 16d3b61..1bb2979 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,7 +20,7 @@ services: ckan_gather_consumer: build: . depends_on: - # - db + - db - solr - redis env_file: From 3038899c0aae9748c368dec6ecad9d0cf8c4efbb Mon Sep 17 00:00:00 2001 From: Xavier Frankline Date: Fri, 28 Jul 2023 15:51:29 +0300 Subject: [PATCH 10/20] revert entrypoint Dockerfile --- Dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index e75454f..0805db2 100755 --- a/Dockerfile +++ b/Dockerfile @@ -17,7 +17,5 @@ ADD Procfile /Procfile ADD entrypoint.sh /entrypoint.sh -RUN chmod +x /entrypoint.sh +CMD ["gunicorn", "--workers", "3", "--worker-class", "gevent", "--ckan", "ckan.ini", "-t", "600"] -# CMD ["gunicorn", "--workers", "3", "--worker-class", "gevent", "--paste", "ckan.ini", "-t", "600"] -CMD ["./entrypoint.sh"] From 75ccf9dc7b1909fad7cbede8faf8cbff45ce6bbb Mon Sep 17 00:00:00 2001 From: Xavier Frankline Date: Fri, 28 Jul 2023 15:52:05 +0300 Subject: [PATCH 11/20] revert entrypoint Dockerfile --- Dockerfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0805db2..3693473 100755 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,5 @@ ADD ckan.ini /ckan.ini ADD Procfile /Procfile -ADD entrypoint.sh /entrypoint.sh - CMD ["gunicorn", "--workers", "3", "--worker-class", "gevent", "--ckan", "ckan.ini", "-t", "600"] From e9fdfe5d9908273f95f24aac3a292b75b26e5c02 Mon Sep 17 00:00:00 2001 From: =xavierfrank4 <=> Date: Thu, 3 Aug 2023 15:25:50 +0300 Subject: [PATCH 12/20] add wsgi.py file --- Dockerfile | 5 +- contrib/ckan/Dockerfile | 2 - contrib/ckan/meta.py | 147 ---------------------------------------- wsgi.py | 20 ++++++ 4 files changed, 23 insertions(+), 151 deletions(-) delete mode 100644 contrib/ckan/meta.py create mode 100644 wsgi.py diff --git a/Dockerfile b/Dockerfile index 3693473..9be8c19 100755 --- a/Dockerfile +++ b/Dockerfile @@ -10,10 +10,11 @@ RUN pip install -q -r /requirements.txt && \ pip install -q -r /src/ckanext-harvest/pip-requirements.txt && \ pip install -q -r /src/ckanext-dcat/requirements.txt -# RUN ln -s ./src/ckan/ckan/config/who.ini /who.ini ADD ckan.ini /ckan.ini +ADD wsgi.py /wsgi.py + ADD Procfile /Procfile -CMD ["gunicorn", "--workers", "3", "--worker-class", "gevent", "--ckan", "ckan.ini", "-t", "600"] +CMD ["gunicorn", "--workers", "3", "-b", ":5000", "wsgi.py", "-t", "600"] diff --git a/contrib/ckan/Dockerfile b/contrib/ckan/Dockerfile index ee8c024..2e4bae0 100644 --- a/contrib/ckan/Dockerfile +++ b/contrib/ckan/Dockerfile @@ -8,7 +8,5 @@ RUN pip install -U -q pip "setuptools<46.0.0" RUN pip install -q -e "git+https://github.com/ckan/ckan.git@ckan-${CKAN_VERSION}#egg=ckan" -# ADD meta.py /src/ckan/ckan/model/meta.py - RUN pip install -q -r /src/ckan/requirements.txt RUN pip install -q vdm==0.15 sqlalchemy==1.2.19 diff --git a/contrib/ckan/meta.py b/contrib/ckan/meta.py deleted file mode 100644 index 0b89c75..0000000 --- a/contrib/ckan/meta.py +++ /dev/null @@ -1,147 +0,0 @@ -# encoding: utf-8 - -import datetime - -from paste.deploy.converters import asbool -from ckan.common import config -"""SQLAlchemy Metadata and Session object""" -from sqlalchemy import MetaData, and_ -import sqlalchemy.orm as orm -from sqlalchemy.orm.session import SessionExtension - -import extension -import ckan.lib.activity_streams_session_extension as activity - -__all__ = ['Session', 'engine_is_sqlite', 'engine_is_pg'] - - -class CkanCacheExtension(SessionExtension): - ''' This extension checks what tables have been affected by - database access and allows us to act on them. Currently this is - used by the page cache to flush the cache when data in the database - is altered. ''' - - def __init__(self, *args, **kw): - super(CkanCacheExtension, self).__init__(*args, **kw) - - def after_commit(self, session): - if hasattr(session, '_object_cache'): - oc = session._object_cache - oc_list = oc['new'] - oc_list.update(oc['changed']) - oc_list.update(oc['deleted']) - objs = set() - for item in oc_list: - objs.add(item.__class__.__name__) - - -class CkanSessionExtension(SessionExtension): - - def before_flush(self, session, flush_context, instances): - if not hasattr(session, '_object_cache'): - session._object_cache= {'new': set(), - 'deleted': set(), - 'changed': set()} - - changed = [obj for obj in session.dirty if - session.is_modified(obj, include_collections=False, passive=True)] - - session._object_cache['new'].update(session.new) - session._object_cache['deleted'].update(session.deleted) - session._object_cache['changed'].update(changed) - - - def before_commit(self, session): - session.flush() - try: - obj_cache = session._object_cache - revision = session.revision - except AttributeError: - return - if getattr(session, 'revisioning_disabled', False): - return - new = obj_cache['new'] - changed = obj_cache['changed'] - deleted = obj_cache['deleted'] - for obj in new | changed | deleted: - if not hasattr(obj, '__revision_class__'): - continue - revision_cls = obj.__revision_class__ - revision_table = orm.class_mapper(revision_cls).mapped_table - ## when a normal active transaction happens - - ### this is an sql statement as we do not want it in object cache - session.execute( - revision_table.update().where( - and_(revision_table.c.id == obj.id, - revision_table.c.current == True) - ).values(current=False) - ) - - q = session.query(revision_cls) - q = q.filter_by(expired_timestamp=datetime.datetime(9999, 12, 31), id=obj.id) - results = q.all() - for rev_obj in results: - values = {} - if rev_obj.revision_id == revision.id: - values['revision_timestamp'] = revision.timestamp - else: - values['expired_timestamp'] = revision.timestamp - session.execute( - revision_table.update().where( - and_(revision_table.c.id == rev_obj.id, - revision_table.c.revision_id == rev_obj.revision_id) - ).values(**values) - ) - - def after_commit(self, session): - if hasattr(session, '_object_cache'): - del session._object_cache - - def after_rollback(self, session): - if hasattr(session, '_object_cache'): - del session._object_cache - -# __all__ = ['Session', 'engine', 'metadata', 'mapper'] - -# SQLAlchemy database engine. Updated by model.init_model() -engine = None - -Session = orm.scoped_session(orm.sessionmaker( - autoflush=False, - autocommit=False, - expire_on_commit=False, - extension=[CkanCacheExtension(), - CkanSessionExtension(), - extension.PluginSessionExtension(), - activity.DatasetActivitySessionExtension()], -)) - -create_local_session = orm.sessionmaker( - autoflush=False, - autocommit=False, - expire_on_commit=False, - extension=[CkanCacheExtension(), - CkanSessionExtension(), - extension.PluginSessionExtension(), - activity.DatasetActivitySessionExtension()], -) - -#mapper = Session.mapper -mapper = orm.mapper - -# Global metadata. If you have multiple databases with overlapping table -# names, you'll need a metadata for each database -metadata = MetaData() - - -def engine_is_sqlite(sa_engine=None): - # Returns true iff the engine is connected to a sqlite database. - return (sa_engine or engine).url.drivername == 'sqlite' - - -def engine_is_pg(sa_engine=None): - # Returns true iff the engine is connected to a postgresql database. - # According to http://docs.sqlalchemy.org/en/latest/core/engines.html#postgresql - # all Postgres driver names start with `postgres` - return (sa_engine or engine).url.drivername.startswith('postgres') diff --git a/wsgi.py b/wsgi.py new file mode 100644 index 0000000..ec0ee78 --- /dev/null +++ b/wsgi.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- + +import os +from ckan.config.middleware import make_app +from ckan.cli import CKANConfigLoader +from logging.config import fileConfig as loggingFileConfig + +if os.environ.get(u'CKAN_INI'): + config_path = os.environ[u'CKAN_INI'] +else: + config_path = os.path.join( + os.path.dirname(os.path.abspath(__file__)), u'ckan.ini') + +if not os.path.exists(config_path): + raise RuntimeError(u'CKAN config option not found: {}'.format(config_path)) + +loggingFileConfig(config_path) +config = CKANConfigLoader(config_path).get_config() + +application = make_app(config) \ No newline at end of file From 15bf17ff07044a5c0fe1787bddd80ee764a8e17c Mon Sep 17 00:00:00 2001 From: =xavierfrank4 <=> Date: Mon, 7 Aug 2023 16:43:01 +0300 Subject: [PATCH 13/20] install pyOpenSSL & cryptography --- requirements.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/requirements.txt b/requirements.txt index 6d3b3f6..e49415e 100755 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,8 @@ gunicorn==19.7.0 gevent==20.9.0 +cryptography +pyOpenSSL + # CKAN Extensions From 9c039e404117a96cff7f109f7d730f6aaa8385d7 Mon Sep 17 00:00:00 2001 From: Xavier Frankline Date: Fri, 2 Feb 2024 09:18:26 +0300 Subject: [PATCH 14/20] chore: changes to support ckan 2.9 --- Dockerfile | 77 ++++++++++++++++++++++++++++++----------- ckan.ini | 16 +++++---- contrib/ckan/Dockerfile | 55 +++++++++++++++++++++++++---- docker-compose.yml | 12 +++---- 4 files changed, 120 insertions(+), 40 deletions(-) mode change 100755 => 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile old mode 100755 new mode 100644 index 9be8c19..17d751d --- a/Dockerfile +++ b/Dockerfile @@ -1,20 +1,57 @@ -FROM codeforafrica/ckan:2.9.0 - -EXPOSE 5000/tcp - -WORKDIR / - -ADD requirements.txt /requirements.txt -RUN pip install -q -r /requirements.txt && \ - pip install -q -r /src/ckanext-s3filestore/requirements.txt && \ - pip install -q -r /src/ckanext-harvest/pip-requirements.txt && \ - pip install -q -r /src/ckanext-dcat/requirements.txt - -ADD ckan.ini /ckan.ini - -ADD wsgi.py /wsgi.py - -ADD Procfile /Procfile - -CMD ["gunicorn", "--workers", "3", "-b", ":5000", "wsgi.py", "-t", "600"] - +FROM ckan/ckan-dev:2.9 + +# Install any extensions needed by your CKAN instance +# - Make sure to add the plugins to CKAN__PLUGINS in the .env file +# - Also make sure all provide all extra configuration options, either by: +# * Adding them to the .env file (check the ckanext-envvars syntax for env vars), or +# * Adding extra configuration scripts to /docker-entrypoint.d folder) to update +# the CKAN config file (ckan.ini) with the `ckan config-tool` command +# +# See README > Extending the base images for more details +# +# For instance: +# +### OpenAfrica ### +RUN pip3 install -e git+https://github.com/CodeForAfrica/ckanext-openafrica.git@ft/ui-changes#egg=ckanext-openafrica + +## s3filestore +RUN pip3 install -e git+https://github.com/qld-gov-au/ckanext-s3filestore.git#egg=ckanext-s3filestore && \ + pip3 install -r ${APP_DIR}/src/ckanext-s3filestore/requirements.txt + +### XLoader ### +#RUN pip3 install -e 'git+https://github.com/ckan/ckanext-xloader.git@master#egg=ckanext-xloader' && \ +# pip3 install -r ${APP_DIR}/src/ckanext-xloader/requirements.txt && \ +# pip3 install -U requests[security] + +### Harvester ### +RUN pip3 install -e 'git+https://github.com/ckan/ckanext-harvest.git@master#egg=ckanext-harvest' && \ + pip3 install -r ${APP_DIR}/src/ckanext-harvest/pip-requirements.txt +# will also require gather_consumer and fetch_consumer processes running (please see https://github.com/ckan/ckanext-harvest) + +### Scheming ### +#RUN pip3 install -e 'git+https://github.com/ckan/ckanext-scheming.git@master#egg=ckanext-scheming' + +### Pages ### +#RUN pip3 install -e git+https://github.com/ckan/ckanext-pages.git#egg=ckanext-pages + +### DCAT ### +#RUN pip3 install -e git+https://github.com/ckan/ckanext-dcat.git@v0.0.6#egg=ckanext-dcat && \ +# pip3 install -r https://raw.githubusercontent.com/ckan/ckanext-dcat/v0.0.6/requirements.txt + +# Clone the extension(s) your are writing for your own project in the `src` folder +# to get them mounted in this image at runtime + +# Copy custom initialization scripts +# COPY docker-entrypoint.d/* /docker-entrypoint.d/ + +# Apply any patches needed to CKAN core or any of the built extensions (not the +# runtime mounted ones) +# COPY patches ${APP_DIR}/patches + +# RUN for d in $APP_DIR/patches/*; do \ +# if [ -d $d ]; then \ +# for f in `ls $d/*.patch | sort -g`; do \ +# cd $SRC_DIR/`basename "$d"` && echo "$0: Applying patch $f to $SRC_DIR/`basename $d`"; patch -p1 < "$f" ; \ +# done ; \ +# fi ; \ +# done \ No newline at end of file diff --git a/ckan.ini b/ckan.ini index 1d3d646..cbc4d87 100755 --- a/ckan.ini +++ b/ckan.ini @@ -14,7 +14,7 @@ [DEFAULT] # WARNING: *THIS SETTING MUST BE SET TO FALSE ON A PRODUCTION ENVIRONMENT* -debug = false +debug = true [server:main] use = egg:Paste#http @@ -22,6 +22,8 @@ host = 0.0.0.0 port = 5000 [app:main] +api_token.jwt.decode.secret=string:poMHskrmngl7Lv92m_XUdYzrxH2OEl1GBoJCVgFhZAQ +api_token.jwt.encode.secret=string:poMHskrmngl7Lv92m_XUdYzrxH2OEl1GBoJCVgFhZAQ use = egg:ckan full_stack = true cache_dir = /tmp/%(ckan.site_id)s/ @@ -30,14 +32,14 @@ beaker.session.key = ckan # This is the secret token that the beaker library uses to hash the cookie sent # to the client. `paster make-config` generates a unique value for this each # time it generates a config file. -beaker.session.secret = +beaker.session.secret=Dtg1m0WXy0VAkM78aOL3l6ZnR-D7Qzu7OBbquHCpQU8 # `paster make-config` generates a unique value for this each time it generates # a config file. app_instance_uuid = # repoze.who config -who.config_file = /src/ckan/ckan/config/who.ini +who.config_file = /etc/ckan/who.ini who.log_level = warning who.log_file = %(cache_dir)s/who_log.ini # Session timeout (user logged out after period of inactivity, in seconds). @@ -101,8 +103,8 @@ ckan.cors.origin_allow_all = true # Add ``resource_proxy`` to enable resorce proxying and get around the # same origin policy -ckan.plugins = stats text_view image_view recline_view recline_grid_view gdoc_view resource_proxy harvest ckan_harvester s3filestore openafrica datarequests showcase datapusher datastore officedocs_view pdf_view dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface structured_data issues sentry envvars - +ckan.plugins = envvars image_view text_view recline_view datastore datapusher openafrica +#ckan.plugins = stats text_view image_view recline_view recline_grid_view gdoc_view resource_proxy openafrica showcase datapusher datastore envvars # Define which views should be created by default # (plugins must be loaded in ckan.plugins) ckan.views.default_views = image_view text_view recline_view recline_grid_view pdf_view gdoc_view @@ -177,8 +179,8 @@ ckan.datapusher.assume_task_stale_after = 3600 ckan.hide_activity_from_users = %(ckan.site_id)s # Bootstrap -ckan.base_public_folder = public-bs2 -ckan.base_templates_folder = templates-bs2 +ckan.base_public_folder = public +ckan.base_templates_folder = templates ## Email settings diff --git a/contrib/ckan/Dockerfile b/contrib/ckan/Dockerfile index 2e4bae0..0b5b5be 100644 --- a/contrib/ckan/Dockerfile +++ b/contrib/ckan/Dockerfile @@ -1,12 +1,53 @@ -FROM python:3.8 +FROM ckan/ckan-dev:2.9 -ENV DEBIAN_FRONTEND noninteractive +# Install any extensions needed by your CKAN instance +# - Make sure to add the plugins to CKAN__PLUGINS in the .env file +# - Also make sure all provide all extra configuration options, either by: +# * Adding them to the .env file (check the ckanext-envvars syntax for env vars), or +# * Adding extra configuration scripts to /docker-entrypoint.d folder) to update +# the CKAN config file (ckan.ini) with the `ckan config-tool` command +# +# See README > Extending the base images for more details +# +# For instance: +# +### OpenAfrica ### +RUN pip3 install -e git+https://github.com/CodeForAfrica/ckanext-openafrica.git@ft/upgrade-ckan-2.9#egg=ckanext-openafrica -ARG CKAN_VERSION=2.9.0 +### XLoader ### +#RUN pip3 install -e 'git+https://github.com/ckan/ckanext-xloader.git@master#egg=ckanext-xloader' && \ +# pip3 install -r ${APP_DIR}/src/ckanext-xloader/requirements.txt && \ +# pip3 install -U requests[security] -RUN pip install -U -q pip "setuptools<46.0.0" +### Harvester ### +#RUN pip3 install -e 'git+https://github.com/ckan/ckanext-harvest.git@master#egg=ckanext-harvest' && \ +# pip3 install -r ${APP_DIR}/src/ckanext-harvest/pip-requirements.txt +# will also require gather_consumer and fetch_consumer processes running (please see https://github.com/ckan/ckanext-harvest) -RUN pip install -q -e "git+https://github.com/ckan/ckan.git@ckan-${CKAN_VERSION}#egg=ckan" +### Scheming ### +#RUN pip3 install -e 'git+https://github.com/ckan/ckanext-scheming.git@master#egg=ckanext-scheming' -RUN pip install -q -r /src/ckan/requirements.txt -RUN pip install -q vdm==0.15 sqlalchemy==1.2.19 +### Pages ### +#RUN pip3 install -e git+https://github.com/ckan/ckanext-pages.git#egg=ckanext-pages + +### DCAT ### +#RUN pip3 install -e git+https://github.com/ckan/ckanext-dcat.git@v0.0.6#egg=ckanext-dcat && \ +# pip3 install -r https://raw.githubusercontent.com/ckan/ckanext-dcat/v0.0.6/requirements.txt + +# Clone the extension(s) your are writing for your own project in the `src` folder +# to get them mounted in this image at runtime + +# Copy custom initialization scripts +COPY docker-entrypoint.d/* /docker-entrypoint.d/ + +# Apply any patches needed to CKAN core or any of the built extensions (not the +# runtime mounted ones) +# COPY patches ${APP_DIR}/patches + +# RUN for d in $APP_DIR/patches/*; do \ +# if [ -d $d ]; then \ +# for f in `ls $d/*.patch | sort -g`; do \ +# cd $SRC_DIR/`basename "$d"` && echo "$0: Applying patch $f to $SRC_DIR/`basename $d`"; patch -p1 < "$f" ; \ +# done ; \ +# fi ; \ +# done \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 1bb2979..06e5745 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,16 +25,16 @@ services: - redis env_file: - .env - command: ckan --config=ckan.ini --plugin=ckanext-harvest harvester gather_consumer + command: ckan --config=ckan.ini --plugin=ckanext-harvest harvester gather-consumer volumes: - ./ckan.ini:/ckan.ini - ckan-filestore:/var/lib/ckan/default - datapusher: - build: contrib/ckan-datapusher - depends_on: - - web + container_name: datapusher + image: clementmouchet/datapusher + ports: + - "8800:8800" db: image: postgres:13.7 @@ -51,7 +51,7 @@ services: - ./contrib/postgres:/docker-entrypoint-initdb.d solr: - image: codeforafrica/ckan-solr:2.9.0 + image: ckan/ckan-solr:2.9 ports: - "8983:8983" volumes: From 2b25a393d3055ab454000b943e6c98a0de775691 Mon Sep 17 00:00:00 2001 From: Xavier Frankline Date: Tue, 6 Feb 2024 10:04:29 +0300 Subject: [PATCH 15/20] update datarequest and GA --- Dockerfile | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/Dockerfile b/Dockerfile index 17d751d..15d681f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,29 +14,20 @@ FROM ckan/ckan-dev:2.9 ### OpenAfrica ### RUN pip3 install -e git+https://github.com/CodeForAfrica/ckanext-openafrica.git@ft/ui-changes#egg=ckanext-openafrica -## s3filestore -RUN pip3 install -e git+https://github.com/qld-gov-au/ckanext-s3filestore.git#egg=ckanext-s3filestore && \ - pip3 install -r ${APP_DIR}/src/ckanext-s3filestore/requirements.txt - -### XLoader ### -#RUN pip3 install -e 'git+https://github.com/ckan/ckanext-xloader.git@master#egg=ckanext-xloader' && \ -# pip3 install -r ${APP_DIR}/src/ckanext-xloader/requirements.txt && \ -# pip3 install -U requests[security] +### Datarequests +RUN pip3 install -e git+https://github.com/qld-gov-au/ckanext-datarequests.git#egg=ckanext-datarequests ### Harvester ### -RUN pip3 install -e 'git+https://github.com/ckan/ckanext-harvest.git@master#egg=ckanext-harvest' && \ +RUN pip3 install -e git+https://github.com/ckan/ckanext-harvest.git@master#egg=ckanext-harvest && \ pip3 install -r ${APP_DIR}/src/ckanext-harvest/pip-requirements.txt -# will also require gather_consumer and fetch_consumer processes running (please see https://github.com/ckan/ckanext-harvest) -### Scheming ### -#RUN pip3 install -e 'git+https://github.com/ckan/ckanext-scheming.git@master#egg=ckanext-scheming' - -### Pages ### -#RUN pip3 install -e git+https://github.com/ckan/ckanext-pages.git#egg=ckanext-pages +## s3filestore +RUN pip3 install -e git+https://github.com/qld-gov-au/ckanext-s3filestore.git#egg=ckanext-s3filestore && \ + pip3 install -r ${APP_DIR}/src/ckanext-s3filestore/requirements.txt -### DCAT ### -#RUN pip3 install -e git+https://github.com/ckan/ckanext-dcat.git@v0.0.6#egg=ckanext-dcat && \ -# pip3 install -r https://raw.githubusercontent.com/ckan/ckanext-dcat/v0.0.6/requirements.txt +## ckan GoogleAnalytics +RUN pip3 install-e -e git+https://github.com/ckan/ckanext-googleanalytics.git#egg=ckanext-googleanalytics && \ + pip3 install -r ${APP_DIR}/src/ckanext-googleanalytics/requirements.txt # Clone the extension(s) your are writing for your own project in the `src` folder # to get them mounted in this image at runtime From 92f56b41c4bef8ca3d0bdb9c2b0b88f6816bf7cf Mon Sep 17 00:00:00 2001 From: Xavier Frankline Date: Tue, 6 Feb 2024 12:36:04 +0300 Subject: [PATCH 16/20] mount ckannext --- Dockerfile | 4 +++- docker-compose.yml | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 15d681f..226b0c5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,11 +26,13 @@ RUN pip3 install -e git+https://github.com/qld-gov-au/ckanext-s3filestore.git#eg pip3 install -r ${APP_DIR}/src/ckanext-s3filestore/requirements.txt ## ckan GoogleAnalytics -RUN pip3 install-e -e git+https://github.com/ckan/ckanext-googleanalytics.git#egg=ckanext-googleanalytics && \ +RUN pip3 install -e git+https://github.com/ckan/ckanext-googleanalytics.git#egg=ckanext-googleanalytics && \ pip3 install -r ${APP_DIR}/src/ckanext-googleanalytics/requirements.txt # Clone the extension(s) your are writing for your own project in the `src` folder # to get them mounted in this image at runtime +# COPY ckanext-openafrica/* {APP_DIR}/src/ckanext-openafrica/ +# RUN cd {APP_DIR}/src/ckanext-openafrica && python3 setup.py develop # Copy custom initialization scripts # COPY docker-entrypoint.d/* /docker-entrypoint.d/ diff --git a/docker-compose.yml b/docker-compose.yml index 06e5745..a774d46 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,6 +16,7 @@ services: volumes: - ./ckan.ini:/ckan.ini - ckan-filestore:/var/lib/ckan/default + - ./ckanext-openafrica/:/srv/app/src/ckanext-openafrica/ ckan_gather_consumer: build: . From 5a46df48e91cbabee3220d987b06f17e118c3067 Mon Sep 17 00:00:00 2001 From: Xavier Frankline Date: Tue, 6 Feb 2024 12:48:21 +0300 Subject: [PATCH 17/20] fix --- Dockerfile | 2 +- wsgi.py | 20 -------------------- 2 files changed, 1 insertion(+), 21 deletions(-) delete mode 100644 wsgi.py diff --git a/Dockerfile b/Dockerfile index 15d681f..b63f486 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,7 +26,7 @@ RUN pip3 install -e git+https://github.com/qld-gov-au/ckanext-s3filestore.git#eg pip3 install -r ${APP_DIR}/src/ckanext-s3filestore/requirements.txt ## ckan GoogleAnalytics -RUN pip3 install-e -e git+https://github.com/ckan/ckanext-googleanalytics.git#egg=ckanext-googleanalytics && \ +RUN pip3 install -e git+https://github.com/ckan/ckanext-googleanalytics.git#egg=ckanext-googleanalytics && \ pip3 install -r ${APP_DIR}/src/ckanext-googleanalytics/requirements.txt # Clone the extension(s) your are writing for your own project in the `src` folder diff --git a/wsgi.py b/wsgi.py deleted file mode 100644 index ec0ee78..0000000 --- a/wsgi.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- - -import os -from ckan.config.middleware import make_app -from ckan.cli import CKANConfigLoader -from logging.config import fileConfig as loggingFileConfig - -if os.environ.get(u'CKAN_INI'): - config_path = os.environ[u'CKAN_INI'] -else: - config_path = os.path.join( - os.path.dirname(os.path.abspath(__file__)), u'ckan.ini') - -if not os.path.exists(config_path): - raise RuntimeError(u'CKAN config option not found: {}'.format(config_path)) - -loggingFileConfig(config_path) -config = CKANConfigLoader(config_path).get_config() - -application = make_app(config) \ No newline at end of file From 59404b1b7d2c92a244bdcf25ef70e932e2608ebf Mon Sep 17 00:00:00 2001 From: Xavier Frankline Date: Tue, 6 Feb 2024 21:23:18 +0300 Subject: [PATCH 18/20] update, customise ckan.ini file --- Dockerfile | 7 +++- ckan.ini | 91 ++++++++++++++++++++++++++++------------------ docker-compose.yml | 4 +- 3 files changed, 61 insertions(+), 41 deletions(-) mode change 100755 => 100644 ckan.ini diff --git a/Dockerfile b/Dockerfile index 226b0c5..6368ac0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,6 @@ FROM ckan/ckan-dev:2.9 # See README > Extending the base images for more details # # For instance: -# ### OpenAfrica ### RUN pip3 install -e git+https://github.com/CodeForAfrica/ckanext-openafrica.git@ft/ui-changes#egg=ckanext-openafrica @@ -29,13 +28,17 @@ RUN pip3 install -e git+https://github.com/qld-gov-au/ckanext-s3filestore.git#eg RUN pip3 install -e git+https://github.com/ckan/ckanext-googleanalytics.git#egg=ckanext-googleanalytics && \ pip3 install -r ${APP_DIR}/src/ckanext-googleanalytics/requirements.txt +## ckanext-showcase +RUN pip3 install -e git+https://github.com/ckan/ckanext-showcase.git#egg=ckanext-showcase && \ + pip3 install -r ${APP_DIR}/src/ckanext-showcase/requirements.txt + # Clone the extension(s) your are writing for your own project in the `src` folder # to get them mounted in this image at runtime # COPY ckanext-openafrica/* {APP_DIR}/src/ckanext-openafrica/ # RUN cd {APP_DIR}/src/ckanext-openafrica && python3 setup.py develop # Copy custom initialization scripts -# COPY docker-entrypoint.d/* /docker-entrypoint.d/ +COPY contrib/ckan/docker-entrypoint.d/* /docker-entrypoint.d/ # Apply any patches needed to CKAN core or any of the built extensions (not the # runtime mounted ones) diff --git a/ckan.ini b/ckan.ini old mode 100755 new mode 100644 index cbc4d87..40c4363 --- a/ckan.ini +++ b/ckan.ini @@ -13,33 +13,33 @@ [DEFAULT] -# WARNING: *THIS SETTING MUST BE SET TO FALSE ON A PRODUCTION ENVIRONMENT* +# WARNING: *THIS SETTING MUST BE SET TO FALSE ON A PUBLIC ENVIRONMENT* +# With debug mode enabled, a visitor to your site could execute malicious commands. debug = true -[server:main] -use = egg:Paste#http -host = 0.0.0.0 -port = 5000 - [app:main] -api_token.jwt.decode.secret=string:poMHskrmngl7Lv92m_XUdYzrxH2OEl1GBoJCVgFhZAQ -api_token.jwt.encode.secret=string:poMHskrmngl7Lv92m_XUdYzrxH2OEl1GBoJCVgFhZAQ use = egg:ckan -full_stack = true + +## Development settings +ckan.devserver.host = localhost +ckan.devserver.port = 5000 + + +## Session settings cache_dir = /tmp/%(ckan.site_id)s/ beaker.session.key = ckan # This is the secret token that the beaker library uses to hash the cookie sent -# to the client. `paster make-config` generates a unique value for this each +# to the client. `ckan generate config` generates a unique value for this each # time it generates a config file. -beaker.session.secret=Dtg1m0WXy0VAkM78aOL3l6ZnR-D7Qzu7OBbquHCpQU8 +beaker.session.secret=6QnZms_5F2tZO8oW3TYYO5ZzrJCYM6TYQxg9aP17QDo -# `paster make-config` generates a unique value for this each time it generates +# `ckan generate config` generates a unique value for this each time it generates # a config file. -app_instance_uuid = +app_instance_uuid = 5e8919ec-96f2-46f9-867b-b3d0b699bb04 # repoze.who config -who.config_file = /etc/ckan/who.ini +who.config_file = %(here)s/who.ini who.log_level = warning who.log_file = %(cache_dir)s/who_log.ini # Session timeout (user logged out after period of inactivity, in seconds). @@ -56,11 +56,16 @@ sqlalchemy.url = postgresql://ckan_default:pass@localhost/ckan_default ckan.datastore.default_fts_lang = english ckan.datastore.default_fts_index_method = gist + ## Site Settings ckan.site_url = #ckan.use_pylons_response_cleanup_middleware = true +# Default timeout for Requests +#ckan.requests.timeout = 10 + + ## Authorization Settings ckan.auth.anon_create_dataset = false @@ -68,12 +73,24 @@ ckan.auth.create_unowned_dataset = false ckan.auth.create_dataset_if_not_in_organization = false ckan.auth.user_create_groups = false ckan.auth.user_create_organizations = false -ckan.auth.user_delete_groups = false -ckan.auth.user_delete_organizations = false +ckan.auth.user_delete_groups = true +ckan.auth.user_delete_organizations = true ckan.auth.create_user_via_api = false -ckan.auth.create_user_via_web = true +ckan.auth.create_user_via_web = false ckan.auth.roles_that_cascade_to_sub_groups = admin +ckan.auth.public_user_details = true +ckan.auth.public_activity_stream_detail = true +ckan.auth.allow_dataset_collaborators = false +ckan.auth.create_default_api_keys = false +## API Token Settings +api_token.nbytes = 60 +api_token.jwt.encode.secret=string:M_BXUHlhDEJZnlkC29My8pgjXJS6SDsy127KTkG1Y1Q +api_token.jwt.decode.secret=string:M_BXUHlhDEJZnlkC29My8pgjXJS6SDsy127KTkG1Y1Q +api_token.jwt.algorithm = HS256 + +## API Token: expire_api_token plugin +expire_api_token.default_lifetime = 3600 ## Search Settings @@ -91,7 +108,7 @@ ckan.site_id = default # If cors.origin_allow_all is true, all origins are allowed. # If false, the cors.origin_whitelist is used. -ckan.cors.origin_allow_all = true +# ckan.cors.origin_allow_all = true # cors.origin_whitelist is a space separated list of allowed domains. # ckan.cors.origin_whitelist = http://example1.com http://example2.com @@ -102,12 +119,11 @@ ckan.cors.origin_allow_all = true # Add ``datapusher`` to enable DataPusher # Add ``resource_proxy`` to enable resorce proxying and get around the # same origin policy +ckan.plugins = envvars image_view text_view recline_view datastore datapusher openafrica s3filestore datarequests -ckan.plugins = envvars image_view text_view recline_view datastore datapusher openafrica -#ckan.plugins = stats text_view image_view recline_view recline_grid_view gdoc_view resource_proxy openafrica showcase datapusher datastore envvars # Define which views should be created by default # (plugins must be loaded in ckan.plugins) -ckan.views.default_views = image_view text_view recline_view recline_grid_view pdf_view gdoc_view +ckan.views.default_views = image_view text_view recline_view # Customize which text formats the text_view plugin will show #ckan.preview.json_formats = json @@ -118,6 +134,7 @@ ckan.views.default_views = image_view text_view recline_view recline_grid_view p #ckan.preview.image_formats = png jpeg jpg gif ## Front-End Settings + ckan.site_title = openAFRICA ckan.site_logo = /base/images/ckan-logo.png ckan.site_description = "Africa's largest independent source for open data" @@ -130,7 +147,6 @@ ckan.display_timezone = server # package_hide_extras = for_search_index_only #package_edit_return_url = http://another.frontend/dataset/ #package_new_return_url = http://another.frontend/dataset/ -#ckan.recaptcha.version = 1 #ckan.recaptcha.publickey = #ckan.recaptcha.privatekey = #licenses_group_url = http://licenses.opendefinition.org/licenses/groups/ckan.json @@ -156,6 +172,11 @@ ckan.feeds.author_link = #ckan.max_resource_size = 10 #ckan.max_image_size = 2 +## Webassets Settings +#ckan.webassets.use_x_sendfile = false +#ckan.webassets.path = /var/lib/ckan/webassets + + ## Datapusher settings # Make sure you have set up the DataStore @@ -169,6 +190,8 @@ ckan.datapusher.assume_task_stale_after = 3600 #ckan.resource_proxy.max_file_size = 1048576 # Size of chunks to read/write. #ckan.resource_proxy.chunk_size = 4096 +# Default timeout for fetching proxied items +#ckan.resource_proxy.timeout = 10 ## Activity Streams Settings @@ -191,24 +214,14 @@ ckan.base_templates_folder = templates #smtp.user = username@example.com #smtp.password = your_password #smtp.mail_from = +#smtp.reply_to = - -## Harvester settings -ckan.harvest.mq.type = redis -ckan.harvest.log_scope = 0 -ckan.harvest.log_level = info - -# Datastore settings -ckan.datastore.write_url = postgresql://ckan_default:pass@localhost/datastore_default -ckan.datastore.read_url = postgresql://datastore_default:pass@localhost/datastore_default - -# Sentry settings -sentry.configure_logging = True -sentry.log_level = WARN +## Background Job Settings +ckan.jobs.timeout = 180 ## Logging configuration [loggers] -keys = root, ckan, ckanext +keys = root, ckan, ckanext, werkzeug [handlers] keys = console @@ -220,6 +233,12 @@ keys = generic level = WARNING handlers = console +[logger_werkzeug] +level = WARNING +handlers = console +qualname = werkzeug +propagate = 0 + [logger_ckan] level = INFO handlers = console diff --git a/docker-compose.yml b/docker-compose.yml index a774d46..e084595 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,8 +14,6 @@ services: env_file: - .env volumes: - - ./ckan.ini:/ckan.ini - - ckan-filestore:/var/lib/ckan/default - ./ckanext-openafrica/:/srv/app/src/ckanext-openafrica/ ckan_gather_consumer: @@ -28,7 +26,7 @@ services: - .env command: ckan --config=ckan.ini --plugin=ckanext-harvest harvester gather-consumer volumes: - - ./ckan.ini:/ckan.ini + # - ./ckan.ini:/ckan.ini - ckan-filestore:/var/lib/ckan/default datapusher: From b6a1c49d926c9424c4625af8c19c98d1e31fe2bf Mon Sep 17 00:00:00 2001 From: Xavier Frankline Date: Tue, 6 Feb 2024 22:09:27 +0300 Subject: [PATCH 19/20] update, customise ckan.ini file --- .../ckan/docker-entrypoint.d/customize_ckan.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 contrib/ckan/docker-entrypoint.d/customize_ckan.sh diff --git a/contrib/ckan/docker-entrypoint.d/customize_ckan.sh b/contrib/ckan/docker-entrypoint.d/customize_ckan.sh new file mode 100644 index 0000000..e436561 --- /dev/null +++ b/contrib/ckan/docker-entrypoint.d/customize_ckan.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +#Setup CKAN Datarequests (Init DB) +if [[ $CKAN__PLUGINS == *"datarequests"* ]]; then + ckan -c /srv/app/ckan.ini datarequests initdb +else + echo "Datarequests extension not available" +fi + +if [[ $CKAN__PLUGINS == *"openafrica"* ]]; then + ckan config-tool ${CKAN_INI} -s app:main -e ckan.site_title="openAFRICA" + ckan config-tool ${CKAN_INI} -s app:main -e ckan.site_description=""Africa's largest independent source for open data" +else + echo "Not appliyng OpenAfrica customizations" +fi \ No newline at end of file From 8cce1c16fd1ac48e5322aa8aa5e9988377245b96 Mon Sep 17 00:00:00 2001 From: Xavier Frankline Date: Tue, 6 Feb 2024 22:24:09 +0300 Subject: [PATCH 20/20] update datarequests init-db --- contrib/ckan/docker-entrypoint.d/customize_ckan.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/ckan/docker-entrypoint.d/customize_ckan.sh b/contrib/ckan/docker-entrypoint.d/customize_ckan.sh index e436561..d1bb4dc 100644 --- a/contrib/ckan/docker-entrypoint.d/customize_ckan.sh +++ b/contrib/ckan/docker-entrypoint.d/customize_ckan.sh @@ -2,7 +2,7 @@ #Setup CKAN Datarequests (Init DB) if [[ $CKAN__PLUGINS == *"datarequests"* ]]; then - ckan -c /srv/app/ckan.ini datarequests initdb + ckan -c /srv/app/ckan.ini datarequests init-db else echo "Datarequests extension not available" fi