diff --git a/.env.tmpl b/.env.tmpl index 564091e..6fbd5b3 100755 --- a/.env.tmpl +++ b/.env.tmpl @@ -12,10 +12,10 @@ CKAN___BEAKER__SESSION__SECRET= CKAN___APP_INSTANCE_UUID= # Database config -CKAN__DATASTORE_WRITE_URL=postgres://ckan_default:pass@db/datastore_default -CKAN__DATASTORE_READ_URL=postgres://datastore_default:pass@db/datastore_default +CKAN_DATASTORE_WRITE_URL=postgres://ckan_default:pass@db/datastore_default +CKAN_DATASTORE_READ_URL=postgres://datastore_default:pass@db/datastore_default -CKAN__REDIS__URL=redis://redis:6379/0 +CKAN_REDIS_URL=redis://redis:6379/0 # Local storage config CKAN__STORAGE_PATH=/var/lib/ckan diff --git a/.gitignore b/.gitignore index bf8d431..d55b30a 100644 --- a/.gitignore +++ b/.gitignore @@ -105,7 +105,6 @@ ENV/ ckan ckan/ ckan-*.tar.gz - # Env env* !env.tmpl diff --git a/Dockerfile b/Dockerfile index d8be8b8..150ae78 100755 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM codeforafrica/ckan:2.7.6 +FROM codeforafrica/ckan:2.8.3 EXPOSE 5000/tcp diff --git a/Makefile b/Makefile index 0e8a06d..88c6b8d 100644 --- a/Makefile +++ b/Makefile @@ -24,18 +24,18 @@ issues-init: ckan: - docker build --no-cache --build-arg CKAN_VERSION=2.7.6 -t codeforafrica/ckan:latest -t codeforafrica/ckan:2.7.6 contrib/ckan + docker build --no-cache --build-arg CKAN_VERSION=2.8.11 -t codeforafrica/ckan:latest -t codeforafrica/ckan:2.8.11 contrib/ckan ckan-publish: docker push codeforafrica/ckan:latest - docker push codeforafrica/ckan:2.7.6 + docker push codeforafrica/ckan:2.8.11 solr: - docker build --no-cache --build-arg CKAN_VERSION=2.7.6 -t codeforafrica/ckan-solr:latest -t codeforafrica/ckan-solr:2.7.6 contrib/solr + docker build --no-cache --build-arg CKAN_VERSION=2.8.11 -t codeforafrica/ckan-solr:latest -t codeforafrica/ckan-solr:2.8.11 contrib/solr solr-publish: docker push codeforafrica/ckan-solr:latest - docker push codeforafrica/ckan-solr:2.7.6 + docker push codeforafrica/ckan-solr:2.8.11 datapusher: diff --git a/README.md b/README.md index 2c77ca8..31824b6 100644 --- a/README.md +++ b/README.md @@ -14,16 +14,14 @@ We use CKAN's own vanilla releases but because they haven't properly adopted Doc The ckan extensions we are using include: - ckanext-openafrica - https://github.com/CodeForAfrica/ckanext-openafrica -- ckanext-datarequests - https://github.com/CodeForAfricaLabs/ckanext-datarequests +- ckanext-datarequests - https://github.com/conwetlab/ckanext-datarequests - ckanext-harvester - https://github.com/ckan/ckanext-harvest -- ckanext-socialite (experimental) - https://github.com/CodeForAfricaLabs/ckanext-socialite -- ckanext-social - https://github.com/CodeForAfricaLabs/ckanext-social -- ckanext-notify - https://github.com/CodeForAfricaLabs/ckanext-notify -- ckanext-s3filestore - https://github.com/CodeForAfricaLabs/ckanext-s3filestore +- ckanext-s3filestore - https://github.com/okfn/ckanext-s3filestore - ckanext-showcase - https://github.com/ckan/ckanext-showcase - ckanext-googleanalytics - https://github.com/ckan/ckanext-googleanalytics - ckanext-issues - https://github.com/ckan/ckanext-issues - ckanext-gdoc - https://github.com/OpenUpSA/ckanext-gdoc +- ckanext-envvars - https://github.com/okfn/ckanext-envvars/ --- diff --git a/ckan.ini b/ckan.ini index ea866d2..1d3d646 100755 --- a/ckan.ini +++ b/ckan.ini @@ -176,6 +176,9 @@ ckan.datapusher.assume_task_stale_after = 3600 #ckan.email_notifications_since = 2 days ckan.hide_activity_from_users = %(ckan.site_id)s +# Bootstrap +ckan.base_public_folder = public-bs2 +ckan.base_templates_folder = templates-bs2 ## Email settings diff --git a/contrib/ckan/Dockerfile b/contrib/ckan/Dockerfile index 53ad733..3f64506 100644 --- a/contrib/ckan/Dockerfile +++ b/contrib/ckan/Dockerfile @@ -2,9 +2,13 @@ FROM python:2.7 ENV DEBIAN_FRONTEND noninteractive -ARG CKAN_VERSION=2.7.6 +ARG CKAN_VERSION=2.8.11 RUN pip install -U -q pip setuptools RUN pip install -q -e "git+https://github.com/ckan/ckan.git@ckan-${CKAN_VERSION}#egg=ckan" + +ADD meta.py /src/ckan/ckan/model/meta.py + RUN pip install -q -r /src/ckan/requirements.txt +RUN pip install -q vdm==0.15 sqlalchemy==1.2.19 diff --git a/contrib/ckan/meta.py b/contrib/ckan/meta.py new file mode 100644 index 0000000..0b89c75 --- /dev/null +++ b/contrib/ckan/meta.py @@ -0,0 +1,147 @@ +# encoding: utf-8 + +import datetime + +from paste.deploy.converters import asbool +from ckan.common import config +"""SQLAlchemy Metadata and Session object""" +from sqlalchemy import MetaData, and_ +import sqlalchemy.orm as orm +from sqlalchemy.orm.session import SessionExtension + +import extension +import ckan.lib.activity_streams_session_extension as activity + +__all__ = ['Session', 'engine_is_sqlite', 'engine_is_pg'] + + +class CkanCacheExtension(SessionExtension): + ''' This extension checks what tables have been affected by + database access and allows us to act on them. Currently this is + used by the page cache to flush the cache when data in the database + is altered. ''' + + def __init__(self, *args, **kw): + super(CkanCacheExtension, self).__init__(*args, **kw) + + def after_commit(self, session): + if hasattr(session, '_object_cache'): + oc = session._object_cache + oc_list = oc['new'] + oc_list.update(oc['changed']) + oc_list.update(oc['deleted']) + objs = set() + for item in oc_list: + objs.add(item.__class__.__name__) + + +class CkanSessionExtension(SessionExtension): + + def before_flush(self, session, flush_context, instances): + if not hasattr(session, '_object_cache'): + session._object_cache= {'new': set(), + 'deleted': set(), + 'changed': set()} + + changed = [obj for obj in session.dirty if + session.is_modified(obj, include_collections=False, passive=True)] + + session._object_cache['new'].update(session.new) + session._object_cache['deleted'].update(session.deleted) + session._object_cache['changed'].update(changed) + + + def before_commit(self, session): + session.flush() + try: + obj_cache = session._object_cache + revision = session.revision + except AttributeError: + return + if getattr(session, 'revisioning_disabled', False): + return + new = obj_cache['new'] + changed = obj_cache['changed'] + deleted = obj_cache['deleted'] + for obj in new | changed | deleted: + if not hasattr(obj, '__revision_class__'): + continue + revision_cls = obj.__revision_class__ + revision_table = orm.class_mapper(revision_cls).mapped_table + ## when a normal active transaction happens + + ### this is an sql statement as we do not want it in object cache + session.execute( + revision_table.update().where( + and_(revision_table.c.id == obj.id, + revision_table.c.current == True) + ).values(current=False) + ) + + q = session.query(revision_cls) + q = q.filter_by(expired_timestamp=datetime.datetime(9999, 12, 31), id=obj.id) + results = q.all() + for rev_obj in results: + values = {} + if rev_obj.revision_id == revision.id: + values['revision_timestamp'] = revision.timestamp + else: + values['expired_timestamp'] = revision.timestamp + session.execute( + revision_table.update().where( + and_(revision_table.c.id == rev_obj.id, + revision_table.c.revision_id == rev_obj.revision_id) + ).values(**values) + ) + + def after_commit(self, session): + if hasattr(session, '_object_cache'): + del session._object_cache + + def after_rollback(self, session): + if hasattr(session, '_object_cache'): + del session._object_cache + +# __all__ = ['Session', 'engine', 'metadata', 'mapper'] + +# SQLAlchemy database engine. Updated by model.init_model() +engine = None + +Session = orm.scoped_session(orm.sessionmaker( + autoflush=False, + autocommit=False, + expire_on_commit=False, + extension=[CkanCacheExtension(), + CkanSessionExtension(), + extension.PluginSessionExtension(), + activity.DatasetActivitySessionExtension()], +)) + +create_local_session = orm.sessionmaker( + autoflush=False, + autocommit=False, + expire_on_commit=False, + extension=[CkanCacheExtension(), + CkanSessionExtension(), + extension.PluginSessionExtension(), + activity.DatasetActivitySessionExtension()], +) + +#mapper = Session.mapper +mapper = orm.mapper + +# Global metadata. If you have multiple databases with overlapping table +# names, you'll need a metadata for each database +metadata = MetaData() + + +def engine_is_sqlite(sa_engine=None): + # Returns true iff the engine is connected to a sqlite database. + return (sa_engine or engine).url.drivername == 'sqlite' + + +def engine_is_pg(sa_engine=None): + # Returns true iff the engine is connected to a postgresql database. + # According to http://docs.sqlalchemy.org/en/latest/core/engines.html#postgresql + # all Postgres driver names start with `postgres` + return (sa_engine or engine).url.drivername.startswith('postgres') diff --git a/contrib/postgres/add-databases.sh b/contrib/postgres/add-databases.sh old mode 100644 new mode 100755 diff --git a/contrib/solr/Dockerfile b/contrib/solr/Dockerfile index f19271c..f73cfd9 100644 --- a/contrib/solr/Dockerfile +++ b/contrib/solr/Dockerfile @@ -1,16 +1,13 @@ -FROM solr:6.2 +FROM solr:6.6.2 MAINTAINER Open Knowledge ## Taken from https://github.com/ckan/ckan/blob/ckan-2.7.0/contrib/docker/solr/Dockerfile # Updated the ARG values -ARG CKAN_VERSION=2.7.6 +ARG CKAN_VERSION=2.8.11 # Enviroment ENV SOLR_CORE ckan -# User -USER root - # Create Directories RUN mkdir -p /opt/solr/server/solr/$SOLR_CORE/conf RUN mkdir -p /opt/solr/server/solr/$SOLR_CORE/data @@ -18,15 +15,19 @@ RUN mkdir -p /opt/solr/server/solr/$SOLR_CORE/data # Adding Files ADD https://raw.githubusercontent.com/ckan/ckan/ckan-${CKAN_VERSION}/contrib/docker/solr/solrconfig.xml \ https://raw.githubusercontent.com/ckan/ckan/ckan-${CKAN_VERSION}/ckan/config/solr/schema.xml \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.0.0/solr/server/solr/configsets/basic_configs/conf/currency.xml \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.0.0/solr/server/solr/configsets/basic_configs/conf/synonyms.txt \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.0.0/solr/server/solr/configsets/basic_configs/conf/stopwords.txt \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.0.0/solr/server/solr/configsets/basic_configs/conf/protwords.txt \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.0.0/solr/server/solr/configsets/data_driven_schema_configs/conf/elevate.xml \ +https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.6.2/solr/server/solr/configsets/basic_configs/conf/currency.xml \ +https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.6.2/solr/server/solr/configsets/basic_configs/conf/synonyms.txt \ +https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.6.2/solr/server/solr/configsets/basic_configs/conf/stopwords.txt \ +https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.6.2/solr/server/solr/configsets/basic_configs/conf/protwords.txt \ +https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.6.2/solr/server/solr/configsets/data_driven_schema_configs/conf/elevate.xml \ /opt/solr/server/solr/$SOLR_CORE/conf/ # Create Core.properties RUN echo name=$SOLR_CORE > /opt/solr/server/solr/$SOLR_CORE/core.properties # Giving ownership to Solr +USER root RUN chown -R $SOLR_USER:$SOLR_USER /opt/solr/server/solr/$SOLR_CORE + +# User +USER $SOLR_USER:$SOLR_USER diff --git a/docker-compose.yml b/docker-compose.yml index ece4824..ef05cb1 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,7 +10,7 @@ services: - solr - redis ports: - - "5000:5000" + - "5050:5000" env_file: - .env volumes: @@ -37,7 +37,7 @@ services: - web db: - image: postgres:9.6 + image: postgres:13.7 ports: - "54321:5432" environment: @@ -51,7 +51,7 @@ services: - ./contrib/postgres:/docker-entrypoint-initdb.d solr: - image: codeforafrica/ckan-solr:2.7.6 + image: codeforafrica/ckan-solr:2.8.11 ports: - "8983:8983" volumes: diff --git a/requirements.txt b/requirements.txt index 8763779..0d049fa 100755 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,7 @@ gevent==1.2.2 -e git+https://github.com/ckan/ckanext-issues.git#egg=ckanext-issues -e git+https://github.com/ckan/ckanext-harvest.git@v1.2.1#egg=ckanext-harvest --e git+https://github.com/ckan/ckanext-dcat.git@v0.0.9#egg=ckanext-dcat +-e git+https://github.com/ckan/ckanext-dcat.git@v1.1.3#egg=ckanext-dcat -e git+https://github.com/ckan/ckanext-pdfview.git@0.0.5#egg=ckanext-pdfview