Merge pull request #30 from CodeForAfrica/chore-upgrade-to-ckan-2.8

Chore upgrade to ckan 2.8

.env.tmpl

@@ -12,10 +12,10 @@ CKAN___BEAKER__SESSION__SECRET=
 CKAN___APP_INSTANCE_UUID=
 
 # Database config
-CKAN__DATASTORE_WRITE_URL=postgres://ckan_default:pass@db/datastore_default
-CKAN__DATASTORE_READ_URL=postgres://datastore_default:pass@db/datastore_default
+CKAN_DATASTORE_WRITE_URL=postgres://ckan_default:pass@db/datastore_default
+CKAN_DATASTORE_READ_URL=postgres://datastore_default:pass@db/datastore_default
 
-CKAN__REDIS__URL=redis://redis:6379/0
+CKAN_REDIS_URL=redis://redis:6379/0
 
 # Local storage config
 CKAN__STORAGE_PATH=/var/lib/ckan

.gitignore

@@ -105,7 +105,6 @@ ENV/
 ckan
 ckan/
 ckan-*.tar.gz
-
 # Env
 env*
 !env.tmpl

Dockerfile

@@ -1,4 +1,4 @@
-FROM codeforafrica/ckan:2.7.6
+FROM codeforafrica/ckan:2.8.3
 
 EXPOSE 5000/tcp
 

Makefile

@@ -24,18 +24,18 @@ issues-init:
 
 
 ckan:
-	docker build --no-cache --build-arg CKAN_VERSION=2.7.6 -t codeforafrica/ckan:latest -t codeforafrica/ckan:2.7.6 contrib/ckan
+	docker build --no-cache --build-arg CKAN_VERSION=2.8.11 -t codeforafrica/ckan:latest -t codeforafrica/ckan:2.8.11 contrib/ckan
 
 ckan-publish:
 	docker push codeforafrica/ckan:latest
-	docker push codeforafrica/ckan:2.7.6
+	docker push codeforafrica/ckan:2.8.11
 
 solr:
-	docker build --no-cache --build-arg CKAN_VERSION=2.7.6 -t codeforafrica/ckan-solr:latest -t codeforafrica/ckan-solr:2.7.6 contrib/solr
+	docker build --no-cache --build-arg CKAN_VERSION=2.8.11 -t codeforafrica/ckan-solr:latest -t codeforafrica/ckan-solr:2.8.11 contrib/solr
 
 solr-publish:
 	docker push codeforafrica/ckan-solr:latest
-	docker push codeforafrica/ckan-solr:2.7.6
+	docker push codeforafrica/ckan-solr:2.8.11
 
 
 datapusher:

README.md

@@ -14,16 +14,14 @@ We use CKAN's own vanilla releases but because they haven't properly adopted Doc
 The ckan extensions we are using include:
 
 - ckanext-openafrica - https://github.com/CodeForAfrica/ckanext-openafrica
-- ckanext-datarequests - https://github.com/CodeForAfricaLabs/ckanext-datarequests
+- ckanext-datarequests - https://github.com/conwetlab/ckanext-datarequests
 - ckanext-harvester - https://github.com/ckan/ckanext-harvest
-- ckanext-socialite (experimental) - https://github.com/CodeForAfricaLabs/ckanext-socialite
-- ckanext-social - https://github.com/CodeForAfricaLabs/ckanext-social
-- ckanext-notify - https://github.com/CodeForAfricaLabs/ckanext-notify
-- ckanext-s3filestore - https://github.com/CodeForAfricaLabs/ckanext-s3filestore
+- ckanext-s3filestore - https://github.com/okfn/ckanext-s3filestore
 - ckanext-showcase - https://github.com/ckan/ckanext-showcase
 - ckanext-googleanalytics - https://github.com/ckan/ckanext-googleanalytics
 - ckanext-issues - https://github.com/ckan/ckanext-issues
 - ckanext-gdoc - https://github.com/OpenUpSA/ckanext-gdoc
+- ckanext-envvars - https://github.com/okfn/ckanext-envvars/
 
 
 ---

ckan.ini

@@ -176,6 +176,9 @@ ckan.datapusher.assume_task_stale_after = 3600
 #ckan.email_notifications_since = 2 days
 ckan.hide_activity_from_users = %(ckan.site_id)s
 
+# Bootstrap
+ckan.base_public_folder = public-bs2
+ckan.base_templates_folder = templates-bs2
 
 ## Email settings
 

contrib/ckan/Dockerfile

@@ -2,9 +2,13 @@ FROM python:2.7
 
 ENV DEBIAN_FRONTEND noninteractive
 
-ARG CKAN_VERSION=2.7.6
+ARG CKAN_VERSION=2.8.11
 
 RUN pip install -U -q pip setuptools
 
 RUN pip install -q -e "git+https://github.com/ckan/ckan.git@ckan-${CKAN_VERSION}#egg=ckan"
+
+ADD meta.py /src/ckan/ckan/model/meta.py
+
 RUN pip install -q -r /src/ckan/requirements.txt
+RUN pip install -q vdm==0.15 sqlalchemy==1.2.19

contrib/ckan/meta.py

@@ -0,0 +1,147 @@
+# encoding: utf-8
+
+import datetime
+
+from paste.deploy.converters import asbool
+from ckan.common import config
+"""SQLAlchemy Metadata and Session object"""
+from sqlalchemy import MetaData, and_
+import sqlalchemy.orm as orm
+from sqlalchemy.orm.session import SessionExtension
+
+import extension
+import ckan.lib.activity_streams_session_extension as activity
+
+__all__ = ['Session', 'engine_is_sqlite', 'engine_is_pg']
+
+
+class CkanCacheExtension(SessionExtension):
+    ''' This extension checks what tables have been affected by
+    database access and allows us to act on them. Currently this is
+    used by the page cache to flush the cache when data in the database
+    is altered. '''
+
+    def __init__(self, *args, **kw):
+        super(CkanCacheExtension, self).__init__(*args, **kw)
+
+    def after_commit(self, session):
+        if hasattr(session, '_object_cache'):
+            oc = session._object_cache
+            oc_list = oc['new']
+            oc_list.update(oc['changed'])
+            oc_list.update(oc['deleted'])
+            objs = set()
+            for item in oc_list:
+                objs.add(item.__class__.__name__)
+
+
+class CkanSessionExtension(SessionExtension):
+
+    def before_flush(self, session, flush_context, instances):
+        if not hasattr(session, '_object_cache'):
+            session._object_cache= {'new': set(),
+                                    'deleted': set(),
+                                    'changed': set()}
+
+        changed = [obj for obj in session.dirty if
+            session.is_modified(obj, include_collections=False, passive=True)]
+
+        session._object_cache['new'].update(session.new)
+        session._object_cache['deleted'].update(session.deleted)
+        session._object_cache['changed'].update(changed)
+
+
+    def before_commit(self, session):
+        session.flush()
+        try:
+            obj_cache = session._object_cache
+            revision = session.revision
+        except AttributeError:
+            return
+        if getattr(session, 'revisioning_disabled', False):
+            return
+        new = obj_cache['new']
+        changed = obj_cache['changed']
+        deleted = obj_cache['deleted']
+        for obj in new | changed | deleted:
+            if not hasattr(obj, '__revision_class__'):
+                continue
+            revision_cls = obj.__revision_class__
+            revision_table = orm.class_mapper(revision_cls).mapped_table
+            ## when a normal active transaction happens
+
+            ### this is an sql statement as we do not want it in object cache
+            session.execute(
+                revision_table.update().where(
+                    and_(revision_table.c.id == obj.id,
+                         revision_table.c.current == True)
+                ).values(current=False)
+            )
+
+            q = session.query(revision_cls)
+            q = q.filter_by(expired_timestamp=datetime.datetime(9999, 12, 31), id=obj.id)
+            results = q.all()
+            for rev_obj in results:
+                values = {}
+                if rev_obj.revision_id == revision.id:
+                    values['revision_timestamp'] = revision.timestamp
+                else:
+                    values['expired_timestamp'] = revision.timestamp
+                session.execute(
+                    revision_table.update().where(
+                        and_(revision_table.c.id == rev_obj.id,
+                             revision_table.c.revision_id == rev_obj.revision_id)
+                    ).values(**values)
+                )
+
+    def after_commit(self, session):
+        if hasattr(session, '_object_cache'):
+            del session._object_cache
+
+    def after_rollback(self, session):
+        if hasattr(session, '_object_cache'):
+            del session._object_cache
+
+# __all__ = ['Session', 'engine', 'metadata', 'mapper']
+
+# SQLAlchemy database engine. Updated by model.init_model()
+engine = None
+
+Session = orm.scoped_session(orm.sessionmaker(
+    autoflush=False,
+    autocommit=False,
+    expire_on_commit=False,
+    extension=[CkanCacheExtension(),
+               CkanSessionExtension(),
+               extension.PluginSessionExtension(),
+               activity.DatasetActivitySessionExtension()],
+))
+
+create_local_session = orm.sessionmaker(
+    autoflush=False,
+    autocommit=False,
+    expire_on_commit=False,
+    extension=[CkanCacheExtension(),
+               CkanSessionExtension(),
+               extension.PluginSessionExtension(),
+               activity.DatasetActivitySessionExtension()],
+)
+
+#mapper = Session.mapper
+mapper = orm.mapper
+
+# Global metadata. If you have multiple databases with overlapping table
+# names, you'll need a metadata for each database
+metadata = MetaData()
+
+
+def engine_is_sqlite(sa_engine=None):
+    # Returns true iff the engine is connected to a sqlite database.
+    return (sa_engine or engine).url.drivername == 'sqlite'
+
+
+def engine_is_pg(sa_engine=None):
+    # Returns true iff the engine is connected to a postgresql database.
+    # According to http://docs.sqlalchemy.org/en/latest/core/engines.html#postgresql
+    # all Postgres driver names start with `postgres`
+    return (sa_engine or engine).url.drivername.startswith('postgres')

contrib/solr/Dockerfile

@@ -1,32 +1,33 @@
-FROM solr:6.2
+FROM solr:6.6.2
 MAINTAINER Open Knowledge
 
 ## Taken from https://github.com/ckan/ckan/blob/ckan-2.7.0/contrib/docker/solr/Dockerfile
 # Updated the ARG values
-ARG CKAN_VERSION=2.7.6
+ARG CKAN_VERSION=2.8.11
 
 # Enviroment
 ENV SOLR_CORE ckan
 
-# User
-USER root
-
 # Create Directories
 RUN mkdir -p /opt/solr/server/solr/$SOLR_CORE/conf
 RUN mkdir -p /opt/solr/server/solr/$SOLR_CORE/data
 
 # Adding Files
 ADD https://raw.githubusercontent.com/ckan/ckan/ckan-${CKAN_VERSION}/contrib/docker/solr/solrconfig.xml \
 https://raw.githubusercontent.com/ckan/ckan/ckan-${CKAN_VERSION}/ckan/config/solr/schema.xml \
-https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.0.0/solr/server/solr/configsets/basic_configs/conf/currency.xml \
-https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.0.0/solr/server/solr/configsets/basic_configs/conf/synonyms.txt \
-https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.0.0/solr/server/solr/configsets/basic_configs/conf/stopwords.txt \
-https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.0.0/solr/server/solr/configsets/basic_configs/conf/protwords.txt \
-https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.0.0/solr/server/solr/configsets/data_driven_schema_configs/conf/elevate.xml \
+https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.6.2/solr/server/solr/configsets/basic_configs/conf/currency.xml \
+https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.6.2/solr/server/solr/configsets/basic_configs/conf/synonyms.txt \
+https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.6.2/solr/server/solr/configsets/basic_configs/conf/stopwords.txt \
+https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.6.2/solr/server/solr/configsets/basic_configs/conf/protwords.txt \
+https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.6.2/solr/server/solr/configsets/data_driven_schema_configs/conf/elevate.xml \
 /opt/solr/server/solr/$SOLR_CORE/conf/
 
 # Create Core.properties
 RUN echo name=$SOLR_CORE > /opt/solr/server/solr/$SOLR_CORE/core.properties
 
 # Giving ownership to Solr
+USER root
 RUN chown -R $SOLR_USER:$SOLR_USER /opt/solr/server/solr/$SOLR_CORE
+
+# User
+USER $SOLR_USER:$SOLR_USER

docker-compose.yml

@@ -10,7 +10,7 @@ services:
         - solr
         - redis
     ports:
-        - "5000:5000"
+        - "5050:5000"
     env_file:
       - .env
     volumes:
@@ -37,7 +37,7 @@ services:
       - web
 
   db:
-    image: postgres:9.6
+    image: postgres:13.7
     ports:
       - "54321:5432"
     environment:
@@ -51,7 +51,7 @@ services:
       - ./contrib/postgres:/docker-entrypoint-initdb.d
 
   solr:
-    image: codeforafrica/ckan-solr:2.7.6
+    image: codeforafrica/ckan-solr:2.8.11
     ports:
         - "8983:8983"
     volumes:

requirements.txt

@@ -14,7 +14,7 @@ gevent==1.2.2
 -e git+https://github.com/ckan/ckanext-issues.git#egg=ckanext-issues
 
 -e git+https://github.com/ckan/ckanext-harvest.git@v1.2.1#egg=ckanext-harvest
--e git+https://github.com/ckan/ckanext-dcat.git@v0.0.9#egg=ckanext-dcat
+-e git+https://github.com/ckan/ckanext-dcat.git@v1.1.3#egg=ckanext-dcat
 
 -e git+https://github.com/ckan/ckanext-pdfview.git@0.0.5#egg=ckanext-pdfview