Merge branch 'ckan_2.9'

Vagrantfile

@@ -16,11 +16,11 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
     end
 
 
-    server.vm.provision "shell", inline: "sudo apt-get update && sudo apt-get -y install python2"
+    server.vm.provision "shell", inline: "sudo apt-get update"
 
     server.vm.provision "ansible_local" do |ansible|
-      ansible.install_mode = "pip"
-      ansible.pip_install_cmd = "sudo apt-get install -y python3-distutils && curl -s https://bootstrap.pypa.io/get-pip.py | sudo python3"
+      ansible.install_mode = "pip3"
+      #ansible.pip_install_cmd = "sudo apt-get install -y python3-distutils && curl -s https://bootstrap.pypa.io/get-pip.py | sudo python3"
       ansible.inventory_path = "inventories/vagrant"
       ansible.limit = "all"
       ansible.playbook = "deploy-all.yml"

ansible/deploy-all.yml

@@ -20,9 +20,6 @@
     - postgres-data
     - solr6
     - redis
-    - nginx
-    - role: acmetool
-      when: deployment_environment_id != "vagrant" and deployment_environment_id != "build"
     - apache
     - postfix
     - ckan-extensions
@@ -32,10 +29,11 @@
     - ckan-database
     - ckan-cron
     - datapusher
+    - supervisor
+    - nginx
     - transifex-install
     - ckan-translations
     - ckan-restart
-    - supervisor
     - solr-reindex
     - php-fpm
     - mysql

ansible/deploy-transitional.yml

@@ -20,9 +20,6 @@
     - postgres-data
     - solr6
     - redis
-    - nginx
-    - role: geerlingguy.certbot
-      when: deployment_environment_id != "vagrant" and deployment_environment_id != "build"
     - apache
     - postfix
     - ckan-extensions
@@ -32,10 +29,13 @@
     - ckan-database
     - ckan-cron
     - datapusher
+    - supervisor
+    - nginx
+    - role: geerlingguy.certbot
+      when: deployment_environment_id != "vagrant" and deployment_environment_id != "build"
     #- transifex-install Only used on Vagrant
     - ckan-translations
     - ckan-restart
-    - supervisor
     - solr-reindex
     - php-fpm
     - mysql

ansible/frontend-build.yml

@@ -12,7 +12,7 @@
     - "{{ secrets_file_path }}"
   roles:
     #- ckan-ui
-    - { role: ckan-extensions, ckan_extensions: ckanext-sixodp_ui }
+    - { role: ckan-extensions, ckan_extensions: ckanext-sixodp }
     - ckan
     - ckan-translations
     - ckan-restart

ansible/inventories/vagrant

@@ -4,5 +4,4 @@
 [mainserver:vars]
 deployment_environment_id=vagrant
 secrets_file_path="vars/secrets-defaults.yml"
-google_analytics_secrets_file_path="{{ cache_path}}/google_analytics_credentials.json"
-ansible_python_interpreter=/usr/bin/python2
+google_analytics_secrets_file_path="{{ cache_path}}/google_analytics_credentials.json"

ansible/roles/apache/tasks/main.yml

@@ -1,29 +1,6 @@
 ---
 
-- name: Install Apache
-  apt: pkg={{ item }} state=latest
-  with_items:
-    - apache2
-    - libapache2-mod-wsgi
-    - libffi-dev
-
-- name: Disable Apache sites
-  file: path=/etc/apache2/sites-enabled/{{ item }} state=absent
-  with_items:
-    - default
-    - default-ssl
-    - 000-default
-    - ckan_default
-
-- name: Copy Apache configuration
-  template: src={{ item.src }} dest={{ item.dest }} mode={{ item.mode }} owner={{ item.owner }} group={{ item.owner }}
-  with_items:
-    - { src: ports.conf.j2, dest: /etc/apache2/ports.conf, mode: "0644", owner: root}
-    - { src: apache2.j2, dest: /etc/apache2/sites-available/ckan_catalog.conf, mode: "0644", owner: root}
-    - { src: logrotate.apache2.conf, dest: /etc/logrotate.d/apache2, mode: "0644", owner: root}
-
-- name: Enable Apache configuration
-  file: src=/etc/apache2/sites-available/ckan_catalog.conf dest=/etc/apache2/sites-enabled/ckan_catalog.conf state=link owner=root group=root
-
-- name: Restart Apache
-  service: name=apache2 state=restarted
+- name: Ensure apache in removed
+  apt:
+    pkg: apache2
+    state: absent

ansible/roles/ckan-config/tasks/main.yml

@@ -10,8 +10,8 @@
   template: src={{ item.src }} dest={{ item.dest }} mode={{ item.mode }} owner={{ item.owner }} group={{ item.group }}
   with_items:
     - { src: ckan.py, dest: /usr/bin/ckan, mode: "0755", owner: root, group: root }
-    - { src: ckan.ini.j2, dest: "{{ ckan_ini }}", mode: "0640", owner: root, group: "{{ www_group }}" }
-    - { src: apache.wsgi.j2, dest: /etc/ckan/default/apache.wsgi, mode: "0640", owner: root, group: "{{ www_group }}" }
+    - { src: ckan.ini.j2, dest: "{{ ckan_ini }}", mode: "0640", owner: "{{ www_user }}", group: "{{ www_group }}" }
+    - { src: wsgi.py.j2, dest: /etc/ckan/default/wsgi.py, mode: "0640", owner: "{{ www_user }}", group: "{{ www_group }}" }
     - { src: robots.txt, dest: "{{ virtualenv }}/src/ckan/ckan/public/robots.txt", mode: "0644", owner: root, group: root }
     - { src: test-core.ini.j2, dest: "/src/ckanext/ckan/test-core.ini", mode: "0644", owner: root, group: "{{ www_group }}"}
     - { src: test-core.ini.j2, dest: "{{ ckan_test_core_ini }}", mode: "0644", owner: root, group: "{{ www_group }}"}

ansible/roles/ckan-config/templates/ckan.ini.j2

@@ -81,24 +81,24 @@ ckan.feeds.author_link =
 
 scheming.presets = ckanext.scheming:presets.json
                    ckanext.fluent:presets.json
-                   ckanext.sixodp_scheming:presets.json
+                   ckanext.sixodp:presets.json
 
-scheming.dataset_schemas = ckanext.sixodp_scheming.schemas:dataset.json
+scheming.dataset_schemas = ckanext.sixodp.schemas:dataset.json
                            ckanext.sixodp_showcase.schemas:showcase.json
 
-scheming.group_schemas = ckanext.sixodp_scheming.schemas:group.json
+scheming.group_schemas = ckanext.sixodp.schemas:group.json
                          ckanext.collection.schemas:collection.json
 
-licenses_group_url = file://{{ckanext_sync_path}}/ckanext-sixodp_scheming/ckanext/sixodp_scheming/licenses/licenses.json
+licenses_group_url = file://{{ckanext_sync_path}}/ckanext-sixodp/ckanext/sixodp/licenses/licenses.json
 
-ckanext.sixodp_ui.cms_site_url = {{ cms_site_url }}
-ckanext.sixodp_ui.wp_api_base_url = /wp-json/wp/v2
+ckanext.sixodp.cms_site_url = {{ cms_site_url }}
+ckanext.sixodp.wp_api_base_url = /wp-json/wp/v2
 
-ckanext.sixodp_ui.wp_api_menus_base_url = /wp-json/wp-api-menus/v2
-ckanext.sixodp_ui.wp_main_menu_location = primary
-ckanext.sixodp_ui.wp_footer_menu_location = footer
-ckanext.sixodp_ui.wp_social_menu_location = socialmedia
-ckanext.sixodp_ui.cookiehub_domain_code = {{ cookiehub_domain_code }}
+ckanext.sixodp.wp_api_menus_base_url = /wp-json/wp-api-menus/v2
+ckanext.sixodp.wp_main_menu_location = primary
+ckanext.sixodp.wp_footer_menu_location = footer
+ckanext.sixodp.wp_social_menu_location = socialmedia
+ckanext.sixodp.cookiehub_domain_code = {{ cookiehub_domain_code }}
 
 # Disable sending error emails. Commented out because the default is None, and "None" is interpreted here as a string.
 # email_to = 

ansible/roles/ckan-config/templates/who.ini.j2

@@ -1,10 +1,10 @@
 [plugin:auth_tkt]
-use = ckan.lib.auth_tkt:make_plugin
+use = ckan.lib.repoze_plugins.auth_tkt:make_plugin
 # If no secret key is defined here, beaker.session.secret will be used
 #secret = somesecret
 
 [plugin:friendlyform]
-use = repoze.who.plugins.friendlyform:FriendlyFormPlugin
+use = ckan.lib.repoze_plugins.friendly_form:FriendlyFormPlugin
 login_form_url= /user/login
 login_handler_path = /login_generic
 logout_handler_path = /data/user/logout

ansible/roles/ckan-config/templates/wsgi.py.j2

@@ -0,0 +1,12 @@
+# -*- coding: utf-8 -*-
+
+import os
+from ckan.config.middleware import make_app
+from ckan.cli import CKANConfigLoader
+from logging.config import fileConfig as loggingFileConfig
+config_filepath = os.path.join(
+    os.path.dirname(os.path.abspath(__file__)), u'ckan.ini')
+abspath = os.path.join(os.path.dirname(os.path.abspath(__file__)))
+loggingFileConfig(config_filepath)
+config = CKANConfigLoader(config_filepath).get_config()
+application = make_app(config)

ansible/roles/ckan-database/tasks/main.yml

@@ -1,58 +1,58 @@
 ---
 
 - name: Initialize CKAN database
-  command: ckan db init
+  command: ./bin/ckan "--config={{ ckan_ini }}" db init chdir={{ virtualenv }}
   ignore_errors: True
 
 - name: Upgrade CKAN database
-  command: ckan db upgrade
+  command: ./bin/ckan "--config={{ ckan_ini }}" db upgrade chdir={{ virtualenv }}
   ignore_errors: True
 
 - name: Upgrade CKAN filestore
-  command: ./bin/paster --plugin=ckan db migrate-filestore "--config={{ ckan_ini }}" chdir={{ virtualenv }}
+  command: ./bin/ckan "--config={{ ckan_ini }}" db migrate-filestore chdir={{ virtualenv }}
   ignore_errors: True
 
 - name: Create initial CKAN users
-  command: ./bin/paster --plugin=ckan user add "{{ item.value.username }}" "password={{ item.value.password }}" "email={{ item.value.email }}" "--config={{ ckan_ini }}" chdir={{ virtualenv }}
+  command: ./bin/ckan  "--config={{ ckan_ini }}" user add "{{ item.value.username }}" "password={{ item.value.password }}" "email={{ item.value.email }}" chdir={{ virtualenv }}
   ignore_errors: True
   with_dict: "{{ ckan_users }}"
 
 - name: Set CKAN sysadmins
-  command: ./bin/paster --plugin=ckan sysadmin add {{ item }} --config={{ ckan_ini }} chdir={{ virtualenv }}
+  command: ./bin/ckan "--config={{ ckan_ini }}" sysadmin add {{ item }} chdir={{ virtualenv }}
   ignore_errors: True
   with_items: "{{ ckan_admins }}"
 
-- name: Initialize Harvest database
-  command: ./bin/paster --plugin=ckanext-harvest harvester initdb "--config={{ ckan_ini }}" chdir={{ virtualenv }}
-  when: "'ckanext-harvest' in ckan_extensions"
+#- name: Initialize Harvest database
+#  command: ./bin/paster --plugin=ckanext-harvest harvester initdb "--config={{ ckan_ini }}" chdir={{ virtualenv }}
+#  when: "'ckanext-harvest' in ckan_extensions"
 
-- name: Initialize archiver database
-  command: ./bin/paster --plugin=ckanext-archiver archiver init "--config={{ ckan_ini }}" chdir={{ virtualenv }}
+#- name: Initialize archiver database
+#  command: ./bin/paster --plugin=ckanext-archiver archiver init "--config={{ ckan_ini }}" chdir={{ virtualenv }}
 
 # This can be removed after run in all environments
-- name: Migrate archiver database
-  command: ./bin/paster --plugin=ckanext-archiver archiver migrate "--config={{ ckan_ini }}" chdir={{ virtualenv }}
+#- name: Migrate archiver database
+#  command: ./bin/paster --plugin=ckanext-archiver archiver migrate "--config={{ ckan_ini }}" chdir={{ virtualenv }}
 
-- name: Initialize report database
-  command: ./bin/paster --plugin=ckanext-report report initdb "--config={{ ckan_ini }}" chdir={{ virtualenv }}
+#- name: Initialize report database
+#  command: ./bin/paster --plugin=ckanext-report report initdb "--config={{ ckan_ini }}" chdir={{ virtualenv }}
 
-- name: Initialize QA database
-  command: ./bin/paster --plugin=ckanext-qa qa init "--config={{ ckan_ini }}" chdir={{ virtualenv }}
+#- name: Initialize QA database
+#  command: ./bin/paster --plugin=ckanext-qa qa init "--config={{ ckan_ini }}" chdir={{ virtualenv }}
 
-- name: Initialize Rating database
-  command: ./bin/paster --plugin=ckanext-rating rating init "--config={{ ckan_ini }}" chdir={{ virtualenv }}
+#- name: Initialize Rating database
+#  command: ./bin/paster --plugin=ckanext-rating rating init "--config={{ ckan_ini }}" chdir={{ virtualenv }}
 
-- name: Initialize Reminder database
-  command: ./bin/paster --plugin=ckanext-reminder reminder init "--config={{ ckan_ini }}" chdir={{ virtualenv }}
+#- name: Initialize Reminder database
+#  command: ./bin/paster --plugin=ckanext-reminder reminder init "--config={{ ckan_ini }}" chdir={{ virtualenv }}
 
-- name: Initialize ckanext-googleanalytics database
-  command: ./bin/paster --plugin=ckanext-googleanalytics initdb "--config={{ ckan_ini }}" chdir={{ virtualenv }}
-  when: "'ckanext-googleanalytics' in ckan_extensions"
+#- name: Initialize ckanext-googleanalytics database
+#  command: ./bin/paster --plugin=ckanext-googleanalytics initdb "--config={{ ckan_ini }}" chdir={{ virtualenv }}
+#  when: "'ckanext-googleanalytics' in ckan_extensions"
 
-- name: Initialize ckanext-ga-report database
-  command: ./bin/paster --plugin=ckanext-ga-report initdb "--config={{ ckan_ini }}" chdir={{ virtualenv }}
-  when: "'ckanext-ga-report' in ckan_extensions"
+#- name: Initialize ckanext-ga-report database
+#  command: ./bin/paster --plugin=ckanext-ga-report initdb "--config={{ ckan_ini }}" chdir={{ virtualenv }}
+#  when: "'ckanext-ga-report' in ckan_extensions"
 
-- name: Initialize ckanext-cloudstorage database
-  command: ./bin/paster --plugin=ckanext-cloudstorage cloudstorage initdb "--config={{ ckan_ini }}" chdir={{ virtualenv }}
-  when: AWS.enabled
+#- name: Initialize ckanext-cloudstorage database
+#  command: ./bin/paster --plugin=ckanext-cloudstorage cloudstorage initdb "--config={{ ckan_ini }}" chdir={{ virtualenv }}
+#  when: AWS.enabled

ansible/roles/ckan-extensions/tasks/main.yml

@@ -23,5 +23,5 @@
   copy: src="/{{playbook_dir}}/../sixodp-secrets/google_analytics_credentials.json" dest="{{ google_analytics_secrets_file_path }}"
   when: deployment_environment_id == "vagrant" and "ckanext-googleanalytics" in ckan_extensions
 
-- name: Restart Apache
-  service: name=apache2 state=restarted
+#- name: Restart ckan
+#  notify: Restart ckan

ansible/roles/ckan-install-patches/tasks/main.yml

@@ -16,11 +16,15 @@
   - patch_ckan
 
 - name: Re-minify CKAN core javascript files after patches
-  shell: ./bin/paster --plugin=ckan minify {{ ckan_source_path }}/ckan/public/base/javascript "--config={{ ckan_ini }}" chdir="{{ virtualenv }}"
+  shell:
+    cmd: "./bin/ckan --config={{ ckan_ini }} minify {{ ckan_source_path }}/ckan/public/base/javascript"
+    chdir: "{{ virtualenv }}"
   tags:
   - patch_ckan
 
 - name: Re-minify CKAN vendor javascript files after patches
-  shell: ./bin/paster --plugin=ckan minify {{ ckan_source_path }}/ckan/public/base/vendor "--config={{ ckan_ini }}" chdir="{{ virtualenv }}"
+  shell:
+    cmd: "./bin/ckan --config={{ ckan_ini }} minify {{ ckan_source_path }}/ckan/public/base/vendor"
+    chdir: "{{ virtualenv }}"
   tags:
     - patch_ckan

ansible/roles/ckan-install-patches/vars/main.yml

@@ -7,21 +7,21 @@ ckan_patches:
   - { file: "alert_fix" }
   - { file: "allow_org_admins_create_users" }
   - { file: "set_attachment_content_disposition" }
-  - { file: "package_group_list_fix" }
+  #- { file: "package_group_list_fix" }
   - { file: "enable_multiple_image_uploads" }
   - { file: "fix-default-search-field" }
   - { file: "make_creating_tags_optional" }
   - { file: "fix-popped-wrong-app-context" }
   - { file: "fix_unicode_decode_error_in_error_page" }
   - { file: "fix-data-viewer-race-condition"}
   - { file: "remove_members_from_group_read"}
-  - { file: "remove_user_activity_from_group_activity_list"}
-  - { file: "optimize_group_show" }
+  #- { file: "remove_user_activity_from_group_activity_list"}
+  #- { file: "optimize_group_show" }
   - { file: "optimize_template_loading" }
   - { file: "group_include_extras" }
-  - { file: "add_group_extra_resource_db_indices" }
+  #- { file: "add_group_extra_resource_db_indices" }
   - { file: "correctly_flatten_extras" }
-  - { file: "change_meta_strings_to_booleans" }
+  #- { file: "upgrade_jquery" }
 
 files_created_by_patches:
   - { file: '/usr/lib/ckan/default/src/ckan/ckan/lib/csrf_token.py'}

ansible/roles/ckan-install/files/ckan-uwsgi.ini

@@ -0,0 +1,17 @@
+[uwsgi]
+
+http            =  127.0.0.1:8080
+uid             =  www-data
+guid            =  www-data
+wsgi-file       =  /etc/ckan/default/wsgi.py
+virtualenv      =  /usr/lib/ckan/default
+chdir           =  /etc/ckan/default
+mount           =  /data=wsgi:application
+manage-script-name = true
+master          =  true
+pidfile         =  /tmp/%n.pid
+harakiri        =  50
+max-requests    =  5000
+vacuum          =  true
+callable        =  application
+buffer-size     =  32768

ansible/roles/ckan-install/files/requirement-setuptools.txt

@@ -1 +1 @@
-setuptools==36.1
+setuptools==44.1.0