From f66c6f2bb12f2fc43e3c0cc884b6aa2f5485e708 Mon Sep 17 00:00:00 2001 From: Kathryn Killebrew Date: Fri, 1 Mar 2019 17:00:07 -0500 Subject: [PATCH 01/21] Support Xenial OTP server Connects azavea/ansible-opentripplanner#35. --- Vagrantfile | 2 +- deployment/ansible/otp.yml | 2 +- .../cac-tripplanner.otp-data/tasks/main.yml | 7 ++++++ .../templates/otp.conf.j2 | 11 ++++++++ .../templates/otp.service.js | 17 +++++++++++++ deployment/packer/cac_packer.py | 25 +++++++++++-------- 6 files changed, 52 insertions(+), 12 deletions(-) create mode 100644 deployment/ansible/roles/cac-tripplanner.otp-data/templates/otp.service.js diff --git a/Vagrantfile b/Vagrantfile index 08d5f652c..23f5fe9b9 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -88,7 +88,7 @@ VAGRANT_PROXYCONF_ENDPOINT = ENV["VAGRANT_PROXYCONF_ENDPOINT"] VAGRANTFILE_API_VERSION = "2" Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - config.vm.box = "ubuntu/trusty64" + config.vm.box = "bento/ubuntu-16.04" # Wire up the proxy if: # diff --git a/deployment/ansible/otp.yml b/deployment/ansible/otp.yml index 99f0224f3..0a8eda6d9 100644 --- a/deployment/ansible/otp.yml +++ b/deployment/ansible/otp.yml @@ -7,4 +7,4 @@ apt: update_cache=yes roles: - - { role: "cac-tripplanner.otp-data"} + - {role: "cac-tripplanner.otp-data"} diff --git a/deployment/ansible/roles/cac-tripplanner.otp-data/tasks/main.yml b/deployment/ansible/roles/cac-tripplanner.otp-data/tasks/main.yml index d0a290708..fcaac0422 100644 --- a/deployment/ansible/roles/cac-tripplanner.otp-data/tasks/main.yml +++ b/deployment/ansible/roles/cac-tripplanner.otp-data/tasks/main.yml @@ -16,6 +16,13 @@ - name: Overwrite upstart service template: src=otp.conf.j2 dest=/etc/init/otp.conf + register: otpconf + +- name: Reload systemd + shell: systemctl daemon-reload + when: otpconf.changed + notify: + - Restart OpenTripPlanner - name: Create data directory (test) file: path=./otp_data/ owner={{ansible_user_id}} group={{ansible_user_id}} mode=0664 state=directory diff --git a/deployment/ansible/roles/cac-tripplanner.otp-data/templates/otp.conf.j2 b/deployment/ansible/roles/cac-tripplanner.otp-data/templates/otp.conf.j2 index 0c21e1576..c7aa4a344 100644 --- a/deployment/ansible/roles/cac-tripplanner.otp-data/templates/otp.conf.j2 +++ b/deployment/ansible/roles/cac-tripplanner.otp-data/templates/otp.conf.j2 @@ -1,5 +1,16 @@ ## NOTICE: This file is written by ansible, and any changes made here will be overwritten on next provision. # Modify cac-tripplanner.otp-data/templates/otp.conf.j2 to make changes stick. + +### BEGIN INIT INFO +# Provides: otp +# Required-Start: $network $local_fs $remote_fs +# Required-Stop: $network $local_fs $remote_fs +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: OpenTripPlanner server daemon +# Description: Enable OpenTripPlanner server +### END INIT INFO + description "OpenTripPlanner" start on {{ otp_upstart_start_on }} diff --git a/deployment/ansible/roles/cac-tripplanner.otp-data/templates/otp.service.js b/deployment/ansible/roles/cac-tripplanner.otp-data/templates/otp.service.js new file mode 100644 index 000000000..9279b4f4e --- /dev/null +++ b/deployment/ansible/roles/cac-tripplanner.otp-data/templates/otp.service.js @@ -0,0 +1,17 @@ +## NOTICE: This file is written by ansible, and any changes made here will be overwritten on +# next provision. +# Modify azavea.opentripplanner/templates/otp.service.j2 to make changes stick. + +[Unit] +Description=Start OpenTripPlanner process +After={{ otp_service_after }} + +[Service] +Type=forking +User={{ otp_user }} +WorkingDirectory={{ otp_bin_dir }} +Restart=on-failure +ExecStart=/usr/bin/authbind /usr/bin/java -Xmx{{otp_process_mem}} -jar {{ otp_bin_dir }}/{{ otp_jar_name }} --server --analyst --port 80 --basePath {{ otp_data_dir}} --graphs {{ otp_data_dir }} --router default + +[Install] +WantedBy={{ otp_service_wantedby }} diff --git a/deployment/packer/cac_packer.py b/deployment/packer/cac_packer.py index 0dcd7cee5..4ed62e2c1 100644 --- a/deployment/packer/cac_packer.py +++ b/deployment/packer/cac_packer.py @@ -21,16 +21,21 @@ def get_ubuntu_ami(region, creds): creds (Dict): Dictionary containing AWS credentials """ - conn = ec2.connect_to_region(region, **creds) - amis = conn.get_all_images(owners=[CANONICAL_ACCOUNT_ID], filters={ - 'name': 'ubuntu/images/hvm-ssd/ubuntu-trusty-14.04-amd64-server-*', - 'architecture': 'x86_64', - 'root-device-type': 'ebs', - 'virtualization-type': 'hvm', - }) - - amis = sorted(amis, key=lambda ami: ami.creationDate, reverse=True) - + response = urllib2.urlopen('http://cloud-images.ubuntu.com/query/xenial/' + 'server/released.current.txt').readlines() + fieldnames = ['version', 'version_type', 'release_status', 'date', + 'storage', 'arch', 'region', 'id', 'kernel', + 'unknown_col', 'virtualization_type'] + reader = csv.DictReader(response, fieldnames=fieldnames, delimiter='\t') + + def ami_filter(ami): + """Helper function to filter AMIs""" + return (ami['region'] == region and + ami['arch'] == 'amd64' and + ami['storage'] == 'ebs-ssd' and + ami['virtualization_type'] == 'hvm') + + amis = [row for row in reader if ami_filter(row)] if len(amis) == 0: raise CacStackException('Did not find any ubuntu AMIs to use') return amis[0].id From 69e7e977c291c6a60f495b81729c770214328d7e Mon Sep 17 00:00:00 2001 From: Kathryn Killebrew Date: Mon, 8 Jul 2019 14:05:28 -0400 Subject: [PATCH 02/21] Bump postgres role versions Newer versions support Xenial. --- deployment/ansible/roles.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deployment/ansible/roles.yml b/deployment/ansible/roles.yml index 4260d9fc5..45f3d210d 100644 --- a/deployment/ansible/roles.yml +++ b/deployment/ansible/roles.yml @@ -23,7 +23,7 @@ version: 0.1.0 - src: azavea.postgresql-support - version: 0.3.2 + version: 0.4.0 - src: azavea.postgresql - version: 0.5.0 + version: 1.0.0 From ac38c433a606e7da69d59d9e48feeda469d65047 Mon Sep 17 00:00:00 2001 From: Kathryn Killebrew Date: Mon, 8 Jul 2019 14:10:56 -0400 Subject: [PATCH 03/21] Upgrade from postgresql 9.4 to 9.5 The version available in the Xenial system packages is 9.5. --- deployment/ansible/group_vars/development_template | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/deployment/ansible/group_vars/development_template b/deployment/ansible/group_vars/development_template index 846d4f305..752480e91 100644 --- a/deployment/ansible/group_vars/development_template +++ b/deployment/ansible/group_vars/development_template @@ -26,8 +26,9 @@ default_admin_username: 'admin' default_admin_password: 'admin' default_admin_email: 'systems+cac@azavea.com' -postgresql_version: "9.4" -postgresql_package_version: "9.4.*.pgdg14.04+1" +postgresql_version: "9.5" +postgresql_package_version: "9.5.*" +postgresql_support_libpq_version: "*" postgresql_listen_addresses: "*" postgresql_hba_mapping: From c232a49a28dcddc392a4c0c69081351043636978 Mon Sep 17 00:00:00 2001 From: Kathryn Killebrew Date: Mon, 8 Jul 2019 14:14:54 -0400 Subject: [PATCH 04/21] Fix python package reference for Xenial Still on 2.7. --- deployment/ansible/group_vars/all | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deployment/ansible/group_vars/all b/deployment/ansible/group_vars/all index ed8274eb7..ca96eec4d 100644 --- a/deployment/ansible/group_vars/all +++ b/deployment/ansible/group_vars/all @@ -11,6 +11,8 @@ postgres_host: "192.168.8.25" packer_version: "1.0.2" +python_version: "2.7.*" + nodejs_version: 10.16.0 nodejs_npm_version: 6.9.0 From 1787af2bdab5a70cc421fd9136d4cf1a553d1797 Mon Sep 17 00:00:00 2001 From: Kathryn Killebrew Date: Mon, 8 Jul 2019 14:37:33 -0400 Subject: [PATCH 05/21] Fix NFS version to 4 Fixes issue on linux of version defaulting to 3, although not overridded in environment. --- Vagrantfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Vagrantfile b/Vagrantfile index 23f5fe9b9..ee817beae 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -5,7 +5,7 @@ Vagrant.require_version ">= 2.0" require "yaml" CAC_SHARED_FOLDER_TYPE = ENV.fetch("CAC_SHARED_FOLDER_TYPE", "nfs") -CAC_NFS_VERSION = ENV.fetch("CAC_NFS_VERSION_3", true) ? 'vers=3': 'vers=4' +CAC_NFS_VERSION = 'vers=4' if CAC_SHARED_FOLDER_TYPE == "nfs" if Vagrant::Util::Platform.linux? then From 4513d83a1378a18c25558b7c8d575beb30361374 Mon Sep 17 00:00:00 2001 From: Kathryn Killebrew Date: Tue, 9 Jul 2019 11:42:31 -0400 Subject: [PATCH 06/21] Use official Ubuntu Xenial vagrant image --- Vagrantfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index ee817beae..434a9a54d 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -88,7 +88,7 @@ VAGRANT_PROXYCONF_ENDPOINT = ENV["VAGRANT_PROXYCONF_ENDPOINT"] VAGRANTFILE_API_VERSION = "2" Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - config.vm.box = "bento/ubuntu-16.04" + config.vm.box = "ubuntu/xenial64" # Wire up the proxy if: # @@ -136,7 +136,9 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| if testing? app.vm.synced_folder ".", "/opt/app" else - app.vm.synced_folder ".", "/opt/app", type: CAC_SHARED_FOLDER_TYPE, mount_options: CAC_MOUNT_OPTIONS + app.vm.synced_folder ".", "/opt/app", + type: CAC_SHARED_FOLDER_TYPE, + mount_options: CAC_MOUNT_OPTIONS end # Web From 832f4af1686783dc79c2459f380e67752339ffc1 Mon Sep 17 00:00:00 2001 From: Kathryn Killebrew Date: Tue, 9 Jul 2019 11:43:01 -0400 Subject: [PATCH 07/21] Add PyYAML and requests as dependencies Fixes migrations failing to run. --- deployment/ansible/roles/cac-tripplanner.app/defaults/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deployment/ansible/roles/cac-tripplanner.app/defaults/main.yml b/deployment/ansible/roles/cac-tripplanner.app/defaults/main.yml index da96e7c1f..3655b337d 100644 --- a/deployment/ansible/roles/cac-tripplanner.app/defaults/main.yml +++ b/deployment/ansible/roles/cac-tripplanner.app/defaults/main.yml @@ -25,6 +25,8 @@ cac_python_dependencies: - { name: 'Pillow', version: '6.1.0' } - { name: 'psycopg2-binary', version: '2.8.3' } - { name: 'pytz', version: '2019.1' } + - { name: 'PyYAML', version: '5.1.1' } + - { name: 'requests', version: '2.22.0' } - { name: 'troposphere', version: '1.8.1' } - { name: 'majorkirby', version: '0.2.1' } # Note: django-wpadmin is installed manually to work around the fact that ansible-pip From ab511c61c8df33525a774f1e3a569f86fb92f6f2 Mon Sep 17 00:00:00 2001 From: Kathryn Killebrew Date: Tue, 9 Jul 2019 13:15:04 -0400 Subject: [PATCH 08/21] Use python2 on OTP VM Necessary because transitfeed package does not support python3 yet. --- .../roles/cac-tripplanner.transitfeed/meta/main.yml | 1 + .../roles/cac-tripplanner.transitfeed/tasks/main.yml | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/deployment/ansible/roles/cac-tripplanner.transitfeed/meta/main.yml b/deployment/ansible/roles/cac-tripplanner.transitfeed/meta/main.yml index d7ee68014..f45b16e16 100644 --- a/deployment/ansible/roles/cac-tripplanner.transitfeed/meta/main.yml +++ b/deployment/ansible/roles/cac-tripplanner.transitfeed/meta/main.yml @@ -1,4 +1,5 @@ --- dependencies: + - { role: "azavea.python", python_development: True } - { role: "azavea.pip" } - { role: "azavea.git" } diff --git a/deployment/ansible/roles/cac-tripplanner.transitfeed/tasks/main.yml b/deployment/ansible/roles/cac-tripplanner.transitfeed/tasks/main.yml index 57645fca1..6bda05223 100644 --- a/deployment/ansible/roles/cac-tripplanner.transitfeed/tasks/main.yml +++ b/deployment/ansible/roles/cac-tripplanner.transitfeed/tasks/main.yml @@ -1,6 +1,10 @@ --- - name: Install pytz - pip: name=pytz + pip: + name: pytz + executable: pip2 - name: Install transitfeed - pip: name='git+https://github.com/google/transitfeed@1.2.15#egg=transitfeed' + pip: + name: 'git+https://github.com/google/transitfeed@1.2.15#egg=transitfeed' + executable: pip2 From cdc08f6335d01c0c7a02106cdd89956b0dcab8aa Mon Sep 17 00:00:00 2001 From: Kathryn Killebrew Date: Tue, 9 Jul 2019 19:59:16 -0400 Subject: [PATCH 09/21] Update OTP role for Xenial Use systemd for service. --- .../cac-tripplanner.otp-data/tasks/main.yml | 4 +-- .../templates/otp.conf.j2 | 25 ------------------- .../{otp.service.js => otp.service.j2} | 2 +- 3 files changed, 3 insertions(+), 28 deletions(-) delete mode 100644 deployment/ansible/roles/cac-tripplanner.otp-data/templates/otp.conf.j2 rename deployment/ansible/roles/cac-tripplanner.otp-data/templates/{otp.service.js => otp.service.j2} (97%) diff --git a/deployment/ansible/roles/cac-tripplanner.otp-data/tasks/main.yml b/deployment/ansible/roles/cac-tripplanner.otp-data/tasks/main.yml index fcaac0422..5f510a92f 100644 --- a/deployment/ansible/roles/cac-tripplanner.otp-data/tasks/main.yml +++ b/deployment/ansible/roles/cac-tripplanner.otp-data/tasks/main.yml @@ -14,8 +14,8 @@ - name: Change owner and mode of byport file file: path=/etc/authbind/byport/80 owner={{ otp_user }} mode=0755 -- name: Overwrite upstart service - template: src=otp.conf.j2 dest=/etc/init/otp.conf +- name: Overwrite systemd service + template: src=otp.service.j2 dest=/etc/systemd/system/otp.service register: otpconf - name: Reload systemd diff --git a/deployment/ansible/roles/cac-tripplanner.otp-data/templates/otp.conf.j2 b/deployment/ansible/roles/cac-tripplanner.otp-data/templates/otp.conf.j2 deleted file mode 100644 index c7aa4a344..000000000 --- a/deployment/ansible/roles/cac-tripplanner.otp-data/templates/otp.conf.j2 +++ /dev/null @@ -1,25 +0,0 @@ -## NOTICE: This file is written by ansible, and any changes made here will be overwritten on next provision. -# Modify cac-tripplanner.otp-data/templates/otp.conf.j2 to make changes stick. - -### BEGIN INIT INFO -# Provides: otp -# Required-Start: $network $local_fs $remote_fs -# Required-Stop: $network $local_fs $remote_fs -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: OpenTripPlanner server daemon -# Description: Enable OpenTripPlanner server -### END INIT INFO - -description "OpenTripPlanner" - -start on {{ otp_upstart_start_on }} -stop on shutdown - -respawn -setuid {{ otp_user }} -chdir {{ otp_bin_dir }} - -script - exec /usr/bin/authbind /usr/bin/java -Xmx{{otp_process_mem}} -jar {{ otp_bin_dir }}/{{ otp_jar_name }} --server --analyst --port 80 --basePath {{ otp_data_dir}} --graphs {{ otp_data_dir }} --router default -end script diff --git a/deployment/ansible/roles/cac-tripplanner.otp-data/templates/otp.service.js b/deployment/ansible/roles/cac-tripplanner.otp-data/templates/otp.service.j2 similarity index 97% rename from deployment/ansible/roles/cac-tripplanner.otp-data/templates/otp.service.js rename to deployment/ansible/roles/cac-tripplanner.otp-data/templates/otp.service.j2 index 9279b4f4e..00d898c90 100644 --- a/deployment/ansible/roles/cac-tripplanner.otp-data/templates/otp.service.js +++ b/deployment/ansible/roles/cac-tripplanner.otp-data/templates/otp.service.j2 @@ -7,7 +7,7 @@ Description=Start OpenTripPlanner process After={{ otp_service_after }} [Service] -Type=forking +Type=simple User={{ otp_user }} WorkingDirectory={{ otp_bin_dir }} Restart=on-failure From b14f593c054c01f5159b109344342b40da1da277 Mon Sep 17 00:00:00 2001 From: Kathryn Killebrew Date: Thu, 11 Jul 2019 10:06:50 -0400 Subject: [PATCH 10/21] Use bento image --- Vagrantfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Vagrantfile b/Vagrantfile index 434a9a54d..c614ebd5e 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -88,7 +88,7 @@ VAGRANT_PROXYCONF_ENDPOINT = ENV["VAGRANT_PROXYCONF_ENDPOINT"] VAGRANTFILE_API_VERSION = "2" Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - config.vm.box = "ubuntu/xenial64" + config.vm.box = "bento/ubuntu-16.04" # Wire up the proxy if: # From 6288a346135de8043ec9aa131ffd9ab9c3abfa58 Mon Sep 17 00:00:00 2001 From: Kathryn Killebrew Date: Fri, 12 Jul 2019 10:37:02 -0400 Subject: [PATCH 11/21] Install python dependencies from requirements.txt --- .../cac-tripplanner.app/defaults/main.yml | 20 ------------------- .../roles/cac-tripplanner.app/tasks/main.yml | 7 +++---- python/cac_tripplanner/requirements.txt | 16 +++++++++++++++ 3 files changed, 19 insertions(+), 24 deletions(-) create mode 100644 python/cac_tripplanner/requirements.txt diff --git a/deployment/ansible/roles/cac-tripplanner.app/defaults/main.yml b/deployment/ansible/roles/cac-tripplanner.app/defaults/main.yml index 3655b337d..6835fc53e 100644 --- a/deployment/ansible/roles/cac-tripplanner.app/defaults/main.yml +++ b/deployment/ansible/roles/cac-tripplanner.app/defaults/main.yml @@ -11,23 +11,3 @@ root_conf_dir: "/etc/cac_tripplanner.d" root_src_dir: "/opt/app/src" root_static_dir: "/srv/cac" root_media_dir: "/media/cac" - -cac_python_dependencies: - - { name: 'base58', version: '1.0.3' } - - { name: 'boto', version: '2.49.0' } - - { name: 'django', version: '1.11.21' } - - { name: 'django-ckeditor', version: '5.7.1' } - - { name: 'django-image-cropping', version: '1.2.0' } - - { name: 'django-extensions', version: '1.9.9' } - - { name: 'django-storages', version: '1.7.1' } - - { name: 'easy-thumbnails', version: '2.6.0' } - - { name: 'gunicorn', version: '19.9.0' } - - { name: 'Pillow', version: '6.1.0' } - - { name: 'psycopg2-binary', version: '2.8.3' } - - { name: 'pytz', version: '2019.1' } - - { name: 'PyYAML', version: '5.1.1' } - - { name: 'requests', version: '2.22.0' } - - { name: 'troposphere', version: '1.8.1' } - - { name: 'majorkirby', version: '0.2.1' } - # Note: django-wpadmin is installed manually to work around the fact that ansible-pip - # ignores editable=False diff --git a/deployment/ansible/roles/cac-tripplanner.app/tasks/main.yml b/deployment/ansible/roles/cac-tripplanner.app/tasks/main.yml index cc720d29b..d4bb39790 100644 --- a/deployment/ansible/roles/cac-tripplanner.app/tasks/main.yml +++ b/deployment/ansible/roles/cac-tripplanner.app/tasks/main.yml @@ -13,8 +13,7 @@ - libjpeg-dev - name: Install pip packages - pip: name={{ item.name }} version={{ item.version }} - with_items: "{{ cac_python_dependencies }}" + pip: requirements={{ root_app_dir }}/requirements.txt # Putting 'editable: false' in the entry in cac_python_dependencies should make it install # non-editable, but it's getting ignored @@ -60,13 +59,13 @@ - name: Run migrations django_manage: command=migrate - app_path=/opt/app/python/cac_tripplanner + app_path="{{ root_app_dir }}" notify: Restart cac-tripplanner-app when: develop or test - name: Run collectstatic django_manage: command=collectstatic - app_path=/opt/app/python/cac_tripplanner + app_path="{{ root_app_dir }}" - name: Copy nginx config template: src=nginx-default.j2 dest=/etc/nginx/sites-available/default diff --git a/python/cac_tripplanner/requirements.txt b/python/cac_tripplanner/requirements.txt new file mode 100644 index 000000000..f3686faad --- /dev/null +++ b/python/cac_tripplanner/requirements.txt @@ -0,0 +1,16 @@ +base58==1.0.3 +boto3==1.9.187 +Django==1.11.22 +django-ckeditor==5.7.1 +django-image-cropping==1.2.0 +django-extensions==1.9.9 +django-storages==1.7.1 +easy-thumbnails==2.6.0 +gunicorn==19.9.0 +Pillow==6.1.0 +psycopg2-binary==2.8.3 +pytz==2019.1 +PyYAML==5.1.1 +requests==2.22.0 +troposphere==1.8.1 +majorkirby==0.2.1 From eb32422249cda8cf2371e6b1db890b2e02f43b01 Mon Sep 17 00:00:00 2001 From: Kathryn Killebrew Date: Fri, 12 Jul 2019 11:52:32 -0400 Subject: [PATCH 12/21] Switch app service to systemd Also update ansible package install syntax to address deprecation warning. --- .../cac-tripplanner.app/defaults/main.yml | 1 + .../cac-tripplanner.app/handlers/main.yml | 4 +-- .../roles/cac-tripplanner.app/tasks/main.yml | 36 +++++++++++-------- .../systemd-cac-tripplanner-app.conf.j2 | 20 +++++++++++ .../upstart-cac-tripplanner-app.conf.j2 | 14 -------- 5 files changed, 45 insertions(+), 30 deletions(-) create mode 100644 deployment/ansible/roles/cac-tripplanner.app/templates/systemd-cac-tripplanner-app.conf.j2 delete mode 100644 deployment/ansible/roles/cac-tripplanner.app/templates/upstart-cac-tripplanner-app.conf.j2 diff --git a/deployment/ansible/roles/cac-tripplanner.app/defaults/main.yml b/deployment/ansible/roles/cac-tripplanner.app/defaults/main.yml index 6835fc53e..3896018c5 100644 --- a/deployment/ansible/roles/cac-tripplanner.app/defaults/main.yml +++ b/deployment/ansible/roles/cac-tripplanner.app/defaults/main.yml @@ -6,6 +6,7 @@ app_sass_version: "3.4.22" app_log: "/var/log/cac-tripplanner-app.log" +gunicorn_app_name: "cac-tripplanner-app" root_app_dir: "/opt/app/python/cac_tripplanner" root_conf_dir: "/etc/cac_tripplanner.d" root_src_dir: "/opt/app/src" diff --git a/deployment/ansible/roles/cac-tripplanner.app/handlers/main.yml b/deployment/ansible/roles/cac-tripplanner.app/handlers/main.yml index b864a010e..ec6baed2c 100644 --- a/deployment/ansible/roles/cac-tripplanner.app/handlers/main.yml +++ b/deployment/ansible/roles/cac-tripplanner.app/handlers/main.yml @@ -1,6 +1,6 @@ --- - name: Restart cac-tripplanner-app - service: name=cac-tripplanner-app state=restarted + service: name=cac-tripplanner-app use=sysv state=restarted - name: Restart nginx - service: name=nginx state=restarted + service: name=nginx use=sysv state=restarted diff --git a/deployment/ansible/roles/cac-tripplanner.app/tasks/main.yml b/deployment/ansible/roles/cac-tripplanner.app/tasks/main.yml index d4bb39790..2039cd4c0 100644 --- a/deployment/ansible/roles/cac-tripplanner.app/tasks/main.yml +++ b/deployment/ansible/roles/cac-tripplanner.app/tasks/main.yml @@ -2,15 +2,16 @@ # # Note bzip2 only installed here as workaround for unresolved but closed phantomjs # install issue: https://github.com/Medium/phantomjs/issues/659 - name: Install packages - apt: name={{ item }} state=present - with_items: - - binutils - - bzip2 - - gdal-bin - - libpq-dev - - libproj-dev - - python-dev - - libjpeg-dev + apt: + state: present + pkg: + - binutils + - bzip2 + - gdal-bin + - libpq-dev + - libproj-dev + - python-dev + - libjpeg-dev - name: Install pip packages pip: requirements={{ root_app_dir }}/requirements.txt @@ -40,15 +41,22 @@ - name: Write secrets file template: src=cac_secrets.j2 dest=/etc/cac_secrets when: develop or production - notify: Restart cac-tripplanner-app + notify: Restart {{ gunicorn_app_name }} - name: Configure Gunicorn settings template: src=gunicorn-cac-tripplanner.py.j2 dest={{ root_conf_dir }}/gunicorn.py - notify: Restart cac-tripplanner-app + notify: Restart {{ gunicorn_app_name }} - name: Configure service definition - template: src=upstart-cac-tripplanner-app.conf.j2 dest=/etc/init/cac-tripplanner-app.conf - notify: Restart cac-tripplanner-app + template: src=systemd-{{ gunicorn_app_name }}.conf.j2 + dest=/etc/systemd/system/{{ gunicorn_app_name }}.service + notify: Restart {{ gunicorn_app_name }} + +- name: Enable gunicorn service + systemd: + name: "{{ gunicorn_app_name }}.service" + enabled: yes + daemon_reload: yes - name: Copy media assets copy: src=../../python/cac_tripplanner/default_media @@ -60,7 +68,7 @@ - name: Run migrations django_manage: command=migrate app_path="{{ root_app_dir }}" - notify: Restart cac-tripplanner-app + notify: Restart {{ gunicorn_app_name }} when: develop or test - name: Run collectstatic diff --git a/deployment/ansible/roles/cac-tripplanner.app/templates/systemd-cac-tripplanner-app.conf.j2 b/deployment/ansible/roles/cac-tripplanner.app/templates/systemd-cac-tripplanner-app.conf.j2 new file mode 100644 index 000000000..2dee4fb20 --- /dev/null +++ b/deployment/ansible/roles/cac-tripplanner.app/templates/systemd-cac-tripplanner-app.conf.j2 @@ -0,0 +1,20 @@ +[Unit] +Description = {{ gunicorn_app_name }} +After = network-online.target + +[Service] +PermissionsStartOnly = true +User = {{ app_username }} +Group = {{ app_username }} +WorkingDirectory = {{ root_app_dir }} +ExecStart = /usr/bin/env gunicorn --config {{ root_conf_dir }}/gunicorn.py --timeout {{ otp_session_timeout_s }} cac_tripplanner.wsgi +ExecReload = /bin/kill -s HUP $MAINPID +ExecStop = /bin/kill -s TERM $MAINPID +PrivateTmp = true +StandardOutput = syslog +StandardError = syslog +SyslogIdentifier = {{ gunicorn_app_name }} + +[Install] +WantedBy = network-online.target + diff --git a/deployment/ansible/roles/cac-tripplanner.app/templates/upstart-cac-tripplanner-app.conf.j2 b/deployment/ansible/roles/cac-tripplanner.app/templates/upstart-cac-tripplanner-app.conf.j2 deleted file mode 100644 index 387bc38ee..000000000 --- a/deployment/ansible/roles/cac-tripplanner.app/templates/upstart-cac-tripplanner-app.conf.j2 +++ /dev/null @@ -1,14 +0,0 @@ -description "Gunicorn upstart script for CAC Trip Planner" - -{% if develop or test -%} -start on (vagrant-mounted) -{% else %} -start on (local-filesystems and net-device-up IFACE!=lo) -{% endif %} -stop on shutdown - -respawn -setuid {{ app_username }} -chdir {{ root_app_dir }} - -exec gunicorn --config {{ root_conf_dir }}/gunicorn.py --timeout {{ otp_session_timeout_s }} cac_tripplanner.wsgi From 43a44a050a9df399eb8e2283c64779ee249f485d Mon Sep 17 00:00:00 2001 From: Kathryn Killebrew Date: Fri, 12 Jul 2019 12:42:11 -0400 Subject: [PATCH 13/21] Bump OTP role version Update to use systemd for service management. --- deployment/ansible/roles.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/ansible/roles.yml b/deployment/ansible/roles.yml index 45f3d210d..c27cd6a44 100644 --- a/deployment/ansible/roles.yml +++ b/deployment/ansible/roles.yml @@ -1,5 +1,5 @@ - src: azavea.opentripplanner - version: 1.2.0 + version: 2.0.0 - src: azavea.nginx version: 0.3.1 From 67df9e6d1da89567f3fdd85eea5adc925d90dda7 Mon Sep 17 00:00:00 2001 From: Kathryn Killebrew Date: Fri, 12 Jul 2019 12:51:32 -0400 Subject: [PATCH 14/21] Update ansible apt syntax --- .../tasks/install-postgis-from-source.yml | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/deployment/ansible/roles/cac-tripplanner.database/tasks/install-postgis-from-source.yml b/deployment/ansible/roles/cac-tripplanner.database/tasks/install-postgis-from-source.yml index d936603e9..4d1af4c74 100644 --- a/deployment/ansible/roles/cac-tripplanner.database/tasks/install-postgis-from-source.yml +++ b/deployment/ansible/roles/cac-tripplanner.database/tasks/install-postgis-from-source.yml @@ -1,17 +1,18 @@ --- - name: Install PostGIS build dependencies - apt: pkg="{{ item }}" state=present - with_items: - - build-essential - - libgeos-dev - - libgdal-dev - - libproj-dev - - libjson0-dev - - libxml2-dev - - libxml2-utils - - xsltproc - - docbook-xsl - - docbook-mathml + apt: + state: present + pkg: + - build-essential + - libgeos-dev + - libgdal-dev + - libproj-dev + - libjson0-dev + - libxml2-dev + - libxml2-utils + - xsltproc + - docbook-xsl + - docbook-mathml - name: Download PostGIS source tarball unarchive: | From 0256e3553dd9f592c51e436c0556c6015174681d Mon Sep 17 00:00:00 2001 From: Kathryn Killebrew Date: Fri, 12 Jul 2019 15:31:02 -0400 Subject: [PATCH 15/21] Revert to install boto instead of boto3 --- python/cac_tripplanner/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/cac_tripplanner/requirements.txt b/python/cac_tripplanner/requirements.txt index f3686faad..d90e63f9b 100644 --- a/python/cac_tripplanner/requirements.txt +++ b/python/cac_tripplanner/requirements.txt @@ -1,5 +1,5 @@ base58==1.0.3 -boto3==1.9.187 +boto==2.49.0 Django==1.11.22 django-ckeditor==5.7.1 django-image-cropping==1.2.0 From 0cbfcfe55b690b90a036e13e103f33df0c55cd26 Mon Sep 17 00:00:00 2001 From: Kathryn Killebrew Date: Fri, 12 Jul 2019 15:32:13 -0400 Subject: [PATCH 16/21] Allow version 3 NFS --- Vagrantfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Vagrantfile b/Vagrantfile index c614ebd5e..92a6281b8 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -5,7 +5,7 @@ Vagrant.require_version ">= 2.0" require "yaml" CAC_SHARED_FOLDER_TYPE = ENV.fetch("CAC_SHARED_FOLDER_TYPE", "nfs") -CAC_NFS_VERSION = 'vers=4' +CAC_NFS_VERSION = ENV.fetch("CAC_NFS_VERSION_3", true) ? 'vers=3': 'vers=4' if CAC_SHARED_FOLDER_TYPE == "nfs" if Vagrant::Util::Platform.linux? then From 5145f22fbcc896cea8a482a0d959145f9ae25960 Mon Sep 17 00:00:00 2001 From: Kathryn Killebrew Date: Mon, 22 Jul 2019 10:09:38 -0400 Subject: [PATCH 17/21] Remove vagrant mount options Fix app fileshare mounting permissions on Linux by reverting to defaults. --- Vagrantfile | 26 +------------------------- 1 file changed, 1 insertion(+), 25 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 92a6281b8..20c0d95f9 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -4,23 +4,6 @@ Vagrant.require_version ">= 2.0" require "yaml" -CAC_SHARED_FOLDER_TYPE = ENV.fetch("CAC_SHARED_FOLDER_TYPE", "nfs") -CAC_NFS_VERSION = ENV.fetch("CAC_NFS_VERSION_3", true) ? 'vers=3': 'vers=4' - -if CAC_SHARED_FOLDER_TYPE == "nfs" - if Vagrant::Util::Platform.linux? then - CAC_MOUNT_OPTIONS = ['rw', CAC_NFS_VERSION, 'tcp', 'nolock', 'actimeo=1'] - else - CAC_MOUNT_OPTIONS = [CAC_NFS_VERSION, 'udp', 'actimeo=1'] - end -else - if ENV.has_key?("CAC_MOUNT_OPTIONS") - CAC_MOUNT_OPTIONS = ENV.fetch("CAC_MOUNT_OPTIONS").split - else - CAC_MOUNT_OPTIONS = ["rw"] - end -end - if ENV['CAC_TRIPPLANNER_MEMORY'].nil? # OpenTripPlanner needs > 1GB to build and run CAC_MEMORY_MB = "8192" @@ -132,14 +115,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| config.vm.define "app" do |app| app.vm.hostname = "app" app.vm.network "private_network", ip: "192.168.8.24" - - if testing? - app.vm.synced_folder ".", "/opt/app" - else - app.vm.synced_folder ".", "/opt/app", - type: CAC_SHARED_FOLDER_TYPE, - mount_options: CAC_MOUNT_OPTIONS - end + app.vm.synced_folder ".", "/opt/app" # Web app.vm.network "forwarded_port", guest: 443, host: 8024 From a8ac20455ece7dbe4edd0d9676ec98137a609bdb Mon Sep 17 00:00:00 2001 From: Kathryn Killebrew Date: Mon, 22 Jul 2019 10:12:30 -0400 Subject: [PATCH 18/21] Fix ansible deprecation warning Fixes "Using tests as filters is deprecated" in otp-data role. --- .../ansible/roles/cac-tripplanner.otp-data/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/ansible/roles/cac-tripplanner.otp-data/tasks/main.yml b/deployment/ansible/roles/cac-tripplanner.otp-data/tasks/main.yml index 5f510a92f..06c0d8511 100644 --- a/deployment/ansible/roles/cac-tripplanner.otp-data/tasks/main.yml +++ b/deployment/ansible/roles/cac-tripplanner.otp-data/tasks/main.yml @@ -50,7 +50,7 @@ when: test or (develop and not graph.stat.exists) - name: Copy Built OTP Graph to Host (develop) - when: develop and graph_build|changed + when: develop and graph_build is changed synchronize: mode: pull src: "{{ otp_data_dir }}/Graph.obj" From 3f0be110879d16f5e4c46f9a623c5e5fd817e12a Mon Sep 17 00:00:00 2001 From: Kathryn Killebrew Date: Thu, 25 Jul 2019 13:01:34 -0400 Subject: [PATCH 19/21] Update PostgreSQL package versions in test Match the package versions in development_template. --- deployment/ansible/group_vars/test | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/deployment/ansible/group_vars/test b/deployment/ansible/group_vars/test index eeed3b303..fa3462e8e 100644 --- a/deployment/ansible/group_vars/test +++ b/deployment/ansible/group_vars/test @@ -5,8 +5,9 @@ production: false postgis_version: [2, 1, 8] -postgresql_version: "9.4" -postgresql_package_version: "9.4.*.pgdg14.04+1" +postgresql_version: "9.5" +postgresql_package_version: "9.5.*" +postgresql_support_libpq_version: "*" postgresql_listen_addresses: "*" postgresql_hba_mapping: From c49a25d8d90cd59c8f626ef6795f0ef6cc9548c7 Mon Sep 17 00:00:00 2001 From: Michael Maurizi Date: Fri, 26 Jul 2019 13:07:57 -0400 Subject: [PATCH 20/21] Enable nginx systemd service Required for nginx to start on boot --- deployment/ansible/roles/cac-tripplanner.app/tasks/main.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/deployment/ansible/roles/cac-tripplanner.app/tasks/main.yml b/deployment/ansible/roles/cac-tripplanner.app/tasks/main.yml index 2039cd4c0..0e0d128a5 100644 --- a/deployment/ansible/roles/cac-tripplanner.app/tasks/main.yml +++ b/deployment/ansible/roles/cac-tripplanner.app/tasks/main.yml @@ -58,6 +58,11 @@ enabled: yes daemon_reload: yes +- name: Enable nginx service + systemd: + name: "nginx.service" + enabled: yes + - name: Copy media assets copy: src=../../python/cac_tripplanner/default_media dest=/media/cac From f10a24dfa42dc479a5b04e42bcb13ca328d3e987 Mon Sep 17 00:00:00 2001 From: Michael Maurizi Date: Fri, 26 Jul 2019 13:09:43 -0400 Subject: [PATCH 21/21] Load gunicorn after vagrant mounts in development Without this in-place, gunicorn would frequently fail to load at system-start as the Vagrant synced_folder /opt/app wouldn't be ready yet. --- .../templates/systemd-cac-tripplanner-app.conf.j2 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/deployment/ansible/roles/cac-tripplanner.app/templates/systemd-cac-tripplanner-app.conf.j2 b/deployment/ansible/roles/cac-tripplanner.app/templates/systemd-cac-tripplanner-app.conf.j2 index 2dee4fb20..f848c86dd 100644 --- a/deployment/ansible/roles/cac-tripplanner.app/templates/systemd-cac-tripplanner-app.conf.j2 +++ b/deployment/ansible/roles/cac-tripplanner.app/templates/systemd-cac-tripplanner-app.conf.j2 @@ -16,5 +16,8 @@ StandardError = syslog SyslogIdentifier = {{ gunicorn_app_name }} [Install] +{% if develop or test -%} +WantedBy = opt-app.mount +{% else %} WantedBy = network-online.target - +{% endif %}