WIP Issue 399 part 2: MOU Generation

.github/workflows/django_testing_ci.yml

@@ -37,10 +37,10 @@ jobs:
     - name: Checkout the current commit
       uses: actions/checkout@v3
 
-    - name: Set up Python 3.6.8
+    - name: Set up Python 3.8.13
       uses: actions/setup-python@v3
       with:
-        python-version: 3.6.8
+        python-version: 3.8.13
 
     - name: Cache and/or Install apache2-dev needed for testing suite
       uses: awalsh128/cache-apt-pkgs-action@latest
@@ -57,7 +57,7 @@ jobs:
     - if: ${{ steps.cache-python.outputs.cache-hit != 'true' }} # If a cache is not found
       name: Install Python packages
       run: |
-        python3.6 -m venv ~/venv
+        python3.8 -m venv ~/venv
         source ~/venv/bin/activate
         pip install -r requirements.txt
 
@@ -87,4 +87,6 @@ jobs:
         export django_secret_key=`openssl rand -base64 64`
         python manage.py migrate
         python manage.py test
+      env:
+        GOOGLE_DRIVE_STORAGE_JSON_KEY_FILE_CONTENTS: ${{ secrets.GOOGLE_DRIVE_STORAGE_JSON_KEY_FILE_CONTENTS }}
 

.gitignore

@@ -29,3 +29,6 @@ main.yml
 .vagrant
 .coverage
 htmlcov
+id_coldfront
+fetch_database_backup.sh
+google-drive-key.json

Dockerfile

@@ -3,10 +3,17 @@ FROM centos/python-38-centos7
 LABEL description="coldfront"
 
 USER root
+
+# TODO: comment out these lines if flag_mou_generation_enabled=False
+RUN mkdir /root/.ssh && ssh-keyscan github.com > /root/.ssh/known_hosts
+COPY --chmod=0600 bootstrap/development/id_coldfront /root/.ssh/id_rsa
+RUN pip install git+ssh://git@github.com/ucb-rit/mou-generator.git
+
 WORKDIR /root
 COPY requirements.txt ./
 RUN pip install -r requirements.txt && rm requirements.txt
 RUN pip install jinja2 pyyaml
+RUN yum -y install https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox-0.12.6-1.centos7.x86_64.rpm
 
 # mybrc or mylrc
 ARG PORTAL="mybrc"

bootstrap/ansible/main.copyme

@@ -99,6 +99,9 @@ flag_lrc_enabled: False
 # TODO: For LRC, set the month number to 9 (September).
 flag_next_period_renewal_requestable_month: 5
 flag_multiple_email_addresses_allowed: False
+# Whether to install and enable the MOU generation package.
+# TODO: For Production and Staging, enable.
+flag_mou_generation_enabled: false
 
 # Portal settings.
 # TODO: For LRC, use "MyLRC", "Laboratory Research Computing", "LRC", and
@@ -159,6 +162,19 @@ sentry_dsn: ""
 # app_port: 8000
 # full_host_path: http://scgup-dev.lbl.gov:8000
 
+# # Storage settings.
+# # TODO: fill these out
+# # Note: local_storage currently not implemented
+# file_storage_backend: 'google_drive'
+# # TODO: Change to absolute path
+# google_drive_private_key_file_path: 'google-drive-key.json'
+# # for google_drive, provide only the bottom-level directory name
+# # with a slash at the end. If someone shares a new folder with the same name
+# # to the same service account, new MOUs may go there.
+# new_project_request_mou_path: 'New Project Request MOUs/'
+# secure_directory_request_mou_path: 'Secure Directory Request MOUs/'
+# service_units_purchase_request_mou_path: 'Service Units Purchase Request MOUs/'
+
 # # SSL settings.
 # ssl_enabled: false
 # ssl_certificate_file: /etc/ssl/ssl_certificate.file
@@ -215,6 +231,19 @@ sentry_dsn: ""
 # # TODO: For LRC, use https://mylrc.lbl.gov.
 # full_host_path: https://mybrc.brc.berkeley.edu
 
+# # Storage settings.
+# # TODO: fill these out
+# # Note: local_storage currently not implemented
+# file_storage_backend: 'google_drive'
+# # TODO: Change to absolute path
+# google_drive_private_key_file_path: 'google-drive-key.json'
+# # for google_drive, provide only the bottom-level directory name
+# # with a slash at the end. If someone shares a new folder with the same name
+# # to the same service account, new MOUs may go there.
+# new_project_request_mou_path: 'New Project Request MOUs/'
+# secure_directory_request_mou_path: 'Secure Directory Request MOUs/'
+# service_units_purchase_request_mou_path: 'Service Units Purchase Request MOUs/'
+
 # # SSL settings.
 # ssl_enabled: true
 # ssl_certificate_file: /etc/ssl/ssl_certificate.file
@@ -271,6 +300,19 @@ sentry_dsn: ""
 # app_port: 80
 # full_host_path: http://localhost:8880
 
+# # Storage settings.
+# # TODO: fill these out
+# # Note: local_storage currently not implemented
+# file_storage_backend: 'google_drive'
+# # TODO: Change to absolute path
+# google_drive_private_key_file_path: 'google-drive-key.json'
+# # for google_drive, provide only the bottom-level directory name
+# # with a slash at the end. If someone shares a new folder with the same name
+# # to the same service account, new MOUs may go there.
+# new_project_request_mou_path: 'New Project Request MOUs/'
+# secure_directory_request_mou_path: 'Secure Directory Request MOUs/'
+# service_units_purchase_request_mou_path: 'Service Units Purchase Request MOUs/'
+
 # # SSL settings.
 # ssl_enabled: false
 # ssl_certificate_file: /etc/ssl/ssl_certificate.file

bootstrap/ansible/playbook.yml

@@ -54,8 +54,9 @@
               - git-core
               - gcc-c++
               - libffi-devel
+              - https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox-0.12.6-1.centos7.x86_64.rpm
             state: present
-
+        
         - name: Install SCL
           include_role:
             role: smbambling.scl
@@ -463,13 +464,14 @@
 
         - name: Add dev QOL lines to .bashrc
           blockinfile:
-            path: /home/{{ djangooperator }}/.bashrc
+            path: {{ ansible_env.HOME }}/.bashrc
             block: |
               # Upon login, navigate to the ColdFront directory and source the virtual environment.
               cd {{ git_prefix }}/{{ reponame }}
               source ../venv/bin/activate
               # Restart Apache with a keyword.
               alias reload="sudo service httpd reload"
+          become_user: "{{ djangooperator }}"
 
       when: provisioning_tasks == true and deployment == "dev"
       tags: provisioning
@@ -536,6 +538,36 @@
             state: reloaded
             enabled: true
 
+        - block:
+          # Allow pip installation of private git repo for mou generation.
+
+          - name: Check if github in known_hosts
+            shell: ssh-keygen -F github.com
+            register: github_known_hosts
+            changed_when: false
+            failed_when: false
+
+          - name: Add github.com to known_hosts
+            shell: ssh-keyscan github.com >> {{ ansible_env.HOME }}/.ssh/known_hosts
+            changed_when: github_known_hosts.rc != 0
+            become_user: "{{ djangooperator }}"
+
+          - name: Copy deploy key to root user's .ssh directory
+            copy:
+              src: "{{ git_prefix }}/{{ reponame }}/bootstrap/development/id_coldfront"
+              dest: {{ ansible_env.HOME }}/.ssh/id_rsa
+              owner: "{{ djangooperator }}"
+              group: "{{ djangooperator }}"
+              mode: 0600
+            become_user: "{{ djangooperator }}"
+
+          - name: Install mou generation package
+            pip:
+              name: git+ssh://git@github.com/ucb-rit/mou-generator.git
+              executable: "{{ git_prefix }}/venv/bin/pip3"
+            become_user: "{{ djangooperator }}"
+          when: flag_mou_generation_enabled
+
         # Install Django application dependencies
 
         - name: Install Django application dependencies

bootstrap/ansible/settings_template.tmpl

@@ -65,6 +65,33 @@ REQUEST_APPROVAL_CC_LIST = {{ request_approval_cc_list }}
 ALLOW_ALL_JOBS = True
 {% endif %}
 
+#------------------------------------------------------------------------------
+# File Storage Settings
+#------------------------------------------------------------------------------
+
+# gdstorage (Django Google Drive Storage library) settings.
+# Path to google drive api key json file
+GOOGLE_DRIVE_STORAGE_JSON_KEY_FILE = '{{ google_drive_private_key_file_path }}'
+# Path to prepend to gdstorage file paths. Should be '/'
+GOOGLE_DRIVE_STORAGE_MEDIA_ROOT = '/'
+# Can be used to set permissions e.g "restrict to Berkeley users".
+GOOGLE_DRIVE_PERMISSIONS = ()
+
+FILE_STORAGE = {
+    'backend': '{{ file_storage_backend }}',
+    'details': {
+        'NEW_PROJECT_REQUEST_MOU': {
+            'location': '{{ new_project_request_mou_path }}'
+        },
+        'SECURE_DIRECTORY_REQUEST_MOU': {
+            'location': '{{ secure_directory_request_mou_path }}'
+        },
+        'SERVICE_UNITS_PURCHASE_REQUEST_MOU': {
+            'location': '{{ service_units_purchase_request_mou_path }}'
+        }
+    },
+}
+
 #------------------------------------------------------------------------------
 # Billing settings
 #------------------------------------------------------------------------------
@@ -194,6 +221,7 @@ FLAGS = {
     'SECURE_DIRS_REQUESTABLE': [{'condition': 'boolean', 'value': {{ flag_brc_enabled }}}],
     'SERVICE_UNITS_PURCHASABLE': [{'condition': 'boolean', 'value': {{ flag_brc_enabled }}}],
     'SSO_ENABLED': [{'condition': 'boolean', 'value': {{ flag_sso_enabled }}}],
+    'MOU_GENERATION_ENABLED': [{'condition': 'boolean', 'value': {{ flag_mou_generation_enabled }}}],
 }
 
 # Enforce that boolean flags are consistent with each other.

coldfront/config/settings.py

@@ -75,6 +75,11 @@
     'coldfront.core.socialaccount',
 ]
 
+# Google Cloud Integration
+INSTALLED_APPS += [
+    'gdstorage'
+]
+
 # ------------------------------------------------------------------------------
 # Django Middleware
 # ------------------------------------------------------------------------------

coldfront/config/test_settings.py.sample

@@ -59,6 +59,27 @@ API_LOG_PATH = '/var/log/user_portals/cf_mybrc/cf_mybrc_api.log'
 # A list of admin email addresses to CC when certain requests are approved.
 REQUEST_APPROVAL_CC_LIST = ['test@test.test']
 
+#------------------------------------------------------------------------------
+# File Storage Settings
+#------------------------------------------------------------------------------
+
+GOOGLE_DRIVE_PERMISSIONS = ()
+
+FILE_STORAGE = {
+    'backend': 'local_storage',
+    'details': {
+        'NEW_PROJECT_REQUEST_MOU': {
+            'location': 'New Project Request MOUs/'
+        },
+        'SECURE_DIRECTORY_REQUEST_MOU': {
+            'location': 'Secure Directory Request MOUs/'
+        },
+        'SERVICE_UNITS_PURCHASE_REQUEST_MOU': {
+            'location': 'Service Units Purchase Request MOUs/'
+        }
+    },
+}
+
 #------------------------------------------------------------------------------
 # SSL settings
 #------------------------------------------------------------------------------
@@ -96,4 +117,5 @@ FLAGS = {
     'SECURE_DIRS_REQUESTABLE': [{'condition': 'boolean', 'value': True}],
     'SERVICE_UNITS_PURCHASABLE': [{'condition': 'boolean', 'value': True}],
     'SSO_ENABLED': [{'condition': 'boolean', 'value': False}],
+    'MOU_GENERATION_ENABLED': [{'condition': 'boolean', 'value': False}],
 }

coldfront/core/allocation/forms_/secure_dir_forms.py

@@ -49,6 +49,13 @@ class SecureDirManageUsersRequestCompletionForm(forms.Form):
 
 
 class SecureDirDataDescriptionForm(forms.Form):
+    department = forms.CharField(
+        label=('Specify the full name of the department that this directory ' 
+            'will belong to (e.g "Dept. of Chemistry", "Dept. of Economics").'),
+        validators=[MinLengthValidator(5)],
+        required=True,
+        widget=forms.Textarea(attrs={'rows': 3}))
+
     data_description = forms.CharField(
         label='Please explain the kind of P2/P3 data you are planning to '
               'work with on Savio. Please include: (1) Dataset description '
@@ -200,3 +207,12 @@ class SecureDirRDMConsultationReviewForm(forms.Form):
         validators=[MinLengthValidator(10)],
         required=False,
         widget=forms.Textarea(attrs={'rows': 3}))
+
+class SecureDirRequestEditDepartmentForm(forms.Form):
+
+    department = forms.CharField(
+        label=('Specify the full name of the department that this directory ' 
+            'will belong to (e.g "Dept. of Chemistry", "Dept. of Economics").'),
+        validators=[MinLengthValidator(5)],
+        required=True,
+        widget=forms.Textarea(attrs={'rows': 3}))