Upgrade Python 3.10 -> 3.11 & Node 16 -> 20

.github/workflows/deploy.yml

@@ -19,7 +19,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: '3.10'
+          python-version: '3.11'
           cache: 'pip'
           cache-dependency-path: 'requirements/*/dev.txt'
       - name: Install dependencies

.github/workflows/test.yml

@@ -43,7 +43,7 @@ jobs:
       - run: npm run build
       - uses: actions/setup-python@v4
         with:
-          python-version: '3.10'
+          python-version: '3.11'
           cache: 'pip'
           cache-dependency-path: 'requirements/*/**.txt'
       - name: Cache pre-commit

.nvmrc

@@ -1 +1 @@
-lts/erbium
+lts/iron

.pre-commit-config.yaml

@@ -1,9 +1,9 @@
 repos:
   - repo: https://github.com/psf/black
-    rev: 22.3.0
+    rev: 24.4.2
     hooks:
       - id: black
-        language_version: python3.10
+        language_version: python3.11
         exclude: migrations
   - repo: https://github.com/PyCQA/flake8
     rev: 4.0.1
@@ -29,3 +29,4 @@ repos:
         args: [--allow-missing-credentials]
       - id: detect-private-key
       - id: no-commit-to-branch
+

Dockerfile

@@ -1,12 +1,12 @@
-FROM node:16-alpine as static_files
+FROM node:20-bookworm-slim as static_files
 WORKDIR /code
 ENV PATH /code/node_modules/.bin:$PATH
 COPY package.json package-lock.json webpack.config.js /code/
 RUN npm install --silent
 COPY . /code/
 RUN npm run build
 
-FROM python:3.10-slim-bullseye as base
+FROM python:3.11-slim-bookworm as base
 
 # Install packages needed to run your application (not build deps):
 #   mime-support -- for mime types when serving static files
@@ -101,7 +101,7 @@ ENTRYPOINT ["/code/docker-entrypoint.sh"]
 CMD ["newrelic-admin", "run-program", "uwsgi", "--single-interpreter", "--enable-threads", "--show-config"]
 
 
-FROM python:3.10-slim-bullseye AS dev
+FROM python:3.11-slim-bookworm AS dev
 
 ARG USERNAME=appuser
 ARG USER_UID=1000
@@ -123,6 +123,7 @@ RUN groupadd --gid $USER_GID $USERNAME \
 #   vim -- enhanced vi editor for commits
 ENV KUBE_CLIENT_VERSION="v1.29.3"
 ENV HELM_VERSION="3.14.4"
+ENV POSTGRESQL_CLIENT_VERSION="15"
 RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/apt \
     --mount=type=cache,mode=0755,target=/root/.cache/pip \
     set -ex \
@@ -161,7 +162,7 @@ RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/
     && curl https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor | tee /etc/apt/trusted.gpg.d/docker.gpg >/dev/null \
     && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/trusted.gpg.d/docker.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null \
     # nodejs
-    && sh -c 'echo "deb https://deb.nodesource.com/node_16.x $(lsb_release -cs) main" > /etc/apt/sources.list.d/nodesource.list' \
+    && sh -c 'echo "deb https://deb.nodesource.com/node_20.x $(lsb_release -cs) main" > /etc/apt/sources.list.d/nodesource.list' \
     && wget --quiet -O- https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - \
     # PostgreSQL
     && sh -c 'echo "deb https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' \

README.md

@@ -4,9 +4,9 @@
 ## ✏️ **Develop**
 To begin you should have the following applications installed on your local development system:
 
-- Python >= 3.10
-- NodeJS == 16.13.x
-- npm == 8.1.x (comes with node 16)
+- Python >= 3.11
+- NodeJS == 20.5.x
+- npm == 9.8.x (comes with node 16)
 - [nvm](https://github.com/nvm-sh/nvm/blob/master/README.md) is not strictly _required_, but will almost certainly be necessary unless you just happen to have Node.js 16.x installed on your machine.
 - [pip](http://www.pip-installer.org/) >= 20
 - [virtualenv](http://www.virtualenv.org/) >= 1.10

apps/common/context_processors.py

@@ -32,7 +32,7 @@ def previous_url(request):
     # Note: we use the try/except logic here (rather than request.META.get...)
     # in order to avoid calling get_home_page_url() every time.
     try:
-        return {"previous_url": request.META["HTTP_REFERER"]}
+        return {"previous_url": request.headers["referer"]}
     except KeyError:
         return {"previous_url": get_home_page_url()}
 

apps/common/utils.py

@@ -89,7 +89,7 @@ def get_next_url_from_request(request):
     if next_parameter:
         return next_parameter
     else:
-        return request.META.get("HTTP_REFERER", None) or None
+        return request.headers.get("referer", None) or None
 
 
 def closedpod_user_check(user):

apps/disease_control/utils.py

@@ -16,10 +16,14 @@ def get_disease_control_services_qs():
 def get_visible_section_headers():
     return [
         "Topic-specific Guidance" if get_topic_specific_guidance_qs().exists() else "",
-        "Facility-specific Guidance"
-        if get_facility_specific_guidance_qs().exists()
-        else "",
-        "Disease Control Services"
-        if get_disease_control_services_qs().exists()
-        else "",
+        (
+            "Facility-specific Guidance"
+            if get_facility_specific_guidance_qs().exists()
+            else ""
+        ),
+        (
+            "Disease Control Services"
+            if get_disease_control_services_qs().exists()
+            else ""
+        ),
     ]

apps/hip/tests/test_forms.py

@@ -84,7 +84,7 @@ def test_only_pdfs_are_scaned(db, mocker):
     mock_scan_function = mocker.patch("apps.hip.forms.scan_pdf_for_malicious_content")
 
     # Uploading a non-PDF file does not call scan_pdf_for_malicious_content()
-    for (extension, content_type) in [
+    for extension, content_type in [
         ("png", "image/png"),
         ("jpg", "image/jpeg"),
         ("jpeg", "image/jpeg"),

apps/reports/models.py

@@ -66,9 +66,11 @@ def get_context(self, request):
                 "title": r.title,
                 "url": r.url,
                 "update_frequency": r.staticpage.datareportdetailpage.update_frequency,
-                "last_updated": r.latest_revision_created_at.date()
-                if r.latest_revision_created_at
-                else None,
+                "last_updated": (
+                    r.latest_revision_created_at.date()
+                    if r.latest_revision_created_at
+                    else None
+                ),
                 "associated_disease": r.staticpage.datareportdetailpage.associated_disease,
                 "external": False,
             }
@@ -135,9 +137,9 @@ class DataReportDetailArchiveListPage(HipBasePage):
 
     def get_context(self, request):
         context = super().get_context(request)
-        context[
-            "archived_reports"
-        ] = DataReportDetailArchiveDetailPage.objects.child_of(self).order_by("-year")
+        context["archived_reports"] = (
+            DataReportDetailArchiveDetailPage.objects.child_of(self).order_by("-year")
+        )
         return context
 
 

apps/reports/tests/factories.py

@@ -8,7 +8,6 @@
 
 class ExternalReportBlockFactory(wagtail_factories.StructBlockFactory):
     title = "title here"
-    # title = factory.faker.Faker("word")
     url = factory.faker.Faker("url")
     update_frequency = random.choice(
         ["Annually", "Warterly", "Monthly", "Weekly", "all the time!!!"]

apps/reports/tests/test_models.py

@@ -68,58 +68,36 @@ def test_datareportslistpage_context_only_internal_reports(db, rf):
 def test_datareportslistpage_context_only_external_reports(db, rf):
     """Test the context for a DataReportListPage with no children, but with external_reports."""
     # Create a DataReportListPage with some external reports.
+    external_report_hiv = {
+        "title": "HIV Report",
+        "url": "https://example.com/hiv",
+        "update_frequency": "Monthly",
+        "last_updated": date(year=2021, month=1, day=1),
+    }
     external_report_hep_a = {
-        "title": "Hepatitis A",
-        "url": "example.com/report-hepatitis-a",
+        "title": "Hepatitis A Report",
+        "url": "https://example.com/hep-a",
         "update_frequency": "Annually",
-        # The context should handle if last_updated is a string
         "last_updated": "2020-01-01",
-        "external": True,
     }
-    external_report_hiv = {
-        "title": "HIV/AIDS",
-        "url": "example.com/report-hiv-aids",
-        "update_frequency": "Sometime",
-        # The context should handle if last_updated is a date object
-        "last_updated": date(year=2021, month=1, day=1),
-        "external": True,
-    }
-    external_reports_data = [
-        {
-            "type": "external_reports",  # Block type
-            "value": {
-                "title": external_report_hiv["title"],
-                "url": external_report_hiv["url"],
-                "update_frequency": external_report_hiv["update_frequency"],
-                "last_updated": external_report_hiv["last_updated"],
-            },
-        },
-        {
-            "type": "external_reports",
-            "value": {
-                "title": external_report_hep_a["title"],
-                "url": external_report_hep_a["url"],
-                "update_frequency": external_report_hep_a["update_frequency"],
-                "last_updated": external_report_hep_a["last_updated"],
-            },
-        },
-    ]
     reports_list_page = DataReportListPageFactory(
-        external_reports=external_reports_data
+        external_reports=[
+            ("external_reports", external_report_hiv),
+            ("external_reports", external_report_hep_a),
+        ]
     )
-
     context = reports_list_page.get_context(rf.get("/someurl/"))
 
     expected_result_hep_a = external_report_hep_a.copy()
-    expected_result_hep_a["last_updated"] = date(
-        year=2020, month=1, day=1
-    )  # format date to be the expected type
+    expected_result_hep_a["last_updated"] = date(year=2020, month=1, day=1)
     expected_result_hiv = external_report_hiv.copy()
-    expected_result_hiv["last_updated"] = date(
-        year=2021, month=1, day=1
-    )  # format date to be the expected type
-    # The results should be data for the external reports alphabetical order.
-    assert [expected_result_hep_a, expected_result_hiv] == context["reports"]
+    expected_result_hiv["last_updated"] = date(year=2021, month=1, day=1)
+    context_hep_a = context["reports"][0]
+    context_hiv = context["reports"][1]
+    del context_hep_a["external"]
+    del context_hiv["external"]
+    assert expected_result_hep_a == context_hep_a
+    assert context_hiv == expected_result_hiv
 
 
 def test_datareportslistpage_context_internal_and_external_reports(db, rf):
@@ -130,7 +108,6 @@ def test_datareportslistpage_context_internal_and_external_reports(db, rf):
         "url": "example.com/report-hiv-aids",
         "update_frequency": "Sometime",
         "last_updated": "2021-01-01",
-        "external": True,
     }
     external_reports_data = [
         {
@@ -156,7 +133,6 @@ def test_datareportslistpage_context_internal_and_external_reports(db, rf):
     )
 
     context = reports_list_page.get_context(rf.get("/someurl/"))
-
     # The results should be data for the internal and external reports in alphabetical order.
     expected_result_external_report_hiv = external_report_hiv.copy()
     expected_result_external_report_hiv["last_updated"] = date(
@@ -181,6 +157,7 @@ def test_datareportslistpage_context_internal_and_external_reports(db, rf):
             "external": False,
         },
     ]
+    del context["reports"][1]["external"]
     assert expected_results == context["reports"]
 
 

apps/users/admin.py

@@ -18,6 +18,7 @@ class Meta:
         exclude = ("password",)
 
 
+@admin.register(User)
 class CustomUserAdmin(ExportMixin, UserAdmin):
     resource_class = UserResourceBlackList
 
@@ -48,6 +49,3 @@ class CustomUserAdmin(ExportMixin, UserAdmin):
     add_fieldsets = (
         (None, {"classes": ("wide",), "fields": ("email", "password1", "password2")}),
     )
-
-
-admin.site.register(User, CustomUserAdmin)

apps/users/models.py

@@ -8,8 +8,11 @@
 
 
 class User(AbstractBaseUser, PermissionsMixin, IndexedTimeStampedModel):
-    # use special Postgres-only Case Insensitive Email Field
-    # https://docs.djangoproject.com/en/dev/ref/contrib/postgres/fields/#django.contrib.postgres.fields.CIEmailField
+    # Support for CIEmailField will be depreacated after Django 4.2 in favor
+    # of EmailField. Below is how to migrate to the vanilla EmailField Whilst
+    # creating a non-deterministic collation. Can't migrate yet because it requires
+    # Postgres 12 or higher. Currently Hip Philly is running Postgres 11.
+    # https://adamj.eu/tech/2023/02/23/migrate-django-postgresql-ci-fields-case-insensitive-collation/
     email = CIEmailField(max_length=255, unique=True)
     first_name = models.CharField(_("first name"), max_length=150, blank=True)
     last_name = models.CharField(_("last name"), max_length=150, blank=True)

deploy/group_vars/all.yml

@@ -58,6 +58,11 @@ cloudformation_stack:
 
   template_parameters:
     UseAES256Encryption: "true"
+    AdministratorIPAddress: "{{ administrator_ip_cidrs[0] }}"  # Stack allows only single IP here to SSH to bastion
+    BastionAMI: "ami-0ad554caf874569d2" # https://cloud-images.ubuntu.com/locator/ec2/ [us-east-1 amd64]
+    BastionKeyName: rluna_hip
+    BastionInstanceType: t3.small # Is this a proper size?
+    BastionType: SSH
     CustomerManagedCmkArn: ""
     DomainName: "{{ app_name }}-prod.caktus-built.com"
     DomainNameAlternates: ""

deploy/host_vars/runner.yml

@@ -0,0 +1,35 @@
+---
+##  users role configuration ##
+users_groups: [adm, dialout, docker, sudo]
+users_shell: /bin/bash
+
+# when removing a user, add their username to this list:
+users_remove:
+  # Remove default user installed by Ubuntu. You might need to comment this out
+  # temporarily when first configuring a server, and possibly even reboot the
+  # server before the user can be removed.
+  - ubuntu
+
+# users to provision on all servers
+# find your ssh key with: `cat ~/.ssh/id_*.pub` (should be one line)
+# optionally generate password via `mkpasswd -m sha-512 -R 2000000`
+users:
+  # in alphabetical order
+  - username: copelco
+    authorized_keys:
+      - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICMtpiB+QFK/YDEx3qiq62zUcxKOiuIOe1CNmD+NQYKt copelco@caktusgroup.com
+  - username: ronardlunagerman
+    authorized_keys:
+      - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEhNwSFktLJpdP/e04FPZxEwXsZyqTi8URd2IBjuw0Je rluna@caktusgroup.com
+  - username: tobias
+    authorized_keys:
+      - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFti2WxKH5TJh6SN44pkvG2V4268sJfirn00YrKLy+lY tobias@red-ed25519
+
+# On GitHub
+github_pat: "{{ lookup('env', 'RUNNER_CFG_PAT') }}"
+github_scope: caktus
+
+# On the VM
+github_runner_user: runner
+github_runner_name: philly-hip-runner
+github_runner_location: philly-hip

hip/settings/base.py

@@ -65,7 +65,6 @@
     "wagtail",
     "modelcluster",
     # Full list of icons available @ https://fontawesome.com/v4.7.0/icons/
-    "wagtailfontawesome",
     "wagtail.contrib.modeladmin",
     "wagtailmenus",
 ]
@@ -120,7 +119,7 @@
 
 DATABASES = {
     "default": {
-        "ENGINE": "django.db.backends.postgresql_psycopg2",
+        "ENGINE": "django.db.backends.postgresql",
         "NAME": "hip",
         "USER": "",
         "PASSWORD": "",
@@ -185,7 +184,6 @@
 
 USE_I18N = True
 
-USE_L10N = True
 
 USE_TZ = True
 
@@ -340,3 +338,9 @@
 # Notifies user in Wagtail Admin only of LTS versions
 # https://docs.wagtail.org/en/stable/reference/settings.html#wagtail-enable-update-check
 WAGTAIL_ENABLE_UPDATE_CHECK = "lts"
+
+# Patches a known bug with wagtailmenus, whereby a series
+# of warnings are displayed. Once this bug is fixed, the var
+# SILENCED_SYSTEM_CHECKS should be removed.
+# https://github.com/jazzband/wagtailmenus/issues/464#issuecomment-1839097355
+SILENCED_SYSTEM_CHECKS = ["wagtailadmin.W002"]