Merge branch 'develop' into CU-86897a79g-redis-monitoring

Author: copelco

.github/workflows/deploy.yaml

@@ -24,7 +24,7 @@ jobs:
           echo "ENV_URL=https://nccopwatch.org/" >> $GITHUB_ENV
       - uses: actions/setup-python@v4
         with:
-          python-version: '3.9'
+          python-version: '3.10'
           cache: 'pip'
           cache-dependency-path: 'requirements/*/*.txt'
       - name: Install dependencies

.github/workflows/test.yaml

@@ -28,7 +28,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: '3.9'
+          python-version: '3.10'
           cache: 'pip'
           cache-dependency-path: 'requirements/*/*.txt'
       - name: Install dependencies

.pre-commit-config.yaml

@@ -3,7 +3,7 @@ repos:
     rev: 22.3.0
     hooks:
       - id: black
-        language_version: python3.8
+        language_version: python3.10
         exclude: migrations
   - repo: https://github.com/PyCQA/flake8
     rev: 4.0.1

Dockerfile

@@ -8,7 +8,7 @@ RUN npm install --silent
 COPY frontend/ /code/
 RUN npm run build
 
-FROM python:3.8-slim-bullseye as base
+FROM python:3.10-slim-bullseye as base
 
 # Create a group and user to run our app
 ARG APP_USER=appuser
@@ -93,7 +93,7 @@ ENTRYPOINT ["/code/docker-entrypoint.sh"]
 # Start uWSGI
 CMD ["newrelic-admin", "run-program", "uwsgi", "--single-interpreter", "--enable-threads", "--show-config"]
 
-FROM python:3.8-slim-bullseye AS dev
+FROM python:3.10-slim-bullseye AS dev
 
 ARG USERNAME=appuser
 ARG USER_UID=1000
@@ -113,8 +113,8 @@ RUN groupadd --gid $USER_GID $USERNAME \
 #   openssh-client -- for git over SSH
 #   sudo -- to run commands as superuser
 #   vim -- enhanced vi editor for commits
-ENV KUBE_CLIENT_VERSION="v1.25.10"
-ENV HELM_VERSION="3.12.0"
+ENV KUBE_CLIENT_VERSION="v1.29.4"
+ENV HELM_VERSION="3.14.4"
 RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/apt \
     --mount=type=cache,mode=0755,target=/root/.cache/pip \
     set -ex \

deploy/group_vars/all.yml

@@ -64,7 +64,7 @@ k8s_install_descheduler: yes
 # You must set the k8s_descheduler_chart_version to match the Kubernetes
 # node version (0.23.x -> K8s 1.23.x); see:
 # https://github.com/kubernetes-sigs/descheduler#compatibility-matrix
-k8s_descheduler_chart_version: v0.25.2
+k8s_descheduler_chart_version: v0.29.0
 # See values.yaml for options:
 # https://github.com/kubernetes-sigs/descheduler/blob/master/charts/descheduler/values.yaml#L63
 k8s_descheduler_release_values:
@@ -94,9 +94,9 @@ k8s_iam_users: [copelco]
 # Pin ingress-nginx and cert-manager to current versions so future upgrades of this
 # role will not upgrade these charts without your intervention:
 # https://github.com/kubernetes/ingress-nginx/releases
-k8s_ingress_nginx_chart_version: "4.4.2"
+k8s_ingress_nginx_chart_version: "4.9.1"
 # https://github.com/jetstack/cert-manager/releases
-k8s_cert_manager_chart_version: "v1.11.1"
+k8s_cert_manager_chart_version: "v1.14.3"
 # AWS only:
 # Use the newer load balancer type (NLB). DO NOT edit k8s_aws_load_balancer_type after
 # creating your Service.
@@ -108,7 +108,7 @@ k8s_aws_load_balancer_type: nlb
 
 # New Relic Account: forwardjustice-team@caktusgroup.com
 # https://github.com/newrelic/helm-charts/releases
-k8s_newrelic_chart_version: "5.0.4"
+k8s_newrelic_chart_version: "5.0.68"
 k8s_newrelic_logging_enabled: true
 k8s_newrelic_license_key: !vault |
   $ANSIBLE_VAULT;1.1;AES256

deploy/requirements.yml

@@ -2,16 +2,16 @@
 
 - src: https://github.com/caktus/ansible-role-django-k8s
   name: caktus.django-k8s
-  version: v1.6.0
+  version: v1.9.0
 
 - src: https://github.com/caktus/ansible-role-aws-web-stacks
   name: caktus.aws-web-stacks
   version: ''
 
 - src: https://github.com/caktus/ansible-role-k8s-web-cluster
   name: caktus.k8s-web-cluster
-  version: v1.5.0
+  version: v1.6.0
 
 - src: https://github.com/caktus/ansible-role-k8s-hosting-services
   name: caktus.k8s-hosting-services
-  version: v0.11.0
+  version: v0.12.0

docs/dev-setup.rst

@@ -5,7 +5,7 @@ Below you will find basic setup and deployment instructions for the NC Traffic
 Stops project. To begin you should have the following applications installed on
 your local development system:
 
-- Python 3.8
+- Python 3.10
 - NodeJS >= 12.6.0
 - `pip >= 8 or so <http://www.pip-installer.org/>`_
 - Postgres >= 12
@@ -85,8 +85,8 @@ To use ``psql`` locally, make sure you have the following env variables loaded
 To setup your local environment you should create a virtualenv and install the
 necessary requirements::
 
-    $ which python3.8  # make sure you have Python 3.8 installed
-    $ mkvirtualenv --python=`which python3.8` traffic-stops
+    $ which python3.11  # make sure you have Python 3.11 installed
+    $ mkvirtualenv --python=`which python3.11` traffic-stops
     (traffic-stops)$ pip install -U pip
     (traffic-stops)$ make setup
 

docs/hosting-services.md

@@ -4,7 +4,7 @@ The services configured for this project are:
 * PostgreSQL database backups to S3 (within Caktus AWS account)
     * Currently, this is only `traffic_stops`, which contains users, census data, etc.
     * `traffic_stops_nc` is not backed up since the entire dataset is re-imported daily.
-* Papertrail logging (to Caktus account)
+* New Relic logging (to Caktus account)
 * New Relic Infrastructure monitoring (Account: `admin+newrelic@caktusgroup.com`)
 
 

nc/tests/api/test_arrests.py

@@ -46,7 +46,7 @@ def test_sort_by_stop_purpose(self):
         self.assertEqual(sort_by_stop_purpose(df)["stop_purpose"].tolist(), StopPurpose.values)
 
 
-@pytest.mark.django_db
+@pytest.mark.django_db(databases=["traffic_stops_nc"])
 class TestArrests:
     def test_arrest_contraband_missing_race(self, client, durham):
         """A single stop will result no data for other races"""

nc/tests/api/test_basic_search.py

@@ -16,11 +16,13 @@ def test_no_agency(client, search_url):
     assert response.status_code == status.HTTP_400_BAD_REQUEST
 
 
+@pytest.mark.django_db(databases=["traffic_stops_nc"])
 def test_agency_success(client, search_url, durham):
     response = client.get(search_url, data={"agency": durham.pk}, format="json")
     assert response.status_code == status.HTTP_200_OK
 
 
+@pytest.mark.django_db(databases=["traffic_stops_nc"])
 def test_response_person_fields(client, search_url, durham):
     person = factories.PersonFactory(stop__agency=durham)
     response = client.get(search_url, data={"agency": durham.pk}, format="json")
@@ -42,6 +44,7 @@ def test_response_person_fields(client, search_url, durham):
     assert result == expected
 
 
+@pytest.mark.django_db(databases=["traffic_stops_nc"])
 @pytest.mark.parametrize("race", RACE_VALUES)
 def test_race_filtering(client, search_url, durham, race):
     other_races = RACE_VALUES - set(race)

nc/tests/api/test_timezones.py

@@ -23,6 +23,7 @@ def august_person(durham):
     return factories.PersonFactory(stop__agency=durham, stop__date=stop_date)
 
 
+@pytest.mark.django_db(databases=["traffic_stops_nc"])
 def test_stop_date_after_august_excludes_july_stop(client, search_url, durham, july_person):
     response = client.get(
         search_url,
@@ -33,6 +34,7 @@ def test_stop_date_after_august_excludes_july_stop(client, search_url, durham, j
     assert july_person.stop.stop_id not in stop_ids
 
 
+@pytest.mark.django_db(databases=["traffic_stops_nc"])
 def test_stop_date_after_august_includes_august_stop(client, search_url, durham, august_person):
     response = client.get(
         search_url,
@@ -44,6 +46,7 @@ def test_stop_date_after_august_includes_august_stop(client, search_url, durham,
     assert august_person.stop.date == response.data["results"][0]["date"]
 
 
+@pytest.mark.django_db(databases=["traffic_stops_nc"])
 def test_stop_date_after_july_includes_both(client, search_url, durham, july_person, august_person):
     response = client.get(
         search_url,

nc/tests/test_api.py

@@ -26,9 +26,10 @@ def test_list_agencies(self):
         agency = factories.AgencyFactory()
         url = reverse("nc:agency-api-list")
         response = self.client.get(url, format="json")
+        response_data = response.json()
         self.assertEqual(response.status_code, status.HTTP_200_OK)
         # Other Agencies may have been left around from other tests
-        self.assertIn((agency.pk, agency.name), [(a["id"], a["name"]) for a in response.data])
+        self.assertIn((agency.pk, agency.name), [(a["id"], a["name"]) for a in response_data])
 
     def test_agency_census_data(self):
         """
@@ -39,19 +40,21 @@ def test_agency_census_data(self):
         agency = factories.AgencyFactory(census_profile_id=census_profile.id)
         url = reverse("nc:agency-api-detail", args=[agency.pk])
         response = self.client.get(url, format="json")
+        response_data = response.json()
         self.assertEqual(response.status_code, status.HTTP_200_OK)
-        self.assertIn("census_profile", response.data)
+        self.assertIn("census_profile", response_data)
         # CensusProfile tests check census data in more detail
         for attr in ("hispanic", "non_hispanic", "total"):
-            self.assertEqual(response.data["census_profile"][attr], getattr(census_profile, attr))
+            self.assertEqual(response_data["census_profile"][attr], getattr(census_profile, attr))
 
     def test_stops_api(self):
         """Test Agency stops API endpoint with no stops"""
         agency = factories.AgencyFactory()
         url = reverse("nc:agency-api-stops", args=[agency.pk])
         response = self.client.get(url, format="json")
+        response_data = response.json()
         self.assertEqual(response.status_code, status.HTTP_200_OK)
-        self.assertEqual(len(response.data), 0)
+        self.assertEqual(len(response_data), 0)
 
     def test_stops_count(self):
         """Test Agency stop counts"""
@@ -73,18 +76,19 @@ def test_stops_count(self):
         factories.PersonFactory(race="I", stop__agency=agency, ethnicity="H", stop__year=2012)
 
         url = reverse("nc:agency-api-stops", args=[agency.pk])
-        response = self.client.get(url, format="json")
-        self.assertEqual(len(response.data), 2)
+        response = self.client.get(url, format="application/json")
+        response_data = response.json()
+        self.assertEqual(len(response_data), 2)
         self.assertEqual(response.status_code, status.HTTP_200_OK)
-        self.assertEqual(response.data[0]["year"], 2010)
-        self.assertEqual(response.data[0]["black"], 2)
-        self.assertEqual(response.data[0]["white"], 1)
-        self.assertEqual(response.data[0]["asian"], 0)
-        self.assertEqual(response.data[0]["hispanic"], 3)
-        self.assertEqual(response.data[1]["year"], 2012)
-        self.assertEqual(response.data[1]["black"], 0)
-        self.assertEqual(response.data[1]["white"], 1)
-        self.assertEqual(response.data[1]["hispanic"], 4)
+        self.assertEqual(response_data[0]["year"], 2010)
+        self.assertEqual(response_data[0]["black"], 2)
+        self.assertEqual(response_data[0]["white"], 1)
+        self.assertEqual(response_data[0]["asian"], 0)
+        self.assertEqual(response_data[0]["hispanic"], 3)
+        self.assertEqual(response_data[1]["year"], 2012)
+        self.assertEqual(response_data[1]["black"], 0)
+        self.assertEqual(response_data[1]["white"], 1)
+        self.assertEqual(response_data[1]["hispanic"], 4)
 
     def test_grouping_by_year(self):
         """
@@ -111,12 +115,13 @@ def test_grouping_by_year(self):
         )
         url = reverse("nc:agency-api-stops", args=[agency.pk])
         response = self.client.get(url, format="json")
+        response_data = response.json()
         self.assertEqual(response.status_code, status.HTTP_200_OK)
-        self.assertEqual(len(response.data), 2)
-        self.assertEqual(response.data[0]["year"], year)
-        self.assertEqual(response.data[0][race_label], 1)
-        self.assertEqual(response.data[1]["year"], year + 1)
-        self.assertEqual(response.data[1]["hispanic"], 1)
+        self.assertEqual(len(response_data), 2)
+        self.assertEqual(response_data[0]["year"], year)
+        self.assertEqual(response_data[0][race_label], 1)
+        self.assertEqual(response_data[1]["year"], year + 1)
+        self.assertEqual(response_data[1]["hispanic"], 1)
 
     def test_officer_stops_count(self):
         """Test officer (within an agency) stop counts"""
@@ -131,12 +136,12 @@ def test_officer_stops_count(self):
         url = reverse("nc:agency-api-stops", args=[agency.pk])
         url = "{}?officer={}".format(url, p1.stop.officer_id)
         response = self.client.get(url, format="json")
+        response_data = response.json()
         self.assertEqual(response.status_code, status.HTTP_200_OK)
-        self.assertEqual(len(response.data), 2)
-        self.assertEqual(response.data[0]["year"], p1.stop.date.year)
-        self.assertEqual(response.data[0][GROUPS[p1.race]], 1)
-        self.assertEqual(response.data[1]["year"], p2.stop.date.year)
-        self.assertEqual(response.data[1]["hispanic"], 2)
+        self.assertEqual(len(response_data), 2)
+        self.assertEqual(response_data[0]["year"], p1.stop.date.year)
+        self.assertEqual(response_data[1]["year"], p2.stop.date.year)
+        self.assertEqual(response_data[1]["hispanic"], 2)
 
     def test_stops_by_reason(self):
         """Test Agency stops_by_reason API endpoint"""
@@ -189,10 +194,11 @@ def test_stops_by_reason(self):
         factories.SearchFactory(stop=p5.stop)
 
         response = self.client.get(url, format="json")
+        response_data = response.json()
         self.assertEqual(response.status_code, status.HTTP_200_OK)
-        self.assertEqual(len(response.data.keys()), 2)
+        self.assertEqual(len(response_data.keys()), 2)
 
-        searches = response.data["searches"]
+        searches = response_data["searches"]
         self.assertEqual(searches[0]["year"], 2010)
         self.assertEqual(searches[0]["black"], 0)
         self.assertEqual(searches[0]["hispanic"], 3)
@@ -201,7 +207,7 @@ def test_stops_by_reason(self):
         self.assertEqual(searches[1]["black"], 1)
         self.assertEqual(searches[1]["purpose"], purpose_label)
 
-        stops = response.data["stops"]
+        stops = response_data["stops"]
         self.assertEqual(stops[0]["year"], 2010)
         self.assertEqual(stops[0]["black"], 1)
         self.assertEqual(stops[0]["hispanic"], 3)
@@ -227,16 +233,17 @@ def test_searches(self):
         factories.SearchFactory(person=p5, stop=p5.stop)
         url = reverse("nc:agency-api-searches", args=[agency.pk])
         response = self.client.get(url, format="json")
+        response_data = response.json()
         self.assertEqual(response.status_code, status.HTTP_200_OK)
-        self.assertEqual(len(response.data), 2)
+        self.assertEqual(len(response_data), 2)
         # Everyone got searched, so the expected racial data for 2015 are: 1 black,
         # and for 2016 are: 1 native american, 3 hispanic
-        self.assertEqual(response.data[0]["year"], s1.stop.date.year)
-        self.assertEqual(response.data[0]["black"], 1)
-        self.assertEqual(response.data[1]["year"], s2.stop.date.year)
-        self.assertEqual(response.data[1]["black"], 0)
-        self.assertEqual(response.data[1]["native_american"], 1)
-        self.assertEqual(response.data[1]["hispanic"], 3)
+        self.assertEqual(response_data[0]["year"], s1.stop.date.year)
+        self.assertEqual(response_data[0]["black"], 1)
+        self.assertEqual(response_data[1]["year"], s2.stop.date.year)
+        self.assertEqual(response_data[1]["black"], 0)
+        self.assertEqual(response_data[1]["native_american"], 1)
+        self.assertEqual(response_data[1]["hispanic"], 3)
 
     def test_searches_by_reason(self):
         agency = factories.AgencyFactory()
@@ -259,18 +266,18 @@ def test_searches_by_reason(self):
         factories.SearchFactory(person=p5, stop=p5.stop, type=type_code)
 
         response = self.client.get(url, format="json")
+        response_data = response.json()
         self.assertEqual(response.status_code, status.HTTP_200_OK)
         # Two years = two items
-        self.assertEqual(len(response.data), 2)
+        self.assertEqual(len(response_data), 2)
 
-        searches = response.data
-        self.assertEqual(searches[0]["year"], 2015)
-        self.assertEqual(searches[0]["black"], 1)
-        self.assertEqual(searches[0]["search_type"], type_label)
-        self.assertEqual(searches[1]["year"], 2016)
-        self.assertEqual(searches[1]["hispanic"], 3)
-        self.assertEqual(searches[1]["native_american"], 1)
-        self.assertEqual(searches[1]["search_type"], type_label)
+        self.assertEqual(response_data[0]["year"], 2015)
+        self.assertEqual(response_data[0]["black"], 1)
+        self.assertEqual(response_data[0]["search_type"], type_label)
+        self.assertEqual(response_data[1]["year"], 2016)
+        self.assertEqual(response_data[1]["hispanic"], 3)
+        self.assertEqual(response_data[1]["native_american"], 1)
+        self.assertEqual(response_data[1]["search_type"], type_label)
 
     def test_use_of_force(self):
         pass