switch to httpx for http2 support

Author: copelco

nc/prime_cache.py

@@ -2,7 +2,7 @@
 import time
 
 import boto3
-import requests
+import httpx
 
 from django.conf import settings
 from django.db.models import F, Q, Sum
@@ -117,34 +117,41 @@ def get_group_urls(agency_id: int, officer_id: int = None) -> list[str]:
 def prime_group_cache(agency_id: int, num_stops: int, officer_id: int = None):
     """Prime the cache for an agency (and optionally officer)"""
     logger.debug(f"Priming group cache ({agency_id=}, {officer_id=}, {num_stops=})...")
-    session = requests.Session()
     # Attempt to match Browser behavior
-    session.headers["Accept"] = "application/json, text/plain, */*"
-    session.headers["Accept-Encoding"] = "gzip, deflate, br, zstd"
-    # Configure basic auth if provided
+    headers = {
+        "Accpet": "application/json, text/plain, */*",
+        "Accept-Encoding": "gzip, deflate, br, zstd",
+    }
+    auth = None
     if settings.CACHE_BASICAUTH_USERNAME and settings.CACHE_BASICAUTH_PASSWORD:
-        session.auth = (settings.CACHE_BASICAUTH_USERNAME, settings.CACHE_BASICAUTH_PASSWORD)
-    logger.info(
-        f"Priming cache ({agency_id=}, {officer_id=}, {num_stops=}, {bool(session.auth)=})..."
-    )
-    urls = get_group_urls(agency_id=agency_id, officer_id=officer_id)
-    with Timer() as group_timer:
-        for url in urls:
-            with Timer(threshold_seconds=CLOUDFRONT_RESPONSE_TIMEOUT - 1) as endpoint_timer:
-                response = session.get(url)
-            logger.debug(
-                f"Queried {url=} ({response.headers=}, {response.request.headers=}, "
-                f"{endpoint_timer.elapsed=})"
-            )
-            if endpoint_timer.exceeded_threshold:
-                logger.warning(
-                    f"Slow response possibly not cached: {url} ({endpoint_timer.elapsed})"
+        auth = httpx.BasicAuth(
+            username=settings.CACHE_BASICAUTH_USERNAME, password=settings.CACHE_BASICAUTH_PASSWORD
+        )
+    with httpx.Client(auth=auth, headers=headers, http2=True) as client:
+        # Configure basic auth if provided
+        logger.info(
+            f"Priming cache ({agency_id=}, {officer_id=}, {num_stops=}, {bool(client.auth)=})..."
+        )
+        urls = get_group_urls(agency_id=agency_id, officer_id=officer_id)
+        with Timer() as group_timer:
+            for url in urls:
+                with Timer(threshold_seconds=CLOUDFRONT_RESPONSE_TIMEOUT - 1) as endpoint_timer:
+                    response = client.get(url)
+                logger.debug(
+                    f"Queried {url=} ({response.headers=}, {response.request.headers=}, "
+                    f"{endpoint_timer.elapsed=})"
                 )
-                raise Exception(f"Slow prime cache response possibly not cached {url}")
-            if response.status_code != 200:
-                logger.warning(f"Status not OK: {url} ({response.status_code})")
-                raise Exception(f"Request to {url} failed: {response.status_code}")
-    logger.info(f"Primed cache ({agency_id=}, {officer_id=}, {num_stops=}, {group_timer.elapsed=})")
+                if endpoint_timer.exceeded_threshold:
+                    logger.warning(
+                        f"Slow response possibly not cached: {url} ({endpoint_timer.elapsed})"
+                    )
+                    raise Exception(f"Slow prime cache response possibly not cached {url}")
+                if response.status_code != 200:
+                    logger.warning(f"Status not OK: {url} ({response.status_code})")
+                    raise Exception(f"Request to {url} failed: {response.status_code}")
+        logger.info(
+            f"Primed cache ({agency_id=}, {officer_id=}, {num_stops=}, {group_timer.elapsed=})"
+        )
 
 
 def invalidate_cloudfront_cache(sleep_seconds: int = 30) -> dict:

requirements/base/base.in

@@ -24,6 +24,7 @@ dj-database-url
 drf-extensions==0.7.1
 psycopg2==2.9.9
 brotli==1.1.0
+httpx[http2]
 requests==2.32.3
 urllib3==2.2.1
 six

requirements/base/base.txt

@@ -6,6 +6,8 @@
 #
 amqp==5.2.0
     # via kombu
+anyio==4.6.2.post1
+    # via httpx
 asgiref==3.5.2
     # via django
 billiard==4.2.0
@@ -26,7 +28,10 @@ celery==5.4.0
 census==0.8.22
     # via -r requirements/base/base.in
 certifi==2020.6.20
-    # via requests
+    # via
+    #   httpcore
+    #   httpx
+    #   requests
 charset-normalizer==3.0.1
     # via requests
 click==8.1.7
@@ -85,8 +90,25 @@ djangorestframework==3.12.4
     #   drf-extensions
 drf-extensions==0.7.1
     # via -r requirements/base/base.in
+exceptiongroup==1.2.2
+    # via anyio
+h11==0.14.0
+    # via httpcore
+h2==4.1.0
+    # via httpx
+hpack==4.0.0
+    # via h2
+httpcore==1.0.6
+    # via httpx
+httpx[http2]==0.27.2
+    # via -r requirements/base/base.in
+hyperframe==6.0.1
+    # via h2
 idna==2.10
-    # via requests
+    # via
+    #   anyio
+    #   httpx
+    #   requests
 jellyfish==0.6.1
     # via us
 jmespath==1.0.1
@@ -125,8 +147,14 @@ six==1.15.0
     #   -r requirements/base/base.in
     #   django-extensions
     #   python-dateutil
+sniffio==1.3.1
+    # via
+    #   anyio
+    #   httpx
 sqlparse==0.3.1
     # via django
+typing-extensions==4.12.2
+    # via anyio
 tzdata==2024.1
     # via
     #   celery

requirements/dev/dev.txt

@@ -16,6 +16,7 @@ ansible-core==2.17.5
     # via ansible
 anyio==4.6.2.post1
     # via
+    #   -c requirements/dev/../base/base.txt
     #   starlette
     #   watchfiles
 appnope==0.1.4
@@ -49,7 +50,6 @@ cachetools==5.5.0
 certifi==2020.6.20
     # via
     #   -c requirements/dev/../base/base.txt
-    #   -c requirements/dev/../test/test.txt
     #   kubernetes
     #   requests
 cffi==1.17.1
@@ -61,7 +61,6 @@ cfn-flip==1.3.0
 charset-normalizer==3.0.1
     # via
     #   -c requirements/dev/../base/base.txt
-    #   -c requirements/dev/../test/test.txt
     #   requests
 click==8.1.7
     # via
@@ -96,6 +95,7 @@ durationpy==0.9
     # via kubernetes
 exceptiongroup==1.2.2
     # via
+    #   -c requirements/dev/../base/base.txt
     #   -c requirements/dev/../test/test.txt
     #   anyio
     #   ipython
@@ -104,11 +104,12 @@ executing==2.1.0
 google-auth==2.35.0
     # via kubernetes
 h11==0.14.0
-    # via uvicorn
+    # via
+    #   -c requirements/dev/../base/base.txt
+    #   uvicorn
 idna==2.10
     # via
     #   -c requirements/dev/../base/base.txt
-    #   -c requirements/dev/../test/test.txt
     #   anyio
     #   requests
 imagesize==1.4.1
@@ -229,7 +230,6 @@ referencing==0.35.1
 requests==2.32.3
     # via
     #   -c requirements/dev/../base/base.txt
-    #   -c requirements/dev/../test/test.txt
     #   kubernetes
     #   requests-oauthlib
     #   sphinx
@@ -268,7 +268,9 @@ six==1.15.0
     #   openshift
     #   python-dateutil
 sniffio==1.3.1
-    # via anyio
+    # via
+    #   -c requirements/dev/../base/base.txt
+    #   anyio
 snowballstemmer==2.2.0
     # via sphinx
 sphinx==5.3.0
@@ -308,6 +310,7 @@ typer==0.12.5
     # via rstcheck
 typing-extensions==4.12.2
     # via
+    #   -c requirements/dev/../base/base.txt
     #   -c requirements/dev/../test/test.txt
     #   anyio
     #   ipython
@@ -321,7 +324,6 @@ typing-extensions==4.12.2
 urllib3==2.2.1
     # via
     #   -c requirements/dev/../base/base.txt
-    #   -c requirements/dev/../test/test.txt
     #   botocore
     #   kubernetes
     #   requests

requirements/test/test.in

@@ -6,7 +6,6 @@ isort
 pytest
 pytest-cov
 pytest-django
-requests-mock
 factory_boy
 coverage
 # Linting

requirements/test/test.txt

@@ -6,16 +6,8 @@
 #
 black==24.10.0
     # via -r requirements/test/test.in
-certifi==2020.6.20
-    # via
-    #   -c requirements/test/../base/base.txt
-    #   requests
 cfgv==3.4.0
     # via pre-commit
-charset-normalizer==3.0.1
-    # via
-    #   -c requirements/test/../base/base.txt
-    #   requests
 click==8.1.7
     # via
     #   -c requirements/test/../base/base.txt
@@ -27,7 +19,9 @@ coverage[toml]==7.6.4
 distlib==0.3.9
     # via virtualenv
 exceptiongroup==1.2.2
-    # via pytest
+    # via
+    #   -c requirements/test/../base/base.txt
+    #   pytest
 factory-boy==3.3.1
     # via -r requirements/test/test.in
 faker==30.8.1
@@ -38,10 +32,6 @@ flake8==7.1.1
     # via -r requirements/test/test.in
 identify==2.6.1
     # via pre-commit
-idna==2.10
-    # via
-    #   -c requirements/test/../base/base.txt
-    #   requests
 iniconfig==2.0.0
     # via pytest
 isort==5.13.2
@@ -87,12 +77,6 @@ pyyaml==6.0.2
     # via
     #   -r requirements/test/test.in
     #   pre-commit
-requests==2.32.3
-    # via
-    #   -c requirements/test/../base/base.txt
-    #   requests-mock
-requests-mock==1.12.1
-    # via -r requirements/test/test.in
 six==1.15.0
     # via
     #   -c requirements/test/../base/base.txt
@@ -104,11 +88,8 @@ tomli==2.0.2
     #   pytest
 typing-extensions==4.12.2
     # via
+    #   -c requirements/test/../base/base.txt
     #   black
     #   faker
-urllib3==2.2.1
-    # via
-    #   -c requirements/test/../base/base.txt
-    #   requests
 virtualenv==20.27.0
     # via pre-commit