Introduce and use new API domain environment variables (#3822)

* Introduce and use new API domain environment variables * Print better error messages in example tests * Match host port inside container to simplify inner-container references * Fix ALLOWED_HOSTS port and remove duplicative `localhost` DEBUG entry * Fix nginx API host port * Update use of API_URL in API schema tests --------- Co-authored-by: Madison Swain-Bowden <bowdenm@spu.edu>
WordPress · Mar 9, 2024 · eae8c19 · eae8c19
1 parent 9460b14
commit eae8c19
Show file tree

Hide file tree

Showing 22 changed files with 100 additions and 119 deletions.
diff --git a/api/Dockerfile b/api/Dockerfile
@@ -102,6 +102,7 @@ COPY --chown=opener . /api/
 RUN env \
   SETUP_ES="False" \
   DJANGO_SECRET_KEY="any string" \
+  CANONICAL_DOMAIN="" \
   python manage.py collectstatic
 
 # Add the release version to the docker container

diff --git a/api/api/docs/base_docs.py b/api/api/docs/base_docs.py
@@ -2,6 +2,7 @@
 from textwrap import dedent
 from typing import Literal
 
+from django.conf import settings
 from rest_framework.exceptions import (
     NotAuthenticated,
     NotFound,
@@ -114,7 +115,7 @@ def get_operation_id(self) -> str:
 def build_source_path_parameter(media_type: MediaType):
     valid_description = (
         f"Valid values are source_names from the stats endpoint: "
-        f"https://api.openverse.engineering/v1/{media_type}/stats/."
+        f"{settings.CANONICAL_ORIGIN}/v1/{media_type}/stats/."
     )
 
     return OpenApiParameter(

diff --git a/api/api/examples/audio_requests.py b/api/api/examples/audio_requests.py
@@ -1,10 +1,7 @@
-import os
+from api.examples.environment import ORIGIN, TOKEN
 
 
-token = os.getenv("AUDIO_REQ_TOKEN", "DLBYIcfnKfolaXKcmMC8RIDCavc2hW")
-origin = os.getenv("AUDIO_REQ_ORIGIN", "https://api.openverse.engineering")
-
-auth = f'-H "Authorization: Bearer {token}"' if token else ""
+auth = f'-H "Authorization: Bearer {TOKEN}"' if TOKEN else ""
 identifier = "8624ba61-57f1-4f98-8a85-ece206c319cf"
 
 syntax_examples = {
@@ -25,7 +22,7 @@
 # Example {index}: Search for audio {purpose}
 curl \\
   {auth} \\
-  "{origin}/v1/audio/?q={syntax}"
+  "{ORIGIN}/v1/audio/?q={syntax}"
 """
     for (index, (purpose, syntax)) in enumerate(syntax_examples.items())
 )
@@ -34,28 +31,28 @@
 # Search for music titled "Wish You Were Here" by The.madpix.project
 curl \\
   {auth} \\
-  "{origin}/v1/audio/?title=Wish%20You%20Were%20Here&creator=The.madpix.project"
+  "{ORIGIN}/v1/audio/?title=Wish%20You%20Were%20Here&creator=The.madpix.project"
 """
 
 audio_stats_curl = f"""
 # Get the statistics for audio sources
 curl \\
   {auth} \\
-  "{origin}/v1/audio/stats/"
+  "{ORIGIN}/v1/audio/stats/"
 """
 
 audio_detail_curl = f"""
 # Get the details of audio ID {identifier}
 curl \\
   {auth} \\
-  "{origin}/v1/audio/{identifier}/"
+  "{ORIGIN}/v1/audio/{identifier}/"
 """
 
 audio_related_curl = f"""
 # Get related audio files for audio ID {identifier}
 curl \\
   {auth} \\
-  "{origin}/v1/audio/{identifier}/related/"
+  "{ORIGIN}/v1/audio/{identifier}/related/"
 """
 
 audio_complain_curl = f"""
@@ -65,12 +62,12 @@
   -H "Content-Type: application/json" \\
   {auth} \\
   -d '{{"reason": "mature", "description": "This audio contains sensitive content"}}' \\
-  "{origin}/v1/audio/{identifier}/report/"
+  "{ORIGIN}/v1/audio/{identifier}/report/"
 """
 
 audio_waveform_curl = f"""
 # Get the waveform of audio ID {identifier}
 curl \\
   {auth} \\
-  "{origin}/v1/audio/{identifier}/waveform/"
+  "{ORIGIN}/v1/audio/{identifier}/waveform/"
 """
diff --git a/api/api/examples/audio_responses.py b/api/api/examples/audio_responses.py
@@ -1,8 +1,6 @@
-import os
+from api.examples.environment import ORIGIN
 
 
-origin = os.getenv("AUDIO_REQ_ORIGIN", "https://api.openverse.engineering")
-
 identifier = "8624ba61-57f1-4f98-8a85-ece206c319cf"
 
 base_audio = {
@@ -50,10 +48,10 @@
     "duration": 270000,
     "bit_rate": 128000,
     "sample_rate": 44100,
-    "thumbnail": f"{origin}/v1/audio/{identifier}/thumb/",
-    "detail_url": f"{origin}/v1/audio/{identifier}/",
-    "related_url": f"{origin}/v1/audio/{identifier}/related/",
-    "waveform": f"{origin}/v1/audio/{identifier}/waveform/",
+    "thumbnail": f"{ORIGIN}/v1/audio/{identifier}/thumb/",
+    "detail_url": f"{ORIGIN}/v1/audio/{identifier}/",
+    "related_url": f"{ORIGIN}/v1/audio/{identifier}/related/",
+    "waveform": f"{ORIGIN}/v1/audio/{identifier}/waveform/",
     "unstable__sensitivity": [],
 }
 
@@ -124,8 +122,8 @@
                 "license_version": "2.0",
                 "license_url": "https://creativecommons.org/licenses/by-sa/2.0/",
                 "foreign_landing_url": "https://commons.wikimedia.org/w/index.php?curid=3536953",  # noqa: E501
-                "detail_url": "http://api.openverse.engineering/v1/audio/36537842-b067-4ca0-ad67-e00ff2e06b2e",  # noqa: E501
-                "related_url": "http://api.openverse.engineering/v1/recommendations/audio/36537842-b067-4ca0-ad67-e00ff2e06b2e",  # noqa: E501
+                "detail_url": f"{ORIGIN}/v1/audio/36537842-b067-4ca0-ad67-e00ff2e06b2e",  # noqa: E501
+                "related_url": f"{ORIGIN}/v1/recommendations/audio/36537842-b067-4ca0-ad67-e00ff2e06b2e",  # noqa: E501
                 "fields_matched": ["description", "title"],
                 "tags": [{"name": "exam"}, {"name": "tactics"}],
             }

diff --git a/api/api/examples/environment.py b/api/api/examples/environment.py
@@ -0,0 +1,7 @@
+import os
+
+
+TOKEN = os.getenv("REQUEST_TOKEN", "<Openverse API token>")
+DOMAIN = os.getenv("CANONICAL_DOMAIN")
+_proto = "http" if "localhost" in DOMAIN else "https"
+ORIGIN = f"{_proto}://{DOMAIN}"
diff --git a/api/api/examples/image_requests.py b/api/api/examples/image_requests.py
@@ -1,10 +1,7 @@
-import os
+from api.examples.environment import ORIGIN, TOKEN
 
 
-token = os.getenv("AUDIO_REQ_TOKEN", "DLBYIcfnKfolaXKcmMC8RIDCavc2hW")
-origin = os.getenv("AUDIO_REQ_ORIGIN", "https://api.openverse.engineering")
-
-auth = f'-H "Authorization: Bearer {token}"' if token else ""
+auth = f'-H "Authorization: Bearer {TOKEN}"' if TOKEN else ""
 identifier = "4bc43a04-ef46-4544-a0c1-63c63f56e276"
 
 syntax_examples = {
@@ -25,7 +22,7 @@
 # Example {index}: Search for images {purpose}
 curl \\
   {auth} \\
-  "{origin}/v1/images/?q={syntax}"
+  "{ORIGIN}/v1/images/?q={syntax}"
 """
     for (index, (purpose, syntax)) in enumerate(syntax_examples.items())
 )
@@ -34,28 +31,28 @@
 # Search for images titled "Bark" by Sullivan
 curl \\
   {auth} \\
-  "{origin}/v1/images/?title=Bark&creator=Sullivan"
+  "{ORIGIN}/v1/images/?title=Bark&creator=Sullivan"
 """
 
 image_stats_curl = f"""
 # Get the statistics for image sources
 curl \\
   {auth} \\
-  "{origin}/v1/images/stats/"
+  "{ORIGIN}/v1/images/stats/"
 """
 
 image_detail_curl = f"""
 # Get the details of image ID {identifier}
 curl \\
   {auth} \\
-  "{origin}/v1/images/{identifier}/"
+  "{ORIGIN}/v1/images/{identifier}/"
 """
 
 image_related_curl = f"""
 # Get related images for image ID {identifier}
 curl \\
   {auth} \\
-  "{origin}/v1/images/{identifier}/related/"
+  "{ORIGIN}/v1/images/{identifier}/related/"
 """
 
 image_complain_curl = f"""
@@ -65,12 +62,12 @@
   -H "Content-Type: application/json" \\
   {auth} \\
   -d '{{"reason": "mature", "description": "Image contains sensitive content"}}' \\
-  "{origin}/v1/images/{identifier}/report/"
+  "{ORIGIN}/v1/images/{identifier}/report/"
 """
 
 image_oembed_curl = f"""
 # Retrieve embedded content from an image's URL
 curl \\
   {auth} \\
-  "{origin}/v1/images/oembed/?url=https://wordpress.org/openverse/photos/{identifier}"
+  "{ORIGIN}/v1/images/oembed/?url=https://wordpress.org/openverse/photos/{identifier}"
 """
diff --git a/api/api/examples/image_responses.py b/api/api/examples/image_responses.py
@@ -1,8 +1,6 @@
-import os
+from api.examples.environment import ORIGIN
 
 
-origin = os.getenv("AUDIO_REQ_ORIGIN", "https://api.openverse.engineering")
-
 identifier = "4bc43a04-ef46-4544-a0c1-63c63f56e276"
 
 base_image = {
@@ -53,9 +51,9 @@
     "mature": False,
     "height": 4016,
     "width": 6016,
-    "thumbnail": f"{origin}/v1/images/{identifier}/thumb/",
-    "detail_url": f"{origin}/v1/images/{identifier}/",
-    "related_url": f"{origin}/v1/images/{identifier}/related/",
+    "thumbnail": f"{ORIGIN}/v1/images/{identifier}/thumb/",
+    "detail_url": f"{ORIGIN}/v1/images/{identifier}/",
+    "related_url": f"{ORIGIN}/v1/images/{identifier}/related/",
     "unstable__sensitivity": [],
 }
 
@@ -120,15 +118,15 @@
                 "creator_url": "https://www.flickr.com/photos/18090920@N07",
                 "tags": [{"name": "exam"}, {"name": "tactics"}],
                 "url": "https://live.staticflickr.com/4065/4459771899_07595dc42e.jpg",  # noqa: E501
-                "thumbnail": "https://api.openverse.engineering/v1/thumbs/610756ec-ae31-4d5e-8f03-8cc52f31b71d",  # noqa: E501
+                "thumbnail": f"{ORIGIN}/v1/thumbs/610756ec-ae31-4d5e-8f03-8cc52f31b71d",  # noqa: E501
                 "provider": "flickr",
                 "source": "flickr",
                 "license": "by",
                 "license_version": "2.0",
                 "license_url": "https://creativecommons.org/licenses/by/2.0/",
                 "foreign_landing_url": "https://www.flickr.com/photos/18090920@N07/4459771899",  # noqa: E501
-                "detail_url": "http://api.openverse.engineering/v1/images/610756ec-ae31-4d5e-8f03-8cc52f31b71d",  # noqa: E501
-                "related_url": "http://api.openverse.engineering/v1/recommendations/images/610756ec-ae31-4d5e-8f03-8cc52f31b71d",  # noqa: E501
+                "detail_url": f"{ORIGIN}/v1/images/610756ec-ae31-4d5e-8f03-8cc52f31b71d",  # noqa: E501
+                "related_url": f"{ORIGIN}/v1/recommendations/images/610756ec-ae31-4d5e-8f03-8cc52f31b71d",  # noqa: E501
             }
         ],
     }

diff --git a/api/api/examples/oauth2_requests.py b/api/api/examples/oauth2_requests.py
@@ -1,31 +1,28 @@
-import os
+from api.examples.environment import ORIGIN, TOKEN
 
 
-token = os.getenv("AUDIO_REQ_TOKEN", "DLBYIcfnKfolaXKcmMC8RIDCavc2hW")
-origin = os.getenv("AUTH_REQ_ORIGIN", "https://api.openverse.engineering")
-
-auth = f'-H "Authorization: Bearer {token}"' if token else ""
+auth = f'-H "Authorization: Bearer {TOKEN}"' if TOKEN else ""
 
 auth_register_curl = f"""
 # Register for a key
 curl \\
   -X POST \\
   -H "Content-Type: application/json" \\
   -d '{{"name": "My amazing project", "description": "To access Openverse API", "email": "user@example.com"}}' \\
-  "{origin}/v1/auth_tokens/register/"
+  "{ORIGIN}/v1/auth_tokens/register/"
 """  # noqa: E501
 
 auth_token_curl = f"""
 # Get an access token token
 curl \\
   -X POST \\
   -H "Content-Type: application/x-www-form-urlencoded" \\
-  -d 'grant_type=client_credentials&client_id=pm8GMaIXIhkjQ4iDfXLOvVUUcIKGYRnMlZYApbda&client_secret=YhVjvIBc7TuRJSvO2wIi344ez5SEreXLksV7GjalLiKDpxfbiM8qfUb5sNvcwFOhBUVzGNdzmmHvfyt6yU3aGrN6TAbMW8EOkRMOwhyXkN1iDetmzMMcxLVELf00BR2e' \\
-  "{origin}/v1/auth_tokens/token/"
+  -d 'grant_type=client_credentials&client_id=<Openverse API client ID>&client_secret=<Openverse API client secret>' \\
+  "{ORIGIN}/v1/auth_tokens/token/"
 """  # noqa: E501
 
 auth_key_info_curl = f"""
 curl \\
   {auth} \\
-  "{origin}/v1/rate_limit/"
+  "{ORIGIN}/v1/rate_limit/"
 """
diff --git a/api/api/examples/oauth2_responses.py b/api/api/examples/oauth2_responses.py
@@ -1,17 +1,14 @@
 auth_register_201_example = {
     "application/json": {
         "name": "My amazing project",
-        "client_id": "pm8GMaIXIhkjQ4iDfXLOvVUUcIKGYRnMlZYApbda",
-        "client_secret": (
-            "YhVjvIBc7TuRJSvO2wIi344ez5SEreXLksV7GjalLiKDpxfbiM8qfUb5sNvcwFOh"
-            "BUVzGNdzmmHvfyt6yU3aGrN6TAbMW8EOkRMOwhyXkN1iDetmzMMcxLVELf00BR2e"
-        ),
+        "client_id": "<Openverse API client ID>",
+        "client_secret": "<Openverse API client secret>",
     }
 }
 
 auth_token_200_example = {
     "application/json": {
-        "access_token": "DLBYIcfnKfolaXKcmMC8RIDCavc2hW",
+        "access_token": "<Openverse API token>",
         "scope": "read write groups",
         "expires_in": 36000,
         "token_type": "Bearer",

diff --git a/api/api/models/media.py b/api/api/models/media.py
@@ -145,8 +145,6 @@ class AbstractMediaReport(models.Model):
     deleted_class: type[models.Model] = None
     """the class storing deleted media e.g. ``DeletedImage`` or ``DeletedAudio``"""
 
-    BASE_URL = settings.BASE_URL
-
     REPORT_CHOICES = [(MATURE, MATURE), (DMCA, DMCA), (OTHER, OTHER)]
 
     STATUS_CHOICES = [
@@ -201,9 +199,7 @@ def clean(self):
             )
 
     def url(self, media_type):
-        url = (
-            f"{AbstractMediaReport.BASE_URL}v1/{media_type}/{self.media_obj.identifier}"
-        )
+        url = f"{settings.CANONICAL_ORIGIN}v1/{media_type}/{self.media_obj.identifier}"
         return format_html(f"<a href={url}>{url}</a>")
 
     def save(self, *args, **kwargs):

diff --git a/api/api/serializers/media_serializers.py b/api/api/serializers/media_serializers.py
@@ -633,7 +633,7 @@ class MediaSearchRequestSourceSerializer(serializers.Serializer):
             "help_text": (
                 "A comma separated list of data sources; valid values are "
                 "``source_name``s from the stats endpoint: "
-                f"https://api.openverse.engineering/v1/{media_path}/stats/."
+                f"{settings.CANONICAL_ORIGIN}/v1/{media_path}/stats/."
             ),
             "required": False,
         }

diff --git a/api/conf/settings/security.py b/api/conf/settings/security.py
@@ -16,27 +16,38 @@
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG = config("DJANGO_DEBUG_ENABLED", default=False, cast=bool)
 
-ALLOWED_HOSTS = config("ALLOWED_HOSTS", default="").split(",") + [
+# The domain we treat as "canonical" for this API instance, e.g., `api.` subdomain for production
+CANONICAL_DOMAIN: str = config("CANONICAL_DOMAIN")  # required
+
+_proto = "http" if "localhost" in CANONICAL_DOMAIN else "https"
+CANONICAL_ORIGIN: str = f"{_proto}://{CANONICAL_DOMAIN}"
+
+# Additional domains we serve for this API instance, e.g., `api-production.` subdomain for production
+ADDITIONAL_DOMAINS: list[str] = config(
+    "ADDITIONAL_DOMAINS", default="", cast=lambda x: x.split(",")
+)
+
+ALL_DOMAINS = [CANONICAL_DOMAIN] + ADDITIONAL_DOMAINS
+
+ALLOWED_HOSTS = [
+    # Strip ports off hosts, as ALLOWED_HOSTS does not work with ports, e.g., `localhost:8000` needs to be just `localhost`
+    domain.split(":")[0]
+    for domain in ALL_DOMAINS
+] + [
     gethostname(),
     gethostbyname(gethostname()),
 ]
 
-if lb_url := config("LOAD_BALANCER_URL", default=""):
-    ALLOWED_HOSTS.append(lb_url)
-
 if DEBUG:
     ALLOWED_HOSTS += [
         "dev.openverse.test",  # used in local development
-        "localhost",
         "127.0.0.1",
         "0.0.0.0",
     ]
 
-BASE_URL = config("BASE_URL", default="https://api.openverse.engineering/")
-
 # Trusted origins for CSRF
 # https://docs.djangoproject.com/en/4.2/ref/settings/#csrf-trusted-origins
-CSRF_TRUSTED_ORIGINS = ["https://*.openverse.engineering"]
+CSRF_TRUSTED_ORIGINS = [f"{_proto}://{domain}" for domain in ALL_DOMAINS]
 
 # Allow anybody to access the API from any domain
 if "corsheaders" not in INSTALLED_APPS:
@@ -57,8 +68,6 @@
 
 # Proxy handling, for production
 if config("IS_PROXIED", default=True, cast=bool):
-    # https://docs.djangoproject.com/en/4.0/ref/settings/#use-x-forwarded-host
-    USE_X_FORWARDED_HOST = True
     # https://docs.djangoproject.com/en/4.0/ref/settings/#secure-proxy-ssl-header
     SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")