Fetch ReferenceItem content from raw GitHub URLs when possible

core/models.py

@@ -4,6 +4,7 @@
 from django.contrib.auth.models import User
 from django.utils import timezone
 from urllib.parse import urlparse
+import requests
 from .utils import guess_specification_language_by_extension, guess_language_by_extension
 
 class BaseModel(models.Model):
@@ -217,6 +218,18 @@ class Meta:
     def __str__(self):
         return self.url
 
+    # TODO: Cache content and add optional param to force a refresh (GitHub issue #157)
+    def get_content(self):
+        if (
+            self.url_provider_info.provider_name == GitHubURLInfo.provider_name and
+            self.url_provider_info.raw_url
+        ):
+            content_url = self.url_provider_info.raw_url
+        else:
+            content_url = self.url
+        response = requests.get(content_url)
+        return response.text
+
     @property
     def url_provider_info(self):
         return URLProviderInfo.from_url(self.url) 

core/views.py

@@ -8,7 +8,6 @@
 from django.utils import timezone
 from django.core.exceptions import PermissionDenied
 from functools import wraps
-import requests
 import cmarkgfm
 import bleach
 from .models import (Schema,
@@ -113,7 +112,7 @@ def schema_detail(request, schema):
     latest_readme = schema.latest_readme()
     latest_readme_content = None
     if latest_readme:
-        response_text = requests.get(latest_readme.url).text
+        response_text = latest_readme.get_content()
         if latest_readme.format == DocumentationItem.DocumentationItemFormat.Markdown:
             latest_readme_content = render_markdown(response_text)
         elif latest_readme.format == DocumentationItem.DocumentationItemFormat.PlainText:
@@ -134,9 +133,7 @@ def schema_detail(request, schema):
 @lookup_schema
 def schema_ref_detail(request, schema, schema_ref_id):
     schema_ref = get_object_or_404(schema.schemaref_set.filter(id=schema_ref_id))
-    # TODO: I feel like we can do better here- e.g. put the get request in the model and
-    # pull from cache
-    text_content = requests.get(schema_ref.url).text
+    text_content = schema_ref.get_content()
     if schema_ref.language == "markdown":
         schema_ref.markdown = render_markdown(text_content)
     else:

tests/test_models.py

@@ -1,5 +1,5 @@
 import pytest
-
+import requests_mock
 from core.models import Schema, SchemaRef
 from factories import UserFactory
 
@@ -52,3 +52,13 @@ def test_reference_item_github_url_info_converts_repo_url_to_raw(repo_url, raw_u
 def test_reference_item_github_url_info_converts_raw_url_to_repo(repo_url, raw_url):
     schema_ref = SchemaRef(url=raw_url)
     assert schema_ref.url_provider_info.repo_url == repo_url
+
+
+def test_reference_item_content_is_fetched_from_raw_url_if_available():
+    schema_ref = SchemaRef(url="https://github.com/userorg/reponame/blob/branch/path/to/file.json")
+    mock_content = '{"MOCK_CONTENT":true}'
+    # Make sure the URL we expect to be fetched isn't the one we provided
+    assert schema_ref.url != schema_ref.url_provider_info.raw_url
+    with requests_mock.Mocker() as m:
+        m.get(schema_ref.url_provider_info.raw_url, text=mock_content)
+        assert schema_ref.get_content() == mock_content