diff --git a/django/cantusdb_project/main_app/forms.py b/django/cantusdb_project/main_app/forms.py index 81a96107f..accbd6528 100644 --- a/django/cantusdb_project/main_app/forms.py +++ b/django/cantusdb_project/main_app/forms.py @@ -1,3 +1,5 @@ +from typing import Any + from django import forms from django.contrib.auth.forms import ReadOnlyPasswordHashField from django.contrib.auth import get_user_model @@ -5,8 +7,8 @@ from django.contrib.admin.widgets import ( FilteredSelectMultiple, ) -from django.forms.widgets import CheckboxSelectMultiple -from dal import autocomplete +from django.forms.widgets import CheckboxSelectMultiple, HiddenInput +from dal import autocomplete # type: ignore[import-untyped] from volpiano_display_utilities.cantus_text_syllabification import syllabify_text from volpiano_display_utilities.latin_word_syllabification import LatinError from .models import ( @@ -885,3 +887,36 @@ class Meta: 'using this form.' ) ) + + +class ImageLinkForm(forms.Form): + """ + Subclass of Django's Form class that creates the form we use for + adding image links to chants in a source. + + Initialize the Form with a field for every folio in the source, + passed as the "initial" parameter, which is a dictionary with a key + for every folio and a blank value. + """ + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + initial = kwargs.get("initial") + if initial: + for folio in initial: + self.fields[folio] = forms.CharField( + widget=HiddenInput(attrs={"class": "img-link-input"}), + required=False, + ) + + def save(self, source: Source) -> None: + """ + Save the image links to the database. + + Args: + source: The source to which the image links belong. + """ + cleaned_data = self.cleaned_data + for folio, image_link in cleaned_data.items(): + if image_link != "": + source.chant_set.filter(folio=folio).update(image_link=image_link) diff --git a/django/cantusdb_project/main_app/permissions.py b/django/cantusdb_project/main_app/permissions.py index 4bbd65d82..f85c4da8d 100644 --- a/django/cantusdb_project/main_app/permissions.py +++ b/django/cantusdb_project/main_app/permissions.py @@ -182,7 +182,7 @@ def user_can_view_user_detail(viewing_user: User, user: User) -> bool: return viewing_user.is_authenticated or user.is_indexer -def user_can_manage_source_editors(user: User) -> bool: +def user_can_manage_source_editors(user: Union[User, AnonymousUser]) -> bool: """ Checks if the user has permission to change the editors assigned to a Source. Used in SourceDetailView. diff --git a/django/cantusdb_project/main_app/templates/source_add_image_links.html b/django/cantusdb_project/main_app/templates/source_add_image_links.html new file mode 100644 index 000000000..d7a4c0901 --- /dev/null +++ b/django/cantusdb_project/main_app/templates/source_add_image_links.html @@ -0,0 +1,133 @@ +{% extends "base.html" %} +{% load static %} +{% block title %} + Add Image Links to Source: {{ source.short_heading }} +{% endblock %} +{% block scripts %} + + +{% endblock %} +{% block content %} +
+
+
+

+ Add Image Links to Source: {{ source.heading }} +

+

+ Use this form to add image links from a csv file to the chants in this source. The csv file should + contain two columns. The first column should be the folio on which the chant appears, matching the values + of the "folio" field in the source inventory. The second column should be a url to an image of the chant, + including the scheme (e.g., "https://") and domain. An optional header may be included. +

+

+ The following checks will be performed on the csv file before it is uploaded: +

+
    +
  1. A check that an image link is defined for every folio in the source.
    2. +
  2. + A check that image links are not defined for any folios that do not exist in the source. +
    3. +
  3. + A check that no folios are duplicated in the file (e.g., a single + folio does not have two image links defined). +
    4. +
  4. + A check that either no image links are duplicated (every folio has a + unique image link) or every image link is the same (image links to + individual folios are not available). +
    5. +
+

+ Note: Files that fail these checks may still be uploaded but are provided as a simple + validation for the user. For example, if a folio appears multiple times in the csv, + the image link appearing last in the csv will be used. Sometimes, a mapping that fails + these checks might be correct, such as when images show two facing folios. +

+
+
+ +
+
+ +
+ {% csrf_token %} + {{ form }} +
+
+
+ +
+
+ + +
+{% endblock %} diff --git a/django/cantusdb_project/main_app/tests/test_views/test_source.py b/django/cantusdb_project/main_app/tests/test_views/test_source.py index 434f4e9b2..3d141154b 100644 --- a/django/cantusdb_project/main_app/tests/test_views/test_source.py +++ b/django/cantusdb_project/main_app/tests/test_views/test_source.py @@ -26,6 +26,7 @@ add_accents_to_string, ) from main_app.tests.mixins import HTMLContentsTestMixin +from users.models import User # Create a Faker instance with locale set to Latin faker = Faker("la") @@ -1198,3 +1199,114 @@ def test_ordering(self) -> None: self.assertEqual( list(reversed(expected_source_order)), list(response_sources_reverse) ) + + +class SourceAddImageLinksViewTest(TestCase): + auth_user: User + non_auth_user: User + source: Source + + @classmethod + def setUpTestData(cls) -> None: + user_model = get_user_model() + cls.auth_user = user_model.objects.create( + email="authuser@test.com", password="12345", is_staff=True + ) + cls.non_auth_user = user_model.objects.create( + email="nonauthuser@test.com", password="12345", is_staff=False + ) + cls.source = make_fake_source(published=True) + for folio in ["001r", "001v", "003", "004A"]: + make_fake_chant(source=cls.source, folio=folio, image_link=None) + # Make a second chant for one of the folios, with an existing image link. + # We'll update this image_link in the process. + make_fake_chant( + source=cls.source, folio="001v", image_link="https://i-already-exist.com" + ) + # Make a final chant for a different folio with an existing image link. We + # won't update this image_link in the process. + make_fake_chant( + source=cls.source, folio="004B", image_link="https://i-already-exist.com/2" + ) + + def test_permissions(self) -> None: + with self.subTest("Test unauthenticated user"): + response = self.client.get( + reverse("source-add-image-links", args=[self.source.id]) + ) + self.assertEqual(response.status_code, 302) + self.assertRedirects( + response, + f"{reverse('login')}?next={reverse('source-add-image-links', args=[self.source.id])}", + status_code=302, + target_status_code=200, + ) + response = self.client.post( + reverse("source-add-image-links", args=[self.source.id]) + ) + self.assertEqual(response.status_code, 302) + self.assertRedirects( + response, + f"{reverse('login')}?next={reverse('source-add-image-links', args=[self.source.id])}", + status_code=302, + target_status_code=200, + ) + with self.subTest("Test non-staff user"): + self.client.force_login(self.non_auth_user) + response = self.client.get( + reverse("source-add-image-links", args=[self.source.id]) + ) + self.assertEqual(response.status_code, 403) + response = self.client.post( + reverse("source-add-image-links", args=[self.source.id]) + ) + self.assertEqual(response.status_code, 403) + with self.subTest("Test staff user"): + self.client.force_login(self.auth_user) + response = self.client.get( + reverse("source-add-image-links", args=[self.source.id]) + ) + self.assertEqual(response.status_code, 200) + # Post redirect is tested in the `test_form` method + + def test_form(self) -> None: + with self.subTest("Test form fields"): + self.client.force_login(self.auth_user) + response = self.client.get( + reverse("source-add-image-links", args=[self.source.id]) + ) + self.assertEqual(response.status_code, 200) + self.assertTemplateUsed(response, "source_add_image_links.html") + form = response.context["form"] + self.assertListEqual( + list(form.fields.keys()), ["001r", "001v", "003", "004A", "004B"] + ) + with self.subTest("Test form submission"): + response = self.client.post( + reverse("source-add-image-links", args=[self.source.id]), + { + "001r": "https://example.com/001r", + "001v": "https://example.com/001v", + "004A": "https://example.com/004A", + }, + ) + self.assertRedirects( + response, + reverse("source-detail", args=[self.source.id]), + status_code=302, + target_status_code=200, + ) + with self.subTest("Test saved data"): + chants_001r = Chant.objects.filter(source=self.source, folio="001r").all() + self.assertEqual(len(chants_001r), 1) + self.assertEqual(chants_001r[0].image_link, "https://example.com/001r") + chants_001v = Chant.objects.filter(source=self.source, folio="001v").all() + self.assertEqual(len(chants_001v), 2) + for chant in chants_001v: + self.assertEqual(chant.image_link, "https://example.com/001v") + chants_003 = Chant.objects.filter(source=self.source, folio="003").all() + self.assertEqual(len(chants_003), 1) + self.assertIsNone(chants_003[0].image_link) + chants_004B = Chant.objects.filter(source=self.source, folio="004B").all() + self.assertEqual(len(chants_004B), 1) + self.assertEqual(chants_004B[0].image_link, "https://i-already-exist.com/2") diff --git a/django/cantusdb_project/main_app/urls.py b/django/cantusdb_project/main_app/urls.py index 97a7bd9de..307380857 100644 --- a/django/cantusdb_project/main_app/urls.py +++ b/django/cantusdb_project/main_app/urls.py @@ -85,6 +85,7 @@ SourceListView, SourceDeleteView, SourceInventoryView, + SourceAddImageLinksView, ) from main_app.views.user import ( CustomLogoutView, @@ -364,6 +365,11 @@ SourceDeleteView.as_view(), name="source-delete", ), + path( + "source//add-image-links", + SourceAddImageLinksView.as_view(), + name="source-add-image-links", + ), # melody path( "melody/", diff --git a/django/cantusdb_project/main_app/views/source.py b/django/cantusdb_project/main_app/views/source.py index 4414f9468..9aa3ef0d6 100644 --- a/django/cantusdb_project/main_app/views/source.py +++ b/django/cantusdb_project/main_app/views/source.py @@ -1,4 +1,4 @@ -from typing import Any +from typing import Any, Optional from django.contrib import messages from django.contrib.auth.mixins import LoginRequiredMixin @@ -6,7 +6,7 @@ from django.core.exceptions import PermissionDenied from django.db.models import Q, Prefetch, Value from django.db.models import QuerySet -from django.http import HttpResponseRedirect, Http404 +from django.http import HttpResponseRedirect, Http404, HttpResponse, HttpRequest from django.shortcuts import get_object_or_404 from django.urls import reverse from django.views.generic import ( @@ -16,12 +16,14 @@ UpdateView, DeleteView, TemplateView, + FormView, ) - +from django.views.generic.detail import SingleObjectMixin from main_app.forms import ( SourceCreateForm, SourceEditForm, SourceBrowseChantsProofreadForm, + ImageLinkForm, ) from main_app.models import ( Century, @@ -39,12 +41,11 @@ user_can_view_source, user_can_manage_source_editors, user_can_proofread_source, -) -from main_app.views.chant import ( - get_feast_selector_options, user_can_edit_chants_in_source, ) +from main_app.views.chant import get_feast_selector_options + CANTUS_SEGMENT_ID = 4063 BOWER_SEGMENT_ID = 4064 @@ -248,7 +249,7 @@ def get_context_data(self, **kwargs): return context -class SourceListView(ListView): # type: ignore +class SourceListView(ListView): model = Source paginate_by = 100 context_object_name = "sources" @@ -554,3 +555,46 @@ def get_context_data(self, **kwargs): context["chants"] = queryset return context + + +class SourceAddImageLinksView(UserPassesTestMixin, SingleObjectMixin, FormView): # type: ignore + template_name = "source_add_image_links.html" + pk_url_kwarg = "source_id" + queryset = Source.objects.select_related("holding_institution") + context_object_name = "source" + form_class = ImageLinkForm + object: Source + http_method_names = ["get", "post"] + + def test_func(self) -> bool: + return user_can_manage_source_editors(self.request.user) + + def get_success_url(self) -> str: + return reverse("source-detail", args=[self.object.id]) + + def get(self, request: HttpRequest, *args: Any, **kwargs: Any) -> HttpResponse: + self.object = self.get_object() + return super().get(request, *args, **kwargs) + + def post(self, request: HttpRequest, *args: Any, **kwargs: Any) -> HttpResponse: + self.object = self.get_object() + return super().post(request, *args, **kwargs) + + def get_initial(self) -> dict[str, Any]: + """ + Set the initial data required by the ImageLinkForm + on GET requests. + """ + folios: QuerySet[Chant, Optional[str]] = ( + self.object.chant_set.values_list("folio", flat=True) + .distinct() + .order_by("folio") + ) + return {folio: "" for folio in folios if folio} + + def form_valid(self, form: ImageLinkForm) -> HttpResponseRedirect: + """ + Save the image links to the database. + """ + form.save(self.object) + return HttpResponseRedirect(self.get_success_url()) diff --git a/django/cantusdb_project/static/css/source_add_image_links.css b/django/cantusdb_project/static/css/source_add_image_links.css new file mode 100644 index 000000000..7fc531c5f --- /dev/null +++ b/django/cantusdb_project/static/css/source_add_image_links.css @@ -0,0 +1,27 @@ +td.img-link-preview-cell { + padding-left: 4rem; + padding-right: 4rem; +} + +.csv-testing-table { + table-layout: fixed; + width: 100%; +} + +.csv-testing-table-test { + width: 30%; +} + +.csv-testing-table-icon { + width: 15%; +} + +.csv-testing-table-instances { + max-height: 5em; + overflow-y: scroll; +} + +.csv-preview-table { + max-height: 50em; + overflow-y: scroll; +} \ No newline at end of file diff --git a/django/cantusdb_project/static/js/source_add_image_links.js b/django/cantusdb_project/static/js/source_add_image_links.js new file mode 100644 index 000000000..f25fbf3f5 --- /dev/null +++ b/django/cantusdb_project/static/js/source_add_image_links.js @@ -0,0 +1,147 @@ +function addPreviewTableRow(tableBody, folio, imageLink) { + // Add a row to the preview table with the folio and image link. + const tr = document.createElement('tr'); + const tdFolio = document.createElement('td'); + tdFolio.textContent = folio; + tdFolio.classList.add('img-link-preview-cell'); + tr.appendChild(tdFolio); + const tdLink = document.createElement('td'); + const a = document.createElement('a'); + a.href = imageLink; + a.textContent = imageLink; + a.target = '_blank'; + tdLink.appendChild(a); + tdLink.classList.add('img-link-preview-cell'); + tr.appendChild(tdLink); + tableBody.appendChild(tr); +}; + +function getFormImgLinkInputs(form) { + // Get all the input elements in the form that have the + // 'img-link-input' class and return them as a map of the form + // {folio: inputElement}. + const imgLinkInputs = form.getElementsByClassName('img-link-input'); + const imgLinkMap = {}; + for (let i = 0; i < imgLinkInputs.length; i++) { + const folio = imgLinkInputs[i].name; + imgLinkMap[folio] = imgLinkInputs[i]; + } + return imgLinkMap; +}; + + +function parseAndPreviewImageLinkCSV(csv, imgLinkInputs) { + // Parse the passed CSV file and display it in a table. + // Return two arrays: one with the folios and one with the image links + // for use in testing functions. + const rows = csv.split('\n'); + const columns = rows[0].split(','); + // Check if a header row is present, by checking whether + // the second column is a URL + const start = columns[1].startsWith('http') ? 0 : 1; + const tableBody = document.getElementById('csvPreviewBody'); + // Clear the table and fill in the new data + // using this rough CSV parser. Note that since this + // function is meant to be used by just a few administrators, + // we aren't going to worry too much about parsing edge cases. + // For example, note that we just parse at the first comma to split + // columns. + tableBody.innerHTML = ''; + const parsedCSV = []; + for (let i = start; i < rows.length; i++) { + const row = rows[i]; + let [folio, imageLink] = row.split(',', 2); + folio = folio.trim(); + imageLink = imageLink.trim(); + addPreviewTableRow(tableBody, folio, imageLink); + const folioInput = imgLinkInputs[folio]; + if (folioInput) { + folioInput.value = imageLink; + } + parsedCSV.push({ "folio": folio, "imageLink": imageLink }); + } + document.getElementById("csvPreviewDiv").hidden = false; + return parsedCSV; +} + +function getFoliosAtDuplicatedValues(array) { + // Given an array of objects with imageLink and folio keys, + // parsed from the CSV file, return an array of folios that have + // been duplicated and an array of folios that have duplicated image links. + const folioCounts = {}; + const imageLinkCounts = {}; + for (let i = 0; i < array.length; i++) { + const folio = array[i].folio; + const imageLink = array[i].imageLink; + folioCounts[folio] = (folioCounts[folio] || 0) + 1; + imageLinkCounts[imageLink] = (imageLinkCounts[imageLink] || 0) + 1; + } + const folioDuplicates = Object.keys(folioCounts).filter(folio => folioCounts[folio] > 1); + const imageLinkDuplicates = Object.keys(imageLinkCounts).filter(imageLink => imageLinkCounts[imageLink] > 1); + const folioWImageDuplicates = []; + for (let i = 0; i < array.length; i++) { + if (imageLinkDuplicates.includes(array[i].imageLink)) { + folioWImageDuplicates.push(array[i].folio); + } + } + return [folioDuplicates.sort(), folioWImageDuplicates.sort()]; +} + +function displayCheckResults(checkName, failingFolios, error_message, success_message = '') { + const iconCell = document.getElementById(`${checkName}Icon`); + const instancesCell = document.getElementById(`${checkName}Instances`); + if (failingFolios.length === 0) { + iconCell.className = 'bi bi-check-circle-fill text-success'; + instancesCell.textContent = success_message; + } else { + iconCell.className = 'bi bi-exclamation-circle-fill text-warning'; + instancesCell.textContent = `${error_message}: ${failingFolios.join(', ')}`; + } +}; + +function csvLoadCallback(csv) { + // Callback function for when a CSV file is loaded. + // Parse the CSV file and display it in the table, + // then run checks for completeness and uniqueness. + const imgLinkInputs = getFormImgLinkInputs(document.getElementById('imgLinkForm')); + const sourceFolios = Object.keys(imgLinkInputs); + const parsedCSV = parseAndPreviewImageLinkCSV(csv, imgLinkInputs); + const [dupFolios, foliosWDupImageLinks] = getFoliosAtDuplicatedValues(parsedCSV); + // Display duplicated folios, if they exist. + displayCheckResults('folioDuplication', dupFolios, "The following folios are duplicated in the CSV"); + // Check whether there are any folios in the source that are not in the CSV + // Display these folios with missing Links in the preview table. + const csvFolios = parsedCSV.map(x => x.folio); + const missingLinks = sourceFolios.filter(folio => !csvFolios.includes(folio)); + displayCheckResults('folioCompleteness', missingLinks, "Image links missing for the following folios"); + // Check whether there are any folios in the CSV that are not in the source + // Display these folios as extra folios in the preview table. + const extraFolios = csvFolios.filter(folio => !sourceFolios.includes(folio)); + displayCheckResults('extraFolios', extraFolios.sort(), "The following folios do not exist in the source"); + // We expect one of two cases for the value of image links: + // 1. All image links are identical + // 2. All image links are unique + // Note that a mapping might be valid that does not conform to these cases + // (for example, if every image link shows two facing folios). In that case, the + // check will fail and we rely on the administrator to check the data. + if (foliosWDupImageLinks.length === csvFolios.length) { + displayCheckResults('imageLinkDuplication', [], "", "All image links identical"); + } else { + displayCheckResults('imageLinkDuplication', foliosWDupImageLinks, "Image links duplicated on the following subset of folios", "No image links duplicated"); + } + document.getElementById('csvTestingDiv').hidden = false; +} + +document.addEventListener('DOMContentLoaded', function () { + // Add listener to the file input field to parse and display the CSV file + document.getElementById('imgLinksCSV').addEventListener('change', function (e) { + const file = e.target.files[0]; + const reader = new FileReader(); + reader.onload = function (e) { + const csv = e.target.result; + csvLoadCallback(csv); + }; + reader.readAsText(file); + }); +} +);