diff --git a/.github/workflows/cd_prod.yml b/.github/workflows/cd_prod.yml new file mode 100644 index 00000000..f8244a87 --- /dev/null +++ b/.github/workflows/cd_prod.yml @@ -0,0 +1,45 @@ +name: Deploy to Production Server +on: workflow_dispatch + +concurrency: + group: production + cancel-in-progress: true + +jobs: + deploy-production: + runs-on: ubuntu-latest + steps: + - name: Checkout the current repo + uses: actions/checkout@v5 + - name: Install SSH Key + uses: shimataro/ssh-key-action@v2 + with: + key: ${{ secrets.KEY }} + known_hosts: ${{ secrets.KNOWN_HOSTS }} + - name: Stop the current Docker containers running + run: | + ssh -J ${{ secrets.PROXY_USERNAME }}@${{ secrets.PROXY_HOST1 }},${{ secrets.PROXY_USERNAME }}@${{ secrets.PROXY_HOST_PROD }} ${{ secrets.USERNAME }}@${{ secrets.HOST_PROD }} \ + "cd /virtual-instrument-museum && + sudo docker compose stop" + - name: Fetch new updates from remote `origin` and log current branches + run: | + ssh -J ${{ secrets.PROXY_USERNAME }}@${{ secrets.PROXY_HOST1 }},${{ secrets.PROXY_USERNAME }}@${{ secrets.PROXY_HOST_PROD }} ${{ secrets.USERNAME }}@${{ secrets.HOST_PROD }} \ + "cd /virtual-instrument-museum && + sudo git fetch origin -v && + sudo git branch -v" + - name: Checkout to `main` branch and pull new changes + run: | + ssh -J ${{ secrets.PROXY_USERNAME }}@${{ secrets.PROXY_HOST1 }},${{ secrets.PROXY_USERNAME }}@${{ secrets.PROXY_HOST_PROD }} ${{ secrets.USERNAME }}@${{ secrets.HOST_PROD }} \ + "cd /virtual-instrument-museum && + sudo git checkout main && + sudo git pull origin main" + - name: Build docker images + run: | + ssh -J ${{ secrets.PROXY_USERNAME }}@${{ secrets.PROXY_HOST1 }},${{ secrets.PROXY_USERNAME }}@${{ secrets.PROXY_HOST_PROD }} ${{ secrets.USERNAME }}@${{ secrets.HOST_PROD }} \ + "cd /virtual-instrument-museum && + sudo docker compose build --no-cache" + - name: Start the services + run: | + ssh -J ${{ secrets.PROXY_USERNAME }}@${{ secrets.PROXY_HOST1 }},${{ secrets.PROXY_USERNAME }}@${{ secrets.PROXY_HOST_PROD }} ${{ secrets.USERNAME }}@${{ secrets.HOST_PROD }} \ + "cd /virtual-instrument-museum && + sudo docker compose up -d" diff --git a/.github/workflows/cd_stage.yml b/.github/workflows/cd_stage.yml index 0c227550..bf374615 100644 --- a/.github/workflows/cd_stage.yml +++ b/.github/workflows/cd_stage.yml @@ -1,5 +1,5 @@ name: Deploy to staging Server -on: +on: push: branches: - "develop" @@ -21,16 +21,27 @@ jobs: known_hosts: ${{ secrets.KNOWN_HOSTS }} - name: Stop docker swarm and cleanup services run: | - ssh -J ${{ secrets.PROXY_USERNAME }}@${{ secrets.PROXY_HOST1 }},${{ secrets.PROXY_USERNAME }}@${{ secrets.PROXY_HOST2 }} ${{ secrets.USERNAME }}@${{ secrets.HOST }} "cd /virtual_instrument_museum && sudo docker compose stop" + ssh -J ${{ secrets.PROXY_USERNAME }}@${{ secrets.PROXY_HOST1 }},${{ secrets.PROXY_USERNAME }}@${{ secrets.PROXY_HOST2 }} ${{ secrets.USERNAME }}@${{ secrets.HOST }} \ + "cd /virtual_instrument_museum && + sudo docker compose stop" - name: Log git branch we're on run: | - ssh -J ${{ secrets.PROXY_USERNAME }}@${{ secrets.PROXY_HOST1 }},${{ secrets.PROXY_USERNAME }}@${{ secrets.PROXY_HOST2 }} ${{ secrets.USERNAME }}@${{ secrets.HOST }} "cd /virtual_instrument_museum && sudo git branch --all -v" + ssh -J ${{ secrets.PROXY_USERNAME }}@${{ secrets.PROXY_HOST1 }},${{ secrets.PROXY_USERNAME }}@${{ secrets.PROXY_HOST2 }} ${{ secrets.USERNAME }}@${{ secrets.HOST }} \ + "cd /virtual_instrument_museum && + sudo git branch --all -v" - name: Fetch on dev branch run: | - ssh -J ${{ secrets.PROXY_USERNAME }}@${{ secrets.PROXY_HOST1 }},${{ secrets.PROXY_USERNAME }}@${{ secrets.PROXY_HOST2 }} ${{ secrets.USERNAME }}@${{ secrets.HOST }} "cd /virtual_instrument_museum && sudo git fetch origin develop && sudo git merge origin/develop" + ssh -J ${{ secrets.PROXY_USERNAME }}@${{ secrets.PROXY_HOST1 }},${{ secrets.PROXY_USERNAME }}@${{ secrets.PROXY_HOST2 }} ${{ secrets.USERNAME }}@${{ secrets.HOST }} \ + "cd /virtual_instrument_museum && + sudo git fetch origin develop && + sudo git merge origin/develop" - name: Build docker images run: | - ssh -J ${{ secrets.PROXY_USERNAME }}@${{ secrets.PROXY_HOST1 }},${{ secrets.PROXY_USERNAME }}@${{ secrets.PROXY_HOST2 }} ${{ secrets.USERNAME }}@${{ secrets.HOST }} "cd /virtual_instrument_museum && sudo docker compose build" + ssh -J ${{ secrets.PROXY_USERNAME }}@${{ secrets.PROXY_HOST1 }},${{ secrets.PROXY_USERNAME }}@${{ secrets.PROXY_HOST2 }} ${{ secrets.USERNAME }}@${{ secrets.HOST }} \ + "cd /virtual_instrument_museum && + sudo docker compose build --no-cache" - name: Start the service run: | - ssh -J ${{ secrets.PROXY_USERNAME }}@${{ secrets.PROXY_HOST1 }},${{ secrets.PROXY_USERNAME }}@${{ secrets.PROXY_HOST2 }} ${{ secrets.USERNAME }}@${{ secrets.HOST }} "cd /virtual_instrument_museum && sudo docker compose up -d" \ No newline at end of file + ssh -J ${{ secrets.PROXY_USERNAME }}@${{ secrets.PROXY_HOST1 }},${{ secrets.PROXY_USERNAME }}@${{ secrets.PROXY_HOST2 }} ${{ secrets.USERNAME }}@${{ secrets.HOST }} \ + "cd /virtual_instrument_museum && + sudo docker compose up -d" diff --git a/.github/workflows/frontend_format.yml b/.github/workflows/frontend_format.yml index 36cc57a8..4cb7956d 100644 --- a/.github/workflows/frontend_format.yml +++ b/.github/workflows/frontend_format.yml @@ -20,7 +20,9 @@ jobs: run: poetry install --no-root - name: Run djlint via Poetry - run: poetry run djlint . --check --extension html --exclude "migrations/*" + run: | + cd web-app/django/VIM/templates/ + poetry run djlint . --check --extension html --exclude "migrations/*" - name: Set up Node.js uses: actions/setup-node@v4 diff --git a/prettierrc.yml b/prettierrc.yml index 2206ae1e..b90c68ca 100644 --- a/prettierrc.yml +++ b/prettierrc.yml @@ -10,3 +10,6 @@ overrides: - files: "*.js" options: trailingComma: "es5" + - files: "*.html" + options: + parser: django diff --git a/web-app/django/VIM/apps/instruments/admin.py b/web-app/django/VIM/apps/instruments/admin.py index ccf2f426..fa511e3c 100644 --- a/web-app/django/VIM/apps/instruments/admin.py +++ b/web-app/django/VIM/apps/instruments/admin.py @@ -2,6 +2,31 @@ from VIM.apps.instruments.models import Instrument, InstrumentName, Language, AVResource admin.site.register(Instrument) -admin.site.register(InstrumentName) admin.site.register(Language) admin.site.register(AVResource) + + +@admin.register(InstrumentName) +class InstrumentNameAdmin(admin.ModelAdmin): + list_filter = ("verification_status", "on_wikidata") # Filter by status + search_fields = ( + "name", + "source_name", + "instrument__wikidata_id", + ) # Search by name, source name, and instrument wikidata ID + + def get_readonly_fields(self, request, obj=None): + """ + Make all fields except 'verification_status' read-only for users in the 'reviewer' group. + """ + if request.user.groups.filter(name="reviewer").exists(): + return ( + "instrument", + "language", + "name", + "source_name", + "umil_label", + "contributor", + "on_wikidata", + ) + return super().get_readonly_fields(request, obj) diff --git a/web-app/django/VIM/apps/instruments/management/commands/import_instruments.py b/web-app/django/VIM/apps/instruments/management/commands/import_instruments.py index 9238ef7e..8379034a 100644 --- a/web-app/django/VIM/apps/instruments/management/commands/import_instruments.py +++ b/web-app/django/VIM/apps/instruments/management/commands/import_instruments.py @@ -4,6 +4,8 @@ import os from typing import Optional import requests +from django.conf import settings +from django.contrib.auth import get_user_model from django.core.management.base import BaseCommand from django.db import transaction from VIM.apps.instruments.models import Instrument, InstrumentName, Language, AVResource @@ -23,6 +25,12 @@ class Command(BaseCommand): def __init__(self): super().__init__() self.language_map: dict[str, Language] = {} + User = get_user_model() + self.default_contributor = User.objects.get(username=settings.DDMAL_USERNAME) + if not self.default_contributor: + raise ValueError( + f"Default contributor {settings.DDMAL_USERNAME} not found in the database." + ) def parse_instrument_data( self, instrument_id: str, instrument_data: dict @@ -48,6 +56,11 @@ def parse_instrument_data( ins_names: dict[str, str] = { value["language"]: value["value"] for key, value in ins_labels.items() } + ins_aliases: dict = instrument_data["aliases"] + ins_aliases_dict: dict[str, list[str]] = { + key: [alias["value"] for alias in value] + for key, value in ins_aliases.items() + } # Get Hornbostel-Sachs and MIMO classifications, if available ins_hbs: Optional[list[dict]] = instrument_data["claims"].get("P1762") ins_mimo: Optional[list[dict]] = instrument_data["claims"].get("P3763") @@ -64,6 +77,7 @@ def parse_instrument_data( "ins_names": ins_names, "hornbostel_sachs_class": hbs_class, "mimo_class": mimo_class, + "ins_aliases": ins_aliases_dict, } return parsed_data @@ -80,7 +94,7 @@ def get_instrument_data(self, instrument_ids: list[str]) -> list[dict]: ins_ids_str: str = "|".join(instrument_ids) url = ( "https://www.wikidata.org/w/api.php?action=wbgetentities&" - f"ids={ins_ids_str}&format=json&props=labels|descriptions|claims" + f"ids={ins_ids_str}&format=json&props=labels|descriptions|claims|aliases" ) response = requests.get(url, timeout=10) response_entities = response.json()["entities"] @@ -103,7 +117,10 @@ def create_database_objects( thumbnail_img_path [str]: Path to the thumbnail of the instrument image """ ins_names = instrument_attrs.pop("ins_names") + ins_aliases = instrument_attrs.pop("ins_aliases") instrument, _ = Instrument.objects.update_or_create(**instrument_attrs) + + # Create or update instrument labels in the database (umil_label=True) for lang, name in ins_names.items(): # Skip if the language code is not found in the database. # This commonly happens for codes like "mul" (multiple languages), @@ -121,7 +138,34 @@ def create_database_objects( language=self.language_map[lang], name=name, source_name="Wikidata", + umil_label=True, + contributor=self.default_contributor, + verification_status="verified", + on_wikidata=True, ) + + # Create or update instrument aliases in the database (umil_label=False) + for lang, aliases in ins_aliases.items(): + # Skip if the language code is not found in the database. + if lang not in self.language_map: + self.stdout.write( + self.style.WARNING( + f"Skipping language {lang} for instrument {instrument.wikidata_id} as the language is not found in the database." + ) + ) + continue + for alias in aliases: + InstrumentName.objects.update_or_create( + instrument=instrument, + language=self.language_map[lang], + name=alias, + source_name="Wikidata", + umil_label=False, + contributor=self.default_contributor, + verification_status="verified", + on_wikidata=True, + ) + img_obj = AVResource.objects.create( instrument=instrument, type="image", diff --git a/web-app/django/VIM/apps/instruments/migrations/0003_alter_avresource_instrument_and_more.py b/web-app/django/VIM/apps/instruments/migrations/0003_alter_avresource_instrument_and_more.py index 1bc3f850..990f8ab9 100644 --- a/web-app/django/VIM/apps/instruments/migrations/0003_alter_avresource_instrument_and_more.py +++ b/web-app/django/VIM/apps/instruments/migrations/0003_alter_avresource_instrument_and_more.py @@ -5,7 +5,6 @@ class Migration(migrations.Migration): - dependencies = [ ("instruments", "0002_instrument_thumbnail"), ] diff --git a/web-app/django/VIM/apps/instruments/migrations/0006_instrumentname_contributor_instrumentname_is_alias_and_more.py b/web-app/django/VIM/apps/instruments/migrations/0006_instrumentname_contributor_instrumentname_is_alias_and_more.py new file mode 100644 index 00000000..97291f98 --- /dev/null +++ b/web-app/django/VIM/apps/instruments/migrations/0006_instrumentname_contributor_instrumentname_is_alias_and_more.py @@ -0,0 +1,48 @@ +# Generated by Django 4.2.5 on 2025-05-30 15:29 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ("instruments", "0005_remove_language_wikidata_id"), + ] + + operations = [ + migrations.AddField( + model_name="instrumentname", + name="contributor", + field=models.ForeignKey( + blank=True, + help_text="User who contributed this name", + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to=settings.AUTH_USER_MODEL, + ), + ), + migrations.AddField( + model_name="instrumentname", + name="is_alias", + field=models.BooleanField( + default=False, + help_text="Is this an alias for the instrument? If true, it will not be used as the main name.", + ), + ), + migrations.AddField( + model_name="instrumentname", + name="status", + field=models.CharField( + choices=[ + ("verified", "Verified"), + ("unverified", "Unverified"), + ("uploaded", "Uploaded"), + ], + default="unverified", + help_text="Status of the name entry", + max_length=20, + ), + ), + ] diff --git a/web-app/django/VIM/apps/instruments/migrations/0007_instrumentname_is_approved_and_more.py b/web-app/django/VIM/apps/instruments/migrations/0007_instrumentname_is_approved_and_more.py new file mode 100644 index 00000000..fd71f005 --- /dev/null +++ b/web-app/django/VIM/apps/instruments/migrations/0007_instrumentname_is_approved_and_more.py @@ -0,0 +1,45 @@ +# Generated by Django 4.2.5 on 2025-06-16 13:54 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ( + "instruments", + "0006_instrumentname_contributor_instrumentname_is_alias_and_more", + ), + ] + + operations = [ + migrations.AddField( + model_name="instrumentname", + name="is_approved", + field=models.BooleanField( + default=False, + help_text="When a name is approved to be visible on UMIL and uploaded to Wikidata.", + ), + ), + migrations.AddField( + model_name="instrumentname", + name="on_wikidata", + field=models.BooleanField( + default=False, help_text="Is this name already on Wikidata?" + ), + ), + migrations.AlterField( + model_name="instrumentname", + name="status", + field=models.CharField( + choices=[ + ("verified", "Verified"), + ("unverified", "Unverified"), + ("needs_review", "Needs Review"), + ("rejected", "Rejected"), + ], + default="unverified", + help_text="Status of the name entry", + max_length=20, + ), + ), + ] diff --git a/web-app/django/VIM/apps/instruments/migrations/0008_rename_status_instrumentname_verification_status_and_more.py b/web-app/django/VIM/apps/instruments/migrations/0008_rename_status_instrumentname_verification_status_and_more.py new file mode 100644 index 00000000..476a3efe --- /dev/null +++ b/web-app/django/VIM/apps/instruments/migrations/0008_rename_status_instrumentname_verification_status_and_more.py @@ -0,0 +1,54 @@ +# Generated by Django 4.2.5 on 2025-08-07 17:10 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ("instruments", "0007_instrumentname_is_approved_and_more"), + ] + + operations = [ + migrations.RenameField( + model_name="instrumentname", + old_name="status", + new_name="verification_status", + ), + migrations.RemoveField( + model_name="instrumentname", + name="is_alias", + ), + migrations.RemoveField( + model_name="instrumentname", + name="is_approved", + ), + migrations.AddField( + model_name="instrumentname", + name="umil_label", + field=models.BooleanField( + default=False, + help_text="Is this the label for the instrument? If true, it will be used as the main name.", + ), + ), + migrations.AlterField( + model_name="instrumentname", + name="contributor", + field=models.ForeignKey( + help_text="User who contributed this name", + null=True, + on_delete=django.db.models.deletion.PROTECT, + to=settings.AUTH_USER_MODEL, + ), + ), + migrations.AddConstraint( + model_name="instrumentname", + constraint=models.UniqueConstraint( + condition=models.Q(("umil_label", True)), + fields=("instrument", "language"), + name="unique_umil_label_per_instrument_language", + ), + ), + ] diff --git a/web-app/django/VIM/apps/instruments/migrations/0009_alter_instrumentname_verification_status.py b/web-app/django/VIM/apps/instruments/migrations/0009_alter_instrumentname_verification_status.py new file mode 100644 index 00000000..c2051e52 --- /dev/null +++ b/web-app/django/VIM/apps/instruments/migrations/0009_alter_instrumentname_verification_status.py @@ -0,0 +1,31 @@ +# Generated by Django 4.2.5 on 2025-08-11 16:38 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ( + "instruments", + "0008_rename_status_instrumentname_verification_status_and_more", + ), + ] + + operations = [ + migrations.AlterField( + model_name="instrumentname", + name="verification_status", + field=models.CharField( + choices=[ + ("verified", "Verified"), + ("unverified", "Unverified"), + ("under_review", "Under Review"), + ("needs_additional_review", "Needs Additional Review"), + ("rejected", "Rejected"), + ], + default="unverified", + help_text="Status of the name entry", + max_length=50, + ), + ), + ] diff --git a/web-app/django/VIM/apps/instruments/models/instrument.py b/web-app/django/VIM/apps/instruments/models/instrument.py index da2777a3..bcabbfe9 100644 --- a/web-app/django/VIM/apps/instruments/models/instrument.py +++ b/web-app/django/VIM/apps/instruments/models/instrument.py @@ -25,3 +25,6 @@ class Instrument(models.Model): blank=True, help_text="Musical Instrument Museums Online classification", ) + + def __str__(self): + return f"{self.wikidata_id}" diff --git a/web-app/django/VIM/apps/instruments/models/instrument_name.py b/web-app/django/VIM/apps/instruments/models/instrument_name.py index b9d58c37..11b0bd48 100644 --- a/web-app/django/VIM/apps/instruments/models/instrument_name.py +++ b/web-app/django/VIM/apps/instruments/models/instrument_name.py @@ -8,3 +8,43 @@ class InstrumentName(models.Model): source_name = models.CharField( max_length=50, blank=False, help_text="Who or what called the instrument this?" ) # Stand-in for source data; format TBD + verification_status = models.CharField( + max_length=50, + choices=[ + ("verified", "Verified"), + ("unverified", "Unverified"), + ("under_review", "Under Review"), + ("needs_additional_review", "Needs Additional Review"), + ("rejected", "Rejected"), + ], + default="unverified", + help_text="Status of the name entry", + ) + umil_label = models.BooleanField( + default=False, + help_text="Is this the label for the instrument? If true, it will be used as the main name.", + ) + contributor = models.ForeignKey( + "auth.User", + null=True, + on_delete=models.PROTECT, + help_text="User who contributed this name", + ) + on_wikidata = models.BooleanField( + default=False, + help_text="Is this name already on Wikidata?", + ) + + # Custom validation to ensure at most one UMIL label per instrument language + class Meta: + constraints = [ + models.UniqueConstraint( + fields=["instrument", "language"], + condition=models.Q(umil_label=True), + name="unique_umil_label_per_instrument_language", + ) + ] + + # TODO: add verified_by field to track who verified the name + def __str__(self): + return f"{self.name} ({self.language.en_label}) - {self.instrument.wikidata_id}" diff --git a/web-app/django/VIM/apps/instruments/models/language.py b/web-app/django/VIM/apps/instruments/models/language.py index 656ca416..09f3f6c0 100644 --- a/web-app/django/VIM/apps/instruments/models/language.py +++ b/web-app/django/VIM/apps/instruments/models/language.py @@ -9,3 +9,6 @@ class Language(models.Model): autonym = models.CharField( blank=False, help_text="Language label in the language itself" ) + + def __str__(self): + return f"{self.en_label}" diff --git a/web-app/django/VIM/apps/instruments/views/instrument_detail.py b/web-app/django/VIM/apps/instruments/views/instrument_detail.py index 1aeb9ffa..edc674ba 100644 --- a/web-app/django/VIM/apps/instruments/views/instrument_detail.py +++ b/web-app/django/VIM/apps/instruments/views/instrument_detail.py @@ -1,4 +1,5 @@ from django.views.generic import DetailView +from django.db import models from VIM.apps.instruments.models import Instrument, Language @@ -15,9 +16,17 @@ def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) # Query the instrument names in all languages - context["instrument_names"] = ( - context["instrument"].instrumentname_set.all().select_related("language") + instrument_names = context["instrument"].instrumentname_set.select_related( + "language" ) + if self.request.user.is_authenticated: + # Show all names for authenticated users + context["instrument_names"] = instrument_names.all() + else: + # Show only verified names for unauthenticated users + context["instrument_names"] = instrument_names.filter( + verification_status="verified" + ) # Get the active language active_language_en = self.request.session.get("active_language_en", None) @@ -26,4 +35,19 @@ def get_context_data(self, **kwargs): if active_language_en else Language.objects.get(en_label="English") # default in English ) + + # Get the instrument label in the active language + # Set label to the first instrument name added in the language if there is no "umil_label" set + active_labels = context["instrument_names"].filter( + language=context["active_language"] + ) + umil_label = active_labels.filter(umil_label=True).first() + if umil_label: + context["active_instrument_label"] = umil_label + else: + context["active_instrument_label"] = active_labels.first() + + # Get all languages for the dropdown + context["languages"] = Language.objects.all() + return context diff --git a/web-app/django/VIM/apps/instruments/views/instrument_list.py b/web-app/django/VIM/apps/instruments/views/instrument_list.py index fd6e1e29..08829955 100644 --- a/web-app/django/VIM/apps/instruments/views/instrument_list.py +++ b/web-app/django/VIM/apps/instruments/views/instrument_list.py @@ -1,13 +1,33 @@ from typing import Union +import logging import pysolr import requests from django.conf import settings +from django.core.paginator import Paginator, Page from django.db.models import Prefetch, QuerySet from django.views.generic import ListView from VIM.apps.instruments.models import Instrument, InstrumentName, Language +logger = logging.getLogger(__name__) + +# Constants +SOLR_SEARCH_MARKER = "SOLR_SEARCH" + + +# Custom paginator for Solr search results +class SolrPaginator(Paginator): + """Custom paginator that knows the total count of Solr results.""" + + def __init__(self, object_list, per_page, total_count): + super().__init__(object_list, per_page) + self._count = total_count + + @property + def count(self): + return self._count + # Helper classes to normalize Solr results class SolrInstrument: @@ -73,13 +93,26 @@ def get_active_language_en_label(self) -> str: return language_en return self.request.session.get("active_language_en", "English") + def get_active_language(self) -> Language: + """ + Returns the active Language object. + + Returns: + Language: The active Language object + """ + language_en = self.get_active_language_en_label() + try: + return Language.objects.get(en_label=language_en) + except Language.DoesNotExist: + logger.error(f"Language not found: {language_en}") + raise + def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) context["active_tab"] = "instruments" context["instrument_num"] = context["paginator"].count context["languages"] = Language.objects.all().order_by("en_label") - active_language_en = self.get_active_language_en_label() - context["active_language"] = Language.objects.get(en_label=active_language_en) + context["active_language"] = self.get_active_language() hbs_facet = self.request.GET.get("hbs_facet", None) context["hbs_facet"] = hbs_facet @@ -107,11 +140,98 @@ def get_context_data(self, **kwargs): context["hbs_facet_name"] = next( (x["name"] for x in hbs_facet_list if x["value"] == hbs_facet), "" ) - search_query = self.request.GET.get("q", "").strip() + search_query = self.request.GET.get("query", "").strip() if search_query: context["search_query"] = search_query return context + def _get_solr_connection(self): + """Get a Solr connection with error handling.""" + try: + return pysolr.Solr(settings.SOLR_URL, timeout=10) + except Exception as e: + logger.error(f"Failed to connect to Solr: {e}") + raise + + def _get_solr_search_params(self, search_query: str, language: Language): + """Get common Solr search parameters.""" + lang_code = language.wikidata_code + name_field = f"instrument_name_{lang_code}_ss" + return { + "q": search_query, + "wt": "json", + "facet": "false", + "fl": f"sid, {name_field}, hornbostel_sachs_class_s, mimo_class_s, thumbnail_url", + "lang_code": lang_code, + } + + def _get_solr_total_count(self, solr, search_query: str): + """Get total count of Solr search results.""" + try: + count_params = { + "q": search_query, + "wt": "json", + "rows": 0, # We only want the count + "facet": "false", + } + count_response = solr.search(**count_params) + return count_response.hits + except Exception as e: + logger.error(f"Failed to get Solr count for query '{search_query}': {e}") + return 0 + + def _get_solr_page_results( + self, solr, search_params: dict, page_size: int, start: int + ): + """Get a specific page of Solr search results.""" + try: + solr_params = { + **search_params, + "rows": page_size, + "start": start, + } + # Remove our custom params + lang_code = solr_params.pop("lang_code") + + solr_response = solr.search(**solr_params) + return [ + SolrInstrument(doc, lang_code=lang_code) for doc in solr_response.docs + ] + except Exception as e: + logger.error(f"Failed to get Solr page results: {e}") + return [] + + def paginate_queryset(self, queryset, page_size): + """Custom pagination to handle Solr search results.""" + if queryset == SOLR_SEARCH_MARKER: + return self._paginate_solr_search(page_size) + + # Use default pagination for regular querysets + return super().paginate_queryset(queryset, page_size) + + def _paginate_solr_search(self, page_size): + """Handle Solr search pagination.""" + search_query = self.request.GET.get("query", "").strip() + language = self.get_active_language() + page_number = int(self.request.GET.get("page", 1)) + start = (page_number - 1) * page_size + + # Get Solr connection and search parameters + solr = self._get_solr_connection() + search_params = self._get_solr_search_params(search_query, language) + + # Get total count and page results + total_results = self._get_solr_total_count(solr, search_query) + page_results = self._get_solr_page_results( + solr, search_params, page_size, start + ) + + # Create paginator and page objects + paginator = SolrPaginator(page_results, page_size, total_results) + page = Page(page_results, page_number, paginator) + + return (paginator, page, page_results, page.has_other_pages()) + def get(self, request, *args, **kwargs): language_en = request.GET.get("language", None) if language_en: @@ -119,10 +239,10 @@ def get(self, request, *args, **kwargs): return super().get(request, *args, **kwargs) def get_queryset(self) -> Union[QuerySet[Instrument], list[SolrInstrument]]: - language_en = self.get_active_language_en_label() + language = self.get_active_language() instrumentname_prefetch_manager = Prefetch( "instrumentname_set", - queryset=InstrumentName.objects.filter(language__en_label=language_en), + queryset=InstrumentName.objects.filter(language=language), ) hbs_facet = self.request.GET.get("hbs_facet", None) if hbs_facet: @@ -132,25 +252,11 @@ def get_queryset(self) -> Union[QuerySet[Instrument], list[SolrInstrument]]: .prefetch_related(instrumentname_prefetch_manager) ) - search_query = self.request.GET.get("q", "").strip() + search_query = self.request.GET.get("query", "").strip() if search_query: - solr = pysolr.Solr(settings.SOLR_URL, timeout=10) - lang_code = Language.objects.get(en_label=language_en).wikidata_code - name_field = f"instrument_name_{lang_code}_ss" - solr_params = { - "q": search_query, - "wt": "json", - "rows": 100, - "facet": "false", - "fl": f"sid, {name_field}, hornbostel_sachs_class_s, mimo_class_s, thumbnail_url", - } - - # Send query to Solr and retrieve results - solr_response = solr.search(**solr_params) - return [ - SolrInstrument(doc, lang_code=lang_code) for doc in solr_response.docs - ] + # Return a special marker for Solr search that will be handled in paginate_queryset + return SOLR_SEARCH_MARKER return Instrument.objects.select_related("thumbnail").prefetch_related( instrumentname_prefetch_manager diff --git a/web-app/django/VIM/apps/instruments/views/update_umil_db.py b/web-app/django/VIM/apps/instruments/views/update_umil_db.py new file mode 100644 index 00000000..1a2ed493 --- /dev/null +++ b/web-app/django/VIM/apps/instruments/views/update_umil_db.py @@ -0,0 +1,161 @@ +"""Django view to handle user input to UMIl database""" + +import json +from django.contrib.auth.decorators import login_required +from django.views.decorators.http import require_http_methods +from django.http import HttpRequest, JsonResponse +from django.shortcuts import get_object_or_404 +from VIM.apps.instruments.models import Instrument, Language, InstrumentName +from typing import Any, Dict, List + + +def add_name(request: HttpRequest) -> JsonResponse: + """ + View to add new instrument names to UMIL database. + + This view expects a POST request with the JSON body: + { + "wikidata_id": "Q12345", + "entries": [ + { + "language": "en", + "name": "English label", + "source": "Source name", + }, + { + "language": "fr", + "name": "French label", + ... + } + ], + } + + Returns: + JsonResponse: JSON response with status and message + """ + # Parse the JSON request body, if it fails return missing data error + data: Dict[str, Any] = json.loads(request.body) + wikidata_id: str = data.get("wikidata_id") + entries: List[Dict[str, str]] = data.get("entries", []) + if not wikidata_id or not entries: + return JsonResponse( + { + "status": "error", + "message": "Missing required data", + }, + status=400, + ) + # Fetch the instrument from the database, if it does not exist return does not exist error + instrument = get_object_or_404(Instrument, wikidata_id=wikidata_id) + + # create dictionary to map language codes to Language objects + language = {lang.wikidata_code: lang for lang in Language.objects.all()} + + # considering entries with multiple of the same language, create a dictionary to track if a label has + # been assigned to a previous entry + entry_labels = {entry["language"]: False for entry in entries} + + instrument_names_to_create = [] + + for entry in entries: + language_code: str = entry["language"] + name: str = entry["name"] + source: str = entry["source"] + + # Validate that entry info is provided + if not name or not source or not language_code: + return JsonResponse( + { + "status": "error", + "message": "Missing entry information", + }, + status=400, + ) + + # Find language object from language code dictionary + language_obj: Language = language.get(language_code) + + # Within the entries, check if the language already has a name + # if it does, set umil_label to False + # otherwise, check against the UMILdb + if entry_labels[language_code]: + umil_label = False + else: + umil_label: bool = not ( + instrument.instrumentname_set.filter( + language__wikidata_code=language_code + ).exists() + ) + entry_labels[language_code] = True # Mark that this language now has a name + + # Prepare the InstrumentName object + instrument_names_to_create.append( + InstrumentName( + instrument=instrument, + language=language_obj, + name=name, + source_name=source, + umil_label=umil_label, + contributor=request.user, + ) + ) + + # Bulk create all InstrumentName objects + InstrumentName.objects.bulk_create(instrument_names_to_create) + + return JsonResponse( + { + "status": "success", + "message": "All entries saved successfully", + }, + status=200, + ) + + +def delete_name(request: HttpRequest) -> JsonResponse: + """View to delete an instrument name from UMIL database.""" + + # Parse the JSON request body + data: Dict[str, Any] = json.loads(request.body) + name_id: str = data.get("instrument_name_id") + + # Check if name_id is provided, if not return 400 error + if not name_id: + return JsonResponse( + { + "status": "error", + "message": "Missing required data", + }, + status=400, + ) + + instrument_name = get_object_or_404(InstrumentName, id=name_id) + + # If user is a superuser or created the name, allow deletion + if request.user.is_superuser or instrument_name.contributor == request.user: + instrument_name.delete() + return JsonResponse( + { + "status": "success", + "message": "Instrument name deleted successfully", + }, + status=200, + ) + else: + return JsonResponse( + { + "status": "error", + "message": "You are not allowed to delete this name", + }, + status=403, + ) + + +@login_required +@require_http_methods(["POST", "DELETE"]) +def update_umil_db(request: HttpRequest, pk: int) -> JsonResponse: + if request.method == "POST": + return add_name(request) + + elif request.method == "DELETE": + return delete_name(request) diff --git a/web-app/django/VIM/settings.py b/web-app/django/VIM/settings.py index 4229007b..85280322 100644 --- a/web-app/django/VIM/settings.py +++ b/web-app/django/VIM/settings.py @@ -175,3 +175,6 @@ # SOLR SETTINGS SOLR_URL = "http://solr:8983/solr/virtual-instrument-museum" + +# DDMAL USER ACCOUNT FOR UMIL +DDMAL_USERNAME = "ddmal" diff --git a/web-app/django/VIM/templates/base.html b/web-app/django/VIM/templates/base.html index 642ef72d..c529f1ff 100644 --- a/web-app/django/VIM/templates/base.html +++ b/web-app/django/VIM/templates/base.html @@ -15,15 +15,14 @@ - - - + {% vite_asset 'src/main.ts' %} {% block ts_files %} @@ -34,6 +33,13 @@ + + +
{% endfor %} +
+ {% if user.is_authenticated %} + + {% else %} + Add Instrument Name + {% endif %} +
@@ -173,6 +250,10 @@

View all languages

+ {% include "instruments/includes/deleteName.html" %} + {% include "instruments/includes/addName.html" %} + {% include "instruments/includes/statusInfo.html" %} + diff --git a/web-app/django/VIM/templates/instruments/includes/addName.html b/web-app/django/VIM/templates/instruments/includes/addName.html new file mode 100644 index 00000000..85562b28 --- /dev/null +++ b/web-app/django/VIM/templates/instruments/includes/addName.html @@ -0,0 +1,87 @@ +{% load static %} + + + + + + diff --git a/web-app/django/VIM/templates/instruments/includes/deleteName.html b/web-app/django/VIM/templates/instruments/includes/deleteName.html new file mode 100644 index 00000000..1bdc2b61 --- /dev/null +++ b/web-app/django/VIM/templates/instruments/includes/deleteName.html @@ -0,0 +1,65 @@ +{% load static %} + + + diff --git a/web-app/django/VIM/templates/instruments/includes/paginationOptions.html b/web-app/django/VIM/templates/instruments/includes/paginationOptions.html index 70f30fd2..c4ace5aa 100644 --- a/web-app/django/VIM/templates/instruments/includes/paginationOptions.html +++ b/web-app/django/VIM/templates/instruments/includes/paginationOptions.html @@ -24,7 +24,7 @@ {% if page_obj.has_previous %}
  • @@ -32,7 +32,7 @@
  • @@ -50,7 +50,7 @@ {% if page_obj.has_next %}
  • @@ -58,7 +58,7 @@
  • diff --git a/web-app/django/VIM/templates/instruments/includes/statusInfo.html b/web-app/django/VIM/templates/instruments/includes/statusInfo.html new file mode 100644 index 00000000..ba1b96dc --- /dev/null +++ b/web-app/django/VIM/templates/instruments/includes/statusInfo.html @@ -0,0 +1,53 @@ + diff --git a/web-app/django/VIM/templates/instruments/index.html b/web-app/django/VIM/templates/instruments/index.html index 15853087..98e1cb78 100644 --- a/web-app/django/VIM/templates/instruments/index.html +++ b/web-app/django/VIM/templates/instruments/index.html @@ -80,7 +80,7 @@

    ######

    -
    +