From 4ea8fe3f7227c3acce356718d6defbb8807b7f7d Mon Sep 17 00:00:00 2001 From: Sean Kelly Date: Wed, 15 Jan 2025 16:48:06 -0600 Subject: [PATCH] Support #385 --- .../theme/templates/wagtailadmin/base.html | 2 +- .../theme/templates/wagtailadmin/home.html | 1 - .../theme/templates/wagtailadmin/login.html | 2 +- .../management/commands/edrn_new_explorers.py | 73 ++++++ .../migrations/0035_alter_flexpage_body.py | 160 +++++++++++++ .../migrations/0036_alter_flexpage_body.py | 169 +++++++++++++ .../migrations/0037_alter_flexpage_body.py | 178 ++++++++++++++ .../migrations/0038_alter_flexpage_body.py | 187 +++++++++++++++ .../migrations/0039_alter_flexpage_body.py | 187 +++++++++++++++ .../migrations/0040_alter_flexpage_body.py | 178 ++++++++++++++ .../src/edrnsite/content/models.py | 1 + .../templates/edrnsite.content/cde-node.html | 1 - .../commands/edrn_update_data_models.py | 25 ++ .../src/edrnsite/policy/urls.py | 3 +- src/edrnsite.streams/setup.cfg | 1 + .../src/edrnsite/streams/_cde_blocks.py | 23 ++ .../src/edrnsite/streams/_data_elements.py | 224 ++++++++++++++++++ .../src/edrnsite/streams/blocks.py | 1 + .../streams/migrations/0001_initial.py | 161 +++++++++++++ ...lue_dataelementexplorerpermissiblevalue.py | 17 ++ .../src/edrnsite/streams/models.py | 11 + .../data-element-explorer-block.html | 68 ++++++ .../edrnsite.streams/de-attribute-button.html | 15 ++ .../edrnsite.streams/de-attribute-canvas.html | 63 +++++ .../de-attribute-canvases.html | 8 + .../templates/edrnsite.streams/de-node.html | 41 ++++ .../edrnsite.streams/de-root-canvases.html | 6 + .../templates/edrnsite.streams/de-roots.html | 7 + .../templatetags/edrnsite_streams_tags.py | 56 ++++- .../src/edrnsite/streams/urls.py | 12 + .../src/edrnsite/streams/views.py | 51 ++++ .../src/edrnsite/streams/wagtail_hooks.py | 39 +++ .../eke.knowledge/ingest-controls.html | 11 +- support/cbiit-deploy-prod.sh | 3 +- support/cbiit-deploy.sh | 3 +- support/devrebuild.sh | 1 + 36 files changed, 1977 insertions(+), 12 deletions(-) create mode 100644 src/edrnsite.content/src/edrnsite/content/management/commands/edrn_new_explorers.py create mode 100644 src/edrnsite.content/src/edrnsite/content/migrations/0035_alter_flexpage_body.py create mode 100644 src/edrnsite.content/src/edrnsite/content/migrations/0036_alter_flexpage_body.py create mode 100644 src/edrnsite.content/src/edrnsite/content/migrations/0037_alter_flexpage_body.py create mode 100644 src/edrnsite.content/src/edrnsite/content/migrations/0038_alter_flexpage_body.py create mode 100644 src/edrnsite.content/src/edrnsite/content/migrations/0039_alter_flexpage_body.py create mode 100644 src/edrnsite.content/src/edrnsite/content/migrations/0040_alter_flexpage_body.py create mode 100644 src/edrnsite.policy/src/edrnsite/policy/management/commands/edrn_update_data_models.py create mode 100644 src/edrnsite.streams/src/edrnsite/streams/_cde_blocks.py create mode 100644 src/edrnsite.streams/src/edrnsite/streams/_data_elements.py create mode 100644 src/edrnsite.streams/src/edrnsite/streams/migrations/0001_initial.py create mode 100644 src/edrnsite.streams/src/edrnsite/streams/migrations/0002_rename_cdepermissiblevalue_dataelementexplorerpermissiblevalue.py create mode 100644 src/edrnsite.streams/src/edrnsite/streams/models.py create mode 100644 src/edrnsite.streams/src/edrnsite/streams/templates/edrnsite.streams/data-element-explorer-block.html create mode 100644 src/edrnsite.streams/src/edrnsite/streams/templates/edrnsite.streams/de-attribute-button.html create mode 100644 src/edrnsite.streams/src/edrnsite/streams/templates/edrnsite.streams/de-attribute-canvas.html create mode 100644 src/edrnsite.streams/src/edrnsite/streams/templates/edrnsite.streams/de-attribute-canvases.html create mode 100644 src/edrnsite.streams/src/edrnsite/streams/templates/edrnsite.streams/de-node.html create mode 100644 src/edrnsite.streams/src/edrnsite/streams/templates/edrnsite.streams/de-root-canvases.html create mode 100644 src/edrnsite.streams/src/edrnsite/streams/templates/edrnsite.streams/de-roots.html create mode 100644 src/edrnsite.streams/src/edrnsite/streams/urls.py create mode 100644 src/edrnsite.streams/src/edrnsite/streams/views.py create mode 100644 src/edrnsite.streams/src/edrnsite/streams/wagtail_hooks.py diff --git a/src/edrn.theme/src/edrn/theme/templates/wagtailadmin/base.html b/src/edrn.theme/src/edrn/theme/templates/wagtailadmin/base.html index 3377ed20..65251fd7 100644 --- a/src/edrn.theme/src/edrn/theme/templates/wagtailadmin/base.html +++ b/src/edrn.theme/src/edrn/theme/templates/wagtailadmin/base.html @@ -3,4 +3,4 @@ {% block branding_logo %} Logo of the National Institutes of Health {% endblock %} -{# -*- Django HTML -*- #} +{# -*- HTML (Jinja) -*- #} diff --git a/src/edrn.theme/src/edrn/theme/templates/wagtailadmin/home.html b/src/edrn.theme/src/edrn/theme/templates/wagtailadmin/home.html index a9995d59..1610e835 100644 --- a/src/edrn.theme/src/edrn/theme/templates/wagtailadmin/home.html +++ b/src/edrn.theme/src/edrn/theme/templates/wagtailadmin/home.html @@ -3,4 +3,3 @@ {% block branding_welcome %} Editor's Controls for {{site_name}} {% endblock %} -{# -*- Django HTML -*- #} diff --git a/src/edrn.theme/src/edrn/theme/templates/wagtailadmin/login.html b/src/edrn.theme/src/edrn/theme/templates/wagtailadmin/login.html index 7c669498..619cc2a1 100644 --- a/src/edrn.theme/src/edrn/theme/templates/wagtailadmin/login.html +++ b/src/edrn.theme/src/edrn/theme/templates/wagtailadmin/login.html @@ -20,4 +20,4 @@ {% block branding_logo %} {% endblock %} -{# -*- Django HTML -*- #} +{# -*- HTML (Jinja) -*- #} diff --git a/src/edrnsite.content/src/edrnsite/content/management/commands/edrn_new_explorers.py b/src/edrnsite.content/src/edrnsite/content/management/commands/edrn_new_explorers.py new file mode 100644 index 00000000..743d5b4c --- /dev/null +++ b/src/edrnsite.content/src/edrnsite/content/management/commands/edrn_new_explorers.py @@ -0,0 +1,73 @@ +# encoding: utf-8 + +'''😌 EDRN Site Content: remove old explorer, create new one.''' + +from django.conf import settings +from django.core.management.base import BaseCommand +from edrnsite.content.models import FlexPage +from edrnsite.policy.management.commands.utils import set_site +from edrnsite.streams.views import update_data_element_explorer_trees +from wagtail.models import Page +from wagtail.rich_text import RichText + + +_help_html = '''

To use the data models:

+''' + +_faq_html = '''

For questions about the data models, email +the Informatics Center.''' + + +class Command(BaseCommand): + help = 'Replace old CDE viewer page with new block-based explorer flex page' + + def _update_cde_explorer(self, home_page): + # There should be just one current CDEExplorerPage + count = Page.objects.filter(slug='edrn-data-model').count() + if count > 1: + raise ValueError("There's more than one edrn-data-model! Not sure what to do") + elif count == 0: + self.stdout.write('No edrn-data-model found, so using existing "cde" page as the parent') + parent = FlexPage.objects.filter(slug='cde').first() + assert parent is not None + else: + self.stdout.write('Found the one edrn-data-model, deleting it') + page = Page.objects.filter(slug='edrn-data-model').first() + parent = page.get_parent() + page.delete() + parent.refresh_from_db() + + self.stdout.write('Creating new EDRN Data Model page') + page = FlexPage(title='EDRN Data Model', slug='edrn-data-model', show_in_menus=False) + parent.add_child(instance=page) + + page.body.append(('rich_text', RichText(_help_html))) + page.body.append(('data_explorer', { + 'title': 'Biomarker Data Models', + 'block_id': 'bio', 'spreadsheet_id': '1Kjkvi-bF5GNpAzvq4Kq6r4g1WpceIyP0Srs2OHJJtGs', + })) + page.body.append(('data_explorer', { + 'title': 'Cancer Biomarker Data Commons (LabCAS) Data Model', + 'block_id': 'lab', 'spreadsheet_id': '1btbwoROmVbZlzSLBn3DQ_6rakZ48j24p-NoOkI3OCFg' + })) + page.body.append(('rich_text', RichText(_faq_html))) + page.save() + update_data_element_explorer_trees() + + def handle(self, *args, **options): + self.stdout.write('Updating CDE explorer') + + old = getattr(settings, 'WAGTAILREDIRECTS_AUTO_CREATE', True) + try: + settings.WAGTAILREDIRECTS_AUTO_CREATE = False + settings.WAGTAILSEARCH_BACKENDS['default']['AUTO_UPDATE'] = False + site, home_page = set_site() + self._update_cde_explorer(home_page) + + finally: + settings.WAGTAILREDIRECTS_AUTO_CREATE = old + settings.WAGTAILSEARCH_BACKENDS['default']['AUTO_UPDATE'] = True + self.stdout.write("Job's done!") diff --git a/src/edrnsite.content/src/edrnsite/content/migrations/0035_alter_flexpage_body.py b/src/edrnsite.content/src/edrnsite/content/migrations/0035_alter_flexpage_body.py new file mode 100644 index 00000000..4c98da5a --- /dev/null +++ b/src/edrnsite.content/src/edrnsite/content/migrations/0035_alter_flexpage_body.py @@ -0,0 +1,160 @@ +# Generated by Django 4.2.17 on 2025-01-10 18:20 + +from django.db import migrations +import edrnsite.streams.blocks +import wagtail.blocks +import wagtail.contrib.typed_table_block.blocks +import wagtail.fields +import wagtail.images.blocks + + +class Migration(migrations.Migration): + + dependencies = [ + ("edrnsitecontent", "0034_biomarkersubmissionformpage"), + ] + + operations = [ + migrations.AlterField( + model_name="flexpage", + name="body", + field=wagtail.fields.StreamField( + [ + ( + "rich_text", + wagtail.blocks.RichTextBlock( + help_text="Richly formatted text", + icon="doc-full", + label="Rich Text", + ), + ), + ( + "cards", + wagtail.blocks.StructBlock( + [ + ( + "cards", + wagtail.blocks.ListBlock( + wagtail.blocks.StructBlock( + [ + ( + "title", + wagtail.blocks.CharBlock( + help_text="Title of this card, max 100 chars", + max_length=100, + ), + ) + ] + ) + ), + ) + ] + ), + ), + ("table", edrnsite.streams.blocks.TableBlock()), + ( + "data_explorer", + wagtail.blocks.StructBlock( + [ + ( + "attribute_help_text", + wagtail.blocks.CharBlock( + default="Click/tap on an attribute box below to view details about each attribute", + help_text="Helpful text to tell users that attribute buttons can be clicked", + max_length=300, + required=False, + ), + ) + ] + ), + ), + ( + "block_quote", + edrnsite.streams.blocks.BlockQuoteBlock( + help_text="Block quote" + ), + ), + ( + "typed_table", + wagtail.contrib.typed_table_block.blocks.TypedTableBlock( + [ + ( + "text", + wagtail.blocks.CharBlock( + help_text="Plain text cell" + ), + ), + ( + "rich_text", + wagtail.blocks.RichTextBlock( + help_text="Rich text cell" + ), + ), + ( + "numeric", + wagtail.blocks.FloatBlock(help_text="Numeric cell"), + ), + ( + "integer", + wagtail.blocks.IntegerBlock( + help_text="Integer cell" + ), + ), + ( + "page", + wagtail.blocks.PageChooserBlock( + help_text="Page within the site" + ), + ), + ] + ), + ), + ( + "carousel", + wagtail.blocks.StructBlock( + [ + ( + "media", + wagtail.blocks.ListBlock( + wagtail.blocks.StructBlock( + [ + ( + "image", + wagtail.images.blocks.ImageChooserBlock(), + ), + ( + "label", + wagtail.blocks.CharBlock( + help_text="Overlaid label, if any", + max_length=120, + required=False, + ), + ), + ( + "caption", + wagtail.blocks.CharBlock( + help_text="Overlaid caption, if any", + max_length=400, + required=False, + ), + ), + ] + ) + ), + ) + ] + ), + ), + ( + "raw_html", + wagtail.blocks.RawHTMLBlock( + help_text="Raw HTML (use with care)" + ), + ), + ], + blank=True, + null=True, + use_json_field=True, + ), + ), + ] diff --git a/src/edrnsite.content/src/edrnsite/content/migrations/0036_alter_flexpage_body.py b/src/edrnsite.content/src/edrnsite/content/migrations/0036_alter_flexpage_body.py new file mode 100644 index 00000000..135f948a --- /dev/null +++ b/src/edrnsite.content/src/edrnsite/content/migrations/0036_alter_flexpage_body.py @@ -0,0 +1,169 @@ +# Generated by Django 4.2.17 on 2025-01-10 19:06 + +from django.db import migrations +import edrnsite.streams.blocks +import wagtail.blocks +import wagtail.contrib.typed_table_block.blocks +import wagtail.fields +import wagtail.images.blocks + + +class Migration(migrations.Migration): + + dependencies = [ + ("edrnsitecontent", "0035_alter_flexpage_body"), + ] + + operations = [ + migrations.AlterField( + model_name="flexpage", + name="body", + field=wagtail.fields.StreamField( + [ + ( + "rich_text", + wagtail.blocks.RichTextBlock( + help_text="Richly formatted text", + icon="doc-full", + label="Rich Text", + ), + ), + ( + "cards", + wagtail.blocks.StructBlock( + [ + ( + "cards", + wagtail.blocks.ListBlock( + wagtail.blocks.StructBlock( + [ + ( + "title", + wagtail.blocks.CharBlock( + help_text="Title of this card, max 100 chars", + max_length=100, + ), + ) + ] + ) + ), + ) + ] + ), + ), + ("table", edrnsite.streams.blocks.TableBlock()), + ( + "data_explorer", + wagtail.blocks.StructBlock( + [ + ( + "spreadsheet_id", + wagtail.blocks.CharBlock( + blank=False, + help_text="File ID of a Google Drive spreadsheet", + max_length=250, + null=False, + ), + ), + ( + "attribute_help_text", + wagtail.blocks.CharBlock( + default="Click/tap on an attribute box below to view details about each attribute", + help_text="Helpful text to tell users that attribute buttons can be clicked", + max_length=300, + required=False, + ), + ), + ] + ), + ), + ( + "block_quote", + edrnsite.streams.blocks.BlockQuoteBlock( + help_text="Block quote" + ), + ), + ( + "typed_table", + wagtail.contrib.typed_table_block.blocks.TypedTableBlock( + [ + ( + "text", + wagtail.blocks.CharBlock( + help_text="Plain text cell" + ), + ), + ( + "rich_text", + wagtail.blocks.RichTextBlock( + help_text="Rich text cell" + ), + ), + ( + "numeric", + wagtail.blocks.FloatBlock(help_text="Numeric cell"), + ), + ( + "integer", + wagtail.blocks.IntegerBlock( + help_text="Integer cell" + ), + ), + ( + "page", + wagtail.blocks.PageChooserBlock( + help_text="Page within the site" + ), + ), + ] + ), + ), + ( + "carousel", + wagtail.blocks.StructBlock( + [ + ( + "media", + wagtail.blocks.ListBlock( + wagtail.blocks.StructBlock( + [ + ( + "image", + wagtail.images.blocks.ImageChooserBlock(), + ), + ( + "label", + wagtail.blocks.CharBlock( + help_text="Overlaid label, if any", + max_length=120, + required=False, + ), + ), + ( + "caption", + wagtail.blocks.CharBlock( + help_text="Overlaid caption, if any", + max_length=400, + required=False, + ), + ), + ] + ) + ), + ) + ] + ), + ), + ( + "raw_html", + wagtail.blocks.RawHTMLBlock( + help_text="Raw HTML (use with care)" + ), + ), + ], + blank=True, + null=True, + use_json_field=True, + ), + ), + ] diff --git a/src/edrnsite.content/src/edrnsite/content/migrations/0037_alter_flexpage_body.py b/src/edrnsite.content/src/edrnsite/content/migrations/0037_alter_flexpage_body.py new file mode 100644 index 00000000..8432e446 --- /dev/null +++ b/src/edrnsite.content/src/edrnsite/content/migrations/0037_alter_flexpage_body.py @@ -0,0 +1,178 @@ +# Generated by Django 4.2.17 on 2025-01-10 22:37 + +from django.db import migrations +import edrnsite.streams.blocks +import wagtail.blocks +import wagtail.contrib.typed_table_block.blocks +import wagtail.fields +import wagtail.images.blocks + + +class Migration(migrations.Migration): + + dependencies = [ + ("edrnsitecontent", "0036_alter_flexpage_body"), + ] + + operations = [ + migrations.AlterField( + model_name="flexpage", + name="body", + field=wagtail.fields.StreamField( + [ + ( + "rich_text", + wagtail.blocks.RichTextBlock( + help_text="Richly formatted text", + icon="doc-full", + label="Rich Text", + ), + ), + ( + "cards", + wagtail.blocks.StructBlock( + [ + ( + "cards", + wagtail.blocks.ListBlock( + wagtail.blocks.StructBlock( + [ + ( + "title", + wagtail.blocks.CharBlock( + help_text="Title of this card, max 100 chars", + max_length=100, + ), + ) + ] + ) + ), + ) + ] + ), + ), + ("table", edrnsite.streams.blocks.TableBlock()), + ( + "data_explorer", + wagtail.blocks.StructBlock( + [ + ( + "title", + wagtail.blocks.CharBlock( + blank=True, + help_text="Optional title to display for this explorer block", + max_length=80, + null=False, + ), + ), + ( + "spreadsheet_id", + wagtail.blocks.CharBlock( + blank=False, + help_text="File ID of a Google Drive spreadsheet", + max_length=60, + null=False, + ), + ), + ( + "attribute_help_text", + wagtail.blocks.CharBlock( + default="Click/tap on an attribute box below to view details about each attribute", + help_text="Helpful text to tell users that attribute buttons can be clicked", + max_length=300, + required=False, + ), + ), + ] + ), + ), + ( + "block_quote", + edrnsite.streams.blocks.BlockQuoteBlock( + help_text="Block quote" + ), + ), + ( + "typed_table", + wagtail.contrib.typed_table_block.blocks.TypedTableBlock( + [ + ( + "text", + wagtail.blocks.CharBlock( + help_text="Plain text cell" + ), + ), + ( + "rich_text", + wagtail.blocks.RichTextBlock( + help_text="Rich text cell" + ), + ), + ( + "numeric", + wagtail.blocks.FloatBlock(help_text="Numeric cell"), + ), + ( + "integer", + wagtail.blocks.IntegerBlock( + help_text="Integer cell" + ), + ), + ( + "page", + wagtail.blocks.PageChooserBlock( + help_text="Page within the site" + ), + ), + ] + ), + ), + ( + "carousel", + wagtail.blocks.StructBlock( + [ + ( + "media", + wagtail.blocks.ListBlock( + wagtail.blocks.StructBlock( + [ + ( + "image", + wagtail.images.blocks.ImageChooserBlock(), + ), + ( + "label", + wagtail.blocks.CharBlock( + help_text="Overlaid label, if any", + max_length=120, + required=False, + ), + ), + ( + "caption", + wagtail.blocks.CharBlock( + help_text="Overlaid caption, if any", + max_length=400, + required=False, + ), + ), + ] + ) + ), + ) + ] + ), + ), + ( + "raw_html", + wagtail.blocks.RawHTMLBlock( + help_text="Raw HTML (use with care)" + ), + ), + ], + blank=True, + null=True, + use_json_field=True, + ), + ), + ] diff --git a/src/edrnsite.content/src/edrnsite/content/migrations/0038_alter_flexpage_body.py b/src/edrnsite.content/src/edrnsite/content/migrations/0038_alter_flexpage_body.py new file mode 100644 index 00000000..61ca9b95 --- /dev/null +++ b/src/edrnsite.content/src/edrnsite/content/migrations/0038_alter_flexpage_body.py @@ -0,0 +1,187 @@ +# Generated by Django 4.2.17 on 2025-01-10 22:50 + +from django.db import migrations +import edrnsite.streams.blocks +import wagtail.blocks +import wagtail.contrib.typed_table_block.blocks +import wagtail.fields +import wagtail.images.blocks + + +class Migration(migrations.Migration): + + dependencies = [ + ("edrnsitecontent", "0037_alter_flexpage_body"), + ] + + operations = [ + migrations.AlterField( + model_name="flexpage", + name="body", + field=wagtail.fields.StreamField( + [ + ( + "rich_text", + wagtail.blocks.RichTextBlock( + help_text="Richly formatted text", + icon="doc-full", + label="Rich Text", + ), + ), + ( + "cards", + wagtail.blocks.StructBlock( + [ + ( + "cards", + wagtail.blocks.ListBlock( + wagtail.blocks.StructBlock( + [ + ( + "title", + wagtail.blocks.CharBlock( + help_text="Title of this card, max 100 chars", + max_length=100, + ), + ) + ] + ) + ), + ) + ] + ), + ), + ("table", edrnsite.streams.blocks.TableBlock()), + ( + "data_explorer", + wagtail.blocks.StructBlock( + [ + ( + "block_id", + wagtail.blocks.CharBlock( + blank=False, + help_text="Short alphanumeric ID for this block", + max_length=8, + null=False, + ), + ), + ( + "title", + wagtail.blocks.CharBlock( + blank=True, + help_text="Optional title to display for this explorer block", + max_length=80, + null=False, + ), + ), + ( + "spreadsheet_id", + wagtail.blocks.CharBlock( + blank=False, + help_text="File ID of a Google Drive spreadsheet", + max_length=60, + null=False, + ), + ), + ( + "attribute_help_text", + wagtail.blocks.CharBlock( + default="Click/tap on an attribute box below to view details about each attribute", + help_text="Helpful text to tell users that attribute buttons can be clicked", + max_length=300, + required=False, + ), + ), + ] + ), + ), + ( + "block_quote", + edrnsite.streams.blocks.BlockQuoteBlock( + help_text="Block quote" + ), + ), + ( + "typed_table", + wagtail.contrib.typed_table_block.blocks.TypedTableBlock( + [ + ( + "text", + wagtail.blocks.CharBlock( + help_text="Plain text cell" + ), + ), + ( + "rich_text", + wagtail.blocks.RichTextBlock( + help_text="Rich text cell" + ), + ), + ( + "numeric", + wagtail.blocks.FloatBlock(help_text="Numeric cell"), + ), + ( + "integer", + wagtail.blocks.IntegerBlock( + help_text="Integer cell" + ), + ), + ( + "page", + wagtail.blocks.PageChooserBlock( + help_text="Page within the site" + ), + ), + ] + ), + ), + ( + "carousel", + wagtail.blocks.StructBlock( + [ + ( + "media", + wagtail.blocks.ListBlock( + wagtail.blocks.StructBlock( + [ + ( + "image", + wagtail.images.blocks.ImageChooserBlock(), + ), + ( + "label", + wagtail.blocks.CharBlock( + help_text="Overlaid label, if any", + max_length=120, + required=False, + ), + ), + ( + "caption", + wagtail.blocks.CharBlock( + help_text="Overlaid caption, if any", + max_length=400, + required=False, + ), + ), + ] + ) + ), + ) + ] + ), + ), + ( + "raw_html", + wagtail.blocks.RawHTMLBlock( + help_text="Raw HTML (use with care)" + ), + ), + ], + blank=True, + null=True, + use_json_field=True, + ), + ), + ] diff --git a/src/edrnsite.content/src/edrnsite/content/migrations/0039_alter_flexpage_body.py b/src/edrnsite.content/src/edrnsite/content/migrations/0039_alter_flexpage_body.py new file mode 100644 index 00000000..c16cd004 --- /dev/null +++ b/src/edrnsite.content/src/edrnsite/content/migrations/0039_alter_flexpage_body.py @@ -0,0 +1,187 @@ +# Generated by Django 4.2.17 on 2025-01-13 20:31 + +from django.db import migrations +import edrnsite.streams.blocks +import wagtail.blocks +import wagtail.contrib.typed_table_block.blocks +import wagtail.fields +import wagtail.images.blocks + + +class Migration(migrations.Migration): + + dependencies = [ + ("edrnsitecontent", "0038_alter_flexpage_body"), + ] + + operations = [ + migrations.AlterField( + model_name="flexpage", + name="body", + field=wagtail.fields.StreamField( + [ + ( + "rich_text", + wagtail.blocks.RichTextBlock( + help_text="Richly formatted text", + icon="doc-full", + label="Rich Text", + ), + ), + ( + "cards", + wagtail.blocks.StructBlock( + [ + ( + "cards", + wagtail.blocks.ListBlock( + wagtail.blocks.StructBlock( + [ + ( + "title", + wagtail.blocks.CharBlock( + help_text="Title of this card, max 100 chars", + max_length=100, + ), + ) + ] + ) + ), + ) + ] + ), + ), + ("table", edrnsite.streams.blocks.TableBlock()), + ( + "data_explorer", + wagtail.blocks.StructBlock( + [ + ( + "block_id", + wagtail.blocks.CharBlock( + help_text="Short alphanumeric ID for this block", + max_length=8, + null=False, + required=True, + ), + ), + ( + "title", + wagtail.blocks.CharBlock( + help_text="Optional title to display for this explorer block", + max_length=80, + null=False, + required=False, + ), + ), + ( + "spreadsheet_id", + wagtail.blocks.CharBlock( + help_text="File ID of a Google Drive spreadsheet; don't forget to Update from Google Drive!", + max_length=60, + null=False, + required=False, + ), + ), + ( + "attribute_help_text", + wagtail.blocks.CharBlock( + default="Click/tap on an attribute box below to view details about each attribute", + help_text="Helpful text to tell users that attribute buttons can be clicked", + max_length=300, + required=False, + ), + ), + ] + ), + ), + ( + "block_quote", + edrnsite.streams.blocks.BlockQuoteBlock( + help_text="Block quote" + ), + ), + ( + "typed_table", + wagtail.contrib.typed_table_block.blocks.TypedTableBlock( + [ + ( + "text", + wagtail.blocks.CharBlock( + help_text="Plain text cell" + ), + ), + ( + "rich_text", + wagtail.blocks.RichTextBlock( + help_text="Rich text cell" + ), + ), + ( + "numeric", + wagtail.blocks.FloatBlock(help_text="Numeric cell"), + ), + ( + "integer", + wagtail.blocks.IntegerBlock( + help_text="Integer cell" + ), + ), + ( + "page", + wagtail.blocks.PageChooserBlock( + help_text="Page within the site" + ), + ), + ] + ), + ), + ( + "carousel", + wagtail.blocks.StructBlock( + [ + ( + "media", + wagtail.blocks.ListBlock( + wagtail.blocks.StructBlock( + [ + ( + "image", + wagtail.images.blocks.ImageChooserBlock(), + ), + ( + "label", + wagtail.blocks.CharBlock( + help_text="Overlaid label, if any", + max_length=120, + required=False, + ), + ), + ( + "caption", + wagtail.blocks.CharBlock( + help_text="Overlaid caption, if any", + max_length=400, + required=False, + ), + ), + ] + ) + ), + ) + ] + ), + ), + ( + "raw_html", + wagtail.blocks.RawHTMLBlock( + help_text="Raw HTML (use with care)" + ), + ), + ], + blank=True, + null=True, + use_json_field=True, + ), + ), + ] diff --git a/src/edrnsite.content/src/edrnsite/content/migrations/0040_alter_flexpage_body.py b/src/edrnsite.content/src/edrnsite/content/migrations/0040_alter_flexpage_body.py new file mode 100644 index 00000000..95cf6404 --- /dev/null +++ b/src/edrnsite.content/src/edrnsite/content/migrations/0040_alter_flexpage_body.py @@ -0,0 +1,178 @@ +# Generated by Django 4.2.17 on 2025-01-15 21:37 + +from django.db import migrations +import edrnsite.streams.blocks +import wagtail.blocks +import wagtail.contrib.typed_table_block.blocks +import wagtail.fields +import wagtail.images.blocks + + +class Migration(migrations.Migration): + + dependencies = [ + ("edrnsitecontent", "0039_alter_flexpage_body"), + ] + + operations = [ + migrations.AlterField( + model_name="flexpage", + name="body", + field=wagtail.fields.StreamField( + [ + ( + "rich_text", + wagtail.blocks.RichTextBlock( + help_text="Richly formatted text", + icon="doc-full", + label="Rich Text", + ), + ), + ( + "cards", + wagtail.blocks.StructBlock( + [ + ( + "cards", + wagtail.blocks.ListBlock( + wagtail.blocks.StructBlock( + [ + ( + "title", + wagtail.blocks.CharBlock( + help_text="Title of this card, max 100 chars", + max_length=100, + ), + ) + ] + ) + ), + ) + ] + ), + ), + ("table", edrnsite.streams.blocks.TableBlock()), + ( + "data_explorer", + wagtail.blocks.StructBlock( + [ + ( + "block_id", + wagtail.blocks.CharBlock( + help_text="Short alphanumeric ID for this block", + max_length=8, + null=False, + required=True, + ), + ), + ( + "title", + wagtail.blocks.CharBlock( + help_text="Optional title to display for this explorer block", + max_length=80, + null=False, + required=False, + ), + ), + ( + "spreadsheet_id", + wagtail.blocks.CharBlock( + help_text="File ID of a Google Drive spreadsheet; don't forget to Update from Google Drive!", + max_length=60, + null=False, + required=False, + ), + ), + ] + ), + ), + ( + "block_quote", + edrnsite.streams.blocks.BlockQuoteBlock( + help_text="Block quote" + ), + ), + ( + "typed_table", + wagtail.contrib.typed_table_block.blocks.TypedTableBlock( + [ + ( + "text", + wagtail.blocks.CharBlock( + help_text="Plain text cell" + ), + ), + ( + "rich_text", + wagtail.blocks.RichTextBlock( + help_text="Rich text cell" + ), + ), + ( + "numeric", + wagtail.blocks.FloatBlock(help_text="Numeric cell"), + ), + ( + "integer", + wagtail.blocks.IntegerBlock( + help_text="Integer cell" + ), + ), + ( + "page", + wagtail.blocks.PageChooserBlock( + help_text="Page within the site" + ), + ), + ] + ), + ), + ( + "carousel", + wagtail.blocks.StructBlock( + [ + ( + "media", + wagtail.blocks.ListBlock( + wagtail.blocks.StructBlock( + [ + ( + "image", + wagtail.images.blocks.ImageChooserBlock(), + ), + ( + "label", + wagtail.blocks.CharBlock( + help_text="Overlaid label, if any", + max_length=120, + required=False, + ), + ), + ( + "caption", + wagtail.blocks.CharBlock( + help_text="Overlaid caption, if any", + max_length=400, + required=False, + ), + ), + ] + ) + ), + ) + ] + ), + ), + ( + "raw_html", + wagtail.blocks.RawHTMLBlock( + help_text="Raw HTML (use with care)" + ), + ), + ], + blank=True, + null=True, + use_json_field=True, + ), + ), + ] diff --git a/src/edrnsite.content/src/edrnsite/content/models.py b/src/edrnsite.content/src/edrnsite/content/models.py index 8c1e404a..e3eb41a6 100644 --- a/src/edrnsite.content/src/edrnsite/content/models.py +++ b/src/edrnsite.content/src/edrnsite/content/models.py @@ -82,6 +82,7 @@ class FlexPage(MetadataPageMixin, Page): )), ('cards', blocks.CardsBlock()), ('table', blocks.TableBlock()), + ('data_explorer', blocks.DataElementExplorerBlock()), ('block_quote', blocks.BlockQuoteBlock(help_text='Block quote')), ('typed_table', blocks.TYPED_TABLE_BLOCK), ('carousel', blocks.CarouselBlock()), diff --git a/src/edrnsite.content/src/edrnsite/content/templates/edrnsite.content/cde-node.html b/src/edrnsite.content/src/edrnsite/content/templates/edrnsite.content/cde-node.html index 77ec1796..c03cd049 100644 --- a/src/edrnsite.content/src/edrnsite/content/templates/edrnsite.content/cde-node.html +++ b/src/edrnsite.content/src/edrnsite/content/templates/edrnsite.content/cde-node.html @@ -36,5 +36,4 @@

Attributes

{% endif %} - {# -*- Django HTML -*- #} diff --git a/src/edrnsite.policy/src/edrnsite/policy/management/commands/edrn_update_data_models.py b/src/edrnsite.policy/src/edrnsite/policy/management/commands/edrn_update_data_models.py new file mode 100644 index 00000000..0d87ebb9 --- /dev/null +++ b/src/edrnsite.policy/src/edrnsite/policy/management/commands/edrn_update_data_models.py @@ -0,0 +1,25 @@ +# encoding: utf-8 + +'''🧬 EDRN Site: update data models from Google Drive.''' + +from django.core.management.base import BaseCommand +from edrnsite.streams.views import update_data_element_explorer_trees +import logging + + +class Command(BaseCommand): + help = 'Updates all data models with contents from Google Drive' + + def handle(self, *args, **options): + verbosity = int(options['verbosity']) + root_logger = logging.getLogger('') + if verbosity >= 3: + root_logger.setLevel(logging.DEBUG) + + self.stdout.write('Updating data models') + results = update_data_element_explorer_trees() + if verbosity >= 3: + self.stdout.write('Results:') + for line in results: + self.stdout.write(line.strip()) + self.stdout.write(self.style.SUCCESS('🎉 Done!')) diff --git a/src/edrnsite.policy/src/edrnsite/policy/urls.py b/src/edrnsite.policy/src/edrnsite/policy/urls.py index 8613c3ff..0c3a0a7e 100644 --- a/src/edrnsite.policy/src/edrnsite/policy/urls.py +++ b/src/edrnsite.policy/src/edrnsite/policy/urls.py @@ -13,6 +13,7 @@ from edrn.metrics.urls import urlpatterns as edrn_metrics_urls from edrnsite.controls.urls import urlpatterns as edrnsite_controls_urlpatterns from edrnsite.search.urls import urlpatterns as edrnSiteSearchURLs +from edrnsite.streams.urls import urlpatterns as edrnsite_streams_urlpatterns from eke.knowledge.urls import urlpatterns as ekeKnowledgeURLs from wagtail import urls as wagtail_urls from wagtail.admin import urls as wagtailadmin_urls @@ -21,7 +22,7 @@ from wagtail_favicon.urls import urls as favicon_urls -urlpatterns = edrnsite_controls_urlpatterns + ekeKnowledgeURLs + edrnSiteSearchURLs + edrnAuthURLs + edrn_metrics_urls + [ +urlpatterns = edrnsite_streams_urlpatterns + edrnsite_controls_urlpatterns + ekeKnowledgeURLs + edrnSiteSearchURLs + edrnAuthURLs + edrn_metrics_urls + [ path('clear-caches', clear_caches, name='clear_caches'), path('django-admin/', admin.site.urls), path('admin/', include(wagtailadmin_urls)), diff --git a/src/edrnsite.streams/setup.cfg b/src/edrnsite.streams/setup.cfg index afd8e649..862ca807 100644 --- a/src/edrnsite.streams/setup.cfg +++ b/src/edrnsite.streams/setup.cfg @@ -19,6 +19,7 @@ python_requires = >= 3.9 install_requires = django < 5, > 4 wagtail < 6, > 5 + pandas == 1.5.3 # Must match py3-pands package in Dockerfile [options.packages.find] diff --git a/src/edrnsite.streams/src/edrnsite/streams/_cde_blocks.py b/src/edrnsite.streams/src/edrnsite/streams/_cde_blocks.py new file mode 100644 index 00000000..ac1001b9 --- /dev/null +++ b/src/edrnsite.streams/src/edrnsite/streams/_cde_blocks.py @@ -0,0 +1,23 @@ +# encoding: utf-8 + +'''🦦 EDRN Site streams: CDE blocks.''' + +from wagtail import blocks + + +class DataElementExplorerBlock(blocks.StructBlock): + block_id = blocks.CharBlock( + null=False, required=True, max_length=8, help_text='Short alphanumeric ID for this block' + ) + title = blocks.CharBlock( + null=False, required=False, max_length=80, help_text='Optional title to display for this explorer block' + ) + spreadsheet_id = blocks.CharBlock( + null=False, required=False, max_length=60, + help_text="File ID of a Google Drive spreadsheet; don't forget to Update from Google Drive!" + ) + class Meta: + template = 'edrnsite.streams/data-element-explorer-block.html' + icon = 'form' + label = 'Data Element Explorer' + help_text = 'A block that shows an explorer-like tree view of data elements from a Google spreadsheet' diff --git a/src/edrnsite.streams/src/edrnsite/streams/_data_elements.py b/src/edrnsite.streams/src/edrnsite/streams/_data_elements.py new file mode 100644 index 00000000..d08a9113 --- /dev/null +++ b/src/edrnsite.streams/src/edrnsite/streams/_data_elements.py @@ -0,0 +1,224 @@ +# encoding: utf-8 + +'''🦦 EDRN Site streams: data element trees.''' + + +from django.db import models +from wagtail.admin.panels import FieldPanel +import dataclasses, logging, traceback, tempfile, gdown, pandas, os + +_logger = logging.getLogger(__name__) + + +@dataclasses.dataclass(order=True) +class _Attribute: + '''A temporary attribute of a node before getting serialized into Django objects.''' + text: str + definition: str + required: str + data_type: str + explanatory_note: str + permissible_values: list[str] + inheritance: bool + def __hash__(self): + return hash(self.text) + def instantiate(self, obj): + attr_obj = DataElementExplorerAttribute( + text=self.text, definition=self.definition, required=self.required, data_type=self.data_type, + explanatory_note=self.explanatory_note, obj=obj, inheritance=self.inheritance + ) + attr_obj.save() + for pv in self.permissible_values: + cde_pv = DataElementExplorerPermissibleValue(value=pv, attribute=attr_obj) + cde_pv.save() + return attr_obj + + +@dataclasses.dataclass(order=True) +class _Node: + '''A temporary node in a tree before getting serialized into Django objects.''' + name: str + description: str + stewardship: str + attributes: list[_Attribute] + children: set[object] = dataclasses.field(default_factory=set, init=False, compare=False) + def __hash__(self): + return hash(self.name) + def instantiate(self, parent=None): + explorer_obj = DataElementExplorerObject( + name=self.name, description=self.description, stewardship=self.stewardship, parent=parent + ) + explorer_obj.save() + for c in self.children: + c.instantiate(explorer_obj) + for a in self.attributes: + a.instantiate(explorer_obj) + return explorer_obj + + +class DataElementExplorerObject(models.Model): + name = models.CharField(null=False, blank=False, max_length=200, help_text='Name of this object in a CDE hierarchy') + description = models.TextField(null=False, blank=True, help_text='A nice long description of this object') + stewardship = models.TextField(null=False, blank=True, help_text="Who's responsible for this object") + parent = models.ForeignKey('self', blank=True, null=True, on_delete=models.CASCADE, related_name='children') + spreadsheet_id = models.CharField(null=False, blank=True, help_text='If a root node, the spreadsheet that generates this node') + panels = [FieldPanel('name'), FieldPanel('description'), FieldPanel('parent'), FieldPanel('spreadsheet_id')] + def __str__(self): + return self.name + class Meta: + indexes = [models.Index(fields=['spreadsheet_id'])] + + +class DataElementExplorerAttribute(models.Model): + text = models.CharField(null=False, blank=False, max_length=100, help_text='Name of this common data element') + obj = models.ForeignKey(DataElementExplorerObject, null=True, on_delete=models.CASCADE, related_name='attributes') + definition = models.TextField(null=False, blank=True, help_text='A thorough definition of this CDE') + required = models.CharField(null=False, blank=True, max_length=50, help_text='Required, not, or something else?') + data_type = models.CharField(null=False, blank=True, max_length=30, help_text='Kind of data') + explanatory_note = models.TextField(null=False, blank=True, help_text='Note helping explain use of the CDE') + inheritance = models.BooleanField(null=False, blank=False, default=False, help_text='Attribute inherits values') + panels = [ + FieldPanel('text'), + FieldPanel('obj'), + FieldPanel('definition'), + FieldPanel('required'), + FieldPanel('data_type'), + FieldPanel('explanatory_note'), + FieldPanel('inheritance') + ] + def __str__(self): + return self.text + + +class DataElementExplorerPermissibleValue(models.Model): + value = models.CharField( + null=False, blank=False, max_length=200, + help_text='An enumerated value allowed for a data element that uses permissible values' + ) + attribute = models.ForeignKey(DataElementExplorerAttribute, null=True, on_delete=models.CASCADE, related_name='permissible_values') + panels = [FieldPanel('value'), FieldPanel('attribute')] + def __str__(self): + return self.value + + +class _ExplorerTreeUpdater: + def __init__(self, spreadsheet_id: str): + self.spreadsheet_id = spreadsheet_id + self.log = [] + + def _read_sheet(self, url): + '''Read the spreadsheet at ``url`` and return it.''' + fd, fn = tempfile.mkstemp('.xlsx') + os.close(fd) + self._log(f'Downloading {self.spreadsheet_id} from Gdrive') + # use_cookies must be False to work on tumor.jpl.nasa.gov + fn = gdown.download(id=self.spreadsheet_id, output=fn, quiet=True, use_cookies=False, format='xlsx') + # gdown doesn't raise an exception on error, but returns None as the filename—even when we pass + # in the filename we want to use; see wkentaro/gdown#276 + if fn is None: + raise ValueError('Error reading from gdown; check console log as gdown does not pass this info along') + return fn + + def _parse_attributes(self, name, sheet): + '''Using the data in ``sheet``, find the tab ``name`` and get all the attributes there.''' + self._log(f'Parsing attributes in tab "{name}"') + + frame = sheet[name] + row_number, attrs = 0, [] + for text in frame['Text']: + # Gather data from the spreadsheet tab + pvs_text, defn = frame['Permissible Values'][row_number], frame['Definition'][row_number] + req, dt = frame['Requirement'][row_number], frame['Data Type'][row_number] + note, inheritance = frame['Explanatory Note'][row_number], frame['Inheritance'][row_number] + + # Handle the empty cells + pvs = [] if pandas.isna(pvs_text) else [i.strip() for i in pvs_text.split('\n')] + defn = '' if pandas.isna(defn) else defn + req = '' if pandas.isna(req) else req + dt = '' if pandas.isna(dt) else dt + note = '' if pandas.isna(note) else note + inh = False if pandas.isna(inheritance) else inheritance + + # Create the temporary attribute and add it to the sequence + attrs.append(_Attribute(text, defn, req, dt, note, pvs, inh)) + row_number += 1 + return attrs + + def _parse_structure(self, sheet): + '''Using the data ``sheet``, produce a sequence of tree structure that match.''' + + # First, catalog all the nodes and ensure they all have tabs + self._log('Parsing overall structure in the "Structure" tab and looking for referenced tabs') + row_number, nodes, roots, structure = 0, {}, [], sheet['Structure'] + for name in structure['Object']: + description, stewardship = structure['Description'][row_number], structure['Stewardship'][row_number] + attributes = self._parse_attributes(name, sheet) + stewardship = '' if pandas.isna(stewardship) else stewardship + nodes[name] = _Node(name, description, stewardship, attributes) + row_number += 1 + + # Now connect parents to children and gather the roots + self._log('Connecting child objects to parents') + row_number = 0 + for name in structure['Object']: + node = nodes[name] + parents = structure['Parent'][row_number] + if pandas.isna(parents): + roots.append(node) + else: + # This handles multiple parents? What does that even mean? + for parent_name in [i.strip() for i in parents.split(',')]: + parent = nodes[parent_name] + parent.children.add(node) + row_number += 1 + + # Tell the roots + self._log(f'Total root objects: {len(roots)}: {", ".join([i.name for i in roots])}') + return roots + + def _delete_obj(self, obj): + '''Delete all the CDE explorer objects at and beneath ``obj``. + + It deletes all child objects of ``obj``. It also deletes all the attributes of ``obj`` and + its children, plus all their permissible values. + ''' + for attr in obj.attributes.all(): + for pv in attr.permissible_values.all(): + pv.delete() + attr.delete() + for child in obj.children.all(): + self._delete_obj(child) + obj.delete() + + def _delete_old_explorer_objects(self): + self._log(f'Deleting old explorer objects for {self.spreadsheet_id}') + for obj in DataElementExplorerObject.objects.filter(spreadsheet_id=self.spreadsheet_id): + self._delete_obj(obj) + + def update(self) -> list[str]: + try: + self._log(f'Reading spreadsheet {self.spreadsheet_id}') + sheet_filename = self._read_sheet(self.spreadsheet_id) + sheet = pandas.read_excel(sheet_filename, sheet_name=None) + roots = self._parse_structure(sheet) + + # At this point, the data in the sheet passes muster, so we drop the old objects and + # instantiate the new + self._delete_old_explorer_objects() + + self._log('Installing new explorer objects') + for root in roots: + explorer_obj = root.instantiate() + explorer_obj.spreadsheet_id = self.spreadsheet_id + explorer_obj.save() + + except Exception as ex: + self._log(f'Exception {ex.__class__.__name__}; aborting update on spreadsheet {self.spreadsheet_id}') + self._log(traceback.format_exc()) + _logger.exception('Abort') + finally: + return self.log + + def _log(self, message: str): + _logger.warning(message) + self.log.append(message) diff --git a/src/edrnsite.streams/src/edrnsite/streams/blocks.py b/src/edrnsite.streams/src/edrnsite/streams/blocks.py index fb197c9b..a27ade09 100644 --- a/src/edrnsite.streams/src/edrnsite/streams/blocks.py +++ b/src/edrnsite.streams/src/edrnsite/streams/blocks.py @@ -10,6 +10,7 @@ from wagtail.contrib.table_block.blocks import TableBlock as BaseTableBlock from wagtail.contrib.typed_table_block.blocks import TypedTableBlock as BaseTypedTableBlock from wagtail.images.blocks import ImageChooserBlock +from ._cde_blocks import DataElementExplorerBlock # noqa class TitleBlock(blocks.StructBlock): diff --git a/src/edrnsite.streams/src/edrnsite/streams/migrations/0001_initial.py b/src/edrnsite.streams/src/edrnsite/streams/migrations/0001_initial.py new file mode 100644 index 00000000..95a6bd11 --- /dev/null +++ b/src/edrnsite.streams/src/edrnsite/streams/migrations/0001_initial.py @@ -0,0 +1,161 @@ +# Generated by Django 4.2.17 on 2025-01-10 22:37 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [] + + operations = [ + migrations.CreateModel( + name="DataElementExplorerObject", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "name", + models.CharField( + help_text="Name of this object in a CDE hierarchy", + max_length=200, + ), + ), + ( + "description", + models.TextField( + blank=True, help_text="A nice long description of this object" + ), + ), + ( + "stewardship", + models.TextField( + blank=True, help_text="Who's responsible for this object" + ), + ), + ( + "spreadsheet_id", + models.CharField( + blank=True, + help_text="If a root node, the spreadsheet that generates this node", + ), + ), + ( + "parent", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="children", + to="edrnsitestreams.dataelementexplorerobject", + ), + ), + ], + ), + migrations.CreateModel( + name="DataElementExplorerAttribute", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "text", + models.CharField( + help_text="Name of this common data element", max_length=100 + ), + ), + ( + "definition", + models.TextField( + blank=True, help_text="A thorough definition of this CDE" + ), + ), + ( + "required", + models.CharField( + blank=True, + help_text="Required, not, or something else?", + max_length=50, + ), + ), + ( + "data_type", + models.CharField( + blank=True, help_text="Kind of data", max_length=30 + ), + ), + ( + "explanatory_note", + models.TextField( + blank=True, help_text="Note helping explain use of the CDE" + ), + ), + ( + "inheritance", + models.BooleanField( + default=False, help_text="Attribute inherits values" + ), + ), + ( + "obj", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="attributes", + to="edrnsitestreams.dataelementexplorerobject", + ), + ), + ], + ), + migrations.CreateModel( + name="CDEPermissibleValue", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "value", + models.CharField( + help_text="An enumerated value allowed for a data element that uses permissible values", + max_length=200, + ), + ), + ( + "attribute", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="permissible_values", + to="edrnsitestreams.dataelementexplorerattribute", + ), + ), + ], + ), + migrations.AddIndex( + model_name="dataelementexplorerobject", + index=models.Index( + fields=["spreadsheet_id"], name="edrnsitestr_spreads_e6c691_idx" + ), + ), + ] diff --git a/src/edrnsite.streams/src/edrnsite/streams/migrations/0002_rename_cdepermissiblevalue_dataelementexplorerpermissiblevalue.py b/src/edrnsite.streams/src/edrnsite/streams/migrations/0002_rename_cdepermissiblevalue_dataelementexplorerpermissiblevalue.py new file mode 100644 index 00000000..72cbf541 --- /dev/null +++ b/src/edrnsite.streams/src/edrnsite/streams/migrations/0002_rename_cdepermissiblevalue_dataelementexplorerpermissiblevalue.py @@ -0,0 +1,17 @@ +# Generated by Django 4.2.17 on 2025-01-13 20:38 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("edrnsitestreams", "0001_initial"), + ] + + operations = [ + migrations.RenameModel( + old_name="CDEPermissibleValue", + new_name="DataElementExplorerPermissibleValue", + ), + ] diff --git a/src/edrnsite.streams/src/edrnsite/streams/models.py b/src/edrnsite.streams/src/edrnsite/streams/models.py new file mode 100644 index 00000000..2f2906af --- /dev/null +++ b/src/edrnsite.streams/src/edrnsite/streams/models.py @@ -0,0 +1,11 @@ +# encoding: utf-8 + +'''🦦 EDRN Site streams: models.''' + +from ._data_elements import ( + DataElementExplorerPermissibleValue, DataElementExplorerAttribute, DataElementExplorerObject +) + +__all__ = [ + DataElementExplorerPermissibleValue, DataElementExplorerAttribute, DataElementExplorerObject +] diff --git a/src/edrnsite.streams/src/edrnsite/streams/templates/edrnsite.streams/data-element-explorer-block.html b/src/edrnsite.streams/src/edrnsite/streams/templates/edrnsite.streams/data-element-explorer-block.html new file mode 100644 index 00000000..22404734 --- /dev/null +++ b/src/edrnsite.streams/src/edrnsite/streams/templates/edrnsite.streams/data-element-explorer-block.html @@ -0,0 +1,68 @@ +{% load wagtailcore_tags edrnsite_streams_tags %} +{# 🔮 TODO: caching #} +{% block extra_css %} + +{% endblock extra_css %} +{% block header_scripts %} + +{% endblock header_scripts %} +
+
+ {% if value.title %} +

{{value.title}}

+ {% endif %} + {% if value.spreadsheet_id %} + +  View as a spreadsheet + + {% endif %} +
+
+{% if value.spreadsheet_id %} +
+
+ {% render_de_roots value.spreadsheet_id %} +
+
+
+ + {% render_all_de_attribute_canvases value.spreadsheet_id %} +{% endif %} + +{% block extra_js %} + +{% endblock extra_js %} +{# -*- HTML (Jinja) -*- #} diff --git a/src/edrnsite.streams/src/edrnsite/streams/templates/edrnsite.streams/de-attribute-button.html b/src/edrnsite.streams/src/edrnsite/streams/templates/edrnsite.streams/de-attribute-button.html new file mode 100644 index 00000000..4c69c7eb --- /dev/null +++ b/src/edrnsite.streams/src/edrnsite/streams/templates/edrnsite.streams/de-attribute-button.html @@ -0,0 +1,15 @@ +{% load wagtailcore_tags edrnsite_streams_tags %} + + {% if required %}{% endif %}{% if inheritance %}{% endif %} {{text}} + +{# -*- HTML (Jinja) -*- #} + diff --git a/src/edrnsite.streams/src/edrnsite/streams/templates/edrnsite.streams/de-attribute-canvas.html b/src/edrnsite.streams/src/edrnsite/streams/templates/edrnsite.streams/de-attribute-canvas.html new file mode 100644 index 00000000..50f47725 --- /dev/null +++ b/src/edrnsite.streams/src/edrnsite/streams/templates/edrnsite.streams/de-attribute-canvas.html @@ -0,0 +1,63 @@ +{% load wagtailcore_tags edrnsite_streams_tags %} +
+
+

{{text}}

+ +
+
+
+
Element text
+
{{text}}
+
Definition
+
+ {% if definition %} + {{definition}} + {% else %} + (No definition provided.) + {% endif %} +
+
Requirement
+
+ {% if required %} + {{required}} + {% else %} + (Information not provided.) + {% endif %} +
+
Inheritance
+
+ {% if inheritance %} + If no values are specified, this attribute inherits values from other objects. + {% else %} + This attribute does not inherit values from other objects. + {% endif %} +
+
Data Type
+
+ {% if data_type %} + {{data_type}} + {% else %} + (Type information not available.) + {% endif %} +
+
Explanatory Note
+
+ {% if note %} + {{note}} + {% else %} + (No notes were given for this element.) + {% endif %} +
+
+ {% if pvs %} +
Permissible Values
+
    + {% for pv in pvs %} +
  • {{pv.value}}
    • + {% endfor %} +
+ {% endif %} +
+
+{# -*- HTML (Jinja) -*- #} + diff --git a/src/edrnsite.streams/src/edrnsite/streams/templates/edrnsite.streams/de-attribute-canvases.html b/src/edrnsite.streams/src/edrnsite/streams/templates/edrnsite.streams/de-attribute-canvases.html new file mode 100644 index 00000000..abec1036 --- /dev/null +++ b/src/edrnsite.streams/src/edrnsite/streams/templates/edrnsite.streams/de-attribute-canvases.html @@ -0,0 +1,8 @@ +{% load edrnsite_streams_tags %} +{% for attribute in attributes %} + {% render_de_attribute_canvas attribute %} +{% endfor %} +{% for child in children %} + {% render_de_attribute_canvases child %} +{% endfor %} +{# -*- HTML (Jinja) -*- #} diff --git a/src/edrnsite.streams/src/edrnsite/streams/templates/edrnsite.streams/de-node.html b/src/edrnsite.streams/src/edrnsite/streams/templates/edrnsite.streams/de-node.html new file mode 100644 index 00000000..8f4c9e93 --- /dev/null +++ b/src/edrnsite.streams/src/edrnsite/streams/templates/edrnsite.streams/de-node.html @@ -0,0 +1,41 @@ +{% load wagtailcore_tags edrnsite_streams_tags %} +
  • + {{name}} +
    +

    {{name}}

    + {% if description %} +

    {{description}}

    + {% else %} +

    (No description of this object is available.)

    + {% endif %} + + {% if stewardship %} +

    Stewardship

    +

    {{stewardship}}

    + {% endif %} + + {% if attributes %} +

    Attributes

    +

    + Attributes shown with are required, while + those with inherit values from other objects. +

    +

    + Click/tap on an attribute text box below to view details about that attribute. +

    +

    + {% for attribute in attributes %} + {% render_de_attribute_button attribute %} + {% endfor %} +

    + {% endif %} +
    + {% if children %} + + {% endif %} +
    • +{# -*- HTML (Jinja) -*- #} diff --git a/src/edrnsite.streams/src/edrnsite/streams/templates/edrnsite.streams/de-root-canvases.html b/src/edrnsite.streams/src/edrnsite/streams/templates/edrnsite.streams/de-root-canvases.html new file mode 100644 index 00000000..a4d52e0a --- /dev/null +++ b/src/edrnsite.streams/src/edrnsite/streams/templates/edrnsite.streams/de-root-canvases.html @@ -0,0 +1,6 @@ +{% load wagtailcore_tags edrnsite_streams_tags %} +{# We don't actually need any other surrounding markup so this template isn't strictly necessary #} +{% for root in root_objects %} + {% render_de_attribute_canvases root %} +{% endfor %} +{# -*- HTML (Jinja) -*- #} diff --git a/src/edrnsite.streams/src/edrnsite/streams/templates/edrnsite.streams/de-roots.html b/src/edrnsite.streams/src/edrnsite/streams/templates/edrnsite.streams/de-roots.html new file mode 100644 index 00000000..682102ef --- /dev/null +++ b/src/edrnsite.streams/src/edrnsite/streams/templates/edrnsite.streams/de-roots.html @@ -0,0 +1,7 @@ +{% load wagtailcore_tags edrnsite_streams_tags %} + +{# -*- HTML (Jinja) -*- #} diff --git a/src/edrnsite.streams/src/edrnsite/streams/templatetags/edrnsite_streams_tags.py b/src/edrnsite.streams/src/edrnsite/streams/templatetags/edrnsite_streams_tags.py index 5deea368..0da2f659 100644 --- a/src/edrnsite.streams/src/edrnsite/streams/templatetags/edrnsite_streams_tags.py +++ b/src/edrnsite.streams/src/edrnsite/streams/templatetags/edrnsite_streams_tags.py @@ -2,9 +2,10 @@ '''🦦 EDRN Site streams: template tags.''' +from ..models import DataElementExplorerAttribute, DataElementExplorerObject from django import template from django.utils.safestring import mark_safe - +from django.utils.text import slugify register = template.Library() @@ -13,3 +14,56 @@ def mark_external_link(url: str) -> str: marker = ' ' if url.startswith('http') else '' return mark_safe(marker) + + +@register.inclusion_tag('edrnsite.streams/de-roots.html', takes_context=False) +def render_de_roots(spreadsheet_id: str) -> dict: + return {'root_objects': DataElementExplorerObject.objects.filter(spreadsheet_id=spreadsheet_id)} + + +@register.inclusion_tag('edrnsite.streams/de-root-canvases.html', takes_context=False) +def render_all_de_attribute_canvases(spreadsheet_id: str) -> dict: + return {'root_objects': DataElementExplorerObject.objects.filter(spreadsheet_id=spreadsheet_id)} + + +@register.inclusion_tag('edrnsite.streams/de-node.html', takes_context=False) +def render_de_node(node: DataElementExplorerObject) -> dict: + return { + 'name': node.name, + 'description': node.description, + 'stewardship': node.stewardship, + 'attributes': node.attributes.all(), + 'children': node.children.all().order_by('name') + } + + +@register.inclusion_tag('edrnsite.streams/de-attribute-button.html', takes_context=False) +def render_de_attribute_button(attribute: DataElementExplorerAttribute) -> dict: + return { + 'id': f'de-{slugify(attribute.obj.name)}-{slugify(attribute.text)}', + 'text': attribute.text, + 'required': attribute.required == 'Required', + 'inheritance': attribute.inheritance + } + + +@register.inclusion_tag('edrnsite.streams/de-attribute-canvases.html', takes_context=False) +def render_de_attribute_canvases(node: DataElementExplorerObject) -> dict: + return { + 'attributes': node.attributes.all(), + 'children': node.children.all() + } + + +@register.inclusion_tag('edrnsite.streams/de-attribute-canvas.html', takes_context=False) +def render_de_attribute_canvas(attribute: DataElementExplorerAttribute) -> dict: + return { + 'id': f'de-{slugify(attribute.obj.name)}-{slugify(attribute.text)}', + 'text': attribute.text, + 'definition': attribute.definition, + 'required': attribute.required, + 'data_type': attribute.data_type, + 'note': attribute.explanatory_note, + 'pvs': attribute.permissible_values.all(), + 'inheritance': attribute.inheritance + } diff --git a/src/edrnsite.streams/src/edrnsite/streams/urls.py b/src/edrnsite.streams/src/edrnsite/streams/urls.py new file mode 100644 index 00000000..44172e97 --- /dev/null +++ b/src/edrnsite.streams/src/edrnsite/streams/urls.py @@ -0,0 +1,12 @@ +# encoding: utf-8 + +'''🦦 EDRN Site streams: URLs.''' + + +from .views import update_data_element_explorers +from django.urls import path + + +urlpatterns = [ + path('update_data_element_explorers', update_data_element_explorers, name='update_data_element_explorers'), +] diff --git a/src/edrnsite.streams/src/edrnsite/streams/views.py b/src/edrnsite.streams/src/edrnsite/streams/views.py new file mode 100644 index 00000000..4fba42ee --- /dev/null +++ b/src/edrnsite.streams/src/edrnsite/streams/views.py @@ -0,0 +1,51 @@ +# encoding: utf-8 + +'''🦦 EDRN Site streams: views.''' + +from edrn.auth.views import logged_in_or_basicauth +from django.http import HttpRequest, HttpResponse, HttpResponseForbidden +from django.apps import apps +from wagtail.blocks.stream_block import StreamValue +from ._data_elements import _ExplorerTreeUpdater +from typing import Generator + + +def _get_referrer(request: HttpRequest) -> str: + try: + return request.META['HTTP_REFERER'] + except KeyError: + return '/' + + +def _update_tree_from_spreadsheet(spreadsheet_id: str) -> str: + updater = _ExplorerTreeUpdater(spreadsheet_id) + results = '\n'.join(updater.update()) + return f'

    {spreadsheet_id}

    {results}
    ' + + +def _find_data_explorer_blocks(stream_value: StreamValue) -> Generator[str, None, None]: + for block in stream_value: + if block.block_type == 'data_explorer': + yield block + elif hasattr(block.value, 'stream_data'): + yield from _find_data_explorer_blocks(block.value) + + +def update_data_element_explorer_trees() -> list[str]: + # Cannot import edrnsite.content (circular dependency): + FlexPage = apps.get_model('edrnsitecontent', 'FlexPage') + results = [] + for fp in FlexPage.objects.all(): + for deb in _find_data_explorer_blocks(fp.body): + spreadsheet_id = deb.value.get('spreadsheet_id') + if spreadsheet_id: + results.append(_update_tree_from_spreadsheet(spreadsheet_id)) + return results + + +@logged_in_or_basicauth('edrn') +def update_data_element_explorers(request: HttpRequest) -> HttpResponse: + if request.user.is_staff or request.user.is_superuser: + return HttpResponse(update_data_element_explorer_trees(), content_type='text/html') + else: + return HttpResponseForbidden() diff --git a/src/edrnsite.streams/src/edrnsite/streams/wagtail_hooks.py b/src/edrnsite.streams/src/edrnsite/streams/wagtail_hooks.py new file mode 100644 index 00000000..851211d4 --- /dev/null +++ b/src/edrnsite.streams/src/edrnsite/streams/wagtail_hooks.py @@ -0,0 +1,39 @@ +# encoding: utf-8 + +'''🦦 EDRN Site streams: hooks for Wagtail.''' + + +from .models import DataElementExplorerPermissibleValue, DataElementExplorerAttribute, DataElementExplorerObject +from wagtail.snippets.models import register_snippet +from wagtail.snippets.views.snippets import SnippetViewSet, SnippetViewSetGroup + + +class DEPermissibleValueAdmin(SnippetViewSet): + model = DataElementExplorerPermissibleValue + menu_label = 'Permissible Values' + icon = 'tag' + list_display = ('value', 'attribute') + + +class DEExplorerAttributeAdmin(SnippetViewSet): + model = DataElementExplorerAttribute + menu_label = 'Attributes' + icon = 'list-ul' + list_display = ('text', 'data_type', 'obj') + + +class DEExplorerObjectAdmin(SnippetViewSet): + model = DataElementExplorerObject + menu_label = 'Objects' + icon = 'folder' + list_display = ('name', 'parent') + + +class DEGroup(SnippetViewSetGroup): + menu_label = 'DE Explorer' + icon = 'folder-open-inverse' + menu_order = 475 + items = (DEExplorerObjectAdmin, DEExplorerAttributeAdmin, DEPermissibleValueAdmin) + + +register_snippet(DEGroup) diff --git a/src/eke.knowledge/src/eke/knowledge/templates/eke.knowledge/ingest-controls.html b/src/eke.knowledge/src/eke/knowledge/templates/eke.knowledge/ingest-controls.html index eb3d066a..3009f0fa 100644 --- a/src/eke.knowledge/src/eke/knowledge/templates/eke.knowledge/ingest-controls.html +++ b/src/eke.knowledge/src/eke/knowledge/templates/eke.knowledge/ingest-controls.html @@ -31,10 +31,13 @@

    EDRN Ingest Controls

    Other EDRN Controls

    {# 🔮 TODO: find a better place for these buttons? #} - Reindex Content - Sync LDAP Groups - Clear Caches - Fix Tree +
    diff --git a/support/cbiit-deploy-prod.sh b/support/cbiit-deploy-prod.sh index 3e3b133a..e67373b0 100755 --- a/support/cbiit-deploy-prod.sh +++ b/support/cbiit-deploy-prod.sh @@ -146,7 +146,8 @@ echo "" echo "🆙 Applying upgrades" ssh -q $USER@$WEBSERVER "cd $WEBROOT ; \ docker compose --project-name edrn exec portal /usr/bin/django-admin copy_daily_hits_from_wagtailsearch &&\ -docker compose --project-name edrn exec portal /usr/bin/django-admin edrn_data_disclaimer" || exit 1 +docker compose --project-name edrn exec portal /usr/bin/django-admin edrn_data_disclaimer &&\ +docker compose --project-name edrn exec portal /usr/bin/django-admin edrn_new_explorers" || exit 1 echo "" echo "🤷‍♀️ Final portal restart and restart of search engine" diff --git a/support/cbiit-deploy.sh b/support/cbiit-deploy.sh index ef3ef2fb..d0fafdee 100755 --- a/support/cbiit-deploy.sh +++ b/support/cbiit-deploy.sh @@ -147,7 +147,8 @@ echo "" echo "🆙 Applying upgrades" ssh -q $USER@$WEBSERVER "cd $WEBROOT ; \ docker compose --project-name edrn exec portal /usr/bin/django-admin copy_daily_hits_from_wagtailsearch &&\ -docker compose --project-name edrn exec portal /usr/bin/django-admin edrn_data_disclaimer" || exit 1 +docker compose --project-name edrn exec portal /usr/bin/django-admin edrn_data_disclaimer &&\ +docker compose --project-name edrn exec portal /usr/bin/django-admin edrn_new_explorers" || exit 1 echo "" diff --git a/support/devrebuild.sh b/support/devrebuild.sh index 88a6bf0b..d53fa944 100755 --- a/support/devrebuild.sh +++ b/support/devrebuild.sh @@ -57,6 +57,7 @@ bzip2 --decompress --stdout edrn.sql.bz2 | psql --dbname=edrn --echo-errors --qu ./manage.sh copy_daily_hits_from_wagtailsearch # Wagtail 5 ./manage.sh collectstatic --no-input --clear --link ./manage.sh edrn_data_disclaimer +./manage.sh edrn_new_explorers ./manage.sh edrndevreset # Add additional upgrade steps here: