From 6288294b1e4b0673376165d67935f04a2f4a755f Mon Sep 17 00:00:00 2001
From: goose-life <32566441+goose-life@users.noreply.github.com>
Date: Fri, 13 Dec 2024 16:00:19 +0200
Subject: [PATCH 01/28] enable italicisation of portion

---
 indigo_api/views/documents.py                 | 26 ++++++++++++++++++-
 .../indigo/views/document_italics_terms.js    | 12 ++++++---
 .../indigo_api/document/_toolbar.html         |  2 +-
 3 files changed, 34 insertions(+), 6 deletions(-)

diff --git a/indigo_api/views/documents.py b/indigo_api/views/documents.py
index 831038d87..a018a2a98 100644
--- a/indigo_api/views/documents.py
+++ b/indigo_api/views/documents.py
@@ -14,6 +14,7 @@
 from django.utils.translation import gettext as _
 from django_comments.models import Comment
 
+from lxml import etree
 from rest_framework.exceptions import ValidationError, MethodNotAllowed
 from rest_framework.views import APIView
 from rest_framework import mixins, viewsets, renderers, status
@@ -404,15 +405,38 @@ class MarkUpItalicsTermsView(DocumentResourceView, APIView):
     """ Find and mark up italics terms.
     """
     def post(self, request, document_id):
-        serializer = DocumentAPISerializer(instance=self.document, data=self.request.data)
+        # in provision mode, fake a full e.g. <act> using the portion XML
+        data = self.request.data
+        provision_eid = data['document'].pop('provision_eid', None)
+        if provision_eid:
+            data['document']['content'] = self.switch_doctype(data['document']['content'], self.document.work.work_uri.doctype)
+
+        serializer = DocumentAPISerializer(instance=self.document, data=data)
         serializer.fields['document'].fields['content'].required = True
         serializer.is_valid(raise_exception=True)
         document = serializer.fields['document'].update_document(self.document, serializer.validated_data['document'])
 
         self.mark_up_italics(document)
 
+        # in provision mode, turn the fake full document back into a portion, minus the wrapping akn tag
+        if provision_eid:
+            document.document_xml = self.switch_doctype(document.document_xml, self.document.work.work_uri.doctype, back_to_portion=True)
+
         return Response({'document': {'content': document.document_xml}})
 
+    def switch_doctype(self, xml, doctype, back_to_portion=False):
+        old_doc = StructuredDocument.for_document_type(doctype if back_to_portion else 'portion')(xml)
+        new_doc = StructuredDocument.for_document_type('portion' if back_to_portion else doctype)()
+        # delete any empty section in body, for example
+        for blank in new_doc.main_content.xpath('*'):
+            new_doc.main_content.remove(blank)
+        for elem in old_doc.main_content.xpath('*'):
+            new_doc.main_content.append(elem)
+        if back_to_portion:
+            # no wrapping akn tag in provision mode
+            return etree.tostring(new_doc.main, encoding='unicode')
+        return etree.tostring(new_doc.root, encoding='unicode')
+
     def mark_up_italics(self, document):
         italics_terms_finder = plugins.for_document('italics-terms', document)
         italics_terms = document.work.country.italics_terms
diff --git a/indigo_app/static/javascript/indigo/views/document_italics_terms.js b/indigo_app/static/javascript/indigo/views/document_italics_terms.js
index eb1ac96f8..a1b2a2b27 100644
--- a/indigo_app/static/javascript/indigo/views/document_italics_terms.js
+++ b/indigo_app/static/javascript/indigo/views/document_italics_terms.js
@@ -40,7 +40,12 @@
           $btn = this.$el.find('.mark-up-italics'),
           data = {'document': this.model.document.toJSON()};
 
-      data.document.content = this.model.toXml();
+      let content = this.model.toXml();
+      if (Indigo.Preloads.provisionEid) {
+        content = `<akomaNtoso xmlns="${this.model.xmlDocument.firstChild.getAttribute('xmlns')}">${content}</akomaNtoso>`;
+        data.document.provision_eid = Indigo.Preloads.provisionEid;
+      }
+      data.document.content = content;
 
       this.$('a[href="#this-document-italics-terms"]').click();
 
@@ -66,9 +71,9 @@
 
     removeItalics: function(e) {
       // remove all italics mark-up
-      var self = this,
-          changed = false;
+      let changed = false;
 
+      // TODO: navigate to 'In this document' first (this click() doesn't do anything)
       this.$('a[href="#this-document-italics-terms"]').click();
 
       this.model.xmlDocument.querySelectorAll('i').forEach(function(term) {
@@ -82,7 +87,6 @@
 
       if (changed) {
         this.model.trigger('change:dom');
-        // TODO: refresh content without saving
       }
     }
   });
diff --git a/indigo_app/templates/indigo_api/document/_toolbar.html b/indigo_app/templates/indigo_api/document/_toolbar.html
index 2faa82e90..f943d1cbd 100644
--- a/indigo_app/templates/indigo_api/document/_toolbar.html
+++ b/indigo_app/templates/indigo_api/document/_toolbar.html
@@ -35,7 +35,7 @@
         <a class="dropdown-item {% if provision_eid %}disabled{% endif %}"
            href="#references-modal" data-bs-toggle="modal"
         >{% trans "References..." %}</a>
-        <a class="dropdown-item {% if provision_eid %}disabled{% endif %}"
+        <a class="dropdown-item"
            href="#italics-modal" data-bs-toggle="modal"
         >{% trans "Italicised terms..." %}</a>
         <div class="dropdown-divider"></div>

From a2e64fc382ed386a289147920aa0cb5705de5ced Mon Sep 17 00:00:00 2001
From: goose-life <32566441+goose-life@users.noreply.github.com>
Date: Fri, 10 Jan 2025 17:28:12 +0200
Subject: [PATCH 02/28] include portionBody in TextPatternMarker'a ancestor
 elements

---
 indigo/analysis/markup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/indigo/analysis/markup.py b/indigo/analysis/markup.py
index e01791e94..56df08741 100644
--- a/indigo/analysis/markup.py
+++ b/indigo/analysis/markup.py
@@ -16,7 +16,7 @@ class TextPatternMarker:
     """ Xpath for candidate text nodes that should be tested for matches. Must be defined by subclasses.
     """
 
-    ancestors = ['coverpage', 'preface', 'preamble', 'body', 'mainBody', 'conclusions']
+    ancestors = ['coverpage', 'preface', 'preamble', 'body', 'mainBody', 'portionBody', 'conclusions']
     """ Tags that the candidate_xpath should be run against.
     """
 

From 7e44d4c9c0f33b5336cecb4a341b7a1d4e25798b Mon Sep 17 00:00:00 2001
From: goose-life <32566441+goose-life@users.noreply.github.com>
Date: Fri, 10 Jan 2025 17:42:15 +0200
Subject: [PATCH 03/28] new DocumentAPISerializer

---
 indigo_api/serializers.py | 50 ++++++++++++++++++++++++++-------------
 1 file changed, 33 insertions(+), 17 deletions(-)

diff --git a/indigo_api/serializers.py b/indigo_api/serializers.py
index 340c1ce6a..c2d3a7ffd 100644
--- a/indigo_api/serializers.py
+++ b/indigo_api/serializers.py
@@ -1,26 +1,24 @@
-import logging
 import os.path
-from itertools import groupby
-from typing import List
-
-from actstream.signals import action
 from collections import OrderedDict
 
-from lxml.etree import LxmlError
-
-from django.contrib.auth.models import User
+import logging
+import reversion
+from actstream.signals import action
+from allauth.account.utils import user_display
 from django.conf import settings
+from django.contrib.auth.models import User
+from lxml import etree
+from lxml.etree import LxmlError
 from rest_framework import serializers
-from rest_framework.reverse import reverse
 from rest_framework.exceptions import ValidationError
-from cobalt import StructuredDocument, FrbrUri
-from cobalt.akn import AKN_NAMESPACES, DEFAULT_VERSION
-import reversion
+from rest_framework.reverse import reverse
+from typing import List
 
+from cobalt import StructuredDocument, FrbrUri
+from cobalt.akn import AKN_NAMESPACES
 from indigo_api.models import Document, Attachment, Annotation, DocumentActivity, Work, Amendment, Language, \
     PublicationDocument, Task, Commencement
 from indigo_api.signals import document_published
-from allauth.account.utils import user_display
 
 log = logging.getLogger(__name__)
 
@@ -438,11 +436,29 @@ class DocumentAPISerializer(serializers.Serializer):
     """
     Helper to handle input documents for general document APIs
     """
-    document = DocumentSerializer(required=True)
+    xml = serializers.CharField()
+    language = serializers.CharField(min_length=3, max_length=3)
+    is_portion = serializers.BooleanField()
+
+    def update_document(self, document):
+        # the language could have been changed but not yet saved during editing, which might affect which plugin is used
+        if document.language.code != self.validated_data.get('language'):
+            document.language = Language.for_code(self.validated_data.get('language'))
+        # the FRBR URI doctype needs to match the XML root's child; change it to portion in provision mode
+        if self.validated_data.get('is_portion'):
+            document.work.work_uri.doctype = 'portion'
+        document.content = self.validated_data.get('xml')
+        return document
 
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.fields['document'].instance = self.instance
+    def updated_xml(self, document):
+        xml = document.document_xml
+        # in provision mode, unwrap the akn tag as we don't want the 'Entire document' option for editing
+        if self.validated_data.get('is_portion'):
+            root = etree.fromstring(xml)
+            # akn will always have exactly one child, 'portion'
+            portion = root.xpath('a:portion', namespaces={'a': document.doc.namespace})[0]
+            xml = etree.tostring(portion, encoding='unicode')
+        return xml
 
 
 class NoopSerializer(object):

From 4c702f79ffa8b6f032385cd77dd23f86e09b5b8d Mon Sep 17 00:00:00 2001
From: goose-life <32566441+goose-life@users.noreply.github.com>
Date: Fri, 10 Jan 2025 18:23:54 +0200
Subject: [PATCH 04/28] validate XML, improve updated_xml()

---
 indigo_api/serializers.py | 29 ++++++++++++++++++++---------
 1 file changed, 20 insertions(+), 9 deletions(-)

diff --git a/indigo_api/serializers.py b/indigo_api/serializers.py
index c2d3a7ffd..b6593e07c 100644
--- a/indigo_api/serializers.py
+++ b/indigo_api/serializers.py
@@ -15,7 +15,7 @@
 from typing import List
 
 from cobalt import StructuredDocument, FrbrUri
-from cobalt.akn import AKN_NAMESPACES
+from cobalt.akn import AKN_NAMESPACES, DEFAULT_VERSION
 from indigo_api.models import Document, Attachment, Annotation, DocumentActivity, Work, Amendment, Language, \
     PublicationDocument, Task, Commencement
 from indigo_api.signals import document_published
@@ -440,7 +440,22 @@ class DocumentAPISerializer(serializers.Serializer):
     language = serializers.CharField(min_length=3, max_length=3)
     is_portion = serializers.BooleanField()
 
-    def update_document(self, document):
+    def validate_xml(self, xml):
+        """ mostly copied from DocumentSerializer.validate()
+        """
+        try:
+            doctype = self.instance.work_uri.doctype if not self.initial_data.get('is_portion') else 'portion'
+            doc = StructuredDocument.for_document_type(doctype)(xml)
+        except (LxmlError, ValueError) as e:
+            raise ValidationError("Invalid XML: %s" % str(e))
+        # ensure the correct namespace
+        if doc.namespace != AKN_NAMESPACES[DEFAULT_VERSION]:
+            raise ValidationError(
+                f"Document must have namespace {AKN_NAMESPACES[DEFAULT_VERSION]}, but it has {doc.namespace} instead.")
+        return xml
+
+    def update_document(self):
+        document = self.instance
         # the language could have been changed but not yet saved during editing, which might affect which plugin is used
         if document.language.code != self.validated_data.get('language'):
             document.language = Language.for_code(self.validated_data.get('language'))
@@ -450,15 +465,11 @@ def update_document(self, document):
         document.content = self.validated_data.get('xml')
         return document
 
-    def updated_xml(self, document):
-        xml = document.document_xml
+    def updated_xml(self):
         # in provision mode, unwrap the akn tag as we don't want the 'Entire document' option for editing
         if self.validated_data.get('is_portion'):
-            root = etree.fromstring(xml)
-            # akn will always have exactly one child, 'portion'
-            portion = root.xpath('a:portion', namespaces={'a': document.doc.namespace})[0]
-            xml = etree.tostring(portion, encoding='unicode')
-        return xml
+            return etree.tostring(self.instance.doc.portion, encoding='unicode')
+        return self.instance.document_xml
 
 
 class NoopSerializer(object):

From f9345adef6df1339fd1e89938e61125134cabfaf Mon Sep 17 00:00:00 2001
From: goose-life <32566441+goose-life@users.noreply.github.com>
Date: Fri, 10 Jan 2025 18:26:16 +0200
Subject: [PATCH 05/28] update MarkUpItalicsTermsView to use new serializer

---
 indigo_api/views/documents.py | 35 ++++-------------------------------
 1 file changed, 4 insertions(+), 31 deletions(-)

diff --git a/indigo_api/views/documents.py b/indigo_api/views/documents.py
index a018a2a98..fad4aa930 100644
--- a/indigo_api/views/documents.py
+++ b/indigo_api/views/documents.py
@@ -14,7 +14,6 @@
 from django.utils.translation import gettext as _
 from django_comments.models import Comment
 
-from lxml import etree
 from rest_framework.exceptions import ValidationError, MethodNotAllowed
 from rest_framework.views import APIView
 from rest_framework import mixins, viewsets, renderers, status
@@ -31,7 +30,7 @@
 from indigo.analysis.refs.base import markup_document_refs
 from indigo.plugins import plugins
 from indigo.xmlutils import parse_html_str
-from ..models import Document, Annotation, DocumentActivity, Task
+from ..models import Document, Annotation, DocumentActivity, Task, Language, Work
 from ..serializers import DocumentSerializer, RenderSerializer, ParseSerializer, DocumentAPISerializer, VersionSerializer, AnnotationSerializer, DocumentActivitySerializer, TaskSerializer, DocumentDiffSerializer
 from ..renderers import AkomaNtosoRenderer, PDFRenderer, EPUBRenderer, HTMLRenderer, ZIPRenderer
 from indigo_api.exporters import HTMLExporter
@@ -405,37 +404,11 @@ class MarkUpItalicsTermsView(DocumentResourceView, APIView):
     """ Find and mark up italics terms.
     """
     def post(self, request, document_id):
-        # in provision mode, fake a full e.g. <act> using the portion XML
-        data = self.request.data
-        provision_eid = data['document'].pop('provision_eid', None)
-        if provision_eid:
-            data['document']['content'] = self.switch_doctype(data['document']['content'], self.document.work.work_uri.doctype)
-
-        serializer = DocumentAPISerializer(instance=self.document, data=data)
-        serializer.fields['document'].fields['content'].required = True
+        serializer = DocumentAPISerializer(instance=self.document, data=self.request.data)
         serializer.is_valid(raise_exception=True)
-        document = serializer.fields['document'].update_document(self.document, serializer.validated_data['document'])
-
+        document = serializer.update_document()
         self.mark_up_italics(document)
-
-        # in provision mode, turn the fake full document back into a portion, minus the wrapping akn tag
-        if provision_eid:
-            document.document_xml = self.switch_doctype(document.document_xml, self.document.work.work_uri.doctype, back_to_portion=True)
-
-        return Response({'document': {'content': document.document_xml}})
-
-    def switch_doctype(self, xml, doctype, back_to_portion=False):
-        old_doc = StructuredDocument.for_document_type(doctype if back_to_portion else 'portion')(xml)
-        new_doc = StructuredDocument.for_document_type('portion' if back_to_portion else doctype)()
-        # delete any empty section in body, for example
-        for blank in new_doc.main_content.xpath('*'):
-            new_doc.main_content.remove(blank)
-        for elem in old_doc.main_content.xpath('*'):
-            new_doc.main_content.append(elem)
-        if back_to_portion:
-            # no wrapping akn tag in provision mode
-            return etree.tostring(new_doc.main, encoding='unicode')
-        return etree.tostring(new_doc.root, encoding='unicode')
+        return Response({'xml': serializer.updated_xml()})
 
     def mark_up_italics(self, document):
         italics_terms_finder = plugins.for_document('italics-terms', document)

From 3a24517a2d04afb12fc3979466a455d937bcc31a Mon Sep 17 00:00:00 2001
From: goose-life <32566441+goose-life@users.noreply.github.com>
Date: Fri, 10 Jan 2025 18:28:43 +0200
Subject: [PATCH 06/28] add and use wrapInAkn, toSimplifiedJSON to
 DocumentContent

---
 indigo_app/static/javascript/indigo/models.js | 23 +++++++++++++++----
 .../indigo/views/document_italics_terms.js    | 13 +++--------
 2 files changed, 21 insertions(+), 15 deletions(-)

diff --git a/indigo_app/static/javascript/indigo/models.js b/indigo_app/static/javascript/indigo/models.js
index 74be22db0..35855d3ff 100644
--- a/indigo_app/static/javascript/indigo/models.js
+++ b/indigo_app/static/javascript/indigo/models.js
@@ -73,6 +73,23 @@
       return Indigo.toXml(node || this.xmlDocument);
     },
 
+    // in provision mode, wrap serialized XML in akn tags
+    wrapInAkn: function (xml) {
+      if (Indigo.Preloads.provisionEid) {
+        xml = `<akomaNtoso xmlns="${this.xmlDocument.firstChild.getAttribute('xmlns')}">${xml}</akomaNtoso>`;
+      }
+      return xml;
+    },
+
+    // serialise the absolute basics of the document for server-side XML manipulation
+    toSimplifiedJSON: function() {
+      return {
+        "xml": this.wrapInAkn(this.toXml()),
+        "language": this.document.attributes.language,
+        "is_portion": Indigo.Preloads.provisionEid !== ""
+      };
+    },
+
     /**
      * Replaces (or deletes) an existing node (or the whole tree) with a new node or nodes.
      * Triggers a change event.
@@ -184,11 +201,7 @@
       // When saving document contents, save all document details, so that we capture all
       // changes in a single revision on the server.
       // We do this by delegating to the document object.
-      let content = this.get('content');
-      if (Indigo.Preloads.provisionEid) {
-        content = `<akomaNtoso xmlns="${this.xmlDocument.firstChild.getAttribute('xmlns')}">${content}</akomaNtoso>`;
-      }
-      this.document.attributes.content = content;
+      this.document.attributes.content = this.wrapInAkn(this.get('content'));
       this.document.attributes.provision_eid = Indigo.Preloads.provisionEid;
       var result = this.document.save();
       // XXX works around https://github.com/Code4SA/indigo/issues/20 by not parsing
diff --git a/indigo_app/static/javascript/indigo/views/document_italics_terms.js b/indigo_app/static/javascript/indigo/views/document_italics_terms.js
index a1b2a2b27..b78bc87d9 100644
--- a/indigo_app/static/javascript/indigo/views/document_italics_terms.js
+++ b/indigo_app/static/javascript/indigo/views/document_italics_terms.js
@@ -36,16 +36,9 @@
     },
 
     markUpItalics: function(e) {
-      var self = this,
+      let self = this,
           $btn = this.$el.find('.mark-up-italics'),
-          data = {'document': this.model.document.toJSON()};
-
-      let content = this.model.toXml();
-      if (Indigo.Preloads.provisionEid) {
-        content = `<akomaNtoso xmlns="${this.model.xmlDocument.firstChild.getAttribute('xmlns')}">${content}</akomaNtoso>`;
-        data.document.provision_eid = Indigo.Preloads.provisionEid;
-      }
-      data.document.content = content;
+          data = this.model.toSimplifiedJSON();
 
       this.$('a[href="#this-document-italics-terms"]').click();
 
@@ -60,7 +53,7 @@
         contentType: "application/json; charset=utf-8",
         dataType: "json"})
         .then(function(response) {
-          self.model.set('content', response.document.content);
+          self.model.set('content', response.xml);
         })
         .always(function() {
           $btn

From b3009f45abcce1993edbf0f8e25b673388528a6c Mon Sep 17 00:00:00 2001
From: goose-life <32566441+goose-life@users.noreply.github.com>
Date: Mon, 13 Jan 2025 09:10:31 +0200
Subject: [PATCH 07/28] new ManipulateXmlView superclass for all four views

---
 indigo_api/serializers.py     |  1 -
 indigo_api/views/documents.py | 75 ++++++++++++-----------------------
 2 files changed, 26 insertions(+), 50 deletions(-)

diff --git a/indigo_api/serializers.py b/indigo_api/serializers.py
index b6593e07c..e08a2ab07 100644
--- a/indigo_api/serializers.py
+++ b/indigo_api/serializers.py
@@ -463,7 +463,6 @@ def update_document(self):
         if self.validated_data.get('is_portion'):
             document.work.work_uri.doctype = 'portion'
         document.content = self.validated_data.get('xml')
-        return document
 
     def updated_xml(self):
         # in provision mode, unwrap the akn tag as we don't want the 'Entire document' option for editing
diff --git a/indigo_api/views/documents.py b/indigo_api/views/documents.py
index fad4aa930..7e428d5f9 100644
--- a/indigo_api/views/documents.py
+++ b/indigo_api/views/documents.py
@@ -364,76 +364,53 @@ def post(self, request, document_id):
             return Response({'output': document.to_html()})
 
 
-class LinkTermsView(DocumentResourceView, APIView):
-    """ Support for running term discovery and linking on a document.
-    """
+class ManipulateXmlView(DocumentResourceView, APIView):
     def post(self, request, document_id):
         serializer = DocumentAPISerializer(instance=self.document, data=self.request.data)
-        serializer.fields['document'].fields['content'].required = True
         serializer.is_valid(raise_exception=True)
-        document = serializer.fields['document'].update_document(self.document, serializer.validated_data['document'])
+        serializer.update_document()
+        self.manipulate_xml()
+        return Response({'xml': serializer.updated_xml()})
 
-        self.link_terms(document)
+    def manipulate_xml(self):
+        raise NotImplementedError()
 
-        return Response({'document': {'content': document.document_xml}})
 
-    def link_terms(self, doc):
-        finder = plugins.for_document('terms', doc)
+class LinkTermsView(ManipulateXmlView):
+    """ Support for running term discovery and linking on a document.
+    """
+    def manipulate_xml(self):
+        # TODO: do the two different steps separately and a bit differently in provision mode
+        finder = plugins.for_document('terms', self.document)
         if finder:
-            finder.find_terms_in_document(doc)
+            finder.find_terms_in_document(self.document)
 
 
-class LinkReferencesView(DocumentResourceView, APIView):
+class LinkReferencesView(ManipulateXmlView):
     """ Find and link internal references and references to other works.
     """
-    def post(self, request, document_id):
-        serializer = DocumentAPISerializer(instance=self.document, data=self.request.data)
-        serializer.fields['document'].fields['content'].required = True
-        serializer.is_valid(raise_exception=True)
-        document = serializer.fields['document'].update_document(self.document, serializer.validated_data['document'])
-
-        self.find_references(document)
-
-        return Response({'document': {'content': document.document_xml}})
-
-    def find_references(self, document):
-        markup_document_refs(document)
+    def manipulate_xml(self):
+        # TODO: markup all references and only return the relevant portion in provision mode?
+        markup_document_refs(self.document)
 
 
-class MarkUpItalicsTermsView(DocumentResourceView, APIView):
+class MarkUpItalicsTermsView(ManipulateXmlView):
     """ Find and mark up italics terms.
     """
-    def post(self, request, document_id):
-        serializer = DocumentAPISerializer(instance=self.document, data=self.request.data)
-        serializer.is_valid(raise_exception=True)
-        document = serializer.update_document()
-        self.mark_up_italics(document)
-        return Response({'xml': serializer.updated_xml()})
-
-    def mark_up_italics(self, document):
-        italics_terms_finder = plugins.for_document('italics-terms', document)
-        italics_terms = document.work.country.italics_terms
+    def manipulate_xml(self):
+        italics_terms_finder = plugins.for_document('italics-terms', self.document)
+        italics_terms = self.document.work.country.italics_terms
         if italics_terms_finder and italics_terms:
-            italics_terms_finder.mark_up_italics_in_document(document, italics_terms)
+            italics_terms_finder.mark_up_italics_in_document(self.document, italics_terms)
 
 
-class SentenceCaseHeadingsView(DocumentResourceView, APIView):
+class SentenceCaseHeadingsView(ManipulateXmlView):
     """ Sentence case headings. Also apply accents as needed / relevant.
     """
-    def post(self, request, document_id):
-        serializer = DocumentAPISerializer(instance=self.document, data=self.request.data)
-        serializer.fields['document'].fields['content'].required = True
-        serializer.is_valid(raise_exception=True)
-        document = serializer.fields['document'].update_document(self.document, serializer.validated_data['document'])
-
-        self.sentence_case(document)
-
-        return Response({'document': {'content': document.document_xml}})
-
-    def sentence_case(self, document):
-        sentence_caser = plugins.for_document('sentence-caser', document)
+    def manipulate_xml(self):
+        sentence_caser = plugins.for_document('sentence-caser', self.document)
         if sentence_caser:
-            sentence_caser.sentence_case_headings_in_document(document)
+            sentence_caser.sentence_case_headings_in_document(self.document)
 
 
 class DocumentDiffView(DocumentResourceView, APIView):

From 645c0ba325467e2efb2cc13e1ed8512747763f03 Mon Sep 17 00:00:00 2001
From: goose-life <32566441+goose-life@users.noreply.github.com>
Date: Mon, 13 Jan 2025 09:14:54 +0200
Subject: [PATCH 08/28] other JS views use the new pattern too

---
 .../javascript/indigo/views/document_defined_terms.js      | 7 ++-----
 .../static/javascript/indigo/views/document_references.js  | 6 ++----
 .../javascript/indigo/views/document_sentences_caser.js    | 6 ++----
 .../templates/indigo_api/document/_italics_terms.html      | 2 +-
 indigo_app/templates/indigo_api/document/_toolbar.html     | 6 +++---
 5 files changed, 10 insertions(+), 17 deletions(-)

diff --git a/indigo_app/static/javascript/indigo/views/document_defined_terms.js b/indigo_app/static/javascript/indigo/views/document_defined_terms.js
index c679aeb9d..e54359aa0 100644
--- a/indigo_app/static/javascript/indigo/views/document_defined_terms.js
+++ b/indigo_app/static/javascript/indigo/views/document_defined_terms.js
@@ -38,10 +38,7 @@
     linkTerms: function(e) {
       var self = this,
           $btn = this.$el.find('.link-terms'),
-          data = {};
-
-      data.document = this.model.document.toJSON();
-      data.document.content = this.model.toXml();
+          data = this.model.toSimplifiedJSON();
 
       $btn
         .prop('disabled', true)
@@ -54,7 +51,7 @@
         contentType: "application/json; charset=utf-8",
         dataType: "json"})
         .then(function(response) {
-          self.model.set('content', response.document.content);
+          self.model.set('content', response.xml);
         })
         .always(function() {
           $btn
diff --git a/indigo_app/static/javascript/indigo/views/document_references.js b/indigo_app/static/javascript/indigo/views/document_references.js
index 665e3c4cc..5584d1dad 100644
--- a/indigo_app/static/javascript/indigo/views/document_references.js
+++ b/indigo_app/static/javascript/indigo/views/document_references.js
@@ -53,9 +53,7 @@
     findReferences: function(e) {
       var self = this,
           $btn = this.$el.find('.find-references'),
-          data = {'document': this.model.document.toJSON()};
-
-      data.document.content = this.model.toXml();
+          data = this.model.toSimplifiedJSON();
 
       $btn
         .prop('disabled', true)
@@ -68,7 +66,7 @@
         contentType: "application/json; charset=utf-8",
         dataType: "json"})
         .then(function(response) {
-          self.model.set('content', response.document.content);
+          self.model.set('content', response.xml);
         })
         .always(function() {
           $btn
diff --git a/indigo_app/static/javascript/indigo/views/document_sentences_caser.js b/indigo_app/static/javascript/indigo/views/document_sentences_caser.js
index 18f8db75a..54c4adef9 100644
--- a/indigo_app/static/javascript/indigo/views/document_sentences_caser.js
+++ b/indigo_app/static/javascript/indigo/views/document_sentences_caser.js
@@ -15,9 +15,7 @@
 
     sentenceCaseHeadings: function(e) {
       let self = this,
-          data = {'document': this.model.document.toJSON()};
-
-      data.document.content = this.model.toXml();
+          data = this.model.toSimplifiedJSON();
 
       $.ajax({
         url: this.model.document.url() + '/analysis/sentence-case-headings',
@@ -26,7 +24,7 @@
         contentType: "application/json; charset=utf-8",
         dataType: "json"})
           .then(function(response) {
-            self.model.set('content', response.document.content);
+            self.model.set('content', response.xml);
           });
     },
   });
diff --git a/indigo_app/templates/indigo_api/document/_italics_terms.html b/indigo_app/templates/indigo_api/document/_italics_terms.html
index 06db13d44..93a9d3373 100644
--- a/indigo_app/templates/indigo_api/document/_italics_terms.html
+++ b/indigo_app/templates/indigo_api/document/_italics_terms.html
@@ -70,7 +70,7 @@ <h5 class="modal-title">{% trans 'Italicised terms' %}</h5>
     </tbody>
   </table>
   {{else}}
-  <em>{% endverbatim %}{% trans 'No italicised terms.' %}{% verbatim %}.</em>
+  <em>{% endverbatim %}{% trans 'No italicised terms.' %}{% verbatim %}</em>
   {{/if}}
   {% endverbatim %}
 
diff --git a/indigo_app/templates/indigo_api/document/_toolbar.html b/indigo_app/templates/indigo_api/document/_toolbar.html
index f943d1cbd..8f8df5f17 100644
--- a/indigo_app/templates/indigo_api/document/_toolbar.html
+++ b/indigo_app/templates/indigo_api/document/_toolbar.html
@@ -29,17 +29,17 @@
     <button type="button" class="btn btn-outline-secondary dropdown-toggle" data-bs-toggle="dropdown"><i class="fas fa-magic"></i> {% trans "Analysis" %}</button>
     <div class="dropdown-menu">
       {% block document-analysis-menu-items %}
-        <a class="dropdown-item {% if provision_eid %}disabled{% endif %}"
+        <a class="dropdown-item"
            href="#defined-terms-modal" data-bs-toggle="modal"
         >{% trans "Defined terms..." %}</a>
-        <a class="dropdown-item {% if provision_eid %}disabled{% endif %}"
+        <a class="dropdown-item"
            href="#references-modal" data-bs-toggle="modal"
         >{% trans "References..." %}</a>
         <a class="dropdown-item"
            href="#italics-modal" data-bs-toggle="modal"
         >{% trans "Italicised terms..." %}</a>
         <div class="dropdown-divider"></div>
-        <a class="dropdown-item {% if provision_eid %}disabled{% endif %}"
+        <a class="dropdown-item"
            href="#" id="sentence-caser"
         ><span class="sentence-case-headings">{% trans "Make headings Sentence case" %}</span></a>
       {% endblock %}

From 4dcc5d591a0dedd284dde1b85462270babe2cff5 Mon Sep 17 00:00:00 2001
From: goose-life <32566441+goose-life@users.noreply.github.com>
Date: Mon, 13 Jan 2025 11:56:04 +0200
Subject: [PATCH 09/28] update existing, add new test_link_terms

---
 indigo_api/tests/fixtures.py          | 42 +++++++++++++++
 indigo_api/tests/test_analysis_api.py | 76 ++++++++++++++++++++-------
 2 files changed, 100 insertions(+), 18 deletions(-)

diff --git a/indigo_api/tests/fixtures.py b/indigo_api/tests/fixtures.py
index 377fcb80c..43f6f9b15 100644
--- a/indigo_api/tests/fixtures.py
+++ b/indigo_api/tests/fixtures.py
@@ -33,6 +33,41 @@
 </akomaNtoso>
 """
 
+PORTION_FIXTURE = """<akomaNtoso xmlns="http://docs.oasis-open.org/legaldocml/ns/akn/3.0">
+  <portion name="portion">
+    <meta>
+      <identification source="">
+        <FRBRWork>
+          <FRBRthis value="/akn/za/act/1900/1/!main"/>
+          <FRBRuri value="/akn/za/act/1900/1"/>
+          <FRBRalias value="Untitled"/>
+          <FRBRdate date="1900-01-01" name="Generation"/>
+          <FRBRauthor href="#council" as="#author"/>
+          <FRBRcountry value="za"/>
+        </FRBRWork>
+        <FRBRExpression>
+          <FRBRthis value="/akn/za/act/1900/1/eng@/!main"/>
+          <FRBRuri value="/akn/za/act/1900/1/eng@"/>
+          <FRBRdate date="1900-01-01" name="Generation"/>
+          <FRBRauthor href="#council" as="#author"/>
+          <FRBRlanguage language="eng"/>
+        </FRBRExpression>
+        <FRBRManifestation>
+          <FRBRthis value="/akn/za/act/1900/1/eng@/!main"/>
+          <FRBRuri value="/akn/za/act/1900/1/eng@"/>
+          <FRBRdate date="1900-01-01" name="Generation"/>
+          <FRBRauthor href="#council" as="#author"/>
+        </FRBRManifestation>
+      </identification>
+      <publication date="2005-07-24" name="Province of Western Cape: σπαθιοῦ Gazette" number="6277" showAs="Province of Western Cape: 😀 Provincial Gazette"/>
+    </meta>
+    <portionBody>
+      %s
+    </portionBody>
+  </portion>
+</akomaNtoso>
+"""
+
 BODY_FIXTURE = """
 <body>
   <section eId="sec_1">
@@ -130,6 +165,13 @@ def document_fixture(text=None, xml=None):
     return DOCUMENT_FIXTURE % xml
 
 
+def portion_fixture(text=None, xml=None):
+    if text:
+        xml = """<section eId="sec_1"><content><p>%s</p></content></section>""" % text
+
+    return PORTION_FIXTURE % xml
+
+
 def component_fixture(text=None, xml=None):
     if text:
         xml = """<section eId="sec_1"><content><p>%s</p></content></section>""" % text
diff --git a/indigo_api/tests/test_analysis_api.py b/indigo_api/tests/test_analysis_api.py
index eea0c1109..bd498f56d 100644
--- a/indigo_api/tests/test_analysis_api.py
+++ b/indigo_api/tests/test_analysis_api.py
@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-
 from nose.tools import *  # noqa
 from rest_framework.test import APITestCase
 
@@ -19,17 +17,12 @@ def setUp(self):
 
     def test_link_terms_no_input(self):
         response = self.client.post('/api/documents/1/analysis/link-terms', {
-            'document': {},
         })
         assert_equal(response.status_code, 400)
 
     def test_link_terms(self):
         response = self.client.post('/api/documents/1/analysis/link-terms', {
-            'document': {
-                'frbr_uri': '/za/act/1992/1',
-                'expression_date': '2001-01-01',
-                'language': 'eng',
-                'content': document_fixture(xml="""
+            'xml': document_fixture(xml="""
 <section id="section-1">
   <num>1.</num>
   <heading>Definitions and interpretation</heading>
@@ -64,28 +57,74 @@ def test_link_terms(self):
     </content>
   </subsection>
 </section>
-""")
-            }
+"""),
+            'language': 'eng',
+            'is_portion': False
         })
         assert_equal(response.status_code, 200)
 
-        content = response.data['document']['content']
+        content = response.data['xml']
 
         assert_true(content.startswith('<akomaNtoso'))
         assert_in('<def ', content)
         assert_in('<TLCTerm ', content)
 
+    def test_link_terms_portion(self):
+        response = self.client.post('/api/documents/1/analysis/link-terms', {
+            'xml': portion_fixture(xml="""
+<section id="section-1">
+  <num>1.</num>
+  <heading>Definitions and interpretation</heading>
+  <subsection id="section-1.subsection-0">
+    <content>
+      <p>In these By-laws, any word or expression that has been defined in the National Road Traffic Act, 1996 (Act No. 93 of 1996) has that meaning and, unless the context otherwise indicates –</p>
+    </content>
+  </subsection>
+  <subsection id="section-1.subsection-1">
+    <content>
+      <blockList id="section-1.subsection-1.list1">
+        <listIntroduction>"authorised official" means –</listIntroduction>
+        <item id="section-1.subsection-1.list1.a">
+          <num>(a)</num>
+          <p>a member of the Johannesburg Metropolitan Police established in terms of section 64A of the South African Police Service Act, 1995 (Act No. 68 of 1995); or</p>
+        </item>
+        <item id="section-1.subsection-1.list1.b">
+          <num>(b)</num>
+          <p>any person or official authorised as such, in writing, by the Council;</p>
+        </item>
+      </blockList>
+    </content>
+  </subsection>
+  <subsection id="section-1.subsection-2">
+    <content>
+      <p>"backfill" means to replace the structural layers, including the base, sub-base, sudgrade and subgrade but excluding the surfacing, in a trench dug in, or other excavation of, a road reserve, and “backfilling” is construed accordingly;</p>
+    </content>
+  </subsection>
+  <subsection id="section-1.subsection-3">
+    <content>
+      <p>"these By-Laws" includes the schedules;</p>
+    </content>
+  </subsection>
+</section>
+"""),
+            'language': 'eng',
+            'is_portion': True
+        })
+        assert_equal(response.status_code, 200)
+
+        content = response.data['xml']
+
+        assert_true(content.startswith('<portion '))
+        assert_in('<def ', content)
+        assert_in('<TLCTerm ', content)
+
     def test_link_terms_no_perms(self):
         self.client.logout()
         self.assertTrue(self.client.login(username='no-perms@example.com', password='password'))
         user = User.objects.get(username='no-perms@example.com')
 
         data = {
-            'document': {
-                'frbr_uri': '/za/act/1992/1',
-                'expression_date': '2001-01-01',
-                'language': 'eng',
-                'content': document_fixture(xml="""
+            'xml': document_fixture(xml="""
 <section id="section-1">
   <num>1.</num>
   <heading>Definitions and interpretation</heading>
@@ -93,8 +132,9 @@ def test_link_terms_no_perms(self):
     <p>test</p>
   </content>
 </section>
-""")
-            }
+"""),
+            'language': 'eng',
+            'is_portion': False
         }
 
         # user doesn't have perms

From 24657e56b3a01b77615e2287c22e750d74b0e3b0 Mon Sep 17 00:00:00 2001
From: goose-life <32566441+goose-life@users.noreply.github.com>
Date: Mon, 13 Jan 2025 17:02:19 +0200
Subject: [PATCH 10/28] latest docpipe (portionBody included in ancestors)

---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index 26b52221a..f93021743 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -52,7 +52,7 @@ dependencies = [
     "django-treebeard>=4.5.1",
     "djangorestframework-xml>=1.3.0",
     "djangorestframework>=3.15.0",
-    "docpipe @ git+https://github.com/laws-africa/docpipe@5ceee068e0bb2929e33f080c2d1f3063eff0857d",
+    "docpipe @ git+https://github.com/laws-africa/docpipe@78850974709df154fd2f9ddd318598a644250a74",
     "drf-spectacular==0.27.2",
     "EbookLib>=0.15",
     "google-api-python-client>=1.7.9",

From 2c6d54d95d3c8531005e14b2c0193b42616c187f Mon Sep 17 00:00:00 2001
From: goose-life <32566441+goose-life@users.noreply.github.com>
Date: Thu, 16 Jan 2025 11:20:18 +0200
Subject: [PATCH 11/28] set the content once when removing terms, italics,
 references

---
 .../indigo/views/document_defined_terms.js           | 12 ++++++++----
 .../indigo/views/document_italics_terms.js           |  6 +++++-
 .../javascript/indigo/views/document_references.js   |  6 +++++-
 3 files changed, 18 insertions(+), 6 deletions(-)

diff --git a/indigo_app/static/javascript/indigo/views/document_defined_terms.js b/indigo_app/static/javascript/indigo/views/document_defined_terms.js
index 12f2c2d0b..c166086ca 100644
--- a/indigo_app/static/javascript/indigo/views/document_defined_terms.js
+++ b/indigo_app/static/javascript/indigo/views/document_defined_terms.js
@@ -66,33 +66,37 @@
 
     removeTerms: function(e) {
       if (!Indigo.view.sourceEditorView.confirmAndDiscardChanges()) return;
+      let xml = this.model.xmlDocument.cloneNode(true);
 
       // unwrap all <def>s
-      this.model.xmlDocument.querySelectorAll('def').forEach(function(def) {
+      xml.querySelectorAll('def').forEach(function(def) {
         var parent = def.parentNode;
         while (def.firstChild) parent.insertBefore(def.firstChild, def);
         parent.removeChild(def);
       });
 
       // unwrap all <term>s
-      this.model.xmlDocument.querySelectorAll('term').forEach(function(term) {
+      xml.querySelectorAll('term').forEach(function(term) {
         var parent = term.parentNode;
         while (term.firstChild) parent.insertBefore(term.firstChild, term);
         parent.removeChild(term);
       });
 
       // remove all <TLCTerm>s
-      this.model.xmlDocument.querySelectorAll('TLCTerm').forEach(function(tlcTerm) {
+      xml.querySelectorAll('TLCTerm').forEach(function(tlcTerm) {
         var parent = tlcTerm.parentNode;
         parent.removeChild(tlcTerm);
       });
 
       // remove all refersTo attributes that start with '#term-'
-      this.model.xmlDocument.querySelectorAll('[refersTo]').forEach(function(el) {
+      xml.querySelectorAll('[refersTo]').forEach(function(el) {
         if ((el.getAttribute('refersTo') || '').startsWith('#term-')) {
           el.removeAttribute('refersTo');
         }
       });
+
+      // set the content once for one mutation record
+      this.model.set('content', Indigo.toXml(xml));
     },
   });
 })(window);
diff --git a/indigo_app/static/javascript/indigo/views/document_italics_terms.js b/indigo_app/static/javascript/indigo/views/document_italics_terms.js
index 5897af21d..a8e5182f5 100644
--- a/indigo_app/static/javascript/indigo/views/document_italics_terms.js
+++ b/indigo_app/static/javascript/indigo/views/document_italics_terms.js
@@ -68,17 +68,21 @@
 
     removeItalics: function(e) {
       if (!Indigo.view.sourceEditorView.confirmAndDiscardChanges()) return;
+      let xml = this.model.xmlDocument.cloneNode(true);
 
       // remove all italics mark-up
       // TODO: this doesn't work; nuke this line or fix it
       this.$('a[href="#this-document-italics-terms"]').click();
 
-      this.model.xmlDocument.querySelectorAll('i').forEach(function(term) {
+      xml.querySelectorAll('i').forEach(function(term) {
         // get rid of <i>
         const parent = term.parentNode;
         while (term.firstChild) parent.insertBefore(term.firstChild, term);
         parent.removeChild(term);
       });
+
+      // set the content once for one mutation record
+      this.model.set('content', Indigo.toXml(xml));
     }
   });
 })(window);
diff --git a/indigo_app/static/javascript/indigo/views/document_references.js b/indigo_app/static/javascript/indigo/views/document_references.js
index e51423afe..ec268cd95 100644
--- a/indigo_app/static/javascript/indigo/views/document_references.js
+++ b/indigo_app/static/javascript/indigo/views/document_references.js
@@ -80,9 +80,10 @@
 
     removeReferences: function(e) {
       if (!Indigo.view.sourceEditorView.confirmAndDiscardChanges()) return;
+      let xml = this.model.xmlDocument.cloneNode(true);
 
       // remove all non-absolute refs
-      this.model.xmlDocument.querySelectorAll('ref').forEach(function(ref) {
+      xml.querySelectorAll('ref').forEach(function(ref) {
         if ((ref.getAttribute('href') || "").startsWith('/') || (ref.getAttribute('href') || "").startsWith('#')) {
           // get rid of ref
           const parent = ref.parentNode;
@@ -90,6 +91,9 @@
           parent.removeChild(ref);
         }
       });
+
+      // set the content once for one mutation record
+      this.model.set('content', Indigo.toXml(xml));
     },
   });
 })(window);

From e0d1aed2bcab623d567e4b35d7f8c06d6115da7b Mon Sep 17 00:00:00 2001
From: goose-life <32566441+goose-life@users.noreply.github.com>
Date: Thu, 16 Jan 2025 11:21:42 +0200
Subject: [PATCH 12/28] add get_provision_xml to Document model

---
 indigo_api/models/documents.py | 9 +++++++++
 indigo_app/views/documents.py  | 8 ++------
 2 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/indigo_api/models/documents.py b/indigo_api/models/documents.py
index 772fbb333..228e3afcb 100644
--- a/indigo_api/models/documents.py
+++ b/indigo_api/models/documents.py
@@ -548,6 +548,15 @@ def change_date(self, new_date, user, comment=None):
             self.expression_date = new_date
             self.save_with_revision(user, comment=comment)
 
+    def get_provision_xml(self, provision_eid):
+        provision_xml = self.doc.get_portion_element(provision_eid)
+        if not provision_xml:
+            return None
+        portion = StructuredDocument.for_document_type('portion')()
+        portion.frbr_uri = self.frbr_uri
+        portion.main_content.append(provision_xml)
+        return portion.main
+
     def update_provision_xml(self, provision_eid, provision_xml):
         xml = etree.fromstring(provision_xml)
         # portionBody will always have exactly one child
diff --git a/indigo_app/views/documents.py b/indigo_app/views/documents.py
index 0b8f3862a..8ec25f94c 100644
--- a/indigo_app/views/documents.py
+++ b/indigo_app/views/documents.py
@@ -130,14 +130,10 @@ class DocumentProvisionDetailView(DocumentDetailView):
     def get(self, request, *args, **kwargs):
         document = self.get_object()
         self.eid = self.kwargs.get('eid')
-        provision_xml = document.doc.get_portion_element(self.eid)
-        if not provision_xml:
+        self.provision_xml = document.get_provision_xml(self.eid)
+        if not self.provision_xml:
             messages.error(request, _("No provision with this id found: '%(eid)s'") % {"eid": self.eid})
             return redirect('choose_document_provision', doc_id=document.id)
-        portion = Portion()
-        portion.frbr_uri = document.frbr_uri
-        portion.main_content.append(provision_xml)
-        self.provision_xml = portion.main
         return super().get(request, *args, **kwargs)
 
     def get_context_data(self, **kwargs):

From 00934e92a45438736e7787c288350c9090f17392 Mon Sep 17 00:00:00 2001
From: goose-life <32566441+goose-life@users.noreply.github.com>
Date: Thu, 16 Jan 2025 11:23:10 +0200
Subject: [PATCH 13/28] use provision_eid, not is_portion

---
 indigo_api/serializers.py                     | 4 ++--
 indigo_api/tests/test_analysis_api.py         | 6 +++---
 indigo_app/static/javascript/indigo/models.js | 2 +-
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/indigo_api/serializers.py b/indigo_api/serializers.py
index e08a2ab07..b7f46410f 100644
--- a/indigo_api/serializers.py
+++ b/indigo_api/serializers.py
@@ -438,13 +438,13 @@ class DocumentAPISerializer(serializers.Serializer):
     """
     xml = serializers.CharField()
     language = serializers.CharField(min_length=3, max_length=3)
-    is_portion = serializers.BooleanField()
+    provision_eid = serializers.CharField(allow_blank=True)
 
     def validate_xml(self, xml):
         """ mostly copied from DocumentSerializer.validate()
         """
         try:
-            doctype = self.instance.work_uri.doctype if not self.initial_data.get('is_portion') else 'portion'
+            doctype = self.instance.work_uri.doctype if not self.initial_data.get('provision_eid') else 'portion'
             doc = StructuredDocument.for_document_type(doctype)(xml)
         except (LxmlError, ValueError) as e:
             raise ValidationError("Invalid XML: %s" % str(e))
diff --git a/indigo_api/tests/test_analysis_api.py b/indigo_api/tests/test_analysis_api.py
index bd498f56d..d3cb37f98 100644
--- a/indigo_api/tests/test_analysis_api.py
+++ b/indigo_api/tests/test_analysis_api.py
@@ -59,7 +59,7 @@ def test_link_terms(self):
 </section>
 """),
             'language': 'eng',
-            'is_portion': False
+            'provision_eid': ""
         })
         assert_equal(response.status_code, 200)
 
@@ -108,7 +108,7 @@ def test_link_terms_portion(self):
 </section>
 """),
             'language': 'eng',
-            'is_portion': True
+            'provision_eid': "section-1"
         })
         assert_equal(response.status_code, 200)
 
@@ -134,7 +134,7 @@ def test_link_terms_no_perms(self):
 </section>
 """),
             'language': 'eng',
-            'is_portion': False
+            'provision_eid': ""
         }
 
         # user doesn't have perms
diff --git a/indigo_app/static/javascript/indigo/models.js b/indigo_app/static/javascript/indigo/models.js
index 79c45556d..01349728d 100644
--- a/indigo_app/static/javascript/indigo/models.js
+++ b/indigo_app/static/javascript/indigo/models.js
@@ -159,7 +159,7 @@
       return {
         "xml": this.wrapInAkn(this.toXml()),
         "language": this.document.attributes.language,
-        "is_portion": Indigo.Preloads.provisionEid !== ""
+        "provision_eid": Indigo.Preloads.provisionEid
       };
     },
 

From a0beafb9d16d6e730dc513b508bd059d7e9e18d4 Mon Sep 17 00:00:00 2001
From: goose-life <32566441+goose-life@users.noreply.github.com>
Date: Thu, 16 Jan 2025 11:34:46 +0200
Subject: [PATCH 14/28] introduce use_full_xml for analyses that require the
 whole document

---
 indigo_api/serializers.py     | 41 ++++++++++++++++++++++++++---------
 indigo_api/views/documents.py |  9 ++++++--
 2 files changed, 38 insertions(+), 12 deletions(-)

diff --git a/indigo_api/serializers.py b/indigo_api/serializers.py
index b7f46410f..5499b3179 100644
--- a/indigo_api/serializers.py
+++ b/indigo_api/serializers.py
@@ -457,18 +457,39 @@ def validate_xml(self, xml):
     def update_document(self):
         document = self.instance
         # the language could have been changed but not yet saved during editing, which might affect which plugin is used
-        if document.language.code != self.validated_data.get('language'):
-            document.language = Language.for_code(self.validated_data.get('language'))
-        # the FRBR URI doctype needs to match the XML root's child; change it to portion in provision mode
-        if self.validated_data.get('is_portion'):
-            document.work.work_uri.doctype = 'portion'
-        document.content = self.validated_data.get('xml')
+        language_code = self.validated_data.get('language')
+        if document.language.code != language_code:
+            document.language = Language.for_code(language_code)
+        # update the content with updated but unsaved changes too
+        self.set_content()
+
+    def set_content(self):
+        document = self.instance
+        xml = self.validated_data.get('xml')
+        provision_eid = self.validated_data.get('provision_eid')
+        if not provision_eid:
+            # update the document's full XML with what's in the editor
+            document.content = xml
+        else:
+            if self.use_full_xml:
+                # we'll need the document's full XML for the analysis,
+                # but update the relevant provision with what's in the editor first
+                document.update_provision_xml(provision_eid, xml)
+            else:
+                # update the document to be a 'portion' and use only what's in the editor as the content
+                document.work.work_uri.doctype = 'portion'
+                document.content = xml
 
     def updated_xml(self):
-        # in provision mode, unwrap the akn tag as we don't want the 'Entire document' option for editing
-        if self.validated_data.get('is_portion'):
-            return etree.tostring(self.instance.doc.portion, encoding='unicode')
-        return self.instance.document_xml
+        document = self.instance
+        provision_eid = self.validated_data.get('provision_eid')
+        if not provision_eid:
+            # return the document's full updated XML
+            return document.document_xml
+        # otherwise, return only the provision being edited (NOT including the outer akn tag)
+        # if we used the full XML for the analysis, grab only the appropriate provision as a portion
+        xml = document.get_provision_xml(provision_eid) if self.use_full_xml else document.doc.portion
+        return etree.tostring(xml, encoding='unicode')
 
 
 class NoopSerializer(object):
diff --git a/indigo_api/views/documents.py b/indigo_api/views/documents.py
index 7e428d5f9..0e1e61c05 100644
--- a/indigo_api/views/documents.py
+++ b/indigo_api/views/documents.py
@@ -365,8 +365,11 @@ def post(self, request, document_id):
 
 
 class ManipulateXmlView(DocumentResourceView, APIView):
+    use_full_xml = False
+
     def post(self, request, document_id):
         serializer = DocumentAPISerializer(instance=self.document, data=self.request.data)
+        serializer.use_full_xml = self.use_full_xml
         serializer.is_valid(raise_exception=True)
         serializer.update_document()
         self.manipulate_xml()
@@ -379,8 +382,9 @@ def manipulate_xml(self):
 class LinkTermsView(ManipulateXmlView):
     """ Support for running term discovery and linking on a document.
     """
+    use_full_xml = True
+
     def manipulate_xml(self):
-        # TODO: do the two different steps separately and a bit differently in provision mode
         finder = plugins.for_document('terms', self.document)
         if finder:
             finder.find_terms_in_document(self.document)
@@ -389,8 +393,9 @@ def manipulate_xml(self):
 class LinkReferencesView(ManipulateXmlView):
     """ Find and link internal references and references to other works.
     """
+    use_full_xml = True
+
     def manipulate_xml(self):
-        # TODO: markup all references and only return the relevant portion in provision mode?
         markup_document_refs(self.document)
 
 

From 6bb493101967101d62639339d288800a1598cb8c Mon Sep 17 00:00:00 2001
From: goose-life <32566441+goose-life@users.noreply.github.com>
Date: Thu, 16 Jan 2025 12:04:59 +0200
Subject: [PATCH 15/28] replace XML rather than de- and re-serializing

---
 .../static/javascript/indigo/views/document_defined_terms.js    | 2 +-
 .../static/javascript/indigo/views/document_italics_terms.js    | 2 +-
 .../static/javascript/indigo/views/document_references.js       | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/indigo_app/static/javascript/indigo/views/document_defined_terms.js b/indigo_app/static/javascript/indigo/views/document_defined_terms.js
index c166086ca..26c0a0d8b 100644
--- a/indigo_app/static/javascript/indigo/views/document_defined_terms.js
+++ b/indigo_app/static/javascript/indigo/views/document_defined_terms.js
@@ -96,7 +96,7 @@
       });
 
       // set the content once for one mutation record
-      this.model.set('content', Indigo.toXml(xml));
+      this.model.xmlDocument.documentElement.replace(xml);
     },
   });
 })(window);
diff --git a/indigo_app/static/javascript/indigo/views/document_italics_terms.js b/indigo_app/static/javascript/indigo/views/document_italics_terms.js
index a8e5182f5..4909bc16c 100644
--- a/indigo_app/static/javascript/indigo/views/document_italics_terms.js
+++ b/indigo_app/static/javascript/indigo/views/document_italics_terms.js
@@ -82,7 +82,7 @@
       });
 
       // set the content once for one mutation record
-      this.model.set('content', Indigo.toXml(xml));
+      this.model.xmlDocument.documentElement.replace(xml);
     }
   });
 })(window);
diff --git a/indigo_app/static/javascript/indigo/views/document_references.js b/indigo_app/static/javascript/indigo/views/document_references.js
index ec268cd95..01c2416ba 100644
--- a/indigo_app/static/javascript/indigo/views/document_references.js
+++ b/indigo_app/static/javascript/indigo/views/document_references.js
@@ -93,7 +93,7 @@
       });
 
       // set the content once for one mutation record
-      this.model.set('content', Indigo.toXml(xml));
+      this.model.xmlDocument.documentElement.replace(xml);
     },
   });
 })(window);

From 038a24597e170faaadf569d24a4ce82d7fdfb3ed Mon Sep 17 00:00:00 2001
From: edith <32566441+goose-life@users.noreply.github.com>
Date: Thu, 16 Jan 2025 12:07:47 +0200
Subject: [PATCH 16/28] Update indigo_app/views/documents.py

Co-authored-by: Greg Kempe <greg@kempe.net>
---
 indigo_app/views/documents.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/indigo_app/views/documents.py b/indigo_app/views/documents.py
index 8ec25f94c..442e0d994 100644
--- a/indigo_app/views/documents.py
+++ b/indigo_app/views/documents.py
@@ -131,7 +131,7 @@ def get(self, request, *args, **kwargs):
         document = self.get_object()
         self.eid = self.kwargs.get('eid')
         self.provision_xml = document.get_provision_xml(self.eid)
-        if not self.provision_xml:
+        if self.provision_xml is None:
             messages.error(request, _("No provision with this id found: '%(eid)s'") % {"eid": self.eid})
             return redirect('choose_document_provision', doc_id=document.id)
         return super().get(request, *args, **kwargs)

From 6717916aace52b761b9318a3bd1861c1757ec292 Mon Sep 17 00:00:00 2001
From: edith <32566441+goose-life@users.noreply.github.com>
Date: Thu, 16 Jan 2025 12:08:12 +0200
Subject: [PATCH 17/28] Update indigo_api/models/documents.py

Co-authored-by: Greg Kempe <greg@kempe.net>
---
 indigo_api/models/documents.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/indigo_api/models/documents.py b/indigo_api/models/documents.py
index 228e3afcb..dfdb5f70d 100644
--- a/indigo_api/models/documents.py
+++ b/indigo_api/models/documents.py
@@ -550,7 +550,7 @@ def change_date(self, new_date, user, comment=None):
 
     def get_provision_xml(self, provision_eid):
         provision_xml = self.doc.get_portion_element(provision_eid)
-        if not provision_xml:
+        if provision_xml is None:
             return None
         portion = StructuredDocument.for_document_type('portion')()
         portion.frbr_uri = self.frbr_uri

From 7b1044648820b1b1094caa613fdf4aec0b5692c9 Mon Sep 17 00:00:00 2001
From: goose-life <32566441+goose-life@users.noreply.github.com>
Date: Thu, 16 Jan 2025 12:33:11 +0200
Subject: [PATCH 18/28] update test, provision_eid must (and will always) match
 an existing eid

---
 indigo_api/tests/test_analysis_api.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/indigo_api/tests/test_analysis_api.py b/indigo_api/tests/test_analysis_api.py
index d3cb37f98..369a7dea4 100644
--- a/indigo_api/tests/test_analysis_api.py
+++ b/indigo_api/tests/test_analysis_api.py
@@ -108,7 +108,7 @@ def test_link_terms_portion(self):
 </section>
 """),
             'language': 'eng',
-            'provision_eid': "section-1"
+            'provision_eid': "sec_1"
         })
         assert_equal(response.status_code, 200)
 

From b70569535d67e7e8f6450826b589c7334fda2d31 Mon Sep 17 00:00:00 2001
From: goose-life <32566441+goose-life@users.noreply.github.com>
Date: Thu, 16 Jan 2025 13:10:04 +0200
Subject: [PATCH 19/28] fix replacements

---
 .../static/javascript/indigo/views/document_defined_terms.js    | 2 +-
 .../static/javascript/indigo/views/document_italics_terms.js    | 2 +-
 .../static/javascript/indigo/views/document_references.js       | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/indigo_app/static/javascript/indigo/views/document_defined_terms.js b/indigo_app/static/javascript/indigo/views/document_defined_terms.js
index 26c0a0d8b..cb0a0e172 100644
--- a/indigo_app/static/javascript/indigo/views/document_defined_terms.js
+++ b/indigo_app/static/javascript/indigo/views/document_defined_terms.js
@@ -96,7 +96,7 @@
       });
 
       // set the content once for one mutation record
-      this.model.xmlDocument.documentElement.replace(xml);
+      this.model.xmlDocument.documentElement.replaceWith(xml.documentElement);
     },
   });
 })(window);
diff --git a/indigo_app/static/javascript/indigo/views/document_italics_terms.js b/indigo_app/static/javascript/indigo/views/document_italics_terms.js
index 4909bc16c..2a63501ec 100644
--- a/indigo_app/static/javascript/indigo/views/document_italics_terms.js
+++ b/indigo_app/static/javascript/indigo/views/document_italics_terms.js
@@ -82,7 +82,7 @@
       });
 
       // set the content once for one mutation record
-      this.model.xmlDocument.documentElement.replace(xml);
+      this.model.xmlDocument.documentElement.replaceWith(xml.documentElement);
     }
   });
 })(window);
diff --git a/indigo_app/static/javascript/indigo/views/document_references.js b/indigo_app/static/javascript/indigo/views/document_references.js
index 01c2416ba..41a995ba0 100644
--- a/indigo_app/static/javascript/indigo/views/document_references.js
+++ b/indigo_app/static/javascript/indigo/views/document_references.js
@@ -93,7 +93,7 @@
       });
 
       // set the content once for one mutation record
-      this.model.xmlDocument.documentElement.replace(xml);
+      this.model.xmlDocument.documentElement.replaceWith(xml.documentElement);
     },
   });
 })(window);

From 7b078a88fb006df95ebb70367f2df30f0f512ae1 Mon Sep 17 00:00:00 2001
From: goose-life <32566441+goose-life@users.noreply.github.com>
Date: Thu, 16 Jan 2025 13:13:25 +0200
Subject: [PATCH 20/28] get_provision_xml --> get_provision_element

---
 indigo_api/models/documents.py | 2 +-
 indigo_api/serializers.py      | 2 +-
 indigo_app/views/documents.py  | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/indigo_api/models/documents.py b/indigo_api/models/documents.py
index dfdb5f70d..dd60d8fb0 100644
--- a/indigo_api/models/documents.py
+++ b/indigo_api/models/documents.py
@@ -548,7 +548,7 @@ def change_date(self, new_date, user, comment=None):
             self.expression_date = new_date
             self.save_with_revision(user, comment=comment)
 
-    def get_provision_xml(self, provision_eid):
+    def get_provision_element(self, provision_eid):
         provision_xml = self.doc.get_portion_element(provision_eid)
         if provision_xml is None:
             return None
diff --git a/indigo_api/serializers.py b/indigo_api/serializers.py
index 5499b3179..3e3cd52e9 100644
--- a/indigo_api/serializers.py
+++ b/indigo_api/serializers.py
@@ -488,7 +488,7 @@ def updated_xml(self):
             return document.document_xml
         # otherwise, return only the provision being edited (NOT including the outer akn tag)
         # if we used the full XML for the analysis, grab only the appropriate provision as a portion
-        xml = document.get_provision_xml(provision_eid) if self.use_full_xml else document.doc.portion
+        xml = document.get_provision_element(provision_eid) if self.use_full_xml else document.doc.portion
         return etree.tostring(xml, encoding='unicode')
 
 
diff --git a/indigo_app/views/documents.py b/indigo_app/views/documents.py
index 442e0d994..e0c647a06 100644
--- a/indigo_app/views/documents.py
+++ b/indigo_app/views/documents.py
@@ -130,7 +130,7 @@ class DocumentProvisionDetailView(DocumentDetailView):
     def get(self, request, *args, **kwargs):
         document = self.get_object()
         self.eid = self.kwargs.get('eid')
-        self.provision_xml = document.get_provision_xml(self.eid)
+        self.provision_xml = document.get_provision_element(self.eid)
         if self.provision_xml is None:
             messages.error(request, _("No provision with this id found: '%(eid)s'") % {"eid": self.eid})
             return redirect('choose_document_provision', doc_id=document.id)

From d0bb190f6d7ad8f00292892da1f2c816e4094d78 Mon Sep 17 00:00:00 2001
From: goose-life <32566441+goose-life@users.noreply.github.com>
Date: Thu, 16 Jan 2025 14:40:51 +0200
Subject: [PATCH 21/28] fix terms across the document before saving, update
 test

---
 indigo/analysis/terms/base.py         | 12 ++++++++++++
 indigo_api/models/documents.py        |  3 +++
 indigo_api/tests/test_analysis_api.py |  2 +-
 3 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/indigo/analysis/terms/base.py b/indigo/analysis/terms/base.py
index cf7cb330f..899309506 100644
--- a/indigo/analysis/terms/base.py
+++ b/indigo/analysis/terms/base.py
@@ -51,6 +51,18 @@ def find_terms(self, doc):
         self.find_term_references(doc, terms)
         self.renumber_terms(doc)
 
+    def fix_terms_in_document(self, document):
+        """ Do everything other than guessing at defined terms in a Document.
+            Run this before saving an updated document when a portion was edited separately.
+        """
+        root = etree.fromstring(document.doc.to_xml(encoding='unicode'))
+        self.setup(root)
+        terms = self.find_definitions(root)
+        self.add_terms_to_references(root, terms)
+        self.find_term_references(root, terms)
+        self.renumber_terms(root)
+        document.content = etree.tostring(root, encoding='unicode')
+
     def setup(self, doc):
         self.ns = doc.nsmap[None]
         self.nsmap = {'a': self.ns}
diff --git a/indigo_api/models/documents.py b/indigo_api/models/documents.py
index dd60d8fb0..8019af0ca 100644
--- a/indigo_api/models/documents.py
+++ b/indigo_api/models/documents.py
@@ -566,6 +566,9 @@ def update_provision_xml(self, provision_eid, provision_xml):
         generator = XmlGenerator(self.frbr_uri)
         generator.generate_eids(self.doc.root)
         rewrite_all_attachment_work_components(self.doc)
+        finder = plugins.for_document('terms', self)
+        if finder:
+            finder.fix_terms_in_document(self)
         self.reset_xml(self.doc.to_xml(encoding='unicode'), from_model=True)
 
     def _make_doc(self, xml):
diff --git a/indigo_api/tests/test_analysis_api.py b/indigo_api/tests/test_analysis_api.py
index 369a7dea4..069eb62dd 100644
--- a/indigo_api/tests/test_analysis_api.py
+++ b/indigo_api/tests/test_analysis_api.py
@@ -116,7 +116,7 @@ def test_link_terms_portion(self):
 
         assert_true(content.startswith('<portion '))
         assert_in('<def ', content)
-        assert_in('<TLCTerm ', content)
+        # TLCTerms get set on save only for portions, since they need to go on the document's meta and not the portion's
 
     def test_link_terms_no_perms(self):
         self.client.logout()

From 2ea019140e40870342691ceeaedda119c84fd733 Mon Sep 17 00:00:00 2001
From: goose-life <32566441+goose-life@users.noreply.github.com>
Date: Thu, 16 Jan 2025 19:00:41 +0200
Subject: [PATCH 22/28] do link_terms_in_provision rather

---
 indigo/analysis/terms/base.py  |  6 ++----
 indigo_api/models/documents.py |  3 ---
 indigo_api/views/documents.py  | 16 ++++++++++------
 3 files changed, 12 insertions(+), 13 deletions(-)

diff --git a/indigo/analysis/terms/base.py b/indigo/analysis/terms/base.py
index 899309506..e93b8f020 100644
--- a/indigo/analysis/terms/base.py
+++ b/indigo/analysis/terms/base.py
@@ -51,14 +51,12 @@ def find_terms(self, doc):
         self.find_term_references(doc, terms)
         self.renumber_terms(doc)
 
-    def fix_terms_in_document(self, document):
-        """ Do everything other than guessing at defined terms in a Document.
-            Run this before saving an updated document when a portion was edited separately.
+    def link_terms_in_provision(self, document):
+        """ Passively link defined terms in a Document while editing in provision mode.
         """
         root = etree.fromstring(document.doc.to_xml(encoding='unicode'))
         self.setup(root)
         terms = self.find_definitions(root)
-        self.add_terms_to_references(root, terms)
         self.find_term_references(root, terms)
         self.renumber_terms(root)
         document.content = etree.tostring(root, encoding='unicode')
diff --git a/indigo_api/models/documents.py b/indigo_api/models/documents.py
index 8019af0ca..dd60d8fb0 100644
--- a/indigo_api/models/documents.py
+++ b/indigo_api/models/documents.py
@@ -566,9 +566,6 @@ def update_provision_xml(self, provision_eid, provision_xml):
         generator = XmlGenerator(self.frbr_uri)
         generator.generate_eids(self.doc.root)
         rewrite_all_attachment_work_components(self.doc)
-        finder = plugins.for_document('terms', self)
-        if finder:
-            finder.fix_terms_in_document(self)
         self.reset_xml(self.doc.to_xml(encoding='unicode'), from_model=True)
 
     def _make_doc(self, xml):
diff --git a/indigo_api/views/documents.py b/indigo_api/views/documents.py
index 0e1e61c05..101249d09 100644
--- a/indigo_api/views/documents.py
+++ b/indigo_api/views/documents.py
@@ -365,15 +365,16 @@ def post(self, request, document_id):
 
 
 class ManipulateXmlView(DocumentResourceView, APIView):
+    serializer = None
     use_full_xml = False
 
     def post(self, request, document_id):
-        serializer = DocumentAPISerializer(instance=self.document, data=self.request.data)
-        serializer.use_full_xml = self.use_full_xml
-        serializer.is_valid(raise_exception=True)
-        serializer.update_document()
+        self.serializer = DocumentAPISerializer(instance=self.document, data=self.request.data)
+        self.serializer.use_full_xml = self.use_full_xml
+        self.serializer.is_valid(raise_exception=True)
+        self.serializer.update_document()
         self.manipulate_xml()
-        return Response({'xml': serializer.updated_xml()})
+        return Response({'xml': self.serializer.updated_xml()})
 
     def manipulate_xml(self):
         raise NotImplementedError()
@@ -387,7 +388,10 @@ class LinkTermsView(ManipulateXmlView):
     def manipulate_xml(self):
         finder = plugins.for_document('terms', self.document)
         if finder:
-            finder.find_terms_in_document(self.document)
+            if self.serializer.validated_data.get('provision_eid'):
+                finder.link_terms_in_provision(self.document)
+            else:
+                finder.find_terms_in_document(self.document)
 
 
 class LinkReferencesView(ManipulateXmlView):

From b9e75a8f4538444c64d5322616921cb5ad981f5e Mon Sep 17 00:00:00 2001
From: goose-life <32566441+goose-life@users.noreply.github.com>
Date: Thu, 16 Jan 2025 19:23:26 +0200
Subject: [PATCH 23/28] update template wording in provision mode

---
 .../indigo_api/document/_defined_terms.html   | 24 ++++++++++++++-----
 .../templates/indigo_api/document/show.html   |  2 +-
 2 files changed, 19 insertions(+), 7 deletions(-)

diff --git a/indigo_app/templates/indigo_api/document/_defined_terms.html b/indigo_app/templates/indigo_api/document/_defined_terms.html
index 5ab00e614..502521c9e 100644
--- a/indigo_app/templates/indigo_api/document/_defined_terms.html
+++ b/indigo_app/templates/indigo_api/document/_defined_terms.html
@@ -8,19 +8,31 @@ <h4 class="modal-title">{% trans 'Defined Terms' %}</h4>
         <button type="button" class="btn-close" data-bs-dismiss="modal"></button>
       </div>
       <div class="modal-body">
-
-        <p>{% trans 'These are terms that are explicitly defined in this document. Always refresh defined terms before publishing a document to ensure that all references to terms are captured.' %}</p>
-
-        <div class="card bg-light scrollable">
+        {% if provision_eid == "" %}
+          <p>{% trans 'These are terms that are explicitly defined in this document. Always refresh defined terms before publishing a document to ensure that all references to terms are captured.' %}</p>
+          <div class="card bg-light scrollable">
           <div class="card-body">
             <div class="document-terms-list"></div>
           </div>
         </div>
-
+        {% else %}
+          <p>
+            <i class="fa fa-exclamation-triangle text-warning"></i>
+            {% trans 'In provision mode, only passive references that have already been marked up can be linked.' %}
+            {% trans 'To mark up definitions, even in this provision, you need to edit the full document.' %}
+          </p>
+        {% endif %}
       </div>
       <div class="modal-footer">
         <button type="button" class="btn btn-outline-secondary remove-terms pull-left"><i class="fas fa-times"></i> {% trans 'Remove defined terms' %}</button>
-        <button class="btn btn-outline-secondary link-terms"><i class="fas fa-sync"></i> {% trans 'Find and link defined terms' %}</button>
+        <button class="btn btn-outline-secondary link-terms">
+          <i class="fas fa-sync"></i>
+          {% if provision_eid == "" %}
+            {% trans 'Find and link defined terms' %}
+          {% else %}
+            {% trans 'Link defined terms' %}
+          {% endif %}
+        </button>
         <button type="button" class="btn btn-primary" data-bs-dismiss="modal">{% trans 'Close' %}</button>
       </div>
     </div>
diff --git a/indigo_app/templates/indigo_api/document/show.html b/indigo_app/templates/indigo_api/document/show.html
index 7c0ede85a..2df3f6080 100644
--- a/indigo_app/templates/indigo_api/document/show.html
+++ b/indigo_app/templates/indigo_api/document/show.html
@@ -109,7 +109,7 @@ <h5 class="main-header-title">
   </div>
 
   {% include "indigo_api/document/_revisions.html" %}
-  {% include "indigo_api/document/_defined_terms.html" %}
+  {% include "indigo_api/document/_defined_terms.html" with provision_eid=provision_eid %}
   {% include "indigo_api/document/_references.html" %}
   {% include "indigo_api/document/_annotations.html" %}
   {% include "indigo_api/document/_insert_image_modal.html" %}

From ec891e0cdc1fcc9e49f873eb1a52ca78fc654f5f Mon Sep 17 00:00:00 2001
From: goose-life <32566441+goose-life@users.noreply.github.com>
Date: Fri, 17 Jan 2025 09:15:00 +0200
Subject: [PATCH 24/28] rename method

---
 indigo/analysis/terms/base.py | 2 +-
 indigo_api/views/documents.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/indigo/analysis/terms/base.py b/indigo/analysis/terms/base.py
index e93b8f020..d26b1a0a4 100644
--- a/indigo/analysis/terms/base.py
+++ b/indigo/analysis/terms/base.py
@@ -51,7 +51,7 @@ def find_terms(self, doc):
         self.find_term_references(doc, terms)
         self.renumber_terms(doc)
 
-    def link_terms_in_provision(self, document):
+    def link_terms_in_document(self, document):
         """ Passively link defined terms in a Document while editing in provision mode.
         """
         root = etree.fromstring(document.doc.to_xml(encoding='unicode'))
diff --git a/indigo_api/views/documents.py b/indigo_api/views/documents.py
index 101249d09..294232800 100644
--- a/indigo_api/views/documents.py
+++ b/indigo_api/views/documents.py
@@ -389,7 +389,7 @@ def manipulate_xml(self):
         finder = plugins.for_document('terms', self.document)
         if finder:
             if self.serializer.validated_data.get('provision_eid'):
-                finder.link_terms_in_provision(self.document)
+                finder.link_terms_in_document(self.document)
             else:
                 finder.find_terms_in_document(self.document)
 

From ec209242f74a677244a8ca05c21c3721356dcb6d Mon Sep 17 00:00:00 2001
From: edith <32566441+goose-life@users.noreply.github.com>
Date: Fri, 17 Jan 2025 09:17:54 +0200
Subject: [PATCH 25/28] Update
 indigo_app/templates/indigo_api/document/_defined_terms.html

Co-authored-by: Greg Kempe <greg@kempe.net>
---
 indigo_app/templates/indigo_api/document/_defined_terms.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/indigo_app/templates/indigo_api/document/_defined_terms.html b/indigo_app/templates/indigo_api/document/_defined_terms.html
index 502521c9e..d8bee4faa 100644
--- a/indigo_app/templates/indigo_api/document/_defined_terms.html
+++ b/indigo_app/templates/indigo_api/document/_defined_terms.html
@@ -18,7 +18,7 @@ <h4 class="modal-title">{% trans 'Defined Terms' %}</h4>
         {% else %}
           <p>
             <i class="fa fa-exclamation-triangle text-warning"></i>
-            {% trans 'In provision mode, only passive references that have already been marked up can be linked.' %}
+            {% trans 'Only uses of defined terms can be linked while editing a provision.' %}
             {% trans 'To mark up definitions, even in this provision, you need to edit the full document.' %}
           </p>
         {% endif %}

From 5ed13de0183ca8633521b3af49701fa316f56773 Mon Sep 17 00:00:00 2001
From: edith <32566441+goose-life@users.noreply.github.com>
Date: Fri, 17 Jan 2025 09:18:11 +0200
Subject: [PATCH 26/28] Update
 indigo_app/templates/indigo_api/document/_defined_terms.html

Co-authored-by: Greg Kempe <greg@kempe.net>
---
 indigo_app/templates/indigo_api/document/_defined_terms.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/indigo_app/templates/indigo_api/document/_defined_terms.html b/indigo_app/templates/indigo_api/document/_defined_terms.html
index d8bee4faa..bf902a728 100644
--- a/indigo_app/templates/indigo_api/document/_defined_terms.html
+++ b/indigo_app/templates/indigo_api/document/_defined_terms.html
@@ -19,7 +19,7 @@ <h4 class="modal-title">{% trans 'Defined Terms' %}</h4>
           <p>
             <i class="fa fa-exclamation-triangle text-warning"></i>
             {% trans 'Only uses of defined terms can be linked while editing a provision.' %}
-            {% trans 'To mark up definitions, even in this provision, you need to edit the full document.' %}
+            {% trans 'To mark up definitions, edit the full document.' %}
           </p>
         {% endif %}
       </div>

From 5f8ba8d135b3b3a8ec76804620f087ed5df8ca91 Mon Sep 17 00:00:00 2001
From: goose-life <32566441+goose-life@users.noreply.github.com>
Date: Fri, 17 Jan 2025 09:31:04 +0200
Subject: [PATCH 27/28] fix test

---
 indigo_api/tests/test_analysis_api.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/indigo_api/tests/test_analysis_api.py b/indigo_api/tests/test_analysis_api.py
index 069eb62dd..a49bd5bcb 100644
--- a/indigo_api/tests/test_analysis_api.py
+++ b/indigo_api/tests/test_analysis_api.py
@@ -115,8 +115,7 @@ def test_link_terms_portion(self):
         content = response.data['xml']
 
         assert_true(content.startswith('<portion '))
-        assert_in('<def ', content)
-        # TLCTerms get set on save only for portions, since they need to go on the document's meta and not the portion's
+        # definitions aren't found, only linked, in provision mode
 
     def test_link_terms_no_perms(self):
         self.client.logout()

From a59dc521270c199ec454c14d3a88bd6d91709a42 Mon Sep 17 00:00:00 2001
From: goose-life <32566441+goose-life@users.noreply.github.com>
Date: Fri, 17 Jan 2025 09:38:08 +0200
Subject: [PATCH 28/28] deal with missing provision_eid differently

---
 indigo_app/templates/indigo_api/document/_defined_terms.html | 4 ++--
 indigo_app/templates/indigo_api/document/show.html           | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/indigo_app/templates/indigo_api/document/_defined_terms.html b/indigo_app/templates/indigo_api/document/_defined_terms.html
index bf902a728..c7ca4b170 100644
--- a/indigo_app/templates/indigo_api/document/_defined_terms.html
+++ b/indigo_app/templates/indigo_api/document/_defined_terms.html
@@ -8,7 +8,7 @@ <h4 class="modal-title">{% trans 'Defined Terms' %}</h4>
         <button type="button" class="btn-close" data-bs-dismiss="modal"></button>
       </div>
       <div class="modal-body">
-        {% if provision_eid == "" %}
+        {% if not provision_eid %}
           <p>{% trans 'These are terms that are explicitly defined in this document. Always refresh defined terms before publishing a document to ensure that all references to terms are captured.' %}</p>
           <div class="card bg-light scrollable">
           <div class="card-body">
@@ -27,7 +27,7 @@ <h4 class="modal-title">{% trans 'Defined Terms' %}</h4>
         <button type="button" class="btn btn-outline-secondary remove-terms pull-left"><i class="fas fa-times"></i> {% trans 'Remove defined terms' %}</button>
         <button class="btn btn-outline-secondary link-terms">
           <i class="fas fa-sync"></i>
-          {% if provision_eid == "" %}
+          {% if not provision_eid %}
             {% trans 'Find and link defined terms' %}
           {% else %}
             {% trans 'Link defined terms' %}
diff --git a/indigo_app/templates/indigo_api/document/show.html b/indigo_app/templates/indigo_api/document/show.html
index 2df3f6080..7c0ede85a 100644
--- a/indigo_app/templates/indigo_api/document/show.html
+++ b/indigo_app/templates/indigo_api/document/show.html
@@ -109,7 +109,7 @@ <h5 class="main-header-title">
   </div>
 
   {% include "indigo_api/document/_revisions.html" %}
-  {% include "indigo_api/document/_defined_terms.html" with provision_eid=provision_eid %}
+  {% include "indigo_api/document/_defined_terms.html" %}
   {% include "indigo_api/document/_references.html" %}
   {% include "indigo_api/document/_annotations.html" %}
   {% include "indigo_api/document/_insert_image_modal.html" %}