2024 01 gg qa fixes

RELEASE_NOTES.md

@@ -0,0 +1,13 @@
+* Terminology System: Properly populate property definitions in expansions
+* Validator: Major rework of tx and validation test cases associated with reworking terminology / validator interface and ongoing testing + reconciliation with Ontoserver
+major upgrade to validation - use terminology server to perform more logic, and standardise interface based on agreements with Vocab & Ontoserver
+* Validator: Improve slicing error message
+* Validator: Add warning to user when referencing an extension that doesn't exist in FHIRPath
+* Validator: Fix bugs in FHIRPath implementation of split() and join()
+* Validator: Fix bug handling null objects in JSON (R5)
+* Validator: Fix for missing search parameter definition in R4
+* Validator: fix bug handling XML name extension
+* Renderer: fix presentation issues for formatted xml and json
+* QA: show source server for message in QA
+* QA: add editor comments to errors + be explicit about tooling IG dependency
+* Publication Process: Improved errors publishing IGs

org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/Publisher.java

@@ -1150,7 +1150,7 @@ public void createIg() throws Exception, IOException, EOperationOutcome, FHIRExc
           new DependencyRenderer(pcm, outputDir, npmName, templateManager, dependencyList, context, markdownEngine).render(publishedIg, true, false, false), new HTAAnalysisRenderer(context, outputDir, markdownEngine).render(publishedIg.getPackageId(), fileList, publishedIg.present()),
           new PublicationChecker(repoRoot, historyPage, markdownEngine).check(), renderGlobals(), copyrightYear, context, scanForR5Extensions(), modifierExtensions,
           generateDraftDependencies(),
-          noNarrativeResources, noValidateResources, validationOff, generationOff, dependentIgFinder);
+          noNarrativeResources, noValidateResources, validationOff, generationOff, dependentIgFinder, context.getTxCache().servers());
       tts.end();
       if (isChild()) {
         log("Built. "+tt.report());
@@ -3065,12 +3065,15 @@ else if (vsCache == null) {
       dep.setVersion(pcm.getLatestVersion(dep.getPackageId()));
       dep.addExtension(ToolingExtensions.EXT_IGDEP_COMMENT, new MarkdownType("Automatically added as a dependency - all IGs depend on HL7 Terminology"));
       sourceIg.getDependsOn().add(0, dep);
+    }    
+    if (!"hl7.fhir.uv.tools".equals(sourceIg.getPackageId()) && !dependsOnTooling(sourceIg.getDependsOn())) {
+      if (sourceIg.getDefinition().hasExtension("http://hl7.org/fhir/tools/StructureDefinition/ig-internal-dependency")) {
+        sourceIg.getDefinition().getExtensionByUrl("http://hl7.org/fhir/tools/StructureDefinition/ig-internal-dependency").setValue(new CodeType("hl7.fhir.uv.tools#0.1.0"));      
+      } else {
+        sourceIg.getDefinition().addExtension("http://hl7.org/fhir/tools/StructureDefinition/ig-internal-dependency", new CodeType("hl7.fhir.uv.tools#0.1.0"));
+      }
     }
-    if (sourceIg.getDefinition().hasExtension("http://hl7.org/fhir/tools/StructureDefinition/ig-internal-dependency")) {
-      sourceIg.getDefinition().getExtensionByUrl("http://hl7.org/fhir/tools/StructureDefinition/ig-internal-dependency").setValue(new CodeType("hl7.fhir.uv.tools#current"));      
-    } else {
-      sourceIg.getDefinition().addExtension("http://hl7.org/fhir/tools/StructureDefinition/ig-internal-dependency", new CodeType("hl7.fhir.uv.tools#current"));
-    }
+
     inspector = new HTMLInspector(outputDir, specMaps, linkSpecMaps, this, igpkp.getCanonical(), sourceIg.getPackageId(), trackedFragments);
     inspector.getManual().add("full-ig.zip");
     if (historyPage != null) {
@@ -3087,15 +3090,15 @@ else if (vsCache == null) {
       loadIg(dep, i, !dep.hasUserData("no-load-deps"));
       i++;
     }
-
+    if (!"hl7.fhir.uv.tools".equals(sourceIg.getPackageId()) && !dependsOnTooling(sourceIg.getDependsOn())) {
+      loadIg("igtools", "hl7.fhir.uv.tools", "0.1.0", "http://hl7.org/fhir/tools/ImplementationGuide/hl7.fhir.uv.tools", i, false);   
+    }
+
     // we're also going to look for packages that can be referred to but aren't dependencies
     for (Extension ext : sourceIg.getDefinition().getExtensionsByUrl("http://hl7.org/fhir/tools/StructureDefinition/ig-link-dependency")) {
       loadLinkIg(ext.getValue().primitiveValue());
     }
 
-    if (!"hl7.fhir.uv.tools".equals(sourceIg.getPackageId())) {
-      loadIg("igtools", "hl7.fhir.uv.tools", "current", "http://hl7.org/fhir/tools/ImplementationGuide/hl7.fhir.uv.tools", i, false);   
-    }
 
     if (!VersionUtilities.isR5Plus(context.getVersion())) {
       System.out.println("Load R5 Specials");
@@ -3286,6 +3289,19 @@ private boolean dependsOnUTG(List<ImplementationGuideDependsOnComponent> depends
   }
 
 
+  private boolean dependsOnTooling(List<ImplementationGuideDependsOnComponent> dependsOn) {
+    for (ImplementationGuideDependsOnComponent d : dependsOn) {
+      if (d.hasPackageId() && d.getPackageId().contains("hl7.fhir.uv.tools")) {
+        return true;
+      }
+      if (d.hasUri() && d.getUri().contains("hl7.org/fhir/tools")) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+
   private boolean dependsOnExtensions(List<ImplementationGuideDependsOnComponent> dependsOn) {
     for (ImplementationGuideDependsOnComponent d : dependsOn) {
       if (d.hasPackageId() && d.getPackageId().equals("hl7.fhir.uv.extensions")) {

org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/SuppressedMessageInformation.java

@@ -100,6 +100,7 @@ public boolean contains(String message, ValidationMessage vMsg) {
           if (!vMsg.isMatched()) {
             sm.use();
             vMsg.setMatched(true);
+            vMsg.setComment(c.name);
           }
           return true;
         }

org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/ValidationServices.java

@@ -27,6 +27,7 @@
 import java.net.URL;
 import java.net.URLConnection;
 import java.util.ArrayList;
+import java.util.EnumSet;
 import java.util.List;
 import java.util.Locale;
 
@@ -52,6 +53,10 @@
 import org.hl7.fhir.r5.utils.validation.IResourceValidator;
 import org.hl7.fhir.r5.utils.validation.IValidationPolicyAdvisor;
 import org.hl7.fhir.r5.utils.validation.IValidatorResourceFetcher;
+import org.hl7.fhir.r5.utils.validation.IValidationPolicyAdvisor.AdditionalBindingPurpose;
+import org.hl7.fhir.r5.utils.validation.IValidationPolicyAdvisor.CodedContentValidationAction;
+import org.hl7.fhir.r5.utils.validation.IValidationPolicyAdvisor.ElementValidationAction;
+import org.hl7.fhir.r5.utils.validation.IValidationPolicyAdvisor.ResourceValidationAction;
 import org.hl7.fhir.r5.utils.validation.constants.BindingKind;
 import org.hl7.fhir.r5.utils.validation.constants.CodedContentValidationPolicy;
 import org.hl7.fhir.r5.utils.validation.constants.ContainedReferenceValidationPolicy;
@@ -191,7 +196,15 @@ private Class getResourceType(String url) {
   }
 
   @Override
-  public ContainedReferenceValidationPolicy policyForContained(IResourceValidator validator, Object appContext, String containerType, String containerId, Element.SpecialElement containingResourceType, String path, String url) {
+  public ContainedReferenceValidationPolicy policyForContained(IResourceValidator validator,
+      Object appContext,
+      StructureDefinition structure,
+      ElementDefinition element,
+      String containerType,
+      String containerId,
+      Element.SpecialElement containingResourceType,
+      String path,
+      String url) {
     return ContainedReferenceValidationPolicy.CHECK_VALID;
   }
 
@@ -333,13 +346,31 @@ public boolean fetchesCanonicalResource(IResourceValidator validator, String url
   }
 
   @Override
-  public CodedContentValidationPolicy policyForCodedContent(IResourceValidator validator, Object appContext, String stackPath, ElementDefinition definition,
-      StructureDefinition structure, BindingKind kind, ValueSet valueSet, List<String> systems) {
+  public EnumSet<CodedContentValidationAction> policyForCodedContent(IResourceValidator validator,
+      Object appContext,
+      String stackPath,
+      ElementDefinition definition,
+      StructureDefinition structure,
+      BindingKind kind,
+      AdditionalBindingPurpose purpose,
+      ValueSet valueSet,
+      List<String> systems) {
     if (VersionUtilities.isR4BVer(context.getVersion()) && 
         "ImplementationGuide.definition.parameter.code".equals(definition.getBase().getPath())) {
-      return CodedContentValidationPolicy.IGNORE;
+      return EnumSet.noneOf(CodedContentValidationAction.class);
     }
-    return CodedContentValidationPolicy.VALUESET;
+    return EnumSet.allOf(CodedContentValidationAction.class);
   }
 
+  @Override
+  public EnumSet<ResourceValidationAction> policyForResource(IResourceValidator validator, Object appContext,
+      StructureDefinition type, String path) {
+    return EnumSet.allOf(ResourceValidationAction.class);
+  }
+
+  @Override
+  public EnumSet<ElementValidationAction> policyForElement(IResourceValidator validator, Object appContext,
+      StructureDefinition structure, ElementDefinition element, String path) {
+    return EnumSet.allOf(ElementValidationAction.class);
+  }
 }

org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/renderers/JsonXhtmlRenderer.java

@@ -70,9 +70,9 @@ public void beginObject() throws IOException {
     if (b == null) {
       b = new StringBuilder();
       if (prism) {
-        b.append("<pre class=\"json\" style=\"white-space: pre; overflow: hidden\"><code class=\"language-json\">\r\n");
+        b.append("<pre class=\"json\" style=\"white-space: pre; text-wrap: nowrap;\"><code class=\"language-json\" style=\"white-space: pre; text-wrap: nowrap;\">\r\n");
       } else {
-        b.append("<pre class=\"json\" style=\"white-space: pre; overflow: hidden\"><code>\r\n");
+        b.append("<pre class=\"json\" style=\"white-space: pre; text-wrap: nowrap;\"><code style=\"white-space: pre; text-wrap: nowrap;\">\r\n");
       }
     }
     commitComments();

org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/renderers/StructureDefinitionRenderer.java

@@ -1485,7 +1485,7 @@ public String pseudoJson() throws Exception {
           generateCoreElemExtension(b, sd.getSnapshot().getElement(), child, children, 2, rn, false, child.getType().get(0), ++c == l, complex);
         extDone = true;
       } else if (child.hasSlicing())
-        generateCoreElemSliced(b, sd.getSnapshot().getElement(), child, children, 2, rn, false, child.getType().get(0), ++c == l, complex);
+        generateCoreElemSliced(b, sd.getSnapshot().getElement(), child, children, 2, rn, false, child.getType().isEmpty() ? null : child.getType().get(0), ++c == l, complex);
       else if (wasSliced(child, children))
         ; // nothing
       else if (child.getType().size() == 1 || allTypesAreReference(child))
@@ -1711,8 +1711,12 @@ private void generateCoreElemSliced(StringBuilder b, List<ElementDefinition> ele
 
     String name = tail(elem.getPath());
     String en = asValue ? "value[x]" : name;
-    if (en.contains("[x]"))
+    if (en.contains("[x]")) {
+      if (type == null) {
+        throw new Error("Type cannot be unknown for element with [x] in the name @ "+pathName);
+      }
       en = en.replace("[x]", upFirst(type.getWorkingCode()));
+    }
     boolean unbounded = elem.hasMax() && elem.getMax().equals("*");
 
     String indentS = "";

org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/renderers/ValidationPresenter.java

@@ -34,6 +34,7 @@
 import java.util.Date;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 
 import org.hl7.fhir.igtools.publisher.DependentIGFinder;
@@ -285,12 +286,13 @@ public ValidationMessage getVm() {
   private List<StructureDefinition> modifierExtensions;
   private String globalCheck;
   private String draftDependencies;
+  private Map<String, String> txServers;
 
   public ValidationPresenter(String statedVersion, String igVersion, IGKnowledgeProvider provider, IGKnowledgeProvider altProvider, String root, String packageId, String altPackageId, 
       String toolsVersion, String currentToolsVersion, RealmBusinessRules realm, PreviousVersionComparator previousVersionComparator, IpaComparator ipaComparator, IpsComparator ipsComparator,
       String dependencies, String csAnalysis, String pubReqCheck, String globalCheck, String copyrightYear, IWorkerContext context,
       Set<String> r5Extensions, List<StructureDefinition> modifierExtensions, String draftDependencies,
-      List<FetchedResource> noNarratives, List<FetchedResource> noValidation, boolean noValidate, boolean noGenerate, DependentIGFinder dependentIgs) {
+      List<FetchedResource> noNarratives, List<FetchedResource> noValidation, boolean noValidate, boolean noGenerate, DependentIGFinder dependentIgs, Map<String, String> txServers) {
     super();
     this.statedVersion = statedVersion;
     this.igVersion = igVersion;
@@ -319,6 +321,7 @@ public ValidationPresenter(String statedVersion, String igVersion, IGKnowledgePr
     this.modifierExtensions = modifierExtensions;
     this.draftDependencies = draftDependencies;
     this.globalCheck = globalCheck;
+    this.txServers = txServers;
     ruleDateCutoff = Date.from(LocalDate.now().minusMonths(1).atStartOfDay(ZoneId.systemDefault()).toInstant());
     determineCode();
   }
@@ -632,6 +635,15 @@ public static List<ValidationMessage> filterMessages(List<ValidationMessage> mes
           passesFilter = false;
         }
       }
+      if (message.getLevel().isError()) {
+        if (suppressedMessages.contains(message.getDisplay(), message) || suppressedMessages.contains(message.getMessage(), message) ||
+            suppressedMessages.contains(message.getHtml(), message) || suppressedMessages.contains(message.getMessageId(), message) || 
+            suppressedMessages.contains(message.getInvId(), message)) {
+          // nothing - we're just letting the message get a comment here
+      } else if (msgs.contains(message.getLocation()+"|"+message.getMessage())) {
+        // nothing - we're just letting the message get a comment here
+      }
+    }
       if (NO_FILTER) {
         passesFilter = true;
       }
@@ -751,28 +763,33 @@ public static List<ValidationMessage> filterMessages(List<ValidationMessage> mes
 
   private final String detailsTemplate = 
       "   <tr style=\"background-color: $color$\">\r\n"+
-      "     <td><b>$path$</b></td><td><b>$level$</b></td><td title=\"$mid$\"><b>$msg$</b></td>\r\n"+
+      "     <td><b>$path$</b></td><td><b>$level$</b></td><td title=\"$mid$\"><b>$msg$</b>$comment$</td>\r\n"+
       "   </tr>\r\n";
 
   private final String groupDetailsTemplate = 
       "   <tr style=\"background-color: $halfcolor$\">\r\n"+
-      "     <td><a href=\"$xlink$\">$fpath$</a></td><td><b>$msg$</b></td>\r\n"+
+      "     <td><a href=\"$xlink$\">$fpath$</a></td><td><b>$msg$</b>$comment$</td>\r\n"+
       "   </tr>\r\n";
 
 
-  private final String detailsTemplateTx = 
+  private final String detailsTemplateTxLink = 
+      "   <tr style=\"background-color: $color$\">\r\n"+
+      "     <td><b>$path$</b></td><td><b>$level$</b></td><td><b>$msg$</b>$comment$ (from <a href=\"qa-txsrvr.html#$txsrvr$\">$txsrvr$</ta>, see <a href=\"$tx$\">log</a>)</td>\r\n"+
+      "   </tr>\r\n";
+
+  private final String detailsTemplateTxNoLink = 
       "   <tr style=\"background-color: $color$\">\r\n"+
-      "     <td><b>$path$</b></td><td><b>$level$</b></td><td><b>$msg$</b> (<a href=\"$tx$\">see Tx log</a>)</td>\r\n"+
+      "     <td><b>$path$</b></td><td><b>$level$</b></td><td><b>$msg$</b>$comment$ (from <a href=\"qa-txsrvr.html#$txsrvr$\">$txsrvr$</ta>)</td>\r\n"+
       "   </tr>\r\n";
 
   private final String detailsTemplateWithExtraDetails = 
       "   <tr style=\"background-color: $color$\">\r\n"+
-      "     <td><b><a href=\"$pathlink$\">$path$</a></b></td><td><b>$level$</b></td><td><b>$msg$</b> <span id=\"s$id$\" class=\"flip\" onclick=\"flip('$id$')\">Show Reasoning</span><div id=\"$id$\" style=\"display: none\"><p>&nbsp;</p>$msgdetails$</div></td>\r\n"+
+      "     <td><b><a href=\"$pathlink$\">$path$</a></b></td><td><b>$level$</b></td><td><b>$msg$</b>$comment$ <span id=\"s$id$\" class=\"flip\" onclick=\"flip('$id$')\">Show Reasoning</span><div id=\"$id$\" style=\"display: none\"><p>&nbsp;</p>$msgdetails$</div></td>\r\n"+
       "   </tr>\r\n";
 
   private final String detailsTemplateWithLink = 
       "   <tr style=\"background-color: $color$\">\r\n"+
-      "     <td><b><a href=\"$pathlink$\">$path$</a></b></td><td><b>$level$</b></td><td><b>$msg$</b></td>\r\n"+
+      "     <td><b><a href=\"$pathlink$\">$path$</a></b></td><td><b>$level$</b></td><td><b>$msg$</b>$comment$</td>\r\n"+
       "   </tr>\r\n";
 
   private final String footerTemplate = 
@@ -1163,7 +1180,19 @@ private String genStartTxt(FetchedFile f) {
   }
 
   private String genDetails(ValidationMessage vm, int id) {
-    ST t = template(vm.isSlicingHint() ? detailsTemplateWithExtraDetails : vm.getLocationLink() != null ? detailsTemplateWithLink : vm.getTxLink() != null ? detailsTemplateTx : detailsTemplate);
+    String tid = null;
+    if (vm.isSlicingHint()) {
+      tid = detailsTemplateWithExtraDetails;
+    } else if (vm.getLocationLink() != null) {
+      tid = detailsTemplateWithLink;
+    } else if (vm.getTxLink() != null) {
+      tid = detailsTemplateTxLink;
+    } else if (vm.getServer() != null) {
+      tid = detailsTemplateTxNoLink;
+    } else {
+      tid = detailsTemplate;
+    }
+    ST t = template(tid);
     if (vm.getLocation()!=null) {
       t.add("path", makeBreakable(stripId(makeLocal(vm.getLocation())+lineCol(vm))));
       t.add("pathlink", vm.getLocationLink());
@@ -1178,10 +1207,20 @@ private String genDetails(ValidationMessage vm, int id) {
     t.add("mid", vm.getMessageId());
     t.add("msg", (isNewRule(vm) ? "<img style=\"vertical-align: text-bottom\" src=\"new.png\" height=\"16px\" width=\"36px\" alt=\"New Rule: \"> " : "")+ vm.getHtml());
     t.add("msgdetails", vm.isSlicingHint() ? vm.getSliceHtml() : vm.getHtml());
+    t.add("comment", vm.getComment() == null ? "" : "<br/><br/><span style=\"border: 1px grey solid; border-radius: 5px; background-color: #eeeeee; padding: 3px; margin: 3px \"><i><b>Editor's Comment</b>: "+Utilities.escapeXml(vm.getComment())+"</i></span>");
     t.add("tx", "qa-tx.html#l"+vm.getTxLink());
+    t.add("txsrvr", getServer(vm.getServer()));
     return t.render();
   }
 
+  private String getServer(String server) {
+    if (txServers.containsKey(server)) {
+      return txServers.get(server);
+    } else {
+      return server;
+    }
+  }
+
   private Object makeBreakable(String path) {
     if (path == null || path.length() < MAX_PATH_LENGTH_NO_BREAK) {
       return path;

org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/renderers/XmlXHtmlRenderer.java

@@ -81,9 +81,9 @@ public void start() throws IOException {
     started = true;
     b = new StringBuilder();
     if (prism) {
-      b.append("<pre class=\"xml\" style=\"white-space: pre; overflow: hidden\"><code class=\"language-xml\">\r\n");
+      b.append("<pre class=\"xml\" style=\"white-space: pre; text-wrap: nowrap;\"><code class=\"language-xml\" style=\"white-space: pre; text-wrap: nowrap;\">\r\n");
     } else {
-      b.append("<pre class=\"xml\" style=\"white-space: pre; overflow: hidden\"><code>\r\n");
+      b.append("<pre class=\"xml\" style=\"white-space: pre; text-wrap: nowrap;\"><code style=\"white-space: pre; text-wrap: nowrap;\">\r\n");
     }
   }
 

pom.xml

@@ -24,7 +24,7 @@
     <version>1.5.9-SNAPSHOT</version> <!-- See the note above -->
 
     <properties>
-        <core_version>6.2.7-SNAPSHOT</core_version>
+        <core_version>6.2.8-SNAPSHOT</core_version>
         <maven_surefire_version>3.0.0-M5</maven_surefire_version>
         <apache_poi_version>5.2.1</apache_poi_version>
         <okhttp.version>4.11.0</okhttp.version>