Merge pull request #1003 from HL7/2024-12-gg-many-fixes

2024 12 gg many fixes

RELEASE_NOTES.md

@@ -0,0 +1,42 @@
+* Loader: Fix problem not finding current version of extensions pack for non-R5 versions
+* Loader: New release of pubpack
+* Loader: extend CQLSubsystem to cover Measure, PlanDefinition, ActivityDefinition
+* Loader: fix missing union and intersection for CodeSystem and ValueSet comparisons
+* Loader: Work around flaw in R5 specmaps claiming to have paths for terminology URLs it does not
+* Loader:fix custom resources with profiles
+* Template Manager: Add the SDC template to the list of trusted templates
+* Terminology System: Fix missing username/password for tx server in IG publisher
+* Terminology System: Add support for valueset-version 
+* Snapshot Generator: Change rules around stripping extensions when generating snapshots
+* Snapshot Generator: Do not use metadata from data type profiles on elements when generating snapshots
+* Snapshot Generator: Fix version issues in snapshot generation tests
+* Validator: Add supplements for used systems as well as for value set systems when validating on server
+* Validator: Add versions to message about multiple matching profiles
+* Validator: fix issue missing idrefs validating IPS documents
+* Validator: fix missing port from server when doing tx-registry redirections
+* Validator: fix NPE in validator around Extension context
+* Validator: Fix obscure error on contentReference in profiles in FHIRPath engine
+* Validator: Fix questionnaire response status checking
+* Validator: Fix validation of displays when language is unknown
+* Validator: Fix version conversion issue for validating derived questionnaires
+* Validator: Handle secondary terminology server errors properly
+* Validator: Update FHIRPath validation to handle rootResource type properly
+* Version Comparison: Fix filter comparison logic when comparing valuesets
+* Version Comparison: Fix presentation issues and union and intersection links in previous version comparison
+* Test Subsystem: Add support for test-data-factory parameter
+* Renderer: Update logic for determining key elements to include elements where min, max, minValue, maxValue, bindings, etc. have changed from the base resource
+* Renderer: Add support for using Liquid on plain JSON directly and add support for markdownify filter
+* Renderer: fix bug using wrong reference on uri in liquid renderer
+* Renderer: Questionnaire rendering improvements
+* Renderer: Refactor Liquid engine and add support for forLoop and capture
+* Renderer: Add support for liquid based rendering of json files
+* Renderer: Defined new dependency table that shows all dependencies, while being less technical (and still making technical information available).
+* Renderer: Ensure details about the primary bindings appear after the primary binding, not after additional bindings
+* Renderer: Fix description of new content in difference analysis
+* Renderer: Improve OID error message 
+* Renderer: In key elements, don't just show MS types, show all of them
+* Log: hide API-Key from appearing on the tx log
+* Log: Ensure that if there's an issue with SQL or fragment expansion, at least something shows up in the log, even if debug is not on.
+* QA: fix presentation of tx server errors
+* QA: Improve qa output for sequence in publication checker
+* QA: Show code system fragments in QA summary for improved visiblity

org.hl7.fhir.publisher.cli/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>org.hl7.fhir.publisher</artifactId>
         <groupId>org.hl7.fhir.publisher</groupId>
-        <version>1.7.6-SNAPSHOT</version>
+        <version>1.8.0-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>

org.hl7.fhir.publisher.core/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>org.hl7.fhir.publisher</groupId>
         <artifactId>org.hl7.fhir.publisher</artifactId>
-        <version>1.7.6-SNAPSHOT</version>
+        <version>1.8.0-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>

org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/Publisher.java

@@ -144,9 +144,6 @@
 import org.hl7.fhir.igtools.web.PublisherConsoleLogger;
 import org.hl7.fhir.igtools.web.WebSiteArchiveBuilder;
 import org.hl7.fhir.r4.formats.FormatUtilities;
-import org.hl7.fhir.r5.utils.sql.Runner;
-import org.hl7.fhir.r5.utils.sql.StorageJson;
-import org.hl7.fhir.r5.utils.sql.StorageSqlite3;
 import org.hl7.fhir.r5.conformance.ConstraintJavaGenerator;
 import org.hl7.fhir.r5.conformance.R5ExtensionsLoader;
 import org.hl7.fhir.r5.conformance.profile.ProfileUtilities;
@@ -164,13 +161,13 @@
 import org.hl7.fhir.r5.fhirpath.ExpressionNode;
 import org.hl7.fhir.r5.fhirpath.FHIRPathEngine;
 import org.hl7.fhir.r5.formats.IParser.OutputStyle;
+import org.hl7.fhir.r5.formats.JsonParser;
+import org.hl7.fhir.r5.formats.RdfParser;
+import org.hl7.fhir.r5.formats.XmlParser;
 import org.hl7.fhir.r5.liquid.BaseJsonWrapper;
 import org.hl7.fhir.r5.liquid.GlobalObject.GlobalObjectRandomFunction;
 import org.hl7.fhir.r5.liquid.LiquidEngine;
 import org.hl7.fhir.r5.liquid.LiquidEngine.LiquidDocument;
-import org.hl7.fhir.r5.formats.JsonParser;
-import org.hl7.fhir.r5.formats.RdfParser;
-import org.hl7.fhir.r5.formats.XmlParser;
 import org.hl7.fhir.r5.model.ActivityDefinition;
 import org.hl7.fhir.r5.model.ActorDefinition;
 import org.hl7.fhir.r5.model.Attachment;
@@ -299,7 +296,6 @@
 import org.hl7.fhir.r5.terminologies.ValueSetUtilities;
 import org.hl7.fhir.r5.terminologies.expansion.ValueSetExpansionOutcome;
 import org.hl7.fhir.r5.terminologies.utilities.ValidationResult;
-import org.hl7.fhir.r5.test.utils.TestingUtilities;
 import org.hl7.fhir.r5.utils.EOperationOutcome;
 import org.hl7.fhir.r5.utils.MappingSheetParser;
 import org.hl7.fhir.r5.utils.NPMPackageGenerator;
@@ -312,6 +308,9 @@
 import org.hl7.fhir.r5.utils.XVerExtensionManager;
 import org.hl7.fhir.r5.utils.client.FHIRToolingClient;
 import org.hl7.fhir.r5.utils.formats.CSVWriter;
+import org.hl7.fhir.r5.utils.sql.Runner;
+import org.hl7.fhir.r5.utils.sql.StorageJson;
+import org.hl7.fhir.r5.utils.sql.StorageSqlite3;
 import org.hl7.fhir.r5.utils.structuremap.StructureMapAnalysis;
 import org.hl7.fhir.r5.utils.structuremap.StructureMapUtilities;
 import org.hl7.fhir.r5.utils.validation.IValidationProfileUsageTracker;
@@ -333,11 +332,17 @@
 import org.hl7.fhir.utilities.filesystem.CSFile;
 import org.hl7.fhir.utilities.http.HTTPResult;
 import org.hl7.fhir.utilities.http.ManagedWebAccess;
-import org.hl7.fhir.utilities.i18n.*;
+import org.hl7.fhir.utilities.i18n.I18nConstants;
+import org.hl7.fhir.utilities.i18n.JsonLangFileProducer;
+import org.hl7.fhir.utilities.i18n.LanguageFileProducer;
 import org.hl7.fhir.utilities.i18n.LanguageFileProducer.TranslationUnit;
+import org.hl7.fhir.utilities.i18n.LanguageTag;
+import org.hl7.fhir.utilities.i18n.PoGetTextProducer;
+import org.hl7.fhir.utilities.i18n.RegionToLocaleMapper;
+import org.hl7.fhir.utilities.i18n.RenderingI18nContext;
+import org.hl7.fhir.utilities.i18n.XLIFFProducer;
 import org.hl7.fhir.utilities.i18n.subtag.LanguageSubtagRegistry;
 import org.hl7.fhir.utilities.i18n.subtag.LanguageSubtagRegistryLoader;
-import org.hl7.fhir.utilities.json.JsonException;
 import org.hl7.fhir.utilities.json.model.JsonArray;
 import org.hl7.fhir.utilities.json.model.JsonBoolean;
 import org.hl7.fhir.utilities.json.model.JsonElement;
@@ -1175,12 +1180,20 @@ public void createIg() throws Exception, IOException, EOperationOutcome, FHIRExc
       generate();
       clean();
       dependentIgFinder.finish(outputDir, sourceIg.present());
+      List<FetchedResource> fragments = new ArrayList<>(); 
+      for (var f : fileList) {
+        for (var r : f.getResources()) {
+          if (r.getResource() != null && r.getResource() instanceof CodeSystem && ((CodeSystem) r.getResource()).getContent() == CodeSystemContentMode.FRAGMENT) {
+            fragments.add(r);
+          }
+        }
+      }
       ValidationPresenter val = new ValidationPresenter(version, workingVersion(), igpkp, childPublisher == null? null : childPublisher.getIgpkp(), rootDir, npmName, childPublisher == null? null : childPublisher.npmName,
           IGVersionUtil.getVersion(), fetchCurrentIGPubVersion(), realmRules, previousVersionComparator, ipaComparator, ipsComparator,
           new DependencyRenderer(pcm, outputDir, npmName, templateManager, dependencyList, context, markdownEngine, rc, specMaps).render(publishedIg, true, false, false), new HTAAnalysisRenderer(context, outputDir, markdownEngine).render(publishedIg.getPackageId(), fileList, publishedIg.present()),
           new PublicationChecker(repoRoot, historyPage, markdownEngine, findReleaseLabelString()).check(), renderGlobals(), copyrightYear, context, scanForR5Extensions(), modifierExtensions,
           generateDraftDependencies(),
-          noNarrativeResources, noValidateResources, validationOff, generationOff, dependentIgFinder, context.getTxClientManager(), versionProblems);
+          noNarrativeResources, noValidateResources, validationOff, generationOff, dependentIgFinder, context.getTxClientManager(), versionProblems, fragments);
       val.setValidationFlags(hintAboutNonMustSupport, anyExtensionsAllowed, checkAggregation, autoLoad, showReferenceMessages, noExperimentalContent, displayWarnings);
       tts.end();
       if (isChild()) {
@@ -1583,13 +1596,22 @@ public ResourceWithReference resolve(RenderingContext context, String url, Strin
 
     for (SpecMapManager sp : specMaps) {
       String fp = Utilities.isAbsoluteUrl(url) ? url : sp.getBase()+"/"+url;
+
       String path;
       try {
         path = sp.getPath(fp, null, null, null);
       } catch (Exception e) {
         path = null;
       }
+
+
+      // hack around an error in the R5 specmap file 
+      if (path != null && sp.isCore() && path.startsWith("http://terminology.hl7.org")) {
+        path = null;
+      }
+
       if (path != null)
+
         return new ResourceWithReference(ResourceReferenceKind.EXTERNAL, url, path, null);
     }
 
@@ -7211,7 +7233,7 @@ private void validateExpression(FetchedFile f, StructureDefinition sd, FHIRPathE
           n = fpe.parse(inv.getExpression(), sd.getUrl()+"#"+ed.getId()+" / "+inv.getKey());
           inv.setUserData(UserDataNames.validator_expression_cache, n);
         }
-        fpe.check(null, sd, ed.getPath(), n);
+        fpe.check(null, "Resource", sd, ed.getPath(), n);
       } catch (Exception e) {
         f.getErrors().add(new ValidationMessage(Source.ProfileValidator, IssueType.INVALID, "StructureDefinition.where(url = '"+sd.getUrl()+"').snapshot.element.where('path = '"+ed.getPath()+"').constraint.where(key = '"+inv.getKey()+"')", e.getMessage(), IssueSeverity.ERROR));
         r.getErrors().add(new ValidationMessage(Source.ProfileValidator, IssueType.INVALID, "StructureDefinition.where(url = '"+sd.getUrl()+"').snapshot.element.where('path = '"+ed.getPath()+"').constraint.where(key = '"+inv.getKey()+"')", e.getMessage(), IssueSeverity.ERROR));

org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/SpecMapManager.java

@@ -43,12 +43,8 @@
 import org.hl7.fhir.utilities.json.parser.JsonParser;
 import org.hl7.fhir.utilities.npm.BasePackageCacheManager;
 import org.hl7.fhir.utilities.npm.NpmPackage;
-import org.hl7.fhir.utilities.npm.PackageHacker;
 import org.hl7.fhir.utilities.npm.ToolsVersion;
 
-
-import net.sourceforge.plantuml.salt.element.WrappedElement;
-
 /**
  * TODO: should we call versionless references ok? not sure....
  * (Grahame, 31-Oct 2019)
@@ -511,5 +507,9 @@ public String getNpmVId() {
     return npmVId;
   }
 
+  public boolean isCore() {
+    return "hl7.fhir.core".equals(getNpmName());
+  }
+
 
 }

org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/comparators/ImposesComparator.java

@@ -232,17 +232,7 @@ public void finishChecks() throws IOException {
           }
           Utilities.createDirectory(Utilities.path(dstDir, "ipa-comparison-v"+vi.version));
           ComparisonRenderer cr = new ComparisonRenderer(vi.context, context, Utilities.path(dstDir, "ipa-comparison-v"+vi.version), session);
-          cr.getTemplates().put("CodeSystem", new String(context.getBinaryForKey("template-comparison-CodeSystem.html")));
-          cr.getTemplates().put("CodeSystem-Union", new String(context.getBinaryForKey("template-comparison-CodeSystem-Union.html")));
-          cr.getTemplates().put("CodeSystem-Intersection", new String(context.getBinaryForKey("template-comparison-CodeSystem-Intersection.html")));
-          cr.getTemplates().put("ValueSet", new String(context.getBinaryForKey("template-comparison-ValueSet.html")));
-          cr.getTemplates().put("ValueSet-Union", new String(context.getBinaryForKey("template-comparison-ValueSet-Union.html")));
-          cr.getTemplates().put("ValueSet-Intersection", new String(context.getBinaryForKey("template-comparison-ValueSet-Intersection.html")));
-          cr.getTemplates().put("Profile", new String(context.getBinaryForKey("template-comparison-Profile.html")));
-          cr.getTemplates().put("Profile-Union", new String(context.getBinaryForKey("template-comparison-Profile-Union.html")));
-          cr.getTemplates().put("Profile-Intersection", new String(context.getBinaryForKey("template-comparison-Profile-Intersection.html")));
-          cr.getTemplates().put("CapabilityStatement", new String(context.getBinaryForKey("template-comparison-CapabilityStatement.html")));
-          cr.getTemplates().put("Index", new String(context.getBinaryForKey("template-comparison-index.html")));
+          cr.loadTemplates(context);
           cr.render("Version "+vi.version, "Current Build");
         } catch (Throwable e) {
           errMsg = "Current Version Comparison failed: "+e.getMessage();

org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/comparators/IpaComparator.java

@@ -232,17 +232,7 @@ public void finishChecks() throws IOException {
           }
           Utilities.createDirectory(Utilities.path(dstDir, "ipa-comparison-v"+vi.version));
           ComparisonRenderer cr = new ComparisonRenderer(vi.context, context, Utilities.path(dstDir, "ipa-comparison-v"+vi.version), session);
-          cr.getTemplates().put("CodeSystem", new String(context.getBinaryForKey("template-comparison-CodeSystem.html")));
-          cr.getTemplates().put("CodeSystem-Union", new String(context.getBinaryForKey("template-comparison-CodeSystem-Union.html")));
-          cr.getTemplates().put("CodeSystem-Intersection", new String(context.getBinaryForKey("template-comparison-CodeSystem-Intersection.html")));
-          cr.getTemplates().put("ValueSet", new String(context.getBinaryForKey("template-comparison-ValueSet.html")));
-          cr.getTemplates().put("ValueSet-Union", new String(context.getBinaryForKey("template-comparison-ValueSet-Union.html")));
-          cr.getTemplates().put("ValueSet-Intersection", new String(context.getBinaryForKey("template-comparison-ValueSet-Intersection.html")));
-          cr.getTemplates().put("Profile", new String(context.getBinaryForKey("template-comparison-Profile.html")));
-          cr.getTemplates().put("Profile-Union", new String(context.getBinaryForKey("template-comparison-Profile-Union.html")));
-          cr.getTemplates().put("Profile-Intersection", new String(context.getBinaryForKey("template-comparison-Profile-Intersection.html")));
-          cr.getTemplates().put("CapabilityStatement", new String(context.getBinaryForKey("template-comparison-CapabilityStatement.html")));
-          cr.getTemplates().put("Index", new String(context.getBinaryForKey("template-comparison-index.html")));
+          cr.loadTemplates(context);
           cr.render("Version "+vi.version, "Current Build");
         } catch (Throwable e) {
           errMsg = "Current Version Comparison failed: "+e.getMessage();

org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/comparators/IpsComparator.java

@@ -232,17 +232,7 @@ public void finishChecks() throws IOException {
           }
           Utilities.createDirectory(Utilities.path(dstDir, "ips-comparison-v"+vi.version));
           ComparisonRenderer cr = new ComparisonRenderer(vi.context, context, Utilities.path(dstDir, "ips-comparison-v"+vi.version), session);
-          cr.getTemplates().put("CodeSystem", new String(context.getBinaryForKey("template-comparison-CodeSystem.html")));
-          cr.getTemplates().put("CodeSystem-Union", new String(context.getBinaryForKey("template-comparison-CodeSystem-Union.html")));
-          cr.getTemplates().put("CodeSystem-Intersection", new String(context.getBinaryForKey("template-comparison-CodeSystem-Intersection.html")));
-          cr.getTemplates().put("ValueSet", new String(context.getBinaryForKey("template-comparison-ValueSet.html")));
-          cr.getTemplates().put("ValueSet-Union", new String(context.getBinaryForKey("template-comparison-ValueSet-Union.html")));
-          cr.getTemplates().put("ValueSet-Intersection", new String(context.getBinaryForKey("template-comparison-ValueSet-Intersection.html")));
-          cr.getTemplates().put("Profile", new String(context.getBinaryForKey("template-comparison-Profile.html")));
-          cr.getTemplates().put("Profile-Union", new String(context.getBinaryForKey("template-comparison-Profile-Union.html")));
-          cr.getTemplates().put("Profile-Intersection", new String(context.getBinaryForKey("template-comparison-Profile-Intersection.html")));
-          cr.getTemplates().put("CapabilityStatement", new String(context.getBinaryForKey("template-comparison-CapabilityStatement.html")));
-          cr.getTemplates().put("Index", new String(context.getBinaryForKey("template-comparison-index.html")));
+          cr.loadTemplates(context);
           cr.render("Version "+vi.version, "Current Build");
         } catch (Throwable e) {
           errMsg = "Current Version Comparison failed: "+e.getMessage();

org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/comparators/PreviousVersionComparator.java

@@ -262,17 +262,7 @@ public void finishChecks() throws IOException {
           }
           Utilities.createDirectory(Utilities.path(dstDir, "comparison-v"+vi.version));
           ComparisonRenderer cr = new ComparisonRenderer(vi.context, context, Utilities.path(dstDir, "comparison-v"+vi.version), session);
-          cr.getTemplates().put("CodeSystem", new String(context.getBinaryForKey("template-comparison-CodeSystem.html")));
-          cr.getTemplates().put("CodeSystem-Union", new String(context.getBinaryForKey("template-comparison-CodeSystem-Union.html")));
-          cr.getTemplates().put("CodeSystem-Intersection", new String(context.getBinaryForKey("template-comparison-CodeSystem-Intersection.html")));
-          cr.getTemplates().put("ValueSet", new String(context.getBinaryForKey("template-comparison-ValueSet.html")));
-          cr.getTemplates().put("ValueSet-Union", new String(context.getBinaryForKey("template-comparison-ValueSet-Union.html")));
-          cr.getTemplates().put("ValueSet-Intersection", new String(context.getBinaryForKey("template-comparison-ValueSet-Intersection.html")));
-          cr.getTemplates().put("Profile", new String(context.getBinaryForKey("template-comparison-Profile.html")));
-          cr.getTemplates().put("Profile-Union", new String(context.getBinaryForKey("template-comparison-Profile-Union.html")));
-          cr.getTemplates().put("Profile-Intersection", new String(context.getBinaryForKey("template-comparison-Profile-Intersection.html")));
-          cr.getTemplates().put("Index", new String(context.getBinaryForKey("template-comparison-index.html")));
-          cr.getTemplates().put("CapabilityStatement", new String(context.getBinaryForKey("template-comparison-CapabilityStatement.html")));
+          cr.loadTemplates(context);
           cr.render("Version "+vi.version, "Current Build");
         } catch (Throwable e) {
           errMsg = "Current Version Comparison failed: "+e.getMessage();