diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index e69de29bb..55035ad28 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -0,0 +1,7 @@ +* Renderer: check that pre-existing narratives do not identify as "Generated narrative" +* Renderer: fix broken links in Bundle rendering +* Renderer: fill out narrative for Parameters resource +* Renderer: major overhaul of DiagnosticReport rendering +* Validator: better validation of version specific profiles in meta.profile +* Validator: look for references inside other parameters in Parameters resource +* Previous Version Comparison: Fix NPE bug in value set comparison diff --git a/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/PreviousVersionComparator.java b/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/PreviousVersionComparator.java index c9e42f92e..82c96e495 100644 --- a/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/PreviousVersionComparator.java +++ b/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/PreviousVersionComparator.java @@ -1,5 +1,8 @@ package org.hl7.fhir.igtools.publisher; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.text.DateFormat; @@ -30,6 +33,7 @@ import org.hl7.fhir.r5.model.StructureDefinition; import org.hl7.fhir.r5.model.UsageContext; import org.hl7.fhir.r5.utils.KeyGenerator; +import org.hl7.fhir.utilities.IniFile; import org.hl7.fhir.utilities.Logger; import org.hl7.fhir.utilities.Utilities; import org.hl7.fhir.utilities.VersionUtilities; @@ -76,10 +80,12 @@ private class VersionInstance { private SimpleWorkerContext context; private List resources = new ArrayList<>(); private String errMsg; + private IniFile ini; - public VersionInstance(String version) { + public VersionInstance(String version, IniFile ini) { super(); this.version = version; + this.ini = ini; } } @@ -96,8 +102,9 @@ public VersionInstance(String version) { private ILoggingService logger; private List resources; - public PreviousVersionComparator(SimpleWorkerContext context, String version, String dstDir, String canonical, ProfileKnowledgeProvider pkp, ILoggingService logger, List versions) { + public PreviousVersionComparator(SimpleWorkerContext context, String version, String rootDir, String dstDir, String canonical, ProfileKnowledgeProvider pkp, ILoggingService logger, List versions) { super(); + this.context = context; this.version = version; this.dstDir = dstDir; @@ -105,13 +112,13 @@ public PreviousVersionComparator(SimpleWorkerContext context, String version, St this.pkp = pkp; this.logger = logger; try { - processVersions(canonical, versions); + processVersions(canonical, versions, rootDir); } catch (Exception e) { errMsg = "Unable to find version history at "+canonical+" ("+e.getMessage()+")"; } } - private void processVersions(String canonical, List versions) throws IOException { + private void processVersions(String canonical, List versions, String rootDir) throws IOException { JsonArray publishedVersions = null; for (String v : versions) { if (Utilities.existsInList(v, "{last}", "{current}")) { @@ -137,22 +144,31 @@ private void processVersions(String canonical, List versions) throws IOE if(last == null) { throw new FHIRException("no {last} version found in package-list.json"); } else { - versionList.add(new VersionInstance(last)); + versionList.add(new VersionInstance(last, makeIni(rootDir, last))); } } if ("{current}".equals(v)) { if(last == null) { throw new FHIRException("no {current} version found in package-list.json"); } else { - versionList.add(new VersionInstance(major)); + versionList.add(new VersionInstance(major, makeIni(rootDir, major))); } } } else { - versionList.add(new VersionInstance(v)); + versionList.add(new VersionInstance(v, makeIni(rootDir, v))); } } } + private IniFile makeIni(String rootDir, String v) throws IOException { + File ini = new File(Utilities.path(rootDir, "url-map-v-"+v+".ini")); + if (ini.exists()) { + return new IniFile(new FileInputStream(ini)); + } else { + return null; + } + } + private JsonArray fetchVersionHistory(String canonical) { try { String ppl = Utilities.pathURL(canonical, "package-list.json"); @@ -172,7 +188,7 @@ private JsonArray fetchVersionHistory(String canonical) { } } } catch (Exception e) { - throw new FHIRException("Problem with package-lists.json at "+canonical+": "+e.getMessage(), e); + throw new FHIRException("Problem with package-list.json at "+canonical+": "+e.getMessage(), e); } } @@ -224,12 +240,13 @@ public void finishChecks() throws IOException { for (VersionInstance vi : versionList) { Set set = new HashSet<>(); for (CanonicalResource rl : vi.resources) { - comparisons.add(new ProfilePair(rl, findByUrl(rl.getUrl(), resources))); + comparisons.add(new ProfilePair(rl, findByUrl(rl.getUrl(), resources, vi.ini))); set.add(rl.getUrl()); } for (CanonicalResource rr : resources) { - if (!set.contains(rr.getUrl())) { - comparisons.add(new ProfilePair(findByUrl(rr.getUrl(), vi.resources), rr)); + String url = fixForIniMap(rr.getUrl(), vi.ini); + if (!set.contains(url)) { + comparisons.add(new ProfilePair(findByUrl(url, vi.resources, null), rr)); } } @@ -254,6 +271,16 @@ public void finishChecks() throws IOException { } } } +private String fixForIniMap(String url, IniFile ini) { + if (ini == null) { + return url; + } + if (ini.hasProperty("urls", url)) { + return ini.getStringProperty("urls", url); + } + return url; + } + // // private void buildindexPage(String path) throws IOException { // StringBuilder b = new StringBuilder(); @@ -301,9 +328,9 @@ public void finishChecks() throws IOException { // } - private CanonicalResource findByUrl(String url, List list) { + private CanonicalResource findByUrl(String url, List list, IniFile ini) { for (CanonicalResource r : list) { - if (r.getUrl().equals(url)) { + if (fixForIniMap(r.getUrl(), ini).equals(url)) { return r; } } diff --git a/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/Publisher.java b/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/Publisher.java index 0ab85f95d..21e5212ad 100644 --- a/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/Publisher.java +++ b/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/Publisher.java @@ -225,6 +225,7 @@ import org.hl7.fhir.r5.openapi.OpenApiGenerator; import org.hl7.fhir.r5.openapi.Writer; import org.hl7.fhir.r5.renderers.BundleRenderer; +import org.hl7.fhir.r5.renderers.ParametersRenderer; import org.hl7.fhir.r5.renderers.RendererFactory; import org.hl7.fhir.r5.renderers.utils.BaseWrappers.ResourceWrapper; import org.hl7.fhir.r5.renderers.utils.DirectWrappers; @@ -235,6 +236,7 @@ import org.hl7.fhir.r5.renderers.utils.RenderingContext.ResourceRendererMode; import org.hl7.fhir.r5.renderers.utils.Resolver.IReferenceResolver; import org.hl7.fhir.r5.renderers.utils.Resolver.ResourceContext; +import org.hl7.fhir.r5.renderers.utils.Resolver.ResourceContextType; import org.hl7.fhir.r5.renderers.utils.Resolver.ResourceWithReference; import org.hl7.fhir.r5.terminologies.ValueSetExpander.ValueSetExpansionOutcome; import org.hl7.fhir.r5.test.utils.ToolsHelper; @@ -526,6 +528,7 @@ public enum GenerationTool { private String specPath; private String qaDir; private String version; + private long fshTimeout = FSH_TIMEOUT; private Map suppressedMessages = new HashMap<>(); private String igName; @@ -849,8 +852,6 @@ public void createIg() throws Exception, IOException, EOperationOutcome, FHIRExc long startTime = System.nanoTime(); log("Processing Conformance Resources"); loadConformance(); - log("Generating Narratives"); - generateNarratives(); log("Validating Resources"); try { validate(); @@ -1040,6 +1041,16 @@ public ResourceWithReference resolve(RenderingContext context, String url) { return new ResourceWithReference(path, new DirectWrappers.ResourceWrapperDirect(context, r.getResource())); } } + if (r.getElement().fhirType().equals("Bundle")) { + List entries = r.getElement().getChildrenByName("entry"); + for (Element entry : entries) { + Element res = entry.getNamedChild("resource"); + if (res != null && res.fhirType().equals(parts[0]) && res.getNamedChildValue("id").equals(parts[1])) { + String path = igpkp.getLinkFor(r, true)+"#"+parts[0]+"_"+parts[1]; + return new ResourceWithReference(path, new ElementWrappers.ResourceWrapperMetaElement(context, r.getElement())); + } + } + } } } for (SpecMapManager sp : specMaps) { @@ -1060,7 +1071,7 @@ public ResourceWithReference resolve(RenderingContext context, String url) { private void generateNarratives() throws Exception { logDebugMessage(LogCategory.PROGRESS, "gen narratives"); for (FetchedFile f : fileList) { - for (FetchedResource r : f.getResources()) { + for (FetchedResource r : f.getResources()) { if (r.getExampleUri()==null || genExampleNarratives) { logDebugMessage(LogCategory.PROGRESS, "narrative for "+f.getName()+" : "+r.getId()); if (r.getResource() != null && isConvertableResource(r.getResource().fhirType())) { @@ -1072,6 +1083,12 @@ private void generateNarratives() throws Exception { } else if (r.getResource() instanceof Bundle) { regen = true; new BundleRenderer(rc).render((Bundle) r.getResource()); + } else if (r.getResource() instanceof Parameters) { + regen = true; + Parameters p = (Parameters) r.getResource(); + new ParametersRenderer(rc, new ResourceContext(ResourceContextType.PARAMETERS , p, null)).render(p); + } else if (r.getResource() instanceof DomainResource) { + checkExistingNarrative(f, r, ((DomainResource) r.getResource()).getText().getDiv()); } if (regen) r.setElement(convertToElement(r.getResource())); @@ -1085,9 +1102,11 @@ private void generateNarratives() throws Exception { Element res = e.getNamedChild("resource"); if (res!=null && "http://hl7.org/fhir/StructureDefinition/DomainResource".equals(res.getProperty().getStructure().getBaseDefinition()) && !hasNarrative(res)) { ResourceWrapper rw = new ElementWrappers.ResourceWrapperMetaElement(lrc, res); - RendererFactory.factory(rw, lrc).render(rw); + RendererFactory.factory(rw, lrc, new ResourceContext(ResourceContextType.BUNDLE, r.getElement(), res)).render(rw); } } + } else if ("http://hl7.org/fhir/StructureDefinition/DomainResource".equals(r.getElement().getProperty().getStructure().getBaseDefinition()) && hasNarrative(r.getElement())) { + checkExistingNarrative(f, r, r.getElement().getNamedChild("text").getNamedChild("div").getXhtml()); } } } else { @@ -1097,6 +1116,25 @@ private void generateNarratives() throws Exception { } } + private void checkExistingNarrative(FetchedFile f, FetchedResource r, XhtmlNode xhtml) { + boolean hasGenNarrative = scanForGeneratedNarrative(xhtml); + if (hasGenNarrative) { + f.getErrors().add(new ValidationMessage(Source.Publisher, IssueType.NOTFOUND, r.fhirType()+".text.div", "Resource has provided narrative, but the narrative indicates that it is generated - remove the narrative or fix it up", IssueSeverity.ERROR)); + } + } + + private boolean scanForGeneratedNarrative(XhtmlNode x) { + if (x.getContent() != null && x.getContent().contains("Generated Narrative")) { + return true; + } + for (XhtmlNode c : x.getChildNodes()) { + if (scanForGeneratedNarrative(c)) { + return true; + } + } + return false; + } + private boolean isConvertableResource(String t) { return Utilities.existsInList(t, "StructureDefinition", "ValueSet", "CodeSystem", "Conformance", "CapabilityStatement", "Questionnaire", "NamingSystem", "ConceptMap", "OperationOutcome", "CompartmentDefinition", "OperationDefinition", "ImplementationGuide"); @@ -1445,7 +1483,14 @@ public void write(int b) throws IOException { } } - private void runFsh(File file) throws IOException { + private void runFsh(File file) throws IOException { + File inif = new File(Utilities.path(Utilities.getDirectoryForFile(file.getAbsolutePath()), "fsh.ini")); + if (inif.exists()) { + IniFile ini = new IniFile(new FileInputStream(inif)); + if (ini.hasProperty("FSH", "timeout")) { + fshTimeout = ini.getLongProperty("FSH", "timeout") * 1000; + } + } log("Run Sushi on "+file.getAbsolutePath()); DefaultExecutor exec = new DefaultExecutor(); exec.setExitValue(0); @@ -1453,9 +1498,8 @@ private void runFsh(File file) throws IOException { PumpStreamHandler pump = new PumpStreamHandler(pumpHandler); exec.setStreamHandler(pump); exec.setWorkingDirectory(file); - ExecuteWatchdog watchdog = new ExecuteWatchdog(FSH_TIMEOUT); + ExecuteWatchdog watchdog = new ExecuteWatchdog(fshTimeout); exec.setWatchdog(watchdog); - try { if (SystemUtils.IS_OS_WINDOWS) exec.execute(org.apache.commons.exec.CommandLine.parse("cmd /C sushi ./fsh -o .")); @@ -2284,7 +2328,7 @@ else if (npmName.contains("hl7") || npmName.contains("argonaut") || npmName.cont } private void loadPubPack() throws FHIRException, IOException { - NpmPackage npm = pcm.loadPackage("hl7.fhir.pubpack", "0.0.6"); + NpmPackage npm = pcm.loadPackage("hl7.fhir.pubpack", "0.0.7"); context.loadFromPackage(npm, null); npm = pcm.loadPackage("hl7.fhir.xver-extensions", "0.0.4"); context.loadFromPackage(npm, null); @@ -3643,8 +3687,11 @@ private void loadConformance() throws Exception { loadInfo(); for (String s : metadataResourceNames()) load(s); + log("Generating Snapshots"); generateSnapshots(); + log("Generating Narratives"); generateNarratives(); + log("Validating Conformance Resources"); for (String s : metadataResourceNames()) validate(s); @@ -3742,7 +3789,7 @@ private PreviousVersionComparator makePreviousVersionComparator() throws IOExcep comparisonVersions = new ArrayList<>(); comparisonVersions.add("{last}"); } - return new PreviousVersionComparator(context, version, tempDir, igpkp.getCanonical(), igpkp, logger, comparisonVersions); + return new PreviousVersionComparator(context, version, rootDir, tempDir, igpkp.getCanonical(), igpkp, logger, comparisonVersions); } private void checkJurisdiction(FetchedFile f, CanonicalResource resource, IssueSeverity error, String verb) { @@ -4406,7 +4453,7 @@ private void validateExpression(FetchedFile f, StructureDefinition sd, FHIRPathE } fpe.check(null, sd, ed.getPath(), n); } catch (Exception e) { - f.getErrors().add(new ValidationMessage(Source.ProfileValidator, IssueType.INVALID, "StructureDefinition.where(url = '"+sd.getUrl()+"').snapshot.element.where('apth = '"+ed.getPath()+"').constraint.where(key = '"+inv.getKey()+"')", e.getMessage(), IssueSeverity.ERROR)); + f.getErrors().add(new ValidationMessage(Source.ProfileValidator, IssueType.INVALID, "StructureDefinition.where(url = '"+sd.getUrl()+"').snapshot.element.where('path = '"+ed.getPath()+"').constraint.where(key = '"+inv.getKey()+"')", e.getMessage(), IssueSeverity.ERROR)); } } } @@ -7393,9 +7440,16 @@ private XhtmlNode getXhtml(FetchedFile f, FetchedResource r) throws Exception { Bundle b = (Bundle) r.getResource(); return new BundleRenderer(rc).render(b); } + if (r.getResource() != null && r.getResource() instanceof Parameters) { + Parameters p = (Parameters) r.getResource(); + return new ParametersRenderer(rc, new ResourceContext(ResourceContextType.PARAMETERS, p, null)).render(p); + } if (r.getElement().fhirType().equals("Bundle")) { RenderingContext lrc = rc.copy().setParser(getTypeLoader(f, r)); return new BundleRenderer(lrc).render(new ElementWrappers.ResourceWrapperMetaElement(lrc, r.getElement())); + } else if (r.getElement().fhirType().equals("Parameters")) { + RenderingContext lrc = rc.copy().setParser(getTypeLoader(f, r)); + return new ParametersRenderer(lrc, new ResourceContext(ResourceContextType.PARAMETERS, r.getElement(), r.getElement())).render(new ElementWrappers.ResourceWrapperMetaElement(lrc, r.getElement())); } else { return getHtmlForResource(r.getElement()); } diff --git a/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/utils/TemplateChanger.java b/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/utils/TemplateChanger.java new file mode 100644 index 000000000..7efcb9492 --- /dev/null +++ b/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/utils/TemplateChanger.java @@ -0,0 +1,164 @@ +package org.hl7.fhir.igtools.publisher.utils; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.hl7.fhir.exceptions.FHIRException; +import org.hl7.fhir.igtools.publisher.utils.IGReleaseUpdater.ServerType; +import org.hl7.fhir.utilities.IniFile; +import org.hl7.fhir.utilities.TextFile; +import org.hl7.fhir.utilities.Utilities; + +public class TemplateChanger { + + private String folder; + + private List fixed = new ArrayList<>(); + private List notFixed = new ArrayList<>(); + + private String thb; + + private String the; + + private String ttb; + + private String tte; + + private String tbb; + + private String tbe; + + private File thf; + + private String th; + + private File ttf; + + private String tt; + + private File tbf; + + private String tb; + + public TemplateChanger(String folder) { + this.folder = folder; + } + + public static void main(String[] args) throws Exception { + new TemplateChanger(args[0]).execute("true".equals(args[1])); + } + + private void execute(boolean logNotDone) throws IOException { + // load publish.ini + File iniF = new File(Utilities.path(folder, "publish.ini")); + check(iniF.exists(), "The folder "+folder+" does not contain a publish.ini file"); + IniFile ini = new IniFile(new FileInputStream(iniF)); + + thb = ini.getStringProperty("template", "head-begin"); + the = ini.getStringProperty("template", "head-end"); + ttb = ini.getStringProperty("template", "top-begin"); + tte = ini.getStringProperty("template", "top-end"); + tbb = ini.getStringProperty("template", "bottom-begin"); + tbe = ini.getStringProperty("template", "bottom-end"); + + // load template fragments + check(ini.hasProperty("template", "head"), "The file "+iniF.getAbsolutePath()+" does not contain a head template"); + thf = new File(Utilities.path(folder, ini.getStringProperty("template", "head"))); + check(thf.exists(), "The head template file "+thf+" does not exist"); + th = TextFile.fileToString(thf); + check(thb != null, "The file "+iniF.getAbsolutePath()+" does not contain a template head-begin marker"); + check(the != null, "The file "+iniF.getAbsolutePath()+" does not contain a template head-end marker"); + + check(ini.hasProperty("template", "top"), "The file "+iniF.getAbsolutePath()+" does not contain a top template"); + ttf = new File(Utilities.path(folder, ini.getStringProperty("template", "top"))); + check(ttf.exists(), "The top template file "+ttf+" does not exist"); + tt = TextFile.fileToString(ttf); + check(ttb != null, "The file "+iniF.getAbsolutePath()+" does not contain a template top-begin marker"); + check(tte != null, "The file "+iniF.getAbsolutePath()+" does not contain a template top-end marker"); + + check(ini.hasProperty("template", "bottom"), "The file "+iniF.getAbsolutePath()+" does not contain a bottom template"); + tbf = new File(Utilities.path(folder, ini.getStringProperty("template", "bottom"))); + check(tbf.exists(), "The bottom template file "+tbf+" does not exist"); + tb = TextFile.fileToString(tbf); + check(tbb != null, "The file "+iniF.getAbsolutePath()+" does not contain a template bottom-begin marker"); + check(tbe != null, "The file "+iniF.getAbsolutePath()+" does not contain a template bottom-end marker"); + + System.out.println("Update HTML template in "+folder); + System.out.println("Head: Replace from "+thb+" to "+ the+" with "+thf.getAbsolutePath()); + System.out.println("Top: Replace from "+ttb+" to "+ tte+" with "+thf.getAbsolutePath()); + System.out.println("Bottom: Replace from "+tbb+" to "+ tbe+" with "+thf.getAbsolutePath()); + System.out.println("Go: Y/n?"); + int r = System.in.read(); + if (r != 'n') { + // walk the paths changing the files + processFiles(new File(folder), ""); + + System.out.println("Done. "+fixed.size()+" files changed"); + System.out.println(""+notFixed.size()+" files not changed as they did not meet the criteria"); + + if (logNotDone) { + for (File f : notFixed) { + System.out.println(" "+f.getAbsolutePath().substring(folder.length()+1)); + } + } + } + } + + private void processFiles(File dir, String path) throws IOException { + System.out.print("Process "+dir); + int i = 0; + for (File f : dir.listFiles()) { + if (!f.isDirectory() && f.getName().endsWith(".html") && !Utilities.existsInList(f.getAbsolutePath(), thf.getAbsolutePath(), ttf.getAbsolutePath(), tbf.getAbsolutePath())) { + i++; + if (i % 100 == 0) { + System.out.print("."); + } + String cnt = TextFile.fileToString(f); + if (hasSeps(cnt, thb, the) && hasSeps(cnt, ttb, tte) && hasSeps(cnt, tbb, tbe)) { + fixed.add(f); + cnt = replaceSeps(cnt, prep(th, path), thb, the); + cnt = replaceSeps(cnt, prep(tt, path), ttb, tte); + cnt = replaceSeps(cnt, prep(tb, path), tbb, tbe); + TextFile.stringToFile(cnt, f); + } else { + notFixed.add(f); + } + } + } + System.out.println(""); + for (File f : dir.listFiles()) { + if (f.isDirectory()) { + processFiles(f, path + "../"); + } + } + } + + private String replaceSeps(String cnt, String tmp, String b, String e) { + int ib = cnt.indexOf(b); + int ie = cnt.indexOf(e); + String start = cnt.substring(0, ib+b.length()); + String end = cnt.substring(ie); + return start+"\r\n"+tmp+"\r\n"+end; + } + + private String prep(String tmp, String path) { + return tmp.replace("{{path}}", path); + } + + private boolean hasSeps(String cnt, String b, String e) { + int ib = cnt.indexOf(b); + int ie = cnt.indexOf(e); + return (ib > 0) && (ie > ib); + } + + private void check(boolean test, String msg) { + if (!test) { + throw new FHIRException(msg); + } + } + + +} diff --git a/pom.xml b/pom.xml index 636912e3e..0c2cda0fd 100644 --- a/pom.xml +++ b/pom.xml @@ -18,13 +18,13 @@ ca.uhn.hapi.fhir org.hl7.fhir.core - 5.0.17-SNAPSHOT + 5.0.18 1.1.7-SNAPSHOT - 5.0.17-SNAPSHOT + 5.0.18 true